1 /*******************************************************************************
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007-2009 Intel Corporation.
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26 *******************************************************************************/
28 #include <linux/module.h>
29 #include <linux/types.h>
30 #include <linux/init.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pagemap.h>
33 #include <linux/netdevice.h>
34 #include <linux/ipv6.h>
35 #include <linux/slab.h>
36 #include <net/checksum.h>
37 #include <net/ip6_checksum.h>
38 #include <linux/net_tstamp.h>
39 #include <linux/mii.h>
40 #include <linux/ethtool.h>
41 #include <linux/if_vlan.h>
42 #include <linux/pci.h>
43 #include <linux/pci-aspm.h>
44 #include <linux/delay.h>
45 #include <linux/interrupt.h>
46 #include <linux/if_ether.h>
47 #include <linux/aer.h>
49 #include <linux/dca.h>
53 #define DRV_VERSION "2.1.0-k2"
54 char igb_driver_name[] = "igb";
55 char igb_driver_version[] = DRV_VERSION;
56 static const char igb_driver_string[] =
57 "Intel(R) Gigabit Ethernet Network Driver";
58 static const char igb_copyright[] = "Copyright (c) 2007-2009 Intel Corporation.";
60 static const struct e1000_info *igb_info_tbl[] = {
61 [board_82575] = &e1000_82575_info,
64 static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
85 /* required last entry */
89 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
91 void igb_reset(struct igb_adapter *);
92 static int igb_setup_all_tx_resources(struct igb_adapter *);
93 static int igb_setup_all_rx_resources(struct igb_adapter *);
94 static void igb_free_all_tx_resources(struct igb_adapter *);
95 static void igb_free_all_rx_resources(struct igb_adapter *);
96 static void igb_setup_mrqc(struct igb_adapter *);
97 void igb_update_stats(struct igb_adapter *);
98 static int igb_probe(struct pci_dev *, const struct pci_device_id *);
99 static void __devexit igb_remove(struct pci_dev *pdev);
100 static int igb_sw_init(struct igb_adapter *);
101 static int igb_open(struct net_device *);
102 static int igb_close(struct net_device *);
103 static void igb_configure_tx(struct igb_adapter *);
104 static void igb_configure_rx(struct igb_adapter *);
105 static void igb_clean_all_tx_rings(struct igb_adapter *);
106 static void igb_clean_all_rx_rings(struct igb_adapter *);
107 static void igb_clean_tx_ring(struct igb_ring *);
108 static void igb_clean_rx_ring(struct igb_ring *);
109 static void igb_set_rx_mode(struct net_device *);
110 static void igb_update_phy_info(unsigned long);
111 static void igb_watchdog(unsigned long);
112 static void igb_watchdog_task(struct work_struct *);
113 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
114 static struct net_device_stats *igb_get_stats(struct net_device *);
115 static int igb_change_mtu(struct net_device *, int);
116 static int igb_set_mac(struct net_device *, void *);
117 static void igb_set_uta(struct igb_adapter *adapter);
118 static irqreturn_t igb_intr(int irq, void *);
119 static irqreturn_t igb_intr_msi(int irq, void *);
120 static irqreturn_t igb_msix_other(int irq, void *);
121 static irqreturn_t igb_msix_ring(int irq, void *);
122 #ifdef CONFIG_IGB_DCA
123 static void igb_update_dca(struct igb_q_vector *);
124 static void igb_setup_dca(struct igb_adapter *);
125 #endif /* CONFIG_IGB_DCA */
126 static bool igb_clean_tx_irq(struct igb_q_vector *);
127 static int igb_poll(struct napi_struct *, int);
128 static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
129 static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
130 static void igb_tx_timeout(struct net_device *);
131 static void igb_reset_task(struct work_struct *);
132 static void igb_vlan_rx_register(struct net_device *, struct vlan_group *);
133 static void igb_vlan_rx_add_vid(struct net_device *, u16);
134 static void igb_vlan_rx_kill_vid(struct net_device *, u16);
135 static void igb_restore_vlan(struct igb_adapter *);
136 static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
137 static void igb_ping_all_vfs(struct igb_adapter *);
138 static void igb_msg_task(struct igb_adapter *);
139 static void igb_vmm_control(struct igb_adapter *);
140 static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
141 static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
142 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
143 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
144 int vf, u16 vlan, u8 qos);
145 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
146 static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
147 struct ifla_vf_info *ivi);
150 static int igb_suspend(struct pci_dev *, pm_message_t);
151 static int igb_resume(struct pci_dev *);
153 static void igb_shutdown(struct pci_dev *);
154 #ifdef CONFIG_IGB_DCA
155 static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
156 static struct notifier_block dca_notifier = {
157 .notifier_call = igb_notify_dca,
162 #ifdef CONFIG_NET_POLL_CONTROLLER
163 /* for netdump / net console */
164 static void igb_netpoll(struct net_device *);
166 #ifdef CONFIG_PCI_IOV
167 static unsigned int max_vfs = 0;
168 module_param(max_vfs, uint, 0);
169 MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
170 "per physical function");
171 #endif /* CONFIG_PCI_IOV */
173 static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
174 pci_channel_state_t);
175 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
176 static void igb_io_resume(struct pci_dev *);
178 static struct pci_error_handlers igb_err_handler = {
179 .error_detected = igb_io_error_detected,
180 .slot_reset = igb_io_slot_reset,
181 .resume = igb_io_resume,
185 static struct pci_driver igb_driver = {
186 .name = igb_driver_name,
187 .id_table = igb_pci_tbl,
189 .remove = __devexit_p(igb_remove),
191 /* Power Managment Hooks */
192 .suspend = igb_suspend,
193 .resume = igb_resume,
195 .shutdown = igb_shutdown,
196 .err_handler = &igb_err_handler
199 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
200 MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
201 MODULE_LICENSE("GPL");
202 MODULE_VERSION(DRV_VERSION);
204 struct igb_reg_info {
209 static const struct igb_reg_info igb_reg_info_tbl[] = {
211 /* General Registers */
212 {E1000_CTRL, "CTRL"},
213 {E1000_STATUS, "STATUS"},
214 {E1000_CTRL_EXT, "CTRL_EXT"},
216 /* Interrupt Registers */
220 {E1000_RCTL, "RCTL"},
221 {E1000_RDLEN(0), "RDLEN"},
222 {E1000_RDH(0), "RDH"},
223 {E1000_RDT(0), "RDT"},
224 {E1000_RXDCTL(0), "RXDCTL"},
225 {E1000_RDBAL(0), "RDBAL"},
226 {E1000_RDBAH(0), "RDBAH"},
229 {E1000_TCTL, "TCTL"},
230 {E1000_TDBAL(0), "TDBAL"},
231 {E1000_TDBAH(0), "TDBAH"},
232 {E1000_TDLEN(0), "TDLEN"},
233 {E1000_TDH(0), "TDH"},
234 {E1000_TDT(0), "TDT"},
235 {E1000_TXDCTL(0), "TXDCTL"},
236 {E1000_TDFH, "TDFH"},
237 {E1000_TDFT, "TDFT"},
238 {E1000_TDFHS, "TDFHS"},
239 {E1000_TDFPC, "TDFPC"},
241 /* List Terminator */
246 * igb_regdump - register printout routine
248 static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
254 switch (reginfo->ofs) {
256 for (n = 0; n < 4; n++)
257 regs[n] = rd32(E1000_RDLEN(n));
260 for (n = 0; n < 4; n++)
261 regs[n] = rd32(E1000_RDH(n));
264 for (n = 0; n < 4; n++)
265 regs[n] = rd32(E1000_RDT(n));
267 case E1000_RXDCTL(0):
268 for (n = 0; n < 4; n++)
269 regs[n] = rd32(E1000_RXDCTL(n));
272 for (n = 0; n < 4; n++)
273 regs[n] = rd32(E1000_RDBAL(n));
276 for (n = 0; n < 4; n++)
277 regs[n] = rd32(E1000_RDBAH(n));
280 for (n = 0; n < 4; n++)
281 regs[n] = rd32(E1000_RDBAL(n));
284 for (n = 0; n < 4; n++)
285 regs[n] = rd32(E1000_TDBAH(n));
288 for (n = 0; n < 4; n++)
289 regs[n] = rd32(E1000_TDLEN(n));
292 for (n = 0; n < 4; n++)
293 regs[n] = rd32(E1000_TDH(n));
296 for (n = 0; n < 4; n++)
297 regs[n] = rd32(E1000_TDT(n));
299 case E1000_TXDCTL(0):
300 for (n = 0; n < 4; n++)
301 regs[n] = rd32(E1000_TXDCTL(n));
304 printk(KERN_INFO "%-15s %08x\n",
305 reginfo->name, rd32(reginfo->ofs));
309 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
310 printk(KERN_INFO "%-15s ", rname);
311 for (n = 0; n < 4; n++)
312 printk(KERN_CONT "%08x ", regs[n]);
313 printk(KERN_CONT "\n");
317 * igb_dump - Print registers, tx-rings and rx-rings
319 static void igb_dump(struct igb_adapter *adapter)
321 struct net_device *netdev = adapter->netdev;
322 struct e1000_hw *hw = &adapter->hw;
323 struct igb_reg_info *reginfo;
325 struct igb_ring *tx_ring;
326 union e1000_adv_tx_desc *tx_desc;
327 struct my_u0 { u64 a; u64 b; } *u0;
328 struct igb_buffer *buffer_info;
329 struct igb_ring *rx_ring;
330 union e1000_adv_rx_desc *rx_desc;
334 if (!netif_msg_hw(adapter))
337 /* Print netdevice Info */
339 dev_info(&adapter->pdev->dev, "Net device Info\n");
340 printk(KERN_INFO "Device Name state "
341 "trans_start last_rx\n");
342 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
349 /* Print Registers */
350 dev_info(&adapter->pdev->dev, "Register Dump\n");
351 printk(KERN_INFO " Register Name Value\n");
352 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
353 reginfo->name; reginfo++) {
354 igb_regdump(hw, reginfo);
357 /* Print TX Ring Summary */
358 if (!netdev || !netif_running(netdev))
361 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
362 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
363 " leng ntw timestamp\n");
364 for (n = 0; n < adapter->num_tx_queues; n++) {
365 tx_ring = adapter->tx_ring[n];
366 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
367 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
368 n, tx_ring->next_to_use, tx_ring->next_to_clean,
369 (u64)buffer_info->dma,
371 buffer_info->next_to_watch,
372 (u64)buffer_info->time_stamp);
376 if (!netif_msg_tx_done(adapter))
377 goto rx_ring_summary;
379 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
381 /* Transmit Descriptor Formats
383 * Advanced Transmit Descriptor
384 * +--------------------------------------------------------------+
385 * 0 | Buffer Address [63:0] |
386 * +--------------------------------------------------------------+
387 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
388 * +--------------------------------------------------------------+
389 * 63 46 45 40 39 38 36 35 32 31 24 15 0
392 for (n = 0; n < adapter->num_tx_queues; n++) {
393 tx_ring = adapter->tx_ring[n];
394 printk(KERN_INFO "------------------------------------\n");
395 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
396 printk(KERN_INFO "------------------------------------\n");
397 printk(KERN_INFO "T [desc] [address 63:0 ] "
398 "[PlPOCIStDDM Ln] [bi->dma ] "
399 "leng ntw timestamp bi->skb\n");
401 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
402 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
403 buffer_info = &tx_ring->buffer_info[i];
404 u0 = (struct my_u0 *)tx_desc;
405 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
406 " %04X %3X %016llX %p", i,
409 (u64)buffer_info->dma,
411 buffer_info->next_to_watch,
412 (u64)buffer_info->time_stamp,
414 if (i == tx_ring->next_to_use &&
415 i == tx_ring->next_to_clean)
416 printk(KERN_CONT " NTC/U\n");
417 else if (i == tx_ring->next_to_use)
418 printk(KERN_CONT " NTU\n");
419 else if (i == tx_ring->next_to_clean)
420 printk(KERN_CONT " NTC\n");
422 printk(KERN_CONT "\n");
424 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
425 print_hex_dump(KERN_INFO, "",
427 16, 1, phys_to_virt(buffer_info->dma),
428 buffer_info->length, true);
432 /* Print RX Rings Summary */
434 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
435 printk(KERN_INFO "Queue [NTU] [NTC]\n");
436 for (n = 0; n < adapter->num_rx_queues; n++) {
437 rx_ring = adapter->rx_ring[n];
438 printk(KERN_INFO " %5d %5X %5X\n", n,
439 rx_ring->next_to_use, rx_ring->next_to_clean);
443 if (!netif_msg_rx_status(adapter))
446 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
448 /* Advanced Receive Descriptor (Read) Format
450 * +-----------------------------------------------------+
451 * 0 | Packet Buffer Address [63:1] |A0/NSE|
452 * +----------------------------------------------+------+
453 * 8 | Header Buffer Address [63:1] | DD |
454 * +-----------------------------------------------------+
457 * Advanced Receive Descriptor (Write-Back) Format
459 * 63 48 47 32 31 30 21 20 17 16 4 3 0
460 * +------------------------------------------------------+
461 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
462 * | Checksum Ident | | | | Type | Type |
463 * +------------------------------------------------------+
464 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
465 * +------------------------------------------------------+
466 * 63 48 47 32 31 20 19 0
469 for (n = 0; n < adapter->num_rx_queues; n++) {
470 rx_ring = adapter->rx_ring[n];
471 printk(KERN_INFO "------------------------------------\n");
472 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
473 printk(KERN_INFO "------------------------------------\n");
474 printk(KERN_INFO "R [desc] [ PktBuf A0] "
475 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
476 "<-- Adv Rx Read format\n");
477 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
478 "[vl er S cks ln] ---------------- [bi->skb] "
479 "<-- Adv Rx Write-Back format\n");
481 for (i = 0; i < rx_ring->count; i++) {
482 buffer_info = &rx_ring->buffer_info[i];
483 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
484 u0 = (struct my_u0 *)rx_desc;
485 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
486 if (staterr & E1000_RXD_STAT_DD) {
487 /* Descriptor Done */
488 printk(KERN_INFO "RWB[0x%03X] %016llX "
489 "%016llX ---------------- %p", i,
494 printk(KERN_INFO "R [0x%03X] %016llX "
495 "%016llX %016llX %p", i,
498 (u64)buffer_info->dma,
501 if (netif_msg_pktdata(adapter)) {
502 print_hex_dump(KERN_INFO, "",
505 phys_to_virt(buffer_info->dma),
506 rx_ring->rx_buffer_len, true);
507 if (rx_ring->rx_buffer_len
509 print_hex_dump(KERN_INFO, "",
513 buffer_info->page_dma +
514 buffer_info->page_offset),
519 if (i == rx_ring->next_to_use)
520 printk(KERN_CONT " NTU\n");
521 else if (i == rx_ring->next_to_clean)
522 printk(KERN_CONT " NTC\n");
524 printk(KERN_CONT "\n");
535 * igb_read_clock - read raw cycle counter (to be used by time counter)
537 static cycle_t igb_read_clock(const struct cyclecounter *tc)
539 struct igb_adapter *adapter =
540 container_of(tc, struct igb_adapter, cycles);
541 struct e1000_hw *hw = &adapter->hw;
546 * The timestamp latches on lowest register read. For the 82580
547 * the lowest register is SYSTIMR instead of SYSTIML. However we never
548 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
550 if (hw->mac.type == e1000_82580) {
551 stamp = rd32(E1000_SYSTIMR) >> 8;
552 shift = IGB_82580_TSYNC_SHIFT;
555 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
556 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
561 * igb_get_hw_dev - return device
562 * used by hardware layer to print debugging information
564 struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
566 struct igb_adapter *adapter = hw->back;
567 return adapter->netdev;
571 * igb_init_module - Driver Registration Routine
573 * igb_init_module is the first routine called when the driver is
574 * loaded. All it does is register with the PCI subsystem.
576 static int __init igb_init_module(void)
579 printk(KERN_INFO "%s - version %s\n",
580 igb_driver_string, igb_driver_version);
582 printk(KERN_INFO "%s\n", igb_copyright);
584 #ifdef CONFIG_IGB_DCA
585 dca_register_notify(&dca_notifier);
587 ret = pci_register_driver(&igb_driver);
591 module_init(igb_init_module);
594 * igb_exit_module - Driver Exit Cleanup Routine
596 * igb_exit_module is called just before the driver is removed
599 static void __exit igb_exit_module(void)
601 #ifdef CONFIG_IGB_DCA
602 dca_unregister_notify(&dca_notifier);
604 pci_unregister_driver(&igb_driver);
607 module_exit(igb_exit_module);
609 #define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
611 * igb_cache_ring_register - Descriptor ring to register mapping
612 * @adapter: board private structure to initialize
614 * Once we know the feature-set enabled for the device, we'll cache
615 * the register offset the descriptor ring is assigned to.
617 static void igb_cache_ring_register(struct igb_adapter *adapter)
620 u32 rbase_offset = adapter->vfs_allocated_count;
622 switch (adapter->hw.mac.type) {
624 /* The queues are allocated for virtualization such that VF 0
625 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
626 * In order to avoid collision we start at the first free queue
627 * and continue consuming queues in the same sequence
629 if (adapter->vfs_allocated_count) {
630 for (; i < adapter->rss_queues; i++)
631 adapter->rx_ring[i]->reg_idx = rbase_offset +
638 for (; i < adapter->num_rx_queues; i++)
639 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
640 for (; j < adapter->num_tx_queues; j++)
641 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
646 static void igb_free_queues(struct igb_adapter *adapter)
650 for (i = 0; i < adapter->num_tx_queues; i++) {
651 kfree(adapter->tx_ring[i]);
652 adapter->tx_ring[i] = NULL;
654 for (i = 0; i < adapter->num_rx_queues; i++) {
655 kfree(adapter->rx_ring[i]);
656 adapter->rx_ring[i] = NULL;
658 adapter->num_rx_queues = 0;
659 adapter->num_tx_queues = 0;
663 * igb_alloc_queues - Allocate memory for all rings
664 * @adapter: board private structure to initialize
666 * We allocate one ring per queue at run-time since we don't know the
667 * number of queues at compile-time.
669 static int igb_alloc_queues(struct igb_adapter *adapter)
671 struct igb_ring *ring;
674 for (i = 0; i < adapter->num_tx_queues; i++) {
675 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
678 ring->count = adapter->tx_ring_count;
679 ring->queue_index = i;
680 ring->dev = &adapter->pdev->dev;
681 ring->netdev = adapter->netdev;
682 /* For 82575, context index must be unique per ring. */
683 if (adapter->hw.mac.type == e1000_82575)
684 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
685 adapter->tx_ring[i] = ring;
688 for (i = 0; i < adapter->num_rx_queues; i++) {
689 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
692 ring->count = adapter->rx_ring_count;
693 ring->queue_index = i;
694 ring->dev = &adapter->pdev->dev;
695 ring->netdev = adapter->netdev;
696 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
697 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
698 /* set flag indicating ring supports SCTP checksum offload */
699 if (adapter->hw.mac.type >= e1000_82576)
700 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
701 adapter->rx_ring[i] = ring;
704 igb_cache_ring_register(adapter);
709 igb_free_queues(adapter);
714 #define IGB_N0_QUEUE -1
715 static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
718 struct igb_adapter *adapter = q_vector->adapter;
719 struct e1000_hw *hw = &adapter->hw;
721 int rx_queue = IGB_N0_QUEUE;
722 int tx_queue = IGB_N0_QUEUE;
724 if (q_vector->rx_ring)
725 rx_queue = q_vector->rx_ring->reg_idx;
726 if (q_vector->tx_ring)
727 tx_queue = q_vector->tx_ring->reg_idx;
729 switch (hw->mac.type) {
731 /* The 82575 assigns vectors using a bitmask, which matches the
732 bitmask for the EICR/EIMS/EIMC registers. To assign one
733 or more queues to a vector, we write the appropriate bits
734 into the MSIXBM register for that vector. */
735 if (rx_queue > IGB_N0_QUEUE)
736 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
737 if (tx_queue > IGB_N0_QUEUE)
738 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
739 if (!adapter->msix_entries && msix_vector == 0)
740 msixbm |= E1000_EIMS_OTHER;
741 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
742 q_vector->eims_value = msixbm;
745 /* 82576 uses a table-based method for assigning vectors.
746 Each queue has a single entry in the table to which we write
747 a vector number along with a "valid" bit. Sadly, the layout
748 of the table is somewhat counterintuitive. */
749 if (rx_queue > IGB_N0_QUEUE) {
750 index = (rx_queue & 0x7);
751 ivar = array_rd32(E1000_IVAR0, index);
753 /* vector goes into low byte of register */
754 ivar = ivar & 0xFFFFFF00;
755 ivar |= msix_vector | E1000_IVAR_VALID;
757 /* vector goes into third byte of register */
758 ivar = ivar & 0xFF00FFFF;
759 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
761 array_wr32(E1000_IVAR0, index, ivar);
763 if (tx_queue > IGB_N0_QUEUE) {
764 index = (tx_queue & 0x7);
765 ivar = array_rd32(E1000_IVAR0, index);
767 /* vector goes into second byte of register */
768 ivar = ivar & 0xFFFF00FF;
769 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
771 /* vector goes into high byte of register */
772 ivar = ivar & 0x00FFFFFF;
773 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
775 array_wr32(E1000_IVAR0, index, ivar);
777 q_vector->eims_value = 1 << msix_vector;
781 /* 82580 uses the same table-based approach as 82576 but has fewer
782 entries as a result we carry over for queues greater than 4. */
783 if (rx_queue > IGB_N0_QUEUE) {
784 index = (rx_queue >> 1);
785 ivar = array_rd32(E1000_IVAR0, index);
786 if (rx_queue & 0x1) {
787 /* vector goes into third byte of register */
788 ivar = ivar & 0xFF00FFFF;
789 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
791 /* vector goes into low byte of register */
792 ivar = ivar & 0xFFFFFF00;
793 ivar |= msix_vector | E1000_IVAR_VALID;
795 array_wr32(E1000_IVAR0, index, ivar);
797 if (tx_queue > IGB_N0_QUEUE) {
798 index = (tx_queue >> 1);
799 ivar = array_rd32(E1000_IVAR0, index);
800 if (tx_queue & 0x1) {
801 /* vector goes into high byte of register */
802 ivar = ivar & 0x00FFFFFF;
803 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
805 /* vector goes into second byte of register */
806 ivar = ivar & 0xFFFF00FF;
807 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
809 array_wr32(E1000_IVAR0, index, ivar);
811 q_vector->eims_value = 1 << msix_vector;
818 /* add q_vector eims value to global eims_enable_mask */
819 adapter->eims_enable_mask |= q_vector->eims_value;
821 /* configure q_vector to set itr on first interrupt */
822 q_vector->set_itr = 1;
826 * igb_configure_msix - Configure MSI-X hardware
828 * igb_configure_msix sets up the hardware to properly
829 * generate MSI-X interrupts.
831 static void igb_configure_msix(struct igb_adapter *adapter)
835 struct e1000_hw *hw = &adapter->hw;
837 adapter->eims_enable_mask = 0;
839 /* set vector for other causes, i.e. link changes */
840 switch (hw->mac.type) {
842 tmp = rd32(E1000_CTRL_EXT);
843 /* enable MSI-X PBA support*/
844 tmp |= E1000_CTRL_EXT_PBA_CLR;
846 /* Auto-Mask interrupts upon ICR read. */
847 tmp |= E1000_CTRL_EXT_EIAME;
848 tmp |= E1000_CTRL_EXT_IRCA;
850 wr32(E1000_CTRL_EXT, tmp);
852 /* enable msix_other interrupt */
853 array_wr32(E1000_MSIXBM(0), vector++,
855 adapter->eims_other = E1000_EIMS_OTHER;
862 /* Turn on MSI-X capability first, or our settings
863 * won't stick. And it will take days to debug. */
864 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
865 E1000_GPIE_PBA | E1000_GPIE_EIAME |
868 /* enable msix_other interrupt */
869 adapter->eims_other = 1 << vector;
870 tmp = (vector++ | E1000_IVAR_VALID) << 8;
872 wr32(E1000_IVAR_MISC, tmp);
875 /* do nothing, since nothing else supports MSI-X */
877 } /* switch (hw->mac.type) */
879 adapter->eims_enable_mask |= adapter->eims_other;
881 for (i = 0; i < adapter->num_q_vectors; i++)
882 igb_assign_vector(adapter->q_vector[i], vector++);
888 * igb_request_msix - Initialize MSI-X interrupts
890 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
893 static int igb_request_msix(struct igb_adapter *adapter)
895 struct net_device *netdev = adapter->netdev;
896 struct e1000_hw *hw = &adapter->hw;
897 int i, err = 0, vector = 0;
899 err = request_irq(adapter->msix_entries[vector].vector,
900 igb_msix_other, 0, netdev->name, adapter);
905 for (i = 0; i < adapter->num_q_vectors; i++) {
906 struct igb_q_vector *q_vector = adapter->q_vector[i];
908 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
910 if (q_vector->rx_ring && q_vector->tx_ring)
911 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
912 q_vector->rx_ring->queue_index);
913 else if (q_vector->tx_ring)
914 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
915 q_vector->tx_ring->queue_index);
916 else if (q_vector->rx_ring)
917 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
918 q_vector->rx_ring->queue_index);
920 sprintf(q_vector->name, "%s-unused", netdev->name);
922 err = request_irq(adapter->msix_entries[vector].vector,
923 igb_msix_ring, 0, q_vector->name,
930 igb_configure_msix(adapter);
936 static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
938 if (adapter->msix_entries) {
939 pci_disable_msix(adapter->pdev);
940 kfree(adapter->msix_entries);
941 adapter->msix_entries = NULL;
942 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
943 pci_disable_msi(adapter->pdev);
948 * igb_free_q_vectors - Free memory allocated for interrupt vectors
949 * @adapter: board private structure to initialize
951 * This function frees the memory allocated to the q_vectors. In addition if
952 * NAPI is enabled it will delete any references to the NAPI struct prior
953 * to freeing the q_vector.
955 static void igb_free_q_vectors(struct igb_adapter *adapter)
959 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
960 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
961 adapter->q_vector[v_idx] = NULL;
964 netif_napi_del(&q_vector->napi);
967 adapter->num_q_vectors = 0;
971 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
973 * This function resets the device so that it has 0 rx queues, tx queues, and
974 * MSI-X interrupts allocated.
976 static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
978 igb_free_queues(adapter);
979 igb_free_q_vectors(adapter);
980 igb_reset_interrupt_capability(adapter);
984 * igb_set_interrupt_capability - set MSI or MSI-X if supported
986 * Attempt to configure interrupts using the best available
987 * capabilities of the hardware and kernel.
989 static void igb_set_interrupt_capability(struct igb_adapter *adapter)
994 /* Number of supported queues. */
995 adapter->num_rx_queues = adapter->rss_queues;
996 if (adapter->vfs_allocated_count)
997 adapter->num_tx_queues = 1;
999 adapter->num_tx_queues = adapter->rss_queues;
1001 /* start with one vector for every rx queue */
1002 numvecs = adapter->num_rx_queues;
1004 /* if tx handler is separate add 1 for every tx queue */
1005 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1006 numvecs += adapter->num_tx_queues;
1008 /* store the number of vectors reserved for queues */
1009 adapter->num_q_vectors = numvecs;
1011 /* add 1 vector for link status interrupts */
1013 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1015 if (!adapter->msix_entries)
1018 for (i = 0; i < numvecs; i++)
1019 adapter->msix_entries[i].entry = i;
1021 err = pci_enable_msix(adapter->pdev,
1022 adapter->msix_entries,
1027 igb_reset_interrupt_capability(adapter);
1029 /* If we can't do MSI-X, try MSI */
1031 #ifdef CONFIG_PCI_IOV
1032 /* disable SR-IOV for non MSI-X configurations */
1033 if (adapter->vf_data) {
1034 struct e1000_hw *hw = &adapter->hw;
1035 /* disable iov and allow time for transactions to clear */
1036 pci_disable_sriov(adapter->pdev);
1039 kfree(adapter->vf_data);
1040 adapter->vf_data = NULL;
1041 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1043 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1046 adapter->vfs_allocated_count = 0;
1047 adapter->rss_queues = 1;
1048 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1049 adapter->num_rx_queues = 1;
1050 adapter->num_tx_queues = 1;
1051 adapter->num_q_vectors = 1;
1052 if (!pci_enable_msi(adapter->pdev))
1053 adapter->flags |= IGB_FLAG_HAS_MSI;
1055 /* Notify the stack of the (possibly) reduced Tx Queue count. */
1056 adapter->netdev->real_num_tx_queues = adapter->num_tx_queues;
1060 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1061 * @adapter: board private structure to initialize
1063 * We allocate one q_vector per queue interrupt. If allocation fails we
1066 static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1068 struct igb_q_vector *q_vector;
1069 struct e1000_hw *hw = &adapter->hw;
1072 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1073 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1076 q_vector->adapter = adapter;
1077 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1078 q_vector->itr_val = IGB_START_ITR;
1079 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1080 adapter->q_vector[v_idx] = q_vector;
1085 igb_free_q_vectors(adapter);
1089 static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1090 int ring_idx, int v_idx)
1092 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1094 q_vector->rx_ring = adapter->rx_ring[ring_idx];
1095 q_vector->rx_ring->q_vector = q_vector;
1096 q_vector->itr_val = adapter->rx_itr_setting;
1097 if (q_vector->itr_val && q_vector->itr_val <= 3)
1098 q_vector->itr_val = IGB_START_ITR;
1101 static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1102 int ring_idx, int v_idx)
1104 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1106 q_vector->tx_ring = adapter->tx_ring[ring_idx];
1107 q_vector->tx_ring->q_vector = q_vector;
1108 q_vector->itr_val = adapter->tx_itr_setting;
1109 if (q_vector->itr_val && q_vector->itr_val <= 3)
1110 q_vector->itr_val = IGB_START_ITR;
1114 * igb_map_ring_to_vector - maps allocated queues to vectors
1116 * This function maps the recently allocated queues to vectors.
1118 static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1123 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1124 (adapter->num_q_vectors < adapter->num_tx_queues))
1127 if (adapter->num_q_vectors >=
1128 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1129 for (i = 0; i < adapter->num_rx_queues; i++)
1130 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1131 for (i = 0; i < adapter->num_tx_queues; i++)
1132 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1134 for (i = 0; i < adapter->num_rx_queues; i++) {
1135 if (i < adapter->num_tx_queues)
1136 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1137 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1139 for (; i < adapter->num_tx_queues; i++)
1140 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1146 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1148 * This function initializes the interrupts and allocates all of the queues.
1150 static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1152 struct pci_dev *pdev = adapter->pdev;
1155 igb_set_interrupt_capability(adapter);
1157 err = igb_alloc_q_vectors(adapter);
1159 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1160 goto err_alloc_q_vectors;
1163 err = igb_alloc_queues(adapter);
1165 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1166 goto err_alloc_queues;
1169 err = igb_map_ring_to_vector(adapter);
1171 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1172 goto err_map_queues;
1178 igb_free_queues(adapter);
1180 igb_free_q_vectors(adapter);
1181 err_alloc_q_vectors:
1182 igb_reset_interrupt_capability(adapter);
1187 * igb_request_irq - initialize interrupts
1189 * Attempts to configure interrupts using the best available
1190 * capabilities of the hardware and kernel.
1192 static int igb_request_irq(struct igb_adapter *adapter)
1194 struct net_device *netdev = adapter->netdev;
1195 struct pci_dev *pdev = adapter->pdev;
1198 if (adapter->msix_entries) {
1199 err = igb_request_msix(adapter);
1202 /* fall back to MSI */
1203 igb_clear_interrupt_scheme(adapter);
1204 if (!pci_enable_msi(adapter->pdev))
1205 adapter->flags |= IGB_FLAG_HAS_MSI;
1206 igb_free_all_tx_resources(adapter);
1207 igb_free_all_rx_resources(adapter);
1208 adapter->num_tx_queues = 1;
1209 adapter->num_rx_queues = 1;
1210 adapter->num_q_vectors = 1;
1211 err = igb_alloc_q_vectors(adapter);
1214 "Unable to allocate memory for vectors\n");
1217 err = igb_alloc_queues(adapter);
1220 "Unable to allocate memory for queues\n");
1221 igb_free_q_vectors(adapter);
1224 igb_setup_all_tx_resources(adapter);
1225 igb_setup_all_rx_resources(adapter);
1227 igb_assign_vector(adapter->q_vector[0], 0);
1230 if (adapter->flags & IGB_FLAG_HAS_MSI) {
1231 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
1232 netdev->name, adapter);
1236 /* fall back to legacy interrupts */
1237 igb_reset_interrupt_capability(adapter);
1238 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1241 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
1242 netdev->name, adapter);
1245 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1252 static void igb_free_irq(struct igb_adapter *adapter)
1254 if (adapter->msix_entries) {
1257 free_irq(adapter->msix_entries[vector++].vector, adapter);
1259 for (i = 0; i < adapter->num_q_vectors; i++) {
1260 struct igb_q_vector *q_vector = adapter->q_vector[i];
1261 free_irq(adapter->msix_entries[vector++].vector,
1265 free_irq(adapter->pdev->irq, adapter);
1270 * igb_irq_disable - Mask off interrupt generation on the NIC
1271 * @adapter: board private structure
1273 static void igb_irq_disable(struct igb_adapter *adapter)
1275 struct e1000_hw *hw = &adapter->hw;
1278 * we need to be careful when disabling interrupts. The VFs are also
1279 * mapped into these registers and so clearing the bits can cause
1280 * issues on the VF drivers so we only need to clear what we set
1282 if (adapter->msix_entries) {
1283 u32 regval = rd32(E1000_EIAM);
1284 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1285 wr32(E1000_EIMC, adapter->eims_enable_mask);
1286 regval = rd32(E1000_EIAC);
1287 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1291 wr32(E1000_IMC, ~0);
1293 if (adapter->msix_entries) {
1295 for (i = 0; i < adapter->num_q_vectors; i++)
1296 synchronize_irq(adapter->msix_entries[i].vector);
1298 synchronize_irq(adapter->pdev->irq);
1303 * igb_irq_enable - Enable default interrupt generation settings
1304 * @adapter: board private structure
1306 static void igb_irq_enable(struct igb_adapter *adapter)
1308 struct e1000_hw *hw = &adapter->hw;
1310 if (adapter->msix_entries) {
1311 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
1312 u32 regval = rd32(E1000_EIAC);
1313 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1314 regval = rd32(E1000_EIAM);
1315 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1316 wr32(E1000_EIMS, adapter->eims_enable_mask);
1317 if (adapter->vfs_allocated_count) {
1318 wr32(E1000_MBVFIMR, 0xFF);
1319 ims |= E1000_IMS_VMMB;
1321 if (adapter->hw.mac.type == e1000_82580)
1322 ims |= E1000_IMS_DRSTA;
1324 wr32(E1000_IMS, ims);
1326 wr32(E1000_IMS, IMS_ENABLE_MASK |
1328 wr32(E1000_IAM, IMS_ENABLE_MASK |
1333 static void igb_update_mng_vlan(struct igb_adapter *adapter)
1335 struct e1000_hw *hw = &adapter->hw;
1336 u16 vid = adapter->hw.mng_cookie.vlan_id;
1337 u16 old_vid = adapter->mng_vlan_id;
1339 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1340 /* add VID to filter table */
1341 igb_vfta_set(hw, vid, true);
1342 adapter->mng_vlan_id = vid;
1344 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1347 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1349 !vlan_group_get_device(adapter->vlgrp, old_vid)) {
1350 /* remove VID from filter table */
1351 igb_vfta_set(hw, old_vid, false);
1356 * igb_release_hw_control - release control of the h/w to f/w
1357 * @adapter: address of board private structure
1359 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1360 * For ASF and Pass Through versions of f/w this means that the
1361 * driver is no longer loaded.
1364 static void igb_release_hw_control(struct igb_adapter *adapter)
1366 struct e1000_hw *hw = &adapter->hw;
1369 /* Let firmware take over control of h/w */
1370 ctrl_ext = rd32(E1000_CTRL_EXT);
1371 wr32(E1000_CTRL_EXT,
1372 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1376 * igb_get_hw_control - get control of the h/w from f/w
1377 * @adapter: address of board private structure
1379 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1380 * For ASF and Pass Through versions of f/w this means that
1381 * the driver is loaded.
1384 static void igb_get_hw_control(struct igb_adapter *adapter)
1386 struct e1000_hw *hw = &adapter->hw;
1389 /* Let firmware know the driver has taken over */
1390 ctrl_ext = rd32(E1000_CTRL_EXT);
1391 wr32(E1000_CTRL_EXT,
1392 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1396 * igb_configure - configure the hardware for RX and TX
1397 * @adapter: private board structure
1399 static void igb_configure(struct igb_adapter *adapter)
1401 struct net_device *netdev = adapter->netdev;
1404 igb_get_hw_control(adapter);
1405 igb_set_rx_mode(netdev);
1407 igb_restore_vlan(adapter);
1409 igb_setup_tctl(adapter);
1410 igb_setup_mrqc(adapter);
1411 igb_setup_rctl(adapter);
1413 igb_configure_tx(adapter);
1414 igb_configure_rx(adapter);
1416 igb_rx_fifo_flush_82575(&adapter->hw);
1418 /* call igb_desc_unused which always leaves
1419 * at least 1 descriptor unused to make sure
1420 * next_to_use != next_to_clean */
1421 for (i = 0; i < adapter->num_rx_queues; i++) {
1422 struct igb_ring *ring = adapter->rx_ring[i];
1423 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
1428 * igb_power_up_link - Power up the phy/serdes link
1429 * @adapter: address of board private structure
1431 void igb_power_up_link(struct igb_adapter *adapter)
1433 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1434 igb_power_up_phy_copper(&adapter->hw);
1436 igb_power_up_serdes_link_82575(&adapter->hw);
1440 * igb_power_down_link - Power down the phy/serdes link
1441 * @adapter: address of board private structure
1443 static void igb_power_down_link(struct igb_adapter *adapter)
1445 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1446 igb_power_down_phy_copper_82575(&adapter->hw);
1448 igb_shutdown_serdes_link_82575(&adapter->hw);
1452 * igb_up - Open the interface and prepare it to handle traffic
1453 * @adapter: board private structure
1455 int igb_up(struct igb_adapter *adapter)
1457 struct e1000_hw *hw = &adapter->hw;
1460 /* hardware has been reset, we need to reload some things */
1461 igb_configure(adapter);
1463 clear_bit(__IGB_DOWN, &adapter->state);
1465 for (i = 0; i < adapter->num_q_vectors; i++) {
1466 struct igb_q_vector *q_vector = adapter->q_vector[i];
1467 napi_enable(&q_vector->napi);
1469 if (adapter->msix_entries)
1470 igb_configure_msix(adapter);
1472 igb_assign_vector(adapter->q_vector[0], 0);
1474 /* Clear any pending interrupts. */
1476 igb_irq_enable(adapter);
1478 /* notify VFs that reset has been completed */
1479 if (adapter->vfs_allocated_count) {
1480 u32 reg_data = rd32(E1000_CTRL_EXT);
1481 reg_data |= E1000_CTRL_EXT_PFRSTD;
1482 wr32(E1000_CTRL_EXT, reg_data);
1485 netif_tx_start_all_queues(adapter->netdev);
1487 /* start the watchdog. */
1488 hw->mac.get_link_status = 1;
1489 schedule_work(&adapter->watchdog_task);
1494 void igb_down(struct igb_adapter *adapter)
1496 struct net_device *netdev = adapter->netdev;
1497 struct e1000_hw *hw = &adapter->hw;
1501 /* signal that we're down so the interrupt handler does not
1502 * reschedule our watchdog timer */
1503 set_bit(__IGB_DOWN, &adapter->state);
1505 /* disable receives in the hardware */
1506 rctl = rd32(E1000_RCTL);
1507 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1508 /* flush and sleep below */
1510 netif_tx_stop_all_queues(netdev);
1512 /* disable transmits in the hardware */
1513 tctl = rd32(E1000_TCTL);
1514 tctl &= ~E1000_TCTL_EN;
1515 wr32(E1000_TCTL, tctl);
1516 /* flush both disables and wait for them to finish */
1520 for (i = 0; i < adapter->num_q_vectors; i++) {
1521 struct igb_q_vector *q_vector = adapter->q_vector[i];
1522 napi_disable(&q_vector->napi);
1525 igb_irq_disable(adapter);
1527 del_timer_sync(&adapter->watchdog_timer);
1528 del_timer_sync(&adapter->phy_info_timer);
1530 netif_carrier_off(netdev);
1532 /* record the stats before reset*/
1533 igb_update_stats(adapter);
1535 adapter->link_speed = 0;
1536 adapter->link_duplex = 0;
1538 if (!pci_channel_offline(adapter->pdev))
1540 igb_clean_all_tx_rings(adapter);
1541 igb_clean_all_rx_rings(adapter);
1542 #ifdef CONFIG_IGB_DCA
1544 /* since we reset the hardware DCA settings were cleared */
1545 igb_setup_dca(adapter);
1549 void igb_reinit_locked(struct igb_adapter *adapter)
1551 WARN_ON(in_interrupt());
1552 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1556 clear_bit(__IGB_RESETTING, &adapter->state);
1559 void igb_reset(struct igb_adapter *adapter)
1561 struct pci_dev *pdev = adapter->pdev;
1562 struct e1000_hw *hw = &adapter->hw;
1563 struct e1000_mac_info *mac = &hw->mac;
1564 struct e1000_fc_info *fc = &hw->fc;
1565 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1568 /* Repartition Pba for greater than 9k mtu
1569 * To take effect CTRL.RST is required.
1571 switch (mac->type) {
1574 pba = rd32(E1000_RXPBS);
1575 pba = igb_rxpbs_adjust_82580(pba);
1578 pba = rd32(E1000_RXPBS);
1579 pba &= E1000_RXPBS_SIZE_MASK_82576;
1583 pba = E1000_PBA_34K;
1587 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1588 (mac->type < e1000_82576)) {
1589 /* adjust PBA for jumbo frames */
1590 wr32(E1000_PBA, pba);
1592 /* To maintain wire speed transmits, the Tx FIFO should be
1593 * large enough to accommodate two full transmit packets,
1594 * rounded up to the next 1KB and expressed in KB. Likewise,
1595 * the Rx FIFO should be large enough to accommodate at least
1596 * one full receive packet and is similarly rounded up and
1597 * expressed in KB. */
1598 pba = rd32(E1000_PBA);
1599 /* upper 16 bits has Tx packet buffer allocation size in KB */
1600 tx_space = pba >> 16;
1601 /* lower 16 bits has Rx packet buffer allocation size in KB */
1603 /* the tx fifo also stores 16 bytes of information about the tx
1604 * but don't include ethernet FCS because hardware appends it */
1605 min_tx_space = (adapter->max_frame_size +
1606 sizeof(union e1000_adv_tx_desc) -
1608 min_tx_space = ALIGN(min_tx_space, 1024);
1609 min_tx_space >>= 10;
1610 /* software strips receive CRC, so leave room for it */
1611 min_rx_space = adapter->max_frame_size;
1612 min_rx_space = ALIGN(min_rx_space, 1024);
1613 min_rx_space >>= 10;
1615 /* If current Tx allocation is less than the min Tx FIFO size,
1616 * and the min Tx FIFO size is less than the current Rx FIFO
1617 * allocation, take space away from current Rx allocation */
1618 if (tx_space < min_tx_space &&
1619 ((min_tx_space - tx_space) < pba)) {
1620 pba = pba - (min_tx_space - tx_space);
1622 /* if short on rx space, rx wins and must trump tx
1624 if (pba < min_rx_space)
1627 wr32(E1000_PBA, pba);
1630 /* flow control settings */
1631 /* The high water mark must be low enough to fit one full frame
1632 * (or the size used for early receive) above it in the Rx FIFO.
1633 * Set it to the lower of:
1634 * - 90% of the Rx FIFO size, or
1635 * - the full Rx FIFO size minus one full frame */
1636 hwm = min(((pba << 10) * 9 / 10),
1637 ((pba << 10) - 2 * adapter->max_frame_size));
1639 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1640 fc->low_water = fc->high_water - 16;
1641 fc->pause_time = 0xFFFF;
1643 fc->current_mode = fc->requested_mode;
1645 /* disable receive for all VFs and wait one second */
1646 if (adapter->vfs_allocated_count) {
1648 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1649 adapter->vf_data[i].flags = 0;
1651 /* ping all the active vfs to let them know we are going down */
1652 igb_ping_all_vfs(adapter);
1654 /* disable transmits and receives */
1655 wr32(E1000_VFRE, 0);
1656 wr32(E1000_VFTE, 0);
1659 /* Allow time for pending master requests to run */
1660 hw->mac.ops.reset_hw(hw);
1663 if (hw->mac.ops.init_hw(hw))
1664 dev_err(&pdev->dev, "Hardware Error\n");
1666 if (hw->mac.type == e1000_82580) {
1667 u32 reg = rd32(E1000_PCIEMISC);
1668 wr32(E1000_PCIEMISC,
1669 reg & ~E1000_PCIEMISC_LX_DECISION);
1671 if (!netif_running(adapter->netdev))
1672 igb_power_down_link(adapter);
1674 igb_update_mng_vlan(adapter);
1676 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1677 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1679 igb_get_phy_info(hw);
1682 static const struct net_device_ops igb_netdev_ops = {
1683 .ndo_open = igb_open,
1684 .ndo_stop = igb_close,
1685 .ndo_start_xmit = igb_xmit_frame_adv,
1686 .ndo_get_stats = igb_get_stats,
1687 .ndo_set_rx_mode = igb_set_rx_mode,
1688 .ndo_set_multicast_list = igb_set_rx_mode,
1689 .ndo_set_mac_address = igb_set_mac,
1690 .ndo_change_mtu = igb_change_mtu,
1691 .ndo_do_ioctl = igb_ioctl,
1692 .ndo_tx_timeout = igb_tx_timeout,
1693 .ndo_validate_addr = eth_validate_addr,
1694 .ndo_vlan_rx_register = igb_vlan_rx_register,
1695 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1696 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1697 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1698 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1699 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1700 .ndo_get_vf_config = igb_ndo_get_vf_config,
1701 #ifdef CONFIG_NET_POLL_CONTROLLER
1702 .ndo_poll_controller = igb_netpoll,
1707 * igb_probe - Device Initialization Routine
1708 * @pdev: PCI device information struct
1709 * @ent: entry in igb_pci_tbl
1711 * Returns 0 on success, negative on failure
1713 * igb_probe initializes an adapter identified by a pci_dev structure.
1714 * The OS initialization, configuring of the adapter private structure,
1715 * and a hardware reset occur.
1717 static int __devinit igb_probe(struct pci_dev *pdev,
1718 const struct pci_device_id *ent)
1720 struct net_device *netdev;
1721 struct igb_adapter *adapter;
1722 struct e1000_hw *hw;
1723 u16 eeprom_data = 0;
1724 static int global_quad_port_a; /* global quad port a indication */
1725 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1726 unsigned long mmio_start, mmio_len;
1727 int err, pci_using_dac;
1728 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1731 /* Catch broken hardware that put the wrong VF device ID in
1732 * the PCIe SR-IOV capability.
1734 if (pdev->is_virtfn) {
1735 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1736 pci_name(pdev), pdev->vendor, pdev->device);
1740 err = pci_enable_device_mem(pdev);
1745 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1747 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1751 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1753 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1755 dev_err(&pdev->dev, "No usable DMA "
1756 "configuration, aborting\n");
1762 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1768 pci_enable_pcie_error_reporting(pdev);
1770 pci_set_master(pdev);
1771 pci_save_state(pdev);
1774 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1775 IGB_ABS_MAX_TX_QUEUES);
1777 goto err_alloc_etherdev;
1779 SET_NETDEV_DEV(netdev, &pdev->dev);
1781 pci_set_drvdata(pdev, netdev);
1782 adapter = netdev_priv(netdev);
1783 adapter->netdev = netdev;
1784 adapter->pdev = pdev;
1787 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1789 mmio_start = pci_resource_start(pdev, 0);
1790 mmio_len = pci_resource_len(pdev, 0);
1793 hw->hw_addr = ioremap(mmio_start, mmio_len);
1797 netdev->netdev_ops = &igb_netdev_ops;
1798 igb_set_ethtool_ops(netdev);
1799 netdev->watchdog_timeo = 5 * HZ;
1801 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1803 netdev->mem_start = mmio_start;
1804 netdev->mem_end = mmio_start + mmio_len;
1806 /* PCI config space info */
1807 hw->vendor_id = pdev->vendor;
1808 hw->device_id = pdev->device;
1809 hw->revision_id = pdev->revision;
1810 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1811 hw->subsystem_device_id = pdev->subsystem_device;
1813 /* Copy the default MAC, PHY and NVM function pointers */
1814 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1815 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1816 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1817 /* Initialize skew-specific constants */
1818 err = ei->get_invariants(hw);
1822 /* setup the private structure */
1823 err = igb_sw_init(adapter);
1827 igb_get_bus_info_pcie(hw);
1829 hw->phy.autoneg_wait_to_complete = false;
1831 /* Copper options */
1832 if (hw->phy.media_type == e1000_media_type_copper) {
1833 hw->phy.mdix = AUTO_ALL_MODES;
1834 hw->phy.disable_polarity_correction = false;
1835 hw->phy.ms_type = e1000_ms_hw_default;
1838 if (igb_check_reset_block(hw))
1839 dev_info(&pdev->dev,
1840 "PHY reset is blocked due to SOL/IDER session.\n");
1842 netdev->features = NETIF_F_SG |
1844 NETIF_F_HW_VLAN_TX |
1845 NETIF_F_HW_VLAN_RX |
1846 NETIF_F_HW_VLAN_FILTER;
1848 netdev->features |= NETIF_F_IPV6_CSUM;
1849 netdev->features |= NETIF_F_TSO;
1850 netdev->features |= NETIF_F_TSO6;
1851 netdev->features |= NETIF_F_GRO;
1853 netdev->vlan_features |= NETIF_F_TSO;
1854 netdev->vlan_features |= NETIF_F_TSO6;
1855 netdev->vlan_features |= NETIF_F_IP_CSUM;
1856 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
1857 netdev->vlan_features |= NETIF_F_SG;
1860 netdev->features |= NETIF_F_HIGHDMA;
1862 if (hw->mac.type >= e1000_82576)
1863 netdev->features |= NETIF_F_SCTP_CSUM;
1865 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
1867 /* before reading the NVM, reset the controller to put the device in a
1868 * known good starting state */
1869 hw->mac.ops.reset_hw(hw);
1871 /* make sure the NVM is good */
1872 if (igb_validate_nvm_checksum(hw) < 0) {
1873 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1878 /* copy the MAC address out of the NVM */
1879 if (hw->mac.ops.read_mac_addr(hw))
1880 dev_err(&pdev->dev, "NVM Read Error\n");
1882 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1883 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1885 if (!is_valid_ether_addr(netdev->perm_addr)) {
1886 dev_err(&pdev->dev, "Invalid MAC Address\n");
1891 setup_timer(&adapter->watchdog_timer, igb_watchdog,
1892 (unsigned long) adapter);
1893 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
1894 (unsigned long) adapter);
1896 INIT_WORK(&adapter->reset_task, igb_reset_task);
1897 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
1899 /* Initialize link properties that are user-changeable */
1900 adapter->fc_autoneg = true;
1901 hw->mac.autoneg = true;
1902 hw->phy.autoneg_advertised = 0x2f;
1904 hw->fc.requested_mode = e1000_fc_default;
1905 hw->fc.current_mode = e1000_fc_default;
1907 igb_validate_mdi_setting(hw);
1909 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
1910 * enable the ACPI Magic Packet filter
1913 if (hw->bus.func == 0)
1914 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
1915 else if (hw->mac.type == e1000_82580)
1916 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
1917 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
1919 else if (hw->bus.func == 1)
1920 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
1922 if (eeprom_data & eeprom_apme_mask)
1923 adapter->eeprom_wol |= E1000_WUFC_MAG;
1925 /* now that we have the eeprom settings, apply the special cases where
1926 * the eeprom may be wrong or the board simply won't support wake on
1927 * lan on a particular port */
1928 switch (pdev->device) {
1929 case E1000_DEV_ID_82575GB_QUAD_COPPER:
1930 adapter->eeprom_wol = 0;
1932 case E1000_DEV_ID_82575EB_FIBER_SERDES:
1933 case E1000_DEV_ID_82576_FIBER:
1934 case E1000_DEV_ID_82576_SERDES:
1935 /* Wake events only supported on port A for dual fiber
1936 * regardless of eeprom setting */
1937 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
1938 adapter->eeprom_wol = 0;
1940 case E1000_DEV_ID_82576_QUAD_COPPER:
1941 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
1942 /* if quad port adapter, disable WoL on all but port A */
1943 if (global_quad_port_a != 0)
1944 adapter->eeprom_wol = 0;
1946 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
1947 /* Reset for multiple quad port adapters */
1948 if (++global_quad_port_a == 4)
1949 global_quad_port_a = 0;
1953 /* initialize the wol settings based on the eeprom settings */
1954 adapter->wol = adapter->eeprom_wol;
1955 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
1957 /* reset the hardware with the new settings */
1960 /* let the f/w know that the h/w is now under the control of the
1962 igb_get_hw_control(adapter);
1964 strcpy(netdev->name, "eth%d");
1965 err = register_netdev(netdev);
1969 /* carrier off reporting is important to ethtool even BEFORE open */
1970 netif_carrier_off(netdev);
1972 #ifdef CONFIG_IGB_DCA
1973 if (dca_add_requester(&pdev->dev) == 0) {
1974 adapter->flags |= IGB_FLAG_DCA_ENABLED;
1975 dev_info(&pdev->dev, "DCA enabled\n");
1976 igb_setup_dca(adapter);
1980 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
1981 /* print bus type/speed/width info */
1982 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
1984 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
1985 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
1987 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
1988 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
1989 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
1993 igb_read_part_num(hw, &part_num);
1994 dev_info(&pdev->dev, "%s: PBA No: %06x-%03x\n", netdev->name,
1995 (part_num >> 8), (part_num & 0xff));
1997 dev_info(&pdev->dev,
1998 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
1999 adapter->msix_entries ? "MSI-X" :
2000 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2001 adapter->num_rx_queues, adapter->num_tx_queues);
2006 igb_release_hw_control(adapter);
2008 if (!igb_check_reset_block(hw))
2011 if (hw->flash_address)
2012 iounmap(hw->flash_address);
2014 igb_clear_interrupt_scheme(adapter);
2015 iounmap(hw->hw_addr);
2017 free_netdev(netdev);
2019 pci_release_selected_regions(pdev,
2020 pci_select_bars(pdev, IORESOURCE_MEM));
2023 pci_disable_device(pdev);
2028 * igb_remove - Device Removal Routine
2029 * @pdev: PCI device information struct
2031 * igb_remove is called by the PCI subsystem to alert the driver
2032 * that it should release a PCI device. The could be caused by a
2033 * Hot-Plug event, or because the driver is going to be removed from
2036 static void __devexit igb_remove(struct pci_dev *pdev)
2038 struct net_device *netdev = pci_get_drvdata(pdev);
2039 struct igb_adapter *adapter = netdev_priv(netdev);
2040 struct e1000_hw *hw = &adapter->hw;
2042 /* flush_scheduled work may reschedule our watchdog task, so
2043 * explicitly disable watchdog tasks from being rescheduled */
2044 set_bit(__IGB_DOWN, &adapter->state);
2045 del_timer_sync(&adapter->watchdog_timer);
2046 del_timer_sync(&adapter->phy_info_timer);
2048 flush_scheduled_work();
2050 #ifdef CONFIG_IGB_DCA
2051 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2052 dev_info(&pdev->dev, "DCA disabled\n");
2053 dca_remove_requester(&pdev->dev);
2054 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2055 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
2059 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2060 * would have already happened in close and is redundant. */
2061 igb_release_hw_control(adapter);
2063 unregister_netdev(netdev);
2065 igb_clear_interrupt_scheme(adapter);
2067 #ifdef CONFIG_PCI_IOV
2068 /* reclaim resources allocated to VFs */
2069 if (adapter->vf_data) {
2070 /* disable iov and allow time for transactions to clear */
2071 pci_disable_sriov(pdev);
2074 kfree(adapter->vf_data);
2075 adapter->vf_data = NULL;
2076 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2078 dev_info(&pdev->dev, "IOV Disabled\n");
2082 iounmap(hw->hw_addr);
2083 if (hw->flash_address)
2084 iounmap(hw->flash_address);
2085 pci_release_selected_regions(pdev,
2086 pci_select_bars(pdev, IORESOURCE_MEM));
2088 free_netdev(netdev);
2090 pci_disable_pcie_error_reporting(pdev);
2092 pci_disable_device(pdev);
2096 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2097 * @adapter: board private structure to initialize
2099 * This function initializes the vf specific data storage and then attempts to
2100 * allocate the VFs. The reason for ordering it this way is because it is much
2101 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2102 * the memory for the VFs.
2104 static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2106 #ifdef CONFIG_PCI_IOV
2107 struct pci_dev *pdev = adapter->pdev;
2109 if (adapter->vfs_allocated_count) {
2110 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2111 sizeof(struct vf_data_storage),
2113 /* if allocation failed then we do not support SR-IOV */
2114 if (!adapter->vf_data) {
2115 adapter->vfs_allocated_count = 0;
2116 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2121 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2122 kfree(adapter->vf_data);
2123 adapter->vf_data = NULL;
2124 #endif /* CONFIG_PCI_IOV */
2125 adapter->vfs_allocated_count = 0;
2126 #ifdef CONFIG_PCI_IOV
2128 unsigned char mac_addr[ETH_ALEN];
2130 dev_info(&pdev->dev, "%d vfs allocated\n",
2131 adapter->vfs_allocated_count);
2132 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2133 random_ether_addr(mac_addr);
2134 igb_set_vf_mac(adapter, i, mac_addr);
2137 #endif /* CONFIG_PCI_IOV */
2142 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2143 * @adapter: board private structure to initialize
2145 * igb_init_hw_timer initializes the function pointer and values for the hw
2146 * timer found in hardware.
2148 static void igb_init_hw_timer(struct igb_adapter *adapter)
2150 struct e1000_hw *hw = &adapter->hw;
2152 switch (hw->mac.type) {
2155 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2156 adapter->cycles.read = igb_read_clock;
2157 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2158 adapter->cycles.mult = 1;
2160 * The 82580 timesync updates the system timer every 8ns by 8ns
2161 * and the value cannot be shifted. Instead we need to shift
2162 * the registers to generate a 64bit timer value. As a result
2163 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2164 * 24 in order to generate a larger value for synchronization.
2166 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2167 /* disable system timer temporarily by setting bit 31 */
2168 wr32(E1000_TSAUXC, 0x80000000);
2171 /* Set registers so that rollover occurs soon to test this. */
2172 wr32(E1000_SYSTIMR, 0x00000000);
2173 wr32(E1000_SYSTIML, 0x80000000);
2174 wr32(E1000_SYSTIMH, 0x000000FF);
2177 /* enable system timer by clearing bit 31 */
2178 wr32(E1000_TSAUXC, 0x0);
2181 timecounter_init(&adapter->clock,
2183 ktime_to_ns(ktime_get_real()));
2185 * Synchronize our NIC clock against system wall clock. NIC
2186 * time stamp reading requires ~3us per sample, each sample
2187 * was pretty stable even under load => only require 10
2188 * samples for each offset comparison.
2190 memset(&adapter->compare, 0, sizeof(adapter->compare));
2191 adapter->compare.source = &adapter->clock;
2192 adapter->compare.target = ktime_get_real;
2193 adapter->compare.num_samples = 10;
2194 timecompare_update(&adapter->compare, 0);
2198 * Initialize hardware timer: we keep it running just in case
2199 * that some program needs it later on.
2201 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2202 adapter->cycles.read = igb_read_clock;
2203 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2204 adapter->cycles.mult = 1;
2206 * Scale the NIC clock cycle by a large factor so that
2207 * relatively small clock corrections can be added or
2208 * substracted at each clock tick. The drawbacks of a large
2209 * factor are a) that the clock register overflows more quickly
2210 * (not such a big deal) and b) that the increment per tick has
2211 * to fit into 24 bits. As a result we need to use a shift of
2212 * 19 so we can fit a value of 16 into the TIMINCA register.
2214 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2216 (1 << E1000_TIMINCA_16NS_SHIFT) |
2217 (16 << IGB_82576_TSYNC_SHIFT));
2219 /* Set registers so that rollover occurs soon to test this. */
2220 wr32(E1000_SYSTIML, 0x00000000);
2221 wr32(E1000_SYSTIMH, 0xFF800000);
2224 timecounter_init(&adapter->clock,
2226 ktime_to_ns(ktime_get_real()));
2228 * Synchronize our NIC clock against system wall clock. NIC
2229 * time stamp reading requires ~3us per sample, each sample
2230 * was pretty stable even under load => only require 10
2231 * samples for each offset comparison.
2233 memset(&adapter->compare, 0, sizeof(adapter->compare));
2234 adapter->compare.source = &adapter->clock;
2235 adapter->compare.target = ktime_get_real;
2236 adapter->compare.num_samples = 10;
2237 timecompare_update(&adapter->compare, 0);
2240 /* 82575 does not support timesync */
2248 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2249 * @adapter: board private structure to initialize
2251 * igb_sw_init initializes the Adapter private data structure.
2252 * Fields are initialized based on PCI device information and
2253 * OS network device settings (MTU size).
2255 static int __devinit igb_sw_init(struct igb_adapter *adapter)
2257 struct e1000_hw *hw = &adapter->hw;
2258 struct net_device *netdev = adapter->netdev;
2259 struct pci_dev *pdev = adapter->pdev;
2261 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2263 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2264 adapter->rx_ring_count = IGB_DEFAULT_RXD;
2265 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2266 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2268 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2269 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2271 #ifdef CONFIG_PCI_IOV
2272 if (hw->mac.type == e1000_82576)
2273 adapter->vfs_allocated_count = (max_vfs > 7) ? 7 : max_vfs;
2275 #endif /* CONFIG_PCI_IOV */
2276 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2279 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2280 * then we should combine the queues into a queue pair in order to
2281 * conserve interrupts due to limited supply
2283 if ((adapter->rss_queues > 4) ||
2284 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2285 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2287 /* This call may decrease the number of queues */
2288 if (igb_init_interrupt_scheme(adapter)) {
2289 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2293 igb_init_hw_timer(adapter);
2294 igb_probe_vfs(adapter);
2296 /* Explicitly disable IRQ since the NIC can be in any state. */
2297 igb_irq_disable(adapter);
2299 set_bit(__IGB_DOWN, &adapter->state);
2304 * igb_open - Called when a network interface is made active
2305 * @netdev: network interface device structure
2307 * Returns 0 on success, negative value on failure
2309 * The open entry point is called when a network interface is made
2310 * active by the system (IFF_UP). At this point all resources needed
2311 * for transmit and receive operations are allocated, the interrupt
2312 * handler is registered with the OS, the watchdog timer is started,
2313 * and the stack is notified that the interface is ready.
2315 static int igb_open(struct net_device *netdev)
2317 struct igb_adapter *adapter = netdev_priv(netdev);
2318 struct e1000_hw *hw = &adapter->hw;
2322 /* disallow open during test */
2323 if (test_bit(__IGB_TESTING, &adapter->state))
2326 netif_carrier_off(netdev);
2328 /* allocate transmit descriptors */
2329 err = igb_setup_all_tx_resources(adapter);
2333 /* allocate receive descriptors */
2334 err = igb_setup_all_rx_resources(adapter);
2338 igb_power_up_link(adapter);
2340 /* before we allocate an interrupt, we must be ready to handle it.
2341 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2342 * as soon as we call pci_request_irq, so we have to setup our
2343 * clean_rx handler before we do so. */
2344 igb_configure(adapter);
2346 err = igb_request_irq(adapter);
2350 /* From here on the code is the same as igb_up() */
2351 clear_bit(__IGB_DOWN, &adapter->state);
2353 for (i = 0; i < adapter->num_q_vectors; i++) {
2354 struct igb_q_vector *q_vector = adapter->q_vector[i];
2355 napi_enable(&q_vector->napi);
2358 /* Clear any pending interrupts. */
2361 igb_irq_enable(adapter);
2363 /* notify VFs that reset has been completed */
2364 if (adapter->vfs_allocated_count) {
2365 u32 reg_data = rd32(E1000_CTRL_EXT);
2366 reg_data |= E1000_CTRL_EXT_PFRSTD;
2367 wr32(E1000_CTRL_EXT, reg_data);
2370 netif_tx_start_all_queues(netdev);
2372 /* start the watchdog. */
2373 hw->mac.get_link_status = 1;
2374 schedule_work(&adapter->watchdog_task);
2379 igb_release_hw_control(adapter);
2380 igb_power_down_link(adapter);
2381 igb_free_all_rx_resources(adapter);
2383 igb_free_all_tx_resources(adapter);
2391 * igb_close - Disables a network interface
2392 * @netdev: network interface device structure
2394 * Returns 0, this is not allowed to fail
2396 * The close entry point is called when an interface is de-activated
2397 * by the OS. The hardware is still under the driver's control, but
2398 * needs to be disabled. A global MAC reset is issued to stop the
2399 * hardware, and all transmit and receive resources are freed.
2401 static int igb_close(struct net_device *netdev)
2403 struct igb_adapter *adapter = netdev_priv(netdev);
2405 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2408 igb_free_irq(adapter);
2410 igb_free_all_tx_resources(adapter);
2411 igb_free_all_rx_resources(adapter);
2417 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2418 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2420 * Return 0 on success, negative on failure
2422 int igb_setup_tx_resources(struct igb_ring *tx_ring)
2424 struct device *dev = tx_ring->dev;
2427 size = sizeof(struct igb_buffer) * tx_ring->count;
2428 tx_ring->buffer_info = vmalloc(size);
2429 if (!tx_ring->buffer_info)
2431 memset(tx_ring->buffer_info, 0, size);
2433 /* round up to nearest 4K */
2434 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2435 tx_ring->size = ALIGN(tx_ring->size, 4096);
2437 tx_ring->desc = dma_alloc_coherent(dev,
2445 tx_ring->next_to_use = 0;
2446 tx_ring->next_to_clean = 0;
2450 vfree(tx_ring->buffer_info);
2452 "Unable to allocate memory for the transmit descriptor ring\n");
2457 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2458 * (Descriptors) for all queues
2459 * @adapter: board private structure
2461 * Return 0 on success, negative on failure
2463 static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2465 struct pci_dev *pdev = adapter->pdev;
2468 for (i = 0; i < adapter->num_tx_queues; i++) {
2469 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2472 "Allocation for Tx Queue %u failed\n", i);
2473 for (i--; i >= 0; i--)
2474 igb_free_tx_resources(adapter->tx_ring[i]);
2479 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
2480 int r_idx = i % adapter->num_tx_queues;
2481 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
2487 * igb_setup_tctl - configure the transmit control registers
2488 * @adapter: Board private structure
2490 void igb_setup_tctl(struct igb_adapter *adapter)
2492 struct e1000_hw *hw = &adapter->hw;
2495 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2496 wr32(E1000_TXDCTL(0), 0);
2498 /* Program the Transmit Control Register */
2499 tctl = rd32(E1000_TCTL);
2500 tctl &= ~E1000_TCTL_CT;
2501 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2502 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2504 igb_config_collision_dist(hw);
2506 /* Enable transmits */
2507 tctl |= E1000_TCTL_EN;
2509 wr32(E1000_TCTL, tctl);
2513 * igb_configure_tx_ring - Configure transmit ring after Reset
2514 * @adapter: board private structure
2515 * @ring: tx ring to configure
2517 * Configure a transmit ring after a reset.
2519 void igb_configure_tx_ring(struct igb_adapter *adapter,
2520 struct igb_ring *ring)
2522 struct e1000_hw *hw = &adapter->hw;
2524 u64 tdba = ring->dma;
2525 int reg_idx = ring->reg_idx;
2527 /* disable the queue */
2528 txdctl = rd32(E1000_TXDCTL(reg_idx));
2529 wr32(E1000_TXDCTL(reg_idx),
2530 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2534 wr32(E1000_TDLEN(reg_idx),
2535 ring->count * sizeof(union e1000_adv_tx_desc));
2536 wr32(E1000_TDBAL(reg_idx),
2537 tdba & 0x00000000ffffffffULL);
2538 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2540 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2541 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2542 writel(0, ring->head);
2543 writel(0, ring->tail);
2545 txdctl |= IGB_TX_PTHRESH;
2546 txdctl |= IGB_TX_HTHRESH << 8;
2547 txdctl |= IGB_TX_WTHRESH << 16;
2549 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2550 wr32(E1000_TXDCTL(reg_idx), txdctl);
2554 * igb_configure_tx - Configure transmit Unit after Reset
2555 * @adapter: board private structure
2557 * Configure the Tx unit of the MAC after a reset.
2559 static void igb_configure_tx(struct igb_adapter *adapter)
2563 for (i = 0; i < adapter->num_tx_queues; i++)
2564 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
2568 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2569 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2571 * Returns 0 on success, negative on failure
2573 int igb_setup_rx_resources(struct igb_ring *rx_ring)
2575 struct device *dev = rx_ring->dev;
2578 size = sizeof(struct igb_buffer) * rx_ring->count;
2579 rx_ring->buffer_info = vmalloc(size);
2580 if (!rx_ring->buffer_info)
2582 memset(rx_ring->buffer_info, 0, size);
2584 desc_len = sizeof(union e1000_adv_rx_desc);
2586 /* Round up to nearest 4K */
2587 rx_ring->size = rx_ring->count * desc_len;
2588 rx_ring->size = ALIGN(rx_ring->size, 4096);
2590 rx_ring->desc = dma_alloc_coherent(dev,
2598 rx_ring->next_to_clean = 0;
2599 rx_ring->next_to_use = 0;
2604 vfree(rx_ring->buffer_info);
2605 rx_ring->buffer_info = NULL;
2606 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2612 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2613 * (Descriptors) for all queues
2614 * @adapter: board private structure
2616 * Return 0 on success, negative on failure
2618 static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2620 struct pci_dev *pdev = adapter->pdev;
2623 for (i = 0; i < adapter->num_rx_queues; i++) {
2624 err = igb_setup_rx_resources(adapter->rx_ring[i]);
2627 "Allocation for Rx Queue %u failed\n", i);
2628 for (i--; i >= 0; i--)
2629 igb_free_rx_resources(adapter->rx_ring[i]);
2638 * igb_setup_mrqc - configure the multiple receive queue control registers
2639 * @adapter: Board private structure
2641 static void igb_setup_mrqc(struct igb_adapter *adapter)
2643 struct e1000_hw *hw = &adapter->hw;
2645 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2650 static const u8 rsshash[40] = {
2651 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2652 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2653 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2654 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2656 /* Fill out hash function seeds */
2657 for (j = 0; j < 10; j++) {
2658 u32 rsskey = rsshash[(j * 4)];
2659 rsskey |= rsshash[(j * 4) + 1] << 8;
2660 rsskey |= rsshash[(j * 4) + 2] << 16;
2661 rsskey |= rsshash[(j * 4) + 3] << 24;
2662 array_wr32(E1000_RSSRK(0), j, rsskey);
2665 num_rx_queues = adapter->rss_queues;
2667 if (adapter->vfs_allocated_count) {
2668 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2669 switch (hw->mac.type) {
2686 if (hw->mac.type == e1000_82575)
2690 for (j = 0; j < (32 * 4); j++) {
2691 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2693 reta.bytes[j & 3] |= num_rx_queues << shift2;
2695 wr32(E1000_RETA(j >> 2), reta.dword);
2699 * Disable raw packet checksumming so that RSS hash is placed in
2700 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2701 * offloads as they are enabled by default
2703 rxcsum = rd32(E1000_RXCSUM);
2704 rxcsum |= E1000_RXCSUM_PCSD;
2706 if (adapter->hw.mac.type >= e1000_82576)
2707 /* Enable Receive Checksum Offload for SCTP */
2708 rxcsum |= E1000_RXCSUM_CRCOFL;
2710 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2711 wr32(E1000_RXCSUM, rxcsum);
2713 /* If VMDq is enabled then we set the appropriate mode for that, else
2714 * we default to RSS so that an RSS hash is calculated per packet even
2715 * if we are only using one queue */
2716 if (adapter->vfs_allocated_count) {
2717 if (hw->mac.type > e1000_82575) {
2718 /* Set the default pool for the PF's first queue */
2719 u32 vtctl = rd32(E1000_VT_CTL);
2720 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2721 E1000_VT_CTL_DISABLE_DEF_POOL);
2722 vtctl |= adapter->vfs_allocated_count <<
2723 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2724 wr32(E1000_VT_CTL, vtctl);
2726 if (adapter->rss_queues > 1)
2727 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2729 mrqc = E1000_MRQC_ENABLE_VMDQ;
2731 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2733 igb_vmm_control(adapter);
2736 * Generate RSS hash based on TCP port numbers and/or
2737 * IPv4/v6 src and dst addresses since UDP cannot be
2738 * hashed reliably due to IP fragmentation
2740 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2741 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2742 E1000_MRQC_RSS_FIELD_IPV6 |
2743 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2744 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
2746 wr32(E1000_MRQC, mrqc);
2750 * igb_setup_rctl - configure the receive control registers
2751 * @adapter: Board private structure
2753 void igb_setup_rctl(struct igb_adapter *adapter)
2755 struct e1000_hw *hw = &adapter->hw;
2758 rctl = rd32(E1000_RCTL);
2760 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2761 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2763 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2764 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2767 * enable stripping of CRC. It's unlikely this will break BMC
2768 * redirection as it did with e1000. Newer features require
2769 * that the HW strips the CRC.
2771 rctl |= E1000_RCTL_SECRC;
2773 /* disable store bad packets and clear size bits. */
2774 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2776 /* enable LPE to prevent packets larger than max_frame_size */
2777 rctl |= E1000_RCTL_LPE;
2779 /* disable queue 0 to prevent tail write w/o re-config */
2780 wr32(E1000_RXDCTL(0), 0);
2782 /* Attention!!! For SR-IOV PF driver operations you must enable
2783 * queue drop for all VF and PF queues to prevent head of line blocking
2784 * if an un-trusted VF does not provide descriptors to hardware.
2786 if (adapter->vfs_allocated_count) {
2787 /* set all queue drop enable bits */
2788 wr32(E1000_QDE, ALL_QUEUES);
2791 wr32(E1000_RCTL, rctl);
2794 static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2797 struct e1000_hw *hw = &adapter->hw;
2800 /* if it isn't the PF check to see if VFs are enabled and
2801 * increase the size to support vlan tags */
2802 if (vfn < adapter->vfs_allocated_count &&
2803 adapter->vf_data[vfn].vlans_enabled)
2804 size += VLAN_TAG_SIZE;
2806 vmolr = rd32(E1000_VMOLR(vfn));
2807 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2808 vmolr |= size | E1000_VMOLR_LPE;
2809 wr32(E1000_VMOLR(vfn), vmolr);
2815 * igb_rlpml_set - set maximum receive packet size
2816 * @adapter: board private structure
2818 * Configure maximum receivable packet size.
2820 static void igb_rlpml_set(struct igb_adapter *adapter)
2822 u32 max_frame_size = adapter->max_frame_size;
2823 struct e1000_hw *hw = &adapter->hw;
2824 u16 pf_id = adapter->vfs_allocated_count;
2827 max_frame_size += VLAN_TAG_SIZE;
2829 /* if vfs are enabled we set RLPML to the largest possible request
2830 * size and set the VMOLR RLPML to the size we need */
2832 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
2833 max_frame_size = MAX_JUMBO_FRAME_SIZE;
2836 wr32(E1000_RLPML, max_frame_size);
2839 static inline void igb_set_vmolr(struct igb_adapter *adapter,
2842 struct e1000_hw *hw = &adapter->hw;
2846 * This register exists only on 82576 and newer so if we are older then
2847 * we should exit and do nothing
2849 if (hw->mac.type < e1000_82576)
2852 vmolr = rd32(E1000_VMOLR(vfn));
2853 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2855 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2857 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
2859 /* clear all bits that might not be set */
2860 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
2862 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
2863 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
2865 * for VMDq only allow the VFs and pool 0 to accept broadcast and
2868 if (vfn <= adapter->vfs_allocated_count)
2869 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
2871 wr32(E1000_VMOLR(vfn), vmolr);
2875 * igb_configure_rx_ring - Configure a receive ring after Reset
2876 * @adapter: board private structure
2877 * @ring: receive ring to be configured
2879 * Configure the Rx unit of the MAC after a reset.
2881 void igb_configure_rx_ring(struct igb_adapter *adapter,
2882 struct igb_ring *ring)
2884 struct e1000_hw *hw = &adapter->hw;
2885 u64 rdba = ring->dma;
2886 int reg_idx = ring->reg_idx;
2889 /* disable the queue */
2890 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2891 wr32(E1000_RXDCTL(reg_idx),
2892 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
2894 /* Set DMA base address registers */
2895 wr32(E1000_RDBAL(reg_idx),
2896 rdba & 0x00000000ffffffffULL);
2897 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
2898 wr32(E1000_RDLEN(reg_idx),
2899 ring->count * sizeof(union e1000_adv_rx_desc));
2901 /* initialize head and tail */
2902 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
2903 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
2904 writel(0, ring->head);
2905 writel(0, ring->tail);
2907 /* set descriptor configuration */
2908 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
2909 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
2910 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
2911 #if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
2912 srrctl |= IGB_RXBUFFER_16384 >>
2913 E1000_SRRCTL_BSIZEPKT_SHIFT;
2915 srrctl |= (PAGE_SIZE / 2) >>
2916 E1000_SRRCTL_BSIZEPKT_SHIFT;
2918 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2920 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
2921 E1000_SRRCTL_BSIZEPKT_SHIFT;
2922 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
2924 if (hw->mac.type == e1000_82580)
2925 srrctl |= E1000_SRRCTL_TIMESTAMP;
2926 /* Only set Drop Enable if we are supporting multiple queues */
2927 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
2928 srrctl |= E1000_SRRCTL_DROP_EN;
2930 wr32(E1000_SRRCTL(reg_idx), srrctl);
2932 /* set filtering for VMDQ pools */
2933 igb_set_vmolr(adapter, reg_idx & 0x7, true);
2935 /* enable receive descriptor fetching */
2936 rxdctl = rd32(E1000_RXDCTL(reg_idx));
2937 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
2938 rxdctl &= 0xFFF00000;
2939 rxdctl |= IGB_RX_PTHRESH;
2940 rxdctl |= IGB_RX_HTHRESH << 8;
2941 rxdctl |= IGB_RX_WTHRESH << 16;
2942 wr32(E1000_RXDCTL(reg_idx), rxdctl);
2946 * igb_configure_rx - Configure receive Unit after Reset
2947 * @adapter: board private structure
2949 * Configure the Rx unit of the MAC after a reset.
2951 static void igb_configure_rx(struct igb_adapter *adapter)
2955 /* set UTA to appropriate mode */
2956 igb_set_uta(adapter);
2958 /* set the correct pool for the PF default MAC address in entry 0 */
2959 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
2960 adapter->vfs_allocated_count);
2962 /* Setup the HW Rx Head and Tail Descriptor Pointers and
2963 * the Base and Length of the Rx Descriptor Ring */
2964 for (i = 0; i < adapter->num_rx_queues; i++)
2965 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
2969 * igb_free_tx_resources - Free Tx Resources per Queue
2970 * @tx_ring: Tx descriptor ring for a specific queue
2972 * Free all transmit software resources
2974 void igb_free_tx_resources(struct igb_ring *tx_ring)
2976 igb_clean_tx_ring(tx_ring);
2978 vfree(tx_ring->buffer_info);
2979 tx_ring->buffer_info = NULL;
2981 /* if not set, then don't free */
2985 dma_free_coherent(tx_ring->dev, tx_ring->size,
2986 tx_ring->desc, tx_ring->dma);
2988 tx_ring->desc = NULL;
2992 * igb_free_all_tx_resources - Free Tx Resources for All Queues
2993 * @adapter: board private structure
2995 * Free all transmit software resources
2997 static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3001 for (i = 0; i < adapter->num_tx_queues; i++)
3002 igb_free_tx_resources(adapter->tx_ring[i]);
3005 void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3006 struct igb_buffer *buffer_info)
3008 if (buffer_info->dma) {
3009 if (buffer_info->mapped_as_page)
3010 dma_unmap_page(tx_ring->dev,
3012 buffer_info->length,
3015 dma_unmap_single(tx_ring->dev,
3017 buffer_info->length,
3019 buffer_info->dma = 0;
3021 if (buffer_info->skb) {
3022 dev_kfree_skb_any(buffer_info->skb);
3023 buffer_info->skb = NULL;
3025 buffer_info->time_stamp = 0;
3026 buffer_info->length = 0;
3027 buffer_info->next_to_watch = 0;
3028 buffer_info->mapped_as_page = false;
3032 * igb_clean_tx_ring - Free Tx Buffers
3033 * @tx_ring: ring to be cleaned
3035 static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3037 struct igb_buffer *buffer_info;
3041 if (!tx_ring->buffer_info)
3043 /* Free all the Tx ring sk_buffs */
3045 for (i = 0; i < tx_ring->count; i++) {
3046 buffer_info = &tx_ring->buffer_info[i];
3047 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3050 size = sizeof(struct igb_buffer) * tx_ring->count;
3051 memset(tx_ring->buffer_info, 0, size);
3053 /* Zero out the descriptor ring */
3054 memset(tx_ring->desc, 0, tx_ring->size);
3056 tx_ring->next_to_use = 0;
3057 tx_ring->next_to_clean = 0;
3061 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3062 * @adapter: board private structure
3064 static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3068 for (i = 0; i < adapter->num_tx_queues; i++)
3069 igb_clean_tx_ring(adapter->tx_ring[i]);
3073 * igb_free_rx_resources - Free Rx Resources
3074 * @rx_ring: ring to clean the resources from
3076 * Free all receive software resources
3078 void igb_free_rx_resources(struct igb_ring *rx_ring)
3080 igb_clean_rx_ring(rx_ring);
3082 vfree(rx_ring->buffer_info);
3083 rx_ring->buffer_info = NULL;
3085 /* if not set, then don't free */
3089 dma_free_coherent(rx_ring->dev, rx_ring->size,
3090 rx_ring->desc, rx_ring->dma);
3092 rx_ring->desc = NULL;
3096 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3097 * @adapter: board private structure
3099 * Free all receive software resources
3101 static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3105 for (i = 0; i < adapter->num_rx_queues; i++)
3106 igb_free_rx_resources(adapter->rx_ring[i]);
3110 * igb_clean_rx_ring - Free Rx Buffers per Queue
3111 * @rx_ring: ring to free buffers from
3113 static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3115 struct igb_buffer *buffer_info;
3119 if (!rx_ring->buffer_info)
3122 /* Free all the Rx ring sk_buffs */
3123 for (i = 0; i < rx_ring->count; i++) {
3124 buffer_info = &rx_ring->buffer_info[i];
3125 if (buffer_info->dma) {
3126 dma_unmap_single(rx_ring->dev,
3128 rx_ring->rx_buffer_len,
3130 buffer_info->dma = 0;
3133 if (buffer_info->skb) {
3134 dev_kfree_skb(buffer_info->skb);
3135 buffer_info->skb = NULL;
3137 if (buffer_info->page_dma) {
3138 dma_unmap_page(rx_ring->dev,
3139 buffer_info->page_dma,
3142 buffer_info->page_dma = 0;
3144 if (buffer_info->page) {
3145 put_page(buffer_info->page);
3146 buffer_info->page = NULL;
3147 buffer_info->page_offset = 0;
3151 size = sizeof(struct igb_buffer) * rx_ring->count;
3152 memset(rx_ring->buffer_info, 0, size);
3154 /* Zero out the descriptor ring */
3155 memset(rx_ring->desc, 0, rx_ring->size);
3157 rx_ring->next_to_clean = 0;
3158 rx_ring->next_to_use = 0;
3162 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3163 * @adapter: board private structure
3165 static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3169 for (i = 0; i < adapter->num_rx_queues; i++)
3170 igb_clean_rx_ring(adapter->rx_ring[i]);
3174 * igb_set_mac - Change the Ethernet Address of the NIC
3175 * @netdev: network interface device structure
3176 * @p: pointer to an address structure
3178 * Returns 0 on success, negative on failure
3180 static int igb_set_mac(struct net_device *netdev, void *p)
3182 struct igb_adapter *adapter = netdev_priv(netdev);
3183 struct e1000_hw *hw = &adapter->hw;
3184 struct sockaddr *addr = p;
3186 if (!is_valid_ether_addr(addr->sa_data))
3187 return -EADDRNOTAVAIL;
3189 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3190 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3192 /* set the correct pool for the new PF MAC address in entry 0 */
3193 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3194 adapter->vfs_allocated_count);
3200 * igb_write_mc_addr_list - write multicast addresses to MTA
3201 * @netdev: network interface device structure
3203 * Writes multicast address list to the MTA hash table.
3204 * Returns: -ENOMEM on failure
3205 * 0 on no addresses written
3206 * X on writing X addresses to MTA
3208 static int igb_write_mc_addr_list(struct net_device *netdev)
3210 struct igb_adapter *adapter = netdev_priv(netdev);
3211 struct e1000_hw *hw = &adapter->hw;
3212 struct netdev_hw_addr *ha;
3216 if (netdev_mc_empty(netdev)) {
3217 /* nothing to program, so clear mc list */
3218 igb_update_mc_addr_list(hw, NULL, 0);
3219 igb_restore_vf_multicasts(adapter);
3223 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3227 /* The shared function expects a packed array of only addresses. */
3229 netdev_for_each_mc_addr(ha, netdev)
3230 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3232 igb_update_mc_addr_list(hw, mta_list, i);
3235 return netdev_mc_count(netdev);
3239 * igb_write_uc_addr_list - write unicast addresses to RAR table
3240 * @netdev: network interface device structure
3242 * Writes unicast address list to the RAR table.
3243 * Returns: -ENOMEM on failure/insufficient address space
3244 * 0 on no addresses written
3245 * X on writing X addresses to the RAR table
3247 static int igb_write_uc_addr_list(struct net_device *netdev)
3249 struct igb_adapter *adapter = netdev_priv(netdev);
3250 struct e1000_hw *hw = &adapter->hw;
3251 unsigned int vfn = adapter->vfs_allocated_count;
3252 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3255 /* return ENOMEM indicating insufficient memory for addresses */
3256 if (netdev_uc_count(netdev) > rar_entries)
3259 if (!netdev_uc_empty(netdev) && rar_entries) {
3260 struct netdev_hw_addr *ha;
3262 netdev_for_each_uc_addr(ha, netdev) {
3265 igb_rar_set_qsel(adapter, ha->addr,
3271 /* write the addresses in reverse order to avoid write combining */
3272 for (; rar_entries > 0 ; rar_entries--) {
3273 wr32(E1000_RAH(rar_entries), 0);
3274 wr32(E1000_RAL(rar_entries), 0);
3282 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3283 * @netdev: network interface device structure
3285 * The set_rx_mode entry point is called whenever the unicast or multicast
3286 * address lists or the network interface flags are updated. This routine is
3287 * responsible for configuring the hardware for proper unicast, multicast,
3288 * promiscuous mode, and all-multi behavior.
3290 static void igb_set_rx_mode(struct net_device *netdev)
3292 struct igb_adapter *adapter = netdev_priv(netdev);
3293 struct e1000_hw *hw = &adapter->hw;
3294 unsigned int vfn = adapter->vfs_allocated_count;
3295 u32 rctl, vmolr = 0;
3298 /* Check for Promiscuous and All Multicast modes */
3299 rctl = rd32(E1000_RCTL);
3301 /* clear the effected bits */
3302 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3304 if (netdev->flags & IFF_PROMISC) {
3305 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3306 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3308 if (netdev->flags & IFF_ALLMULTI) {
3309 rctl |= E1000_RCTL_MPE;
3310 vmolr |= E1000_VMOLR_MPME;
3313 * Write addresses to the MTA, if the attempt fails
3314 * then we should just turn on promiscous mode so
3315 * that we can at least receive multicast traffic
3317 count = igb_write_mc_addr_list(netdev);
3319 rctl |= E1000_RCTL_MPE;
3320 vmolr |= E1000_VMOLR_MPME;
3322 vmolr |= E1000_VMOLR_ROMPE;
3326 * Write addresses to available RAR registers, if there is not
3327 * sufficient space to store all the addresses then enable
3328 * unicast promiscous mode
3330 count = igb_write_uc_addr_list(netdev);
3332 rctl |= E1000_RCTL_UPE;
3333 vmolr |= E1000_VMOLR_ROPE;
3335 rctl |= E1000_RCTL_VFE;
3337 wr32(E1000_RCTL, rctl);
3340 * In order to support SR-IOV and eventually VMDq it is necessary to set
3341 * the VMOLR to enable the appropriate modes. Without this workaround
3342 * we will have issues with VLAN tag stripping not being done for frames
3343 * that are only arriving because we are the default pool
3345 if (hw->mac.type < e1000_82576)
3348 vmolr |= rd32(E1000_VMOLR(vfn)) &
3349 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3350 wr32(E1000_VMOLR(vfn), vmolr);
3351 igb_restore_vf_multicasts(adapter);
3354 /* Need to wait a few seconds after link up to get diagnostic information from
3356 static void igb_update_phy_info(unsigned long data)
3358 struct igb_adapter *adapter = (struct igb_adapter *) data;
3359 igb_get_phy_info(&adapter->hw);
3363 * igb_has_link - check shared code for link and determine up/down
3364 * @adapter: pointer to driver private info
3366 bool igb_has_link(struct igb_adapter *adapter)
3368 struct e1000_hw *hw = &adapter->hw;
3369 bool link_active = false;
3372 /* get_link_status is set on LSC (link status) interrupt or
3373 * rx sequence error interrupt. get_link_status will stay
3374 * false until the e1000_check_for_link establishes link
3375 * for copper adapters ONLY
3377 switch (hw->phy.media_type) {
3378 case e1000_media_type_copper:
3379 if (hw->mac.get_link_status) {
3380 ret_val = hw->mac.ops.check_for_link(hw);
3381 link_active = !hw->mac.get_link_status;
3386 case e1000_media_type_internal_serdes:
3387 ret_val = hw->mac.ops.check_for_link(hw);
3388 link_active = hw->mac.serdes_has_link;
3391 case e1000_media_type_unknown:
3399 * igb_watchdog - Timer Call-back
3400 * @data: pointer to adapter cast into an unsigned long
3402 static void igb_watchdog(unsigned long data)
3404 struct igb_adapter *adapter = (struct igb_adapter *)data;
3405 /* Do the rest outside of interrupt context */
3406 schedule_work(&adapter->watchdog_task);
3409 static void igb_watchdog_task(struct work_struct *work)
3411 struct igb_adapter *adapter = container_of(work,
3414 struct e1000_hw *hw = &adapter->hw;
3415 struct net_device *netdev = adapter->netdev;
3419 link = igb_has_link(adapter);
3421 if (!netif_carrier_ok(netdev)) {
3423 hw->mac.ops.get_speed_and_duplex(hw,
3424 &adapter->link_speed,
3425 &adapter->link_duplex);
3427 ctrl = rd32(E1000_CTRL);
3428 /* Links status message must follow this format */
3429 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
3430 "Flow Control: %s\n",
3432 adapter->link_speed,
3433 adapter->link_duplex == FULL_DUPLEX ?
3434 "Full Duplex" : "Half Duplex",
3435 ((ctrl & E1000_CTRL_TFCE) &&
3436 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3437 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3438 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
3440 /* adjust timeout factor according to speed/duplex */
3441 adapter->tx_timeout_factor = 1;
3442 switch (adapter->link_speed) {
3444 adapter->tx_timeout_factor = 14;
3447 /* maybe add some timeout factor ? */
3451 netif_carrier_on(netdev);
3453 igb_ping_all_vfs(adapter);
3455 /* link state has changed, schedule phy info update */
3456 if (!test_bit(__IGB_DOWN, &adapter->state))
3457 mod_timer(&adapter->phy_info_timer,
3458 round_jiffies(jiffies + 2 * HZ));
3461 if (netif_carrier_ok(netdev)) {
3462 adapter->link_speed = 0;
3463 adapter->link_duplex = 0;
3464 /* Links status message must follow this format */
3465 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3467 netif_carrier_off(netdev);
3469 igb_ping_all_vfs(adapter);
3471 /* link state has changed, schedule phy info update */
3472 if (!test_bit(__IGB_DOWN, &adapter->state))
3473 mod_timer(&adapter->phy_info_timer,
3474 round_jiffies(jiffies + 2 * HZ));
3478 igb_update_stats(adapter);
3480 for (i = 0; i < adapter->num_tx_queues; i++) {
3481 struct igb_ring *tx_ring = adapter->tx_ring[i];
3482 if (!netif_carrier_ok(netdev)) {
3483 /* We've lost link, so the controller stops DMA,
3484 * but we've got queued Tx work that's never going
3485 * to get done, so reset controller to flush Tx.
3486 * (Do the reset outside of interrupt context). */
3487 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3488 adapter->tx_timeout_count++;
3489 schedule_work(&adapter->reset_task);
3490 /* return immediately since reset is imminent */
3495 /* Force detection of hung controller every watchdog period */
3496 tx_ring->detect_tx_hung = true;
3499 /* Cause software interrupt to ensure rx ring is cleaned */
3500 if (adapter->msix_entries) {
3502 for (i = 0; i < adapter->num_q_vectors; i++) {
3503 struct igb_q_vector *q_vector = adapter->q_vector[i];
3504 eics |= q_vector->eims_value;
3506 wr32(E1000_EICS, eics);
3508 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3511 /* Reset the timer */
3512 if (!test_bit(__IGB_DOWN, &adapter->state))
3513 mod_timer(&adapter->watchdog_timer,
3514 round_jiffies(jiffies + 2 * HZ));
3517 enum latency_range {
3521 latency_invalid = 255
3525 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3527 * Stores a new ITR value based on strictly on packet size. This
3528 * algorithm is less sophisticated than that used in igb_update_itr,
3529 * due to the difficulty of synchronizing statistics across multiple
3530 * receive rings. The divisors and thresholds used by this fuction
3531 * were determined based on theoretical maximum wire speed and testing
3532 * data, in order to minimize response time while increasing bulk
3534 * This functionality is controlled by the InterruptThrottleRate module
3535 * parameter (see igb_param.c)
3536 * NOTE: This function is called only when operating in a multiqueue
3537 * receive environment.
3538 * @q_vector: pointer to q_vector
3540 static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3542 int new_val = q_vector->itr_val;
3543 int avg_wire_size = 0;
3544 struct igb_adapter *adapter = q_vector->adapter;
3546 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3547 * ints/sec - ITR timer value of 120 ticks.
3549 if (adapter->link_speed != SPEED_1000) {
3554 if (q_vector->rx_ring && q_vector->rx_ring->total_packets) {
3555 struct igb_ring *ring = q_vector->rx_ring;
3556 avg_wire_size = ring->total_bytes / ring->total_packets;
3559 if (q_vector->tx_ring && q_vector->tx_ring->total_packets) {
3560 struct igb_ring *ring = q_vector->tx_ring;
3561 avg_wire_size = max_t(u32, avg_wire_size,
3562 (ring->total_bytes /
3563 ring->total_packets));
3566 /* if avg_wire_size isn't set no work was done */
3570 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3571 avg_wire_size += 24;
3573 /* Don't starve jumbo frames */
3574 avg_wire_size = min(avg_wire_size, 3000);
3576 /* Give a little boost to mid-size frames */
3577 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3578 new_val = avg_wire_size / 3;
3580 new_val = avg_wire_size / 2;
3582 /* when in itr mode 3 do not exceed 20K ints/sec */
3583 if (adapter->rx_itr_setting == 3 && new_val < 196)
3587 if (new_val != q_vector->itr_val) {
3588 q_vector->itr_val = new_val;
3589 q_vector->set_itr = 1;
3592 if (q_vector->rx_ring) {
3593 q_vector->rx_ring->total_bytes = 0;
3594 q_vector->rx_ring->total_packets = 0;
3596 if (q_vector->tx_ring) {
3597 q_vector->tx_ring->total_bytes = 0;
3598 q_vector->tx_ring->total_packets = 0;
3603 * igb_update_itr - update the dynamic ITR value based on statistics
3604 * Stores a new ITR value based on packets and byte
3605 * counts during the last interrupt. The advantage of per interrupt
3606 * computation is faster updates and more accurate ITR for the current
3607 * traffic pattern. Constants in this function were computed
3608 * based on theoretical maximum wire speed and thresholds were set based
3609 * on testing data as well as attempting to minimize response time
3610 * while increasing bulk throughput.
3611 * this functionality is controlled by the InterruptThrottleRate module
3612 * parameter (see igb_param.c)
3613 * NOTE: These calculations are only valid when operating in a single-
3614 * queue environment.
3615 * @adapter: pointer to adapter
3616 * @itr_setting: current q_vector->itr_val
3617 * @packets: the number of packets during this measurement interval
3618 * @bytes: the number of bytes during this measurement interval
3620 static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3621 int packets, int bytes)
3623 unsigned int retval = itr_setting;
3626 goto update_itr_done;
3628 switch (itr_setting) {
3629 case lowest_latency:
3630 /* handle TSO and jumbo frames */
3631 if (bytes/packets > 8000)
3632 retval = bulk_latency;
3633 else if ((packets < 5) && (bytes > 512))
3634 retval = low_latency;
3636 case low_latency: /* 50 usec aka 20000 ints/s */
3637 if (bytes > 10000) {
3638 /* this if handles the TSO accounting */
3639 if (bytes/packets > 8000) {
3640 retval = bulk_latency;
3641 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3642 retval = bulk_latency;
3643 } else if ((packets > 35)) {
3644 retval = lowest_latency;
3646 } else if (bytes/packets > 2000) {
3647 retval = bulk_latency;
3648 } else if (packets <= 2 && bytes < 512) {
3649 retval = lowest_latency;
3652 case bulk_latency: /* 250 usec aka 4000 ints/s */
3653 if (bytes > 25000) {
3655 retval = low_latency;
3656 } else if (bytes < 1500) {
3657 retval = low_latency;
3666 static void igb_set_itr(struct igb_adapter *adapter)
3668 struct igb_q_vector *q_vector = adapter->q_vector[0];
3670 u32 new_itr = q_vector->itr_val;
3672 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3673 if (adapter->link_speed != SPEED_1000) {
3679 adapter->rx_itr = igb_update_itr(adapter,
3681 q_vector->rx_ring->total_packets,
3682 q_vector->rx_ring->total_bytes);
3684 adapter->tx_itr = igb_update_itr(adapter,
3686 q_vector->tx_ring->total_packets,
3687 q_vector->tx_ring->total_bytes);
3688 current_itr = max(adapter->rx_itr, adapter->tx_itr);
3690 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3691 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
3692 current_itr = low_latency;
3694 switch (current_itr) {
3695 /* counts and packets in update_itr are dependent on these numbers */
3696 case lowest_latency:
3697 new_itr = 56; /* aka 70,000 ints/sec */
3700 new_itr = 196; /* aka 20,000 ints/sec */
3703 new_itr = 980; /* aka 4,000 ints/sec */
3710 q_vector->rx_ring->total_bytes = 0;
3711 q_vector->rx_ring->total_packets = 0;
3712 q_vector->tx_ring->total_bytes = 0;
3713 q_vector->tx_ring->total_packets = 0;
3715 if (new_itr != q_vector->itr_val) {
3716 /* this attempts to bias the interrupt rate towards Bulk
3717 * by adding intermediate steps when interrupt rate is
3719 new_itr = new_itr > q_vector->itr_val ?
3720 max((new_itr * q_vector->itr_val) /
3721 (new_itr + (q_vector->itr_val >> 2)),
3724 /* Don't write the value here; it resets the adapter's
3725 * internal timer, and causes us to delay far longer than
3726 * we should between interrupts. Instead, we write the ITR
3727 * value at the beginning of the next interrupt so the timing
3728 * ends up being correct.
3730 q_vector->itr_val = new_itr;
3731 q_vector->set_itr = 1;
3735 #define IGB_TX_FLAGS_CSUM 0x00000001
3736 #define IGB_TX_FLAGS_VLAN 0x00000002
3737 #define IGB_TX_FLAGS_TSO 0x00000004
3738 #define IGB_TX_FLAGS_IPV4 0x00000008
3739 #define IGB_TX_FLAGS_TSTAMP 0x00000010
3740 #define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3741 #define IGB_TX_FLAGS_VLAN_SHIFT 16
3743 static inline int igb_tso_adv(struct igb_ring *tx_ring,
3744 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3746 struct e1000_adv_tx_context_desc *context_desc;
3749 struct igb_buffer *buffer_info;
3750 u32 info = 0, tu_cmd = 0;
3754 if (skb_header_cloned(skb)) {
3755 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3760 l4len = tcp_hdrlen(skb);
3763 if (skb->protocol == htons(ETH_P_IP)) {
3764 struct iphdr *iph = ip_hdr(skb);
3767 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3771 } else if (skb_is_gso_v6(skb)) {
3772 ipv6_hdr(skb)->payload_len = 0;
3773 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3774 &ipv6_hdr(skb)->daddr,
3778 i = tx_ring->next_to_use;
3780 buffer_info = &tx_ring->buffer_info[i];
3781 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3782 /* VLAN MACLEN IPLEN */
3783 if (tx_flags & IGB_TX_FLAGS_VLAN)
3784 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3785 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3786 *hdr_len += skb_network_offset(skb);
3787 info |= skb_network_header_len(skb);
3788 *hdr_len += skb_network_header_len(skb);
3789 context_desc->vlan_macip_lens = cpu_to_le32(info);
3791 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3792 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3794 if (skb->protocol == htons(ETH_P_IP))
3795 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3796 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3798 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3801 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
3802 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
3804 /* For 82575, context index must be unique per ring. */
3805 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3806 mss_l4len_idx |= tx_ring->reg_idx << 4;
3808 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
3809 context_desc->seqnum_seed = 0;
3811 buffer_info->time_stamp = jiffies;
3812 buffer_info->next_to_watch = i;
3813 buffer_info->dma = 0;
3815 if (i == tx_ring->count)
3818 tx_ring->next_to_use = i;
3823 static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
3824 struct sk_buff *skb, u32 tx_flags)
3826 struct e1000_adv_tx_context_desc *context_desc;
3827 struct device *dev = tx_ring->dev;
3828 struct igb_buffer *buffer_info;
3829 u32 info = 0, tu_cmd = 0;
3832 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
3833 (tx_flags & IGB_TX_FLAGS_VLAN)) {
3834 i = tx_ring->next_to_use;
3835 buffer_info = &tx_ring->buffer_info[i];
3836 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
3838 if (tx_flags & IGB_TX_FLAGS_VLAN)
3839 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
3841 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
3842 if (skb->ip_summed == CHECKSUM_PARTIAL)
3843 info |= skb_network_header_len(skb);
3845 context_desc->vlan_macip_lens = cpu_to_le32(info);
3847 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
3849 if (skb->ip_summed == CHECKSUM_PARTIAL) {
3852 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
3853 const struct vlan_ethhdr *vhdr =
3854 (const struct vlan_ethhdr*)skb->data;
3856 protocol = vhdr->h_vlan_encapsulated_proto;
3858 protocol = skb->protocol;
3862 case cpu_to_be16(ETH_P_IP):
3863 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
3864 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
3865 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3866 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
3867 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3869 case cpu_to_be16(ETH_P_IPV6):
3870 /* XXX what about other V6 headers?? */
3871 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3872 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
3873 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
3874 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
3877 if (unlikely(net_ratelimit()))
3879 "partial checksum but proto=%x!\n",
3885 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
3886 context_desc->seqnum_seed = 0;
3887 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
3888 context_desc->mss_l4len_idx =
3889 cpu_to_le32(tx_ring->reg_idx << 4);
3891 buffer_info->time_stamp = jiffies;
3892 buffer_info->next_to_watch = i;
3893 buffer_info->dma = 0;
3896 if (i == tx_ring->count)
3898 tx_ring->next_to_use = i;
3905 #define IGB_MAX_TXD_PWR 16
3906 #define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
3908 static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
3911 struct igb_buffer *buffer_info;
3912 struct device *dev = tx_ring->dev;
3913 unsigned int hlen = skb_headlen(skb);
3914 unsigned int count = 0, i;
3916 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
3918 i = tx_ring->next_to_use;
3920 buffer_info = &tx_ring->buffer_info[i];
3921 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
3922 buffer_info->length = hlen;
3923 /* set time_stamp *before* dma to help avoid a possible race */
3924 buffer_info->time_stamp = jiffies;
3925 buffer_info->next_to_watch = i;
3926 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
3928 if (dma_mapping_error(dev, buffer_info->dma))
3931 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
3932 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
3933 unsigned int len = frag->size;
3937 if (i == tx_ring->count)
3940 buffer_info = &tx_ring->buffer_info[i];
3941 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
3942 buffer_info->length = len;
3943 buffer_info->time_stamp = jiffies;
3944 buffer_info->next_to_watch = i;
3945 buffer_info->mapped_as_page = true;
3946 buffer_info->dma = dma_map_page(dev,
3951 if (dma_mapping_error(dev, buffer_info->dma))
3956 tx_ring->buffer_info[i].skb = skb;
3957 tx_ring->buffer_info[i].tx_flags = skb_shinfo(skb)->tx_flags;
3958 /* multiply data chunks by size of headers */
3959 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
3960 tx_ring->buffer_info[i].gso_segs = gso_segs;
3961 tx_ring->buffer_info[first].next_to_watch = i;
3966 dev_err(dev, "TX DMA map failed\n");
3968 /* clear timestamp and dma mappings for failed buffer_info mapping */
3969 buffer_info->dma = 0;
3970 buffer_info->time_stamp = 0;
3971 buffer_info->length = 0;
3972 buffer_info->next_to_watch = 0;
3973 buffer_info->mapped_as_page = false;
3975 /* clear timestamp and dma mappings for remaining portion of packet */
3980 buffer_info = &tx_ring->buffer_info[i];
3981 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3987 static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
3988 u32 tx_flags, int count, u32 paylen,
3991 union e1000_adv_tx_desc *tx_desc;
3992 struct igb_buffer *buffer_info;
3993 u32 olinfo_status = 0, cmd_type_len;
3994 unsigned int i = tx_ring->next_to_use;
3996 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
3997 E1000_ADVTXD_DCMD_DEXT);
3999 if (tx_flags & IGB_TX_FLAGS_VLAN)
4000 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
4002 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4003 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
4005 if (tx_flags & IGB_TX_FLAGS_TSO) {
4006 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4008 /* insert tcp checksum */
4009 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4011 /* insert ip checksum */
4012 if (tx_flags & IGB_TX_FLAGS_IPV4)
4013 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4015 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4016 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4019 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4020 (tx_flags & (IGB_TX_FLAGS_CSUM |
4022 IGB_TX_FLAGS_VLAN)))
4023 olinfo_status |= tx_ring->reg_idx << 4;
4025 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4028 buffer_info = &tx_ring->buffer_info[i];
4029 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4030 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4031 tx_desc->read.cmd_type_len =
4032 cpu_to_le32(cmd_type_len | buffer_info->length);
4033 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4036 if (i == tx_ring->count)
4038 } while (count > 0);
4040 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
4041 /* Force memory writes to complete before letting h/w
4042 * know there are new descriptors to fetch. (Only
4043 * applicable for weak-ordered memory model archs,
4044 * such as IA-64). */
4047 tx_ring->next_to_use = i;
4048 writel(i, tx_ring->tail);
4049 /* we need this if more than one processor can write to our tail
4050 * at a time, it syncronizes IO on IA64/Altix systems */
4054 static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4056 struct net_device *netdev = tx_ring->netdev;
4058 netif_stop_subqueue(netdev, tx_ring->queue_index);
4060 /* Herbert's original patch had:
4061 * smp_mb__after_netif_stop_queue();
4062 * but since that doesn't exist yet, just open code it. */
4065 /* We need to check again in a case another CPU has just
4066 * made room available. */
4067 if (igb_desc_unused(tx_ring) < size)
4071 netif_wake_subqueue(netdev, tx_ring->queue_index);
4072 tx_ring->tx_stats.restart_queue++;
4076 static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4078 if (igb_desc_unused(tx_ring) >= size)
4080 return __igb_maybe_stop_tx(tx_ring, size);
4083 netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4084 struct igb_ring *tx_ring)
4086 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
4092 /* need: 1 descriptor per page,
4093 * + 2 desc gap to keep tail from touching head,
4094 * + 1 desc for skb->data,
4095 * + 1 desc for context descriptor,
4096 * otherwise try next time */
4097 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4098 /* this is a hard error */
4099 return NETDEV_TX_BUSY;
4102 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4103 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4104 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4107 if (vlan_tx_tag_present(skb) && adapter->vlgrp) {
4108 tx_flags |= IGB_TX_FLAGS_VLAN;
4109 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4112 if (skb->protocol == htons(ETH_P_IP))
4113 tx_flags |= IGB_TX_FLAGS_IPV4;
4115 first = tx_ring->next_to_use;
4116 if (skb_is_gso(skb)) {
4117 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
4120 dev_kfree_skb_any(skb);
4121 return NETDEV_TX_OK;
4126 tx_flags |= IGB_TX_FLAGS_TSO;
4127 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
4128 (skb->ip_summed == CHECKSUM_PARTIAL))
4129 tx_flags |= IGB_TX_FLAGS_CSUM;
4132 * count reflects descriptors mapped, if 0 or less then mapping error
4133 * has occured and we need to rewind the descriptor queue
4135 count = igb_tx_map_adv(tx_ring, skb, first);
4137 dev_kfree_skb_any(skb);
4138 tx_ring->buffer_info[first].time_stamp = 0;
4139 tx_ring->next_to_use = first;
4140 return NETDEV_TX_OK;
4143 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4145 /* Make sure there is space in the ring for the next send. */
4146 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4148 return NETDEV_TX_OK;
4151 static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4152 struct net_device *netdev)
4154 struct igb_adapter *adapter = netdev_priv(netdev);
4155 struct igb_ring *tx_ring;
4158 if (test_bit(__IGB_DOWN, &adapter->state)) {
4159 dev_kfree_skb_any(skb);
4160 return NETDEV_TX_OK;
4163 if (skb->len <= 0) {
4164 dev_kfree_skb_any(skb);
4165 return NETDEV_TX_OK;
4168 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
4169 tx_ring = adapter->multi_tx_table[r_idx];
4171 /* This goes back to the question of how to logically map a tx queue
4172 * to a flow. Right now, performance is impacted slightly negatively
4173 * if using multiple tx queues. If the stack breaks away from a
4174 * single qdisc implementation, we can look at this again. */
4175 return igb_xmit_frame_ring_adv(skb, tx_ring);
4179 * igb_tx_timeout - Respond to a Tx Hang
4180 * @netdev: network interface device structure
4182 static void igb_tx_timeout(struct net_device *netdev)
4184 struct igb_adapter *adapter = netdev_priv(netdev);
4185 struct e1000_hw *hw = &adapter->hw;
4187 /* Do the reset outside of interrupt context */
4188 adapter->tx_timeout_count++;
4190 if (hw->mac.type == e1000_82580)
4191 hw->dev_spec._82575.global_device_reset = true;
4193 schedule_work(&adapter->reset_task);
4195 (adapter->eims_enable_mask & ~adapter->eims_other));
4198 static void igb_reset_task(struct work_struct *work)
4200 struct igb_adapter *adapter;
4201 adapter = container_of(work, struct igb_adapter, reset_task);
4204 netdev_err(adapter->netdev, "Reset adapter\n");
4205 igb_reinit_locked(adapter);
4209 * igb_get_stats - Get System Network Statistics
4210 * @netdev: network interface device structure
4212 * Returns the address of the device statistics structure.
4213 * The statistics are actually updated from the timer callback.
4215 static struct net_device_stats *igb_get_stats(struct net_device *netdev)
4217 /* only return the current stats */
4218 return &netdev->stats;
4222 * igb_change_mtu - Change the Maximum Transfer Unit
4223 * @netdev: network interface device structure
4224 * @new_mtu: new value for maximum frame size
4226 * Returns 0 on success, negative on failure
4228 static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4230 struct igb_adapter *adapter = netdev_priv(netdev);
4231 struct pci_dev *pdev = adapter->pdev;
4232 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4233 u32 rx_buffer_len, i;
4235 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
4236 dev_err(&pdev->dev, "Invalid MTU setting\n");
4240 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
4241 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
4245 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4248 /* igb_down has a dependency on max_frame_size */
4249 adapter->max_frame_size = max_frame;
4251 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4252 * means we reserve 2 more, this pushes us to allocate from the next
4254 * i.e. RXBUFFER_2048 --> size-4096 slab
4257 if (adapter->hw.mac.type == e1000_82580)
4258 max_frame += IGB_TS_HDR_LEN;
4260 if (max_frame <= IGB_RXBUFFER_1024)
4261 rx_buffer_len = IGB_RXBUFFER_1024;
4262 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4263 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
4265 rx_buffer_len = IGB_RXBUFFER_128;
4267 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4268 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4269 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4271 if ((adapter->hw.mac.type == e1000_82580) &&
4272 (rx_buffer_len == IGB_RXBUFFER_128))
4273 rx_buffer_len += IGB_RXBUFFER_64;
4275 if (netif_running(netdev))
4278 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
4279 netdev->mtu, new_mtu);
4280 netdev->mtu = new_mtu;
4282 for (i = 0; i < adapter->num_rx_queues; i++)
4283 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4285 if (netif_running(netdev))
4290 clear_bit(__IGB_RESETTING, &adapter->state);
4296 * igb_update_stats - Update the board statistics counters
4297 * @adapter: board private structure
4300 void igb_update_stats(struct igb_adapter *adapter)
4302 struct net_device_stats *net_stats = igb_get_stats(adapter->netdev);
4303 struct e1000_hw *hw = &adapter->hw;
4304 struct pci_dev *pdev = adapter->pdev;
4310 #define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4313 * Prevent stats update while adapter is being reset, or if the pci
4314 * connection is down.
4316 if (adapter->link_speed == 0)
4318 if (pci_channel_offline(pdev))
4323 for (i = 0; i < adapter->num_rx_queues; i++) {
4324 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
4325 struct igb_ring *ring = adapter->rx_ring[i];
4326 ring->rx_stats.drops += rqdpc_tmp;
4327 net_stats->rx_fifo_errors += rqdpc_tmp;
4328 bytes += ring->rx_stats.bytes;
4329 packets += ring->rx_stats.packets;
4332 net_stats->rx_bytes = bytes;
4333 net_stats->rx_packets = packets;
4337 for (i = 0; i < adapter->num_tx_queues; i++) {
4338 struct igb_ring *ring = adapter->tx_ring[i];
4339 bytes += ring->tx_stats.bytes;
4340 packets += ring->tx_stats.packets;
4342 net_stats->tx_bytes = bytes;
4343 net_stats->tx_packets = packets;
4345 /* read stats registers */
4346 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4347 adapter->stats.gprc += rd32(E1000_GPRC);
4348 adapter->stats.gorc += rd32(E1000_GORCL);
4349 rd32(E1000_GORCH); /* clear GORCL */
4350 adapter->stats.bprc += rd32(E1000_BPRC);
4351 adapter->stats.mprc += rd32(E1000_MPRC);
4352 adapter->stats.roc += rd32(E1000_ROC);
4354 adapter->stats.prc64 += rd32(E1000_PRC64);
4355 adapter->stats.prc127 += rd32(E1000_PRC127);
4356 adapter->stats.prc255 += rd32(E1000_PRC255);
4357 adapter->stats.prc511 += rd32(E1000_PRC511);
4358 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4359 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4360 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4361 adapter->stats.sec += rd32(E1000_SEC);
4363 mpc = rd32(E1000_MPC);
4364 adapter->stats.mpc += mpc;
4365 net_stats->rx_fifo_errors += mpc;
4366 adapter->stats.scc += rd32(E1000_SCC);
4367 adapter->stats.ecol += rd32(E1000_ECOL);
4368 adapter->stats.mcc += rd32(E1000_MCC);
4369 adapter->stats.latecol += rd32(E1000_LATECOL);
4370 adapter->stats.dc += rd32(E1000_DC);
4371 adapter->stats.rlec += rd32(E1000_RLEC);
4372 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4373 adapter->stats.xontxc += rd32(E1000_XONTXC);
4374 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4375 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4376 adapter->stats.fcruc += rd32(E1000_FCRUC);
4377 adapter->stats.gptc += rd32(E1000_GPTC);
4378 adapter->stats.gotc += rd32(E1000_GOTCL);
4379 rd32(E1000_GOTCH); /* clear GOTCL */
4380 adapter->stats.rnbc += rd32(E1000_RNBC);
4381 adapter->stats.ruc += rd32(E1000_RUC);
4382 adapter->stats.rfc += rd32(E1000_RFC);
4383 adapter->stats.rjc += rd32(E1000_RJC);
4384 adapter->stats.tor += rd32(E1000_TORH);
4385 adapter->stats.tot += rd32(E1000_TOTH);
4386 adapter->stats.tpr += rd32(E1000_TPR);
4388 adapter->stats.ptc64 += rd32(E1000_PTC64);
4389 adapter->stats.ptc127 += rd32(E1000_PTC127);
4390 adapter->stats.ptc255 += rd32(E1000_PTC255);
4391 adapter->stats.ptc511 += rd32(E1000_PTC511);
4392 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4393 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4395 adapter->stats.mptc += rd32(E1000_MPTC);
4396 adapter->stats.bptc += rd32(E1000_BPTC);
4398 adapter->stats.tpt += rd32(E1000_TPT);
4399 adapter->stats.colc += rd32(E1000_COLC);
4401 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4402 /* read internal phy specific stats */
4403 reg = rd32(E1000_CTRL_EXT);
4404 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4405 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4406 adapter->stats.tncrs += rd32(E1000_TNCRS);
4409 adapter->stats.tsctc += rd32(E1000_TSCTC);
4410 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4412 adapter->stats.iac += rd32(E1000_IAC);
4413 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4414 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4415 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4416 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4417 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4418 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4419 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4420 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4422 /* Fill out the OS statistics structure */
4423 net_stats->multicast = adapter->stats.mprc;
4424 net_stats->collisions = adapter->stats.colc;
4428 /* RLEC on some newer hardware can be incorrect so build
4429 * our own version based on RUC and ROC */
4430 net_stats->rx_errors = adapter->stats.rxerrc +
4431 adapter->stats.crcerrs + adapter->stats.algnerrc +
4432 adapter->stats.ruc + adapter->stats.roc +
4433 adapter->stats.cexterr;
4434 net_stats->rx_length_errors = adapter->stats.ruc +
4436 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4437 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4438 net_stats->rx_missed_errors = adapter->stats.mpc;
4441 net_stats->tx_errors = adapter->stats.ecol +
4442 adapter->stats.latecol;
4443 net_stats->tx_aborted_errors = adapter->stats.ecol;
4444 net_stats->tx_window_errors = adapter->stats.latecol;
4445 net_stats->tx_carrier_errors = adapter->stats.tncrs;
4447 /* Tx Dropped needs to be maintained elsewhere */
4450 if (hw->phy.media_type == e1000_media_type_copper) {
4451 if ((adapter->link_speed == SPEED_1000) &&
4452 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4453 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4454 adapter->phy_stats.idle_errors += phy_tmp;
4458 /* Management Stats */
4459 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4460 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4461 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4464 static irqreturn_t igb_msix_other(int irq, void *data)
4466 struct igb_adapter *adapter = data;
4467 struct e1000_hw *hw = &adapter->hw;
4468 u32 icr = rd32(E1000_ICR);
4469 /* reading ICR causes bit 31 of EICR to be cleared */
4471 if (icr & E1000_ICR_DRSTA)
4472 schedule_work(&adapter->reset_task);
4474 if (icr & E1000_ICR_DOUTSYNC) {
4475 /* HW is reporting DMA is out of sync */
4476 adapter->stats.doosync++;
4479 /* Check for a mailbox event */
4480 if (icr & E1000_ICR_VMMB)
4481 igb_msg_task(adapter);
4483 if (icr & E1000_ICR_LSC) {
4484 hw->mac.get_link_status = 1;
4485 /* guard against interrupt when we're going down */
4486 if (!test_bit(__IGB_DOWN, &adapter->state))
4487 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4490 if (adapter->vfs_allocated_count)
4491 wr32(E1000_IMS, E1000_IMS_LSC |
4493 E1000_IMS_DOUTSYNC);
4495 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
4496 wr32(E1000_EIMS, adapter->eims_other);
4501 static void igb_write_itr(struct igb_q_vector *q_vector)
4503 struct igb_adapter *adapter = q_vector->adapter;
4504 u32 itr_val = q_vector->itr_val & 0x7FFC;
4506 if (!q_vector->set_itr)
4512 if (adapter->hw.mac.type == e1000_82575)
4513 itr_val |= itr_val << 16;
4515 itr_val |= 0x8000000;
4517 writel(itr_val, q_vector->itr_register);
4518 q_vector->set_itr = 0;
4521 static irqreturn_t igb_msix_ring(int irq, void *data)
4523 struct igb_q_vector *q_vector = data;
4525 /* Write the ITR value calculated from the previous interrupt. */
4526 igb_write_itr(q_vector);
4528 napi_schedule(&q_vector->napi);
4533 #ifdef CONFIG_IGB_DCA
4534 static void igb_update_dca(struct igb_q_vector *q_vector)
4536 struct igb_adapter *adapter = q_vector->adapter;
4537 struct e1000_hw *hw = &adapter->hw;
4538 int cpu = get_cpu();
4540 if (q_vector->cpu == cpu)
4543 if (q_vector->tx_ring) {
4544 int q = q_vector->tx_ring->reg_idx;
4545 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4546 if (hw->mac.type == e1000_82575) {
4547 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4548 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4550 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4551 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4552 E1000_DCA_TXCTRL_CPUID_SHIFT;
4554 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4555 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4557 if (q_vector->rx_ring) {
4558 int q = q_vector->rx_ring->reg_idx;
4559 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4560 if (hw->mac.type == e1000_82575) {
4561 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4562 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4564 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4565 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4566 E1000_DCA_RXCTRL_CPUID_SHIFT;
4568 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4569 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4570 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4571 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4573 q_vector->cpu = cpu;
4578 static void igb_setup_dca(struct igb_adapter *adapter)
4580 struct e1000_hw *hw = &adapter->hw;
4583 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4586 /* Always use CB2 mode, difference is masked in the CB driver. */
4587 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4589 for (i = 0; i < adapter->num_q_vectors; i++) {
4590 adapter->q_vector[i]->cpu = -1;
4591 igb_update_dca(adapter->q_vector[i]);
4595 static int __igb_notify_dca(struct device *dev, void *data)
4597 struct net_device *netdev = dev_get_drvdata(dev);
4598 struct igb_adapter *adapter = netdev_priv(netdev);
4599 struct pci_dev *pdev = adapter->pdev;
4600 struct e1000_hw *hw = &adapter->hw;
4601 unsigned long event = *(unsigned long *)data;
4604 case DCA_PROVIDER_ADD:
4605 /* if already enabled, don't do it again */
4606 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4608 if (dca_add_requester(dev) == 0) {
4609 adapter->flags |= IGB_FLAG_DCA_ENABLED;
4610 dev_info(&pdev->dev, "DCA enabled\n");
4611 igb_setup_dca(adapter);
4614 /* Fall Through since DCA is disabled. */
4615 case DCA_PROVIDER_REMOVE:
4616 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4617 /* without this a class_device is left
4618 * hanging around in the sysfs model */
4619 dca_remove_requester(dev);
4620 dev_info(&pdev->dev, "DCA disabled\n");
4621 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4622 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4630 static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4635 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4638 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4640 #endif /* CONFIG_IGB_DCA */
4642 static void igb_ping_all_vfs(struct igb_adapter *adapter)
4644 struct e1000_hw *hw = &adapter->hw;
4648 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4649 ping = E1000_PF_CONTROL_MSG;
4650 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4651 ping |= E1000_VT_MSGTYPE_CTS;
4652 igb_write_mbx(hw, &ping, 1, i);
4656 static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4658 struct e1000_hw *hw = &adapter->hw;
4659 u32 vmolr = rd32(E1000_VMOLR(vf));
4660 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4662 vf_data->flags |= ~(IGB_VF_FLAG_UNI_PROMISC |
4663 IGB_VF_FLAG_MULTI_PROMISC);
4664 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4666 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4667 vmolr |= E1000_VMOLR_MPME;
4668 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4671 * if we have hashes and we are clearing a multicast promisc
4672 * flag we need to write the hashes to the MTA as this step
4673 * was previously skipped
4675 if (vf_data->num_vf_mc_hashes > 30) {
4676 vmolr |= E1000_VMOLR_MPME;
4677 } else if (vf_data->num_vf_mc_hashes) {
4679 vmolr |= E1000_VMOLR_ROMPE;
4680 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4681 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4685 wr32(E1000_VMOLR(vf), vmolr);
4687 /* there are flags left unprocessed, likely not supported */
4688 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4695 static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4696 u32 *msgbuf, u32 vf)
4698 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4699 u16 *hash_list = (u16 *)&msgbuf[1];
4700 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4703 /* salt away the number of multicast addresses assigned
4704 * to this VF for later use to restore when the PF multi cast
4707 vf_data->num_vf_mc_hashes = n;
4709 /* only up to 30 hash values supported */
4713 /* store the hashes for later use */
4714 for (i = 0; i < n; i++)
4715 vf_data->vf_mc_hashes[i] = hash_list[i];
4717 /* Flush and reset the mta with the new values */
4718 igb_set_rx_mode(adapter->netdev);
4723 static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4725 struct e1000_hw *hw = &adapter->hw;
4726 struct vf_data_storage *vf_data;
4729 for (i = 0; i < adapter->vfs_allocated_count; i++) {
4730 u32 vmolr = rd32(E1000_VMOLR(i));
4731 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4733 vf_data = &adapter->vf_data[i];
4735 if ((vf_data->num_vf_mc_hashes > 30) ||
4736 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
4737 vmolr |= E1000_VMOLR_MPME;
4738 } else if (vf_data->num_vf_mc_hashes) {
4739 vmolr |= E1000_VMOLR_ROMPE;
4740 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4741 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4743 wr32(E1000_VMOLR(i), vmolr);
4747 static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
4749 struct e1000_hw *hw = &adapter->hw;
4750 u32 pool_mask, reg, vid;
4753 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4755 /* Find the vlan filter for this id */
4756 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4757 reg = rd32(E1000_VLVF(i));
4759 /* remove the vf from the pool */
4762 /* if pool is empty then remove entry from vfta */
4763 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
4764 (reg & E1000_VLVF_VLANID_ENABLE)) {
4766 vid = reg & E1000_VLVF_VLANID_MASK;
4767 igb_vfta_set(hw, vid, false);
4770 wr32(E1000_VLVF(i), reg);
4773 adapter->vf_data[vf].vlans_enabled = 0;
4776 static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
4778 struct e1000_hw *hw = &adapter->hw;
4781 /* The vlvf table only exists on 82576 hardware and newer */
4782 if (hw->mac.type < e1000_82576)
4785 /* we only need to do this if VMDq is enabled */
4786 if (!adapter->vfs_allocated_count)
4789 /* Find the vlan filter for this id */
4790 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4791 reg = rd32(E1000_VLVF(i));
4792 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
4793 vid == (reg & E1000_VLVF_VLANID_MASK))
4798 if (i == E1000_VLVF_ARRAY_SIZE) {
4799 /* Did not find a matching VLAN ID entry that was
4800 * enabled. Search for a free filter entry, i.e.
4801 * one without the enable bit set
4803 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
4804 reg = rd32(E1000_VLVF(i));
4805 if (!(reg & E1000_VLVF_VLANID_ENABLE))
4809 if (i < E1000_VLVF_ARRAY_SIZE) {
4810 /* Found an enabled/available entry */
4811 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
4813 /* if !enabled we need to set this up in vfta */
4814 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
4815 /* add VID to filter table */
4816 igb_vfta_set(hw, vid, true);
4817 reg |= E1000_VLVF_VLANID_ENABLE;
4819 reg &= ~E1000_VLVF_VLANID_MASK;
4821 wr32(E1000_VLVF(i), reg);
4823 /* do not modify RLPML for PF devices */
4824 if (vf >= adapter->vfs_allocated_count)
4827 if (!adapter->vf_data[vf].vlans_enabled) {
4829 reg = rd32(E1000_VMOLR(vf));
4830 size = reg & E1000_VMOLR_RLPML_MASK;
4832 reg &= ~E1000_VMOLR_RLPML_MASK;
4834 wr32(E1000_VMOLR(vf), reg);
4837 adapter->vf_data[vf].vlans_enabled++;
4841 if (i < E1000_VLVF_ARRAY_SIZE) {
4842 /* remove vf from the pool */
4843 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
4844 /* if pool is empty then remove entry from vfta */
4845 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
4847 igb_vfta_set(hw, vid, false);
4849 wr32(E1000_VLVF(i), reg);
4851 /* do not modify RLPML for PF devices */
4852 if (vf >= adapter->vfs_allocated_count)
4855 adapter->vf_data[vf].vlans_enabled--;
4856 if (!adapter->vf_data[vf].vlans_enabled) {
4858 reg = rd32(E1000_VMOLR(vf));
4859 size = reg & E1000_VMOLR_RLPML_MASK;
4861 reg &= ~E1000_VMOLR_RLPML_MASK;
4863 wr32(E1000_VMOLR(vf), reg);
4870 static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
4872 struct e1000_hw *hw = &adapter->hw;
4875 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
4877 wr32(E1000_VMVIR(vf), 0);
4880 static int igb_ndo_set_vf_vlan(struct net_device *netdev,
4881 int vf, u16 vlan, u8 qos)
4884 struct igb_adapter *adapter = netdev_priv(netdev);
4886 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
4889 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
4892 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
4893 igb_set_vmolr(adapter, vf, !vlan);
4894 adapter->vf_data[vf].pf_vlan = vlan;
4895 adapter->vf_data[vf].pf_qos = qos;
4896 dev_info(&adapter->pdev->dev,
4897 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
4898 if (test_bit(__IGB_DOWN, &adapter->state)) {
4899 dev_warn(&adapter->pdev->dev,
4900 "The VF VLAN has been set,"
4901 " but the PF device is not up.\n");
4902 dev_warn(&adapter->pdev->dev,
4903 "Bring the PF device up before"
4904 " attempting to use the VF device.\n");
4907 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
4909 igb_set_vmvir(adapter, vlan, vf);
4910 igb_set_vmolr(adapter, vf, true);
4911 adapter->vf_data[vf].pf_vlan = 0;
4912 adapter->vf_data[vf].pf_qos = 0;
4918 static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4920 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4921 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
4923 return igb_vlvf_set(adapter, vid, add, vf);
4926 static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
4929 adapter->vf_data[vf].flags &= ~(IGB_VF_FLAG_PF_SET_MAC);
4930 adapter->vf_data[vf].last_nack = jiffies;
4932 /* reset offloads to defaults */
4933 igb_set_vmolr(adapter, vf, true);
4935 /* reset vlans for device */
4936 igb_clear_vf_vfta(adapter, vf);
4937 if (adapter->vf_data[vf].pf_vlan)
4938 igb_ndo_set_vf_vlan(adapter->netdev, vf,
4939 adapter->vf_data[vf].pf_vlan,
4940 adapter->vf_data[vf].pf_qos);
4942 igb_clear_vf_vfta(adapter, vf);
4944 /* reset multicast table array for vf */
4945 adapter->vf_data[vf].num_vf_mc_hashes = 0;
4947 /* Flush and reset the mta with the new values */
4948 igb_set_rx_mode(adapter->netdev);
4951 static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
4953 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4955 /* generate a new mac address as we were hotplug removed/added */
4956 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
4957 random_ether_addr(vf_mac);
4959 /* process remaining reset events */
4960 igb_vf_reset(adapter, vf);
4963 static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
4965 struct e1000_hw *hw = &adapter->hw;
4966 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
4967 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
4969 u8 *addr = (u8 *)(&msgbuf[1]);
4971 /* process all the same items cleared in a function level reset */
4972 igb_vf_reset(adapter, vf);
4974 /* set vf mac address */
4975 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
4977 /* enable transmit and receive for vf */
4978 reg = rd32(E1000_VFTE);
4979 wr32(E1000_VFTE, reg | (1 << vf));
4980 reg = rd32(E1000_VFRE);
4981 wr32(E1000_VFRE, reg | (1 << vf));
4983 adapter->vf_data[vf].flags = IGB_VF_FLAG_CTS;
4985 /* reply to reset with ack and vf mac address */
4986 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
4987 memcpy(addr, vf_mac, 6);
4988 igb_write_mbx(hw, msgbuf, 3, vf);
4991 static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
4994 * The VF MAC Address is stored in a packed array of bytes
4995 * starting at the second 32 bit word of the msg array
4997 unsigned char *addr = (char *)&msg[1];
5000 if (is_valid_ether_addr(addr))
5001 err = igb_set_vf_mac(adapter, vf, addr);
5006 static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5008 struct e1000_hw *hw = &adapter->hw;
5009 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5010 u32 msg = E1000_VT_MSGTYPE_NACK;
5012 /* if device isn't clear to send it shouldn't be reading either */
5013 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5014 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
5015 igb_write_mbx(hw, &msg, 1, vf);
5016 vf_data->last_nack = jiffies;
5020 static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
5022 struct pci_dev *pdev = adapter->pdev;
5023 u32 msgbuf[E1000_VFMAILBOX_SIZE];
5024 struct e1000_hw *hw = &adapter->hw;
5025 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5028 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
5031 /* if receive failed revoke VF CTS stats and restart init */
5032 dev_err(&pdev->dev, "Error receiving message from VF\n");
5033 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5034 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5039 /* this is a message we already processed, do nothing */
5040 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
5044 * until the vf completes a reset it should not be
5045 * allowed to start any configuration.
5048 if (msgbuf[0] == E1000_VF_RESET) {
5049 igb_vf_reset_msg(adapter, vf);
5053 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
5054 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5060 switch ((msgbuf[0] & 0xFFFF)) {
5061 case E1000_VF_SET_MAC_ADDR:
5062 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5064 case E1000_VF_SET_PROMISC:
5065 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5067 case E1000_VF_SET_MULTICAST:
5068 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5070 case E1000_VF_SET_LPE:
5071 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5073 case E1000_VF_SET_VLAN:
5074 if (adapter->vf_data[vf].pf_vlan)
5077 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
5080 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
5085 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5087 /* notify the VF of the results of what it sent us */
5089 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5091 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5093 igb_write_mbx(hw, msgbuf, 1, vf);
5096 static void igb_msg_task(struct igb_adapter *adapter)
5098 struct e1000_hw *hw = &adapter->hw;
5101 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5102 /* process any reset requests */
5103 if (!igb_check_for_rst(hw, vf))
5104 igb_vf_reset_event(adapter, vf);
5106 /* process any messages pending */
5107 if (!igb_check_for_msg(hw, vf))
5108 igb_rcv_msg_from_vf(adapter, vf);
5110 /* process any acks */
5111 if (!igb_check_for_ack(hw, vf))
5112 igb_rcv_ack_from_vf(adapter, vf);
5117 * igb_set_uta - Set unicast filter table address
5118 * @adapter: board private structure
5120 * The unicast table address is a register array of 32-bit registers.
5121 * The table is meant to be used in a way similar to how the MTA is used
5122 * however due to certain limitations in the hardware it is necessary to
5123 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscous
5124 * enable bit to allow vlan tag stripping when promiscous mode is enabled
5126 static void igb_set_uta(struct igb_adapter *adapter)
5128 struct e1000_hw *hw = &adapter->hw;
5131 /* The UTA table only exists on 82576 hardware and newer */
5132 if (hw->mac.type < e1000_82576)
5135 /* we only need to do this if VMDq is enabled */
5136 if (!adapter->vfs_allocated_count)
5139 for (i = 0; i < hw->mac.uta_reg_count; i++)
5140 array_wr32(E1000_UTA, i, ~0);
5144 * igb_intr_msi - Interrupt Handler
5145 * @irq: interrupt number
5146 * @data: pointer to a network interface device structure
5148 static irqreturn_t igb_intr_msi(int irq, void *data)
5150 struct igb_adapter *adapter = data;
5151 struct igb_q_vector *q_vector = adapter->q_vector[0];
5152 struct e1000_hw *hw = &adapter->hw;
5153 /* read ICR disables interrupts using IAM */
5154 u32 icr = rd32(E1000_ICR);
5156 igb_write_itr(q_vector);
5158 if (icr & E1000_ICR_DRSTA)
5159 schedule_work(&adapter->reset_task);
5161 if (icr & E1000_ICR_DOUTSYNC) {
5162 /* HW is reporting DMA is out of sync */
5163 adapter->stats.doosync++;
5166 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5167 hw->mac.get_link_status = 1;
5168 if (!test_bit(__IGB_DOWN, &adapter->state))
5169 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5172 napi_schedule(&q_vector->napi);
5178 * igb_intr - Legacy Interrupt Handler
5179 * @irq: interrupt number
5180 * @data: pointer to a network interface device structure
5182 static irqreturn_t igb_intr(int irq, void *data)
5184 struct igb_adapter *adapter = data;
5185 struct igb_q_vector *q_vector = adapter->q_vector[0];
5186 struct e1000_hw *hw = &adapter->hw;
5187 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5188 * need for the IMC write */
5189 u32 icr = rd32(E1000_ICR);
5191 return IRQ_NONE; /* Not our interrupt */
5193 igb_write_itr(q_vector);
5195 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5196 * not set, then the adapter didn't send an interrupt */
5197 if (!(icr & E1000_ICR_INT_ASSERTED))
5200 if (icr & E1000_ICR_DRSTA)
5201 schedule_work(&adapter->reset_task);
5203 if (icr & E1000_ICR_DOUTSYNC) {
5204 /* HW is reporting DMA is out of sync */
5205 adapter->stats.doosync++;
5208 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5209 hw->mac.get_link_status = 1;
5210 /* guard against interrupt when we're going down */
5211 if (!test_bit(__IGB_DOWN, &adapter->state))
5212 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5215 napi_schedule(&q_vector->napi);
5220 static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
5222 struct igb_adapter *adapter = q_vector->adapter;
5223 struct e1000_hw *hw = &adapter->hw;
5225 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5226 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
5227 if (!adapter->msix_entries)
5228 igb_set_itr(adapter);
5230 igb_update_ring_itr(q_vector);
5233 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5234 if (adapter->msix_entries)
5235 wr32(E1000_EIMS, q_vector->eims_value);
5237 igb_irq_enable(adapter);
5242 * igb_poll - NAPI Rx polling callback
5243 * @napi: napi polling structure
5244 * @budget: count of how many packets we should handle
5246 static int igb_poll(struct napi_struct *napi, int budget)
5248 struct igb_q_vector *q_vector = container_of(napi,
5249 struct igb_q_vector,
5251 int tx_clean_complete = 1, work_done = 0;
5253 #ifdef CONFIG_IGB_DCA
5254 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5255 igb_update_dca(q_vector);
5257 if (q_vector->tx_ring)
5258 tx_clean_complete = igb_clean_tx_irq(q_vector);
5260 if (q_vector->rx_ring)
5261 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5263 if (!tx_clean_complete)
5266 /* If not enough Rx work done, exit the polling mode */
5267 if (work_done < budget) {
5268 napi_complete(napi);
5269 igb_ring_irq_enable(q_vector);
5276 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
5277 * @adapter: board private structure
5278 * @shhwtstamps: timestamp structure to update
5279 * @regval: unsigned 64bit system time value.
5281 * We need to convert the system time value stored in the RX/TXSTMP registers
5282 * into a hwtstamp which can be used by the upper level timestamping functions
5284 static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5285 struct skb_shared_hwtstamps *shhwtstamps,
5291 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5292 * 24 to match clock shift we setup earlier.
5294 if (adapter->hw.mac.type == e1000_82580)
5295 regval <<= IGB_82580_TSYNC_SHIFT;
5297 ns = timecounter_cyc2time(&adapter->clock, regval);
5298 timecompare_update(&adapter->compare, ns);
5299 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5300 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5301 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5305 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5306 * @q_vector: pointer to q_vector containing needed info
5307 * @buffer: pointer to igb_buffer structure
5309 * If we were asked to do hardware stamping and such a time stamp is
5310 * available, then it must have been for this skb here because we only
5311 * allow only one such packet into the queue.
5313 static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
5315 struct igb_adapter *adapter = q_vector->adapter;
5316 struct e1000_hw *hw = &adapter->hw;
5317 struct skb_shared_hwtstamps shhwtstamps;
5320 /* if skb does not support hw timestamp or TX stamp not valid exit */
5321 if (likely(!(buffer_info->tx_flags & SKBTX_HW_TSTAMP)) ||
5322 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5325 regval = rd32(E1000_TXSTMPL);
5326 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5328 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
5329 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
5333 * igb_clean_tx_irq - Reclaim resources after transmit completes
5334 * @q_vector: pointer to q_vector containing needed info
5335 * returns true if ring is completely cleaned
5337 static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
5339 struct igb_adapter *adapter = q_vector->adapter;
5340 struct igb_ring *tx_ring = q_vector->tx_ring;
5341 struct net_device *netdev = tx_ring->netdev;
5342 struct e1000_hw *hw = &adapter->hw;
5343 struct igb_buffer *buffer_info;
5344 union e1000_adv_tx_desc *tx_desc, *eop_desc;
5345 unsigned int total_bytes = 0, total_packets = 0;
5346 unsigned int i, eop, count = 0;
5347 bool cleaned = false;
5349 i = tx_ring->next_to_clean;
5350 eop = tx_ring->buffer_info[i].next_to_watch;
5351 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5353 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5354 (count < tx_ring->count)) {
5355 rmb(); /* read buffer_info after eop_desc status */
5356 for (cleaned = false; !cleaned; count++) {
5357 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
5358 buffer_info = &tx_ring->buffer_info[i];
5359 cleaned = (i == eop);
5361 if (buffer_info->skb) {
5362 total_bytes += buffer_info->bytecount;
5363 /* gso_segs is currently only valid for tcp */
5364 total_packets += buffer_info->gso_segs;
5365 igb_tx_hwtstamp(q_vector, buffer_info);
5368 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
5369 tx_desc->wb.status = 0;
5372 if (i == tx_ring->count)
5375 eop = tx_ring->buffer_info[i].next_to_watch;
5376 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5379 tx_ring->next_to_clean = i;
5381 if (unlikely(count &&
5382 netif_carrier_ok(netdev) &&
5383 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5384 /* Make sure that anybody stopping the queue after this
5385 * sees the new next_to_clean.
5388 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5389 !(test_bit(__IGB_DOWN, &adapter->state))) {
5390 netif_wake_subqueue(netdev, tx_ring->queue_index);
5391 tx_ring->tx_stats.restart_queue++;
5395 if (tx_ring->detect_tx_hung) {
5396 /* Detect a transmit hang in hardware, this serializes the
5397 * check with the clearing of time_stamp and movement of i */
5398 tx_ring->detect_tx_hung = false;
5399 if (tx_ring->buffer_info[i].time_stamp &&
5400 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
5401 (adapter->tx_timeout_factor * HZ)) &&
5402 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
5404 /* detected Tx unit hang */
5405 dev_err(tx_ring->dev,
5406 "Detected Tx Unit Hang\n"
5410 " next_to_use <%x>\n"
5411 " next_to_clean <%x>\n"
5412 "buffer_info[next_to_clean]\n"
5413 " time_stamp <%lx>\n"
5414 " next_to_watch <%x>\n"
5416 " desc.status <%x>\n",
5417 tx_ring->queue_index,
5418 readl(tx_ring->head),
5419 readl(tx_ring->tail),
5420 tx_ring->next_to_use,
5421 tx_ring->next_to_clean,
5422 tx_ring->buffer_info[eop].time_stamp,
5425 eop_desc->wb.status);
5426 netif_stop_subqueue(netdev, tx_ring->queue_index);
5429 tx_ring->total_bytes += total_bytes;
5430 tx_ring->total_packets += total_packets;
5431 tx_ring->tx_stats.bytes += total_bytes;
5432 tx_ring->tx_stats.packets += total_packets;
5433 return (count < tx_ring->count);
5437 * igb_receive_skb - helper function to handle rx indications
5438 * @q_vector: structure containing interrupt and ring information
5439 * @skb: packet to send up
5440 * @vlan_tag: vlan tag for packet
5442 static void igb_receive_skb(struct igb_q_vector *q_vector,
5443 struct sk_buff *skb,
5446 struct igb_adapter *adapter = q_vector->adapter;
5448 if (vlan_tag && adapter->vlgrp)
5449 vlan_gro_receive(&q_vector->napi, adapter->vlgrp,
5452 napi_gro_receive(&q_vector->napi, skb);
5455 static inline void igb_rx_checksum_adv(struct igb_ring *ring,
5456 u32 status_err, struct sk_buff *skb)
5458 skb_checksum_none_assert(skb);
5460 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
5461 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5462 (status_err & E1000_RXD_STAT_IXSM))
5465 /* TCP/UDP checksum error bit is set */
5467 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
5469 * work around errata with sctp packets where the TCPE aka
5470 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5471 * packets, (aka let the stack check the crc32c)
5473 if ((skb->len == 60) &&
5474 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM))
5475 ring->rx_stats.csum_err++;
5477 /* let the stack verify checksum errors */
5480 /* It must be a TCP or UDP packet with a valid checksum */
5481 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5482 skb->ip_summed = CHECKSUM_UNNECESSARY;
5484 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
5487 static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5488 struct sk_buff *skb)
5490 struct igb_adapter *adapter = q_vector->adapter;
5491 struct e1000_hw *hw = &adapter->hw;
5495 * If this bit is set, then the RX registers contain the time stamp. No
5496 * other packet will be time stamped until we read these registers, so
5497 * read the registers to make them available again. Because only one
5498 * packet can be time stamped at a time, we know that the register
5499 * values must belong to this one here and therefore we don't need to
5500 * compare any of the additional attributes stored for it.
5502 * If nothing went wrong, then it should have a shared tx_flags that we
5503 * can turn into a skb_shared_hwtstamps.
5505 if (staterr & E1000_RXDADV_STAT_TSIP) {
5506 u32 *stamp = (u32 *)skb->data;
5507 regval = le32_to_cpu(*(stamp + 2));
5508 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5509 skb_pull(skb, IGB_TS_HDR_LEN);
5511 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5514 regval = rd32(E1000_RXSTMPL);
5515 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5518 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5520 static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
5521 union e1000_adv_rx_desc *rx_desc)
5523 /* HW will not DMA in data larger than the given buffer, even if it
5524 * parses the (NFS, of course) header to be larger. In that case, it
5525 * fills the header buffer and spills the rest into the page.
5527 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5528 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5529 if (hlen > rx_ring->rx_buffer_len)
5530 hlen = rx_ring->rx_buffer_len;
5534 static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5535 int *work_done, int budget)
5537 struct igb_ring *rx_ring = q_vector->rx_ring;
5538 struct net_device *netdev = rx_ring->netdev;
5539 struct device *dev = rx_ring->dev;
5540 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5541 struct igb_buffer *buffer_info , *next_buffer;
5542 struct sk_buff *skb;
5543 bool cleaned = false;
5544 int cleaned_count = 0;
5545 int current_node = numa_node_id();
5546 unsigned int total_bytes = 0, total_packets = 0;
5552 i = rx_ring->next_to_clean;
5553 buffer_info = &rx_ring->buffer_info[i];
5554 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5555 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5557 while (staterr & E1000_RXD_STAT_DD) {
5558 if (*work_done >= budget)
5561 rmb(); /* read descriptor and rx_buffer_info after status DD */
5563 skb = buffer_info->skb;
5564 prefetch(skb->data - NET_IP_ALIGN);
5565 buffer_info->skb = NULL;
5568 if (i == rx_ring->count)
5571 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5573 next_buffer = &rx_ring->buffer_info[i];
5575 length = le16_to_cpu(rx_desc->wb.upper.length);
5579 if (buffer_info->dma) {
5580 dma_unmap_single(dev, buffer_info->dma,
5581 rx_ring->rx_buffer_len,
5583 buffer_info->dma = 0;
5584 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
5585 skb_put(skb, length);
5588 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
5592 dma_unmap_page(dev, buffer_info->page_dma,
5593 PAGE_SIZE / 2, DMA_FROM_DEVICE);
5594 buffer_info->page_dma = 0;
5596 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
5598 buffer_info->page_offset,
5601 if ((page_count(buffer_info->page) != 1) ||
5602 (page_to_nid(buffer_info->page) != current_node))
5603 buffer_info->page = NULL;
5605 get_page(buffer_info->page);
5608 skb->data_len += length;
5609 skb->truesize += length;
5612 if (!(staterr & E1000_RXD_STAT_EOP)) {
5613 buffer_info->skb = next_buffer->skb;
5614 buffer_info->dma = next_buffer->dma;
5615 next_buffer->skb = skb;
5616 next_buffer->dma = 0;
5620 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5621 dev_kfree_skb_irq(skb);
5625 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5626 igb_rx_hwtstamp(q_vector, staterr, skb);
5627 total_bytes += skb->len;
5630 igb_rx_checksum_adv(rx_ring, staterr, skb);
5632 skb->protocol = eth_type_trans(skb, netdev);
5633 skb_record_rx_queue(skb, rx_ring->queue_index);
5635 vlan_tag = ((staterr & E1000_RXD_STAT_VP) ?
5636 le16_to_cpu(rx_desc->wb.upper.vlan) : 0);
5638 igb_receive_skb(q_vector, skb, vlan_tag);
5641 rx_desc->wb.upper.status_error = 0;
5643 /* return some buffers to hardware, one at a time is too slow */
5644 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
5645 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5649 /* use prefetched values */
5651 buffer_info = next_buffer;
5652 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5655 rx_ring->next_to_clean = i;
5656 cleaned_count = igb_desc_unused(rx_ring);
5659 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5661 rx_ring->total_packets += total_packets;
5662 rx_ring->total_bytes += total_bytes;
5663 rx_ring->rx_stats.packets += total_packets;
5664 rx_ring->rx_stats.bytes += total_bytes;
5669 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5670 * @adapter: address of board private structure
5672 void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
5674 struct net_device *netdev = rx_ring->netdev;
5675 union e1000_adv_rx_desc *rx_desc;
5676 struct igb_buffer *buffer_info;
5677 struct sk_buff *skb;
5681 i = rx_ring->next_to_use;
5682 buffer_info = &rx_ring->buffer_info[i];
5684 bufsz = rx_ring->rx_buffer_len;
5686 while (cleaned_count--) {
5687 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5689 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
5690 if (!buffer_info->page) {
5691 buffer_info->page = netdev_alloc_page(netdev);
5692 if (!buffer_info->page) {
5693 rx_ring->rx_stats.alloc_failed++;
5696 buffer_info->page_offset = 0;
5698 buffer_info->page_offset ^= PAGE_SIZE / 2;
5700 buffer_info->page_dma =
5701 dma_map_page(rx_ring->dev, buffer_info->page,
5702 buffer_info->page_offset,
5705 if (dma_mapping_error(rx_ring->dev,
5706 buffer_info->page_dma)) {
5707 buffer_info->page_dma = 0;
5708 rx_ring->rx_stats.alloc_failed++;
5713 skb = buffer_info->skb;
5715 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
5717 rx_ring->rx_stats.alloc_failed++;
5721 buffer_info->skb = skb;
5723 if (!buffer_info->dma) {
5724 buffer_info->dma = dma_map_single(rx_ring->dev,
5728 if (dma_mapping_error(rx_ring->dev,
5729 buffer_info->dma)) {
5730 buffer_info->dma = 0;
5731 rx_ring->rx_stats.alloc_failed++;
5735 /* Refresh the desc even if buffer_addrs didn't change because
5736 * each write-back erases this info. */
5737 if (bufsz < IGB_RXBUFFER_1024) {
5738 rx_desc->read.pkt_addr =
5739 cpu_to_le64(buffer_info->page_dma);
5740 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
5742 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
5743 rx_desc->read.hdr_addr = 0;
5747 if (i == rx_ring->count)
5749 buffer_info = &rx_ring->buffer_info[i];
5753 if (rx_ring->next_to_use != i) {
5754 rx_ring->next_to_use = i;
5756 i = (rx_ring->count - 1);
5760 /* Force memory writes to complete before letting h/w
5761 * know there are new descriptors to fetch. (Only
5762 * applicable for weak-ordered memory model archs,
5763 * such as IA-64). */
5765 writel(i, rx_ring->tail);
5775 static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5777 struct igb_adapter *adapter = netdev_priv(netdev);
5778 struct mii_ioctl_data *data = if_mii(ifr);
5780 if (adapter->hw.phy.media_type != e1000_media_type_copper)
5785 data->phy_id = adapter->hw.phy.addr;
5788 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
5800 * igb_hwtstamp_ioctl - control hardware time stamping
5805 * Outgoing time stamping can be enabled and disabled. Play nice and
5806 * disable it when requested, although it shouldn't case any overhead
5807 * when no packet needs it. At most one packet in the queue may be
5808 * marked for time stamping, otherwise it would be impossible to tell
5809 * for sure to which packet the hardware time stamp belongs.
5811 * Incoming time stamping has to be configured via the hardware
5812 * filters. Not all combinations are supported, in particular event
5813 * type has to be specified. Matching the kind of event packet is
5814 * not supported, with the exception of "all V2 events regardless of
5818 static int igb_hwtstamp_ioctl(struct net_device *netdev,
5819 struct ifreq *ifr, int cmd)
5821 struct igb_adapter *adapter = netdev_priv(netdev);
5822 struct e1000_hw *hw = &adapter->hw;
5823 struct hwtstamp_config config;
5824 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
5825 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5826 u32 tsync_rx_cfg = 0;
5831 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
5834 /* reserved for future extensions */
5838 switch (config.tx_type) {
5839 case HWTSTAMP_TX_OFF:
5841 case HWTSTAMP_TX_ON:
5847 switch (config.rx_filter) {
5848 case HWTSTAMP_FILTER_NONE:
5851 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
5852 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
5853 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
5854 case HWTSTAMP_FILTER_ALL:
5856 * register TSYNCRXCFG must be set, therefore it is not
5857 * possible to time stamp both Sync and Delay_Req messages
5858 * => fall back to time stamping all packets
5860 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5861 config.rx_filter = HWTSTAMP_FILTER_ALL;
5863 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
5864 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5865 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
5868 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
5869 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
5870 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
5873 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
5874 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
5875 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5876 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
5879 config.rx_filter = HWTSTAMP_FILTER_SOME;
5881 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
5882 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
5883 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
5884 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
5887 config.rx_filter = HWTSTAMP_FILTER_SOME;
5889 case HWTSTAMP_FILTER_PTP_V2_EVENT:
5890 case HWTSTAMP_FILTER_PTP_V2_SYNC:
5891 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
5892 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
5893 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
5900 if (hw->mac.type == e1000_82575) {
5901 if (tsync_rx_ctl | tsync_tx_ctl)
5907 * Per-packet timestamping only works if all packets are
5908 * timestamped, so enable timestamping in all packets as
5909 * long as one rx filter was configured.
5911 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
5912 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
5913 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
5916 /* enable/disable TX */
5917 regval = rd32(E1000_TSYNCTXCTL);
5918 regval &= ~E1000_TSYNCTXCTL_ENABLED;
5919 regval |= tsync_tx_ctl;
5920 wr32(E1000_TSYNCTXCTL, regval);
5922 /* enable/disable RX */
5923 regval = rd32(E1000_TSYNCRXCTL);
5924 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
5925 regval |= tsync_rx_ctl;
5926 wr32(E1000_TSYNCRXCTL, regval);
5928 /* define which PTP packets are time stamped */
5929 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
5931 /* define ethertype filter for timestamped packets */
5934 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
5935 E1000_ETQF_1588 | /* enable timestamping */
5936 ETH_P_1588)); /* 1588 eth protocol type */
5938 wr32(E1000_ETQF(3), 0);
5940 #define PTP_PORT 319
5941 /* L4 Queue Filter[3]: filter by destination port and protocol */
5943 u32 ftqf = (IPPROTO_UDP /* UDP */
5944 | E1000_FTQF_VF_BP /* VF not compared */
5945 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
5946 | E1000_FTQF_MASK); /* mask all inputs */
5947 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
5949 wr32(E1000_IMIR(3), htons(PTP_PORT));
5950 wr32(E1000_IMIREXT(3),
5951 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
5952 if (hw->mac.type == e1000_82576) {
5953 /* enable source port check */
5954 wr32(E1000_SPQF(3), htons(PTP_PORT));
5955 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
5957 wr32(E1000_FTQF(3), ftqf);
5959 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
5963 adapter->hwtstamp_config = config;
5965 /* clear TX/RX time stamp registers, just to be sure */
5966 regval = rd32(E1000_TXSTMPH);
5967 regval = rd32(E1000_RXSTMPH);
5969 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
5979 static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
5985 return igb_mii_ioctl(netdev, ifr, cmd);
5987 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
5993 s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
5995 struct igb_adapter *adapter = hw->back;
5998 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6000 return -E1000_ERR_CONFIG;
6002 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6007 s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6009 struct igb_adapter *adapter = hw->back;
6012 cap_offset = pci_find_capability(adapter->pdev, PCI_CAP_ID_EXP);
6014 return -E1000_ERR_CONFIG;
6016 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6021 static void igb_vlan_rx_register(struct net_device *netdev,
6022 struct vlan_group *grp)
6024 struct igb_adapter *adapter = netdev_priv(netdev);
6025 struct e1000_hw *hw = &adapter->hw;
6028 igb_irq_disable(adapter);
6029 adapter->vlgrp = grp;
6032 /* enable VLAN tag insert/strip */
6033 ctrl = rd32(E1000_CTRL);
6034 ctrl |= E1000_CTRL_VME;
6035 wr32(E1000_CTRL, ctrl);
6037 /* Disable CFI check */
6038 rctl = rd32(E1000_RCTL);
6039 rctl &= ~E1000_RCTL_CFIEN;
6040 wr32(E1000_RCTL, rctl);
6042 /* disable VLAN tag insert/strip */
6043 ctrl = rd32(E1000_CTRL);
6044 ctrl &= ~E1000_CTRL_VME;
6045 wr32(E1000_CTRL, ctrl);
6048 igb_rlpml_set(adapter);
6050 if (!test_bit(__IGB_DOWN, &adapter->state))
6051 igb_irq_enable(adapter);
6054 static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6056 struct igb_adapter *adapter = netdev_priv(netdev);
6057 struct e1000_hw *hw = &adapter->hw;
6058 int pf_id = adapter->vfs_allocated_count;
6060 /* attempt to add filter to vlvf array */
6061 igb_vlvf_set(adapter, vid, true, pf_id);
6063 /* add the filter since PF can receive vlans w/o entry in vlvf */
6064 igb_vfta_set(hw, vid, true);
6067 static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6069 struct igb_adapter *adapter = netdev_priv(netdev);
6070 struct e1000_hw *hw = &adapter->hw;
6071 int pf_id = adapter->vfs_allocated_count;
6074 igb_irq_disable(adapter);
6075 vlan_group_set_device(adapter->vlgrp, vid, NULL);
6077 if (!test_bit(__IGB_DOWN, &adapter->state))
6078 igb_irq_enable(adapter);
6080 /* remove vlan from VLVF table array */
6081 err = igb_vlvf_set(adapter, vid, false, pf_id);
6083 /* if vid was not present in VLVF just remove it from table */
6085 igb_vfta_set(hw, vid, false);
6088 static void igb_restore_vlan(struct igb_adapter *adapter)
6090 igb_vlan_rx_register(adapter->netdev, adapter->vlgrp);
6092 if (adapter->vlgrp) {
6094 for (vid = 0; vid < VLAN_GROUP_ARRAY_LEN; vid++) {
6095 if (!vlan_group_get_device(adapter->vlgrp, vid))
6097 igb_vlan_rx_add_vid(adapter->netdev, vid);
6102 int igb_set_spd_dplx(struct igb_adapter *adapter, u16 spddplx)
6104 struct pci_dev *pdev = adapter->pdev;
6105 struct e1000_mac_info *mac = &adapter->hw.mac;
6110 case SPEED_10 + DUPLEX_HALF:
6111 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6113 case SPEED_10 + DUPLEX_FULL:
6114 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6116 case SPEED_100 + DUPLEX_HALF:
6117 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6119 case SPEED_100 + DUPLEX_FULL:
6120 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6122 case SPEED_1000 + DUPLEX_FULL:
6124 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6126 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6128 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6134 static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
6136 struct net_device *netdev = pci_get_drvdata(pdev);
6137 struct igb_adapter *adapter = netdev_priv(netdev);
6138 struct e1000_hw *hw = &adapter->hw;
6139 u32 ctrl, rctl, status;
6140 u32 wufc = adapter->wol;
6145 netif_device_detach(netdev);
6147 if (netif_running(netdev))
6150 igb_clear_interrupt_scheme(adapter);
6153 retval = pci_save_state(pdev);
6158 status = rd32(E1000_STATUS);
6159 if (status & E1000_STATUS_LU)
6160 wufc &= ~E1000_WUFC_LNKC;
6163 igb_setup_rctl(adapter);
6164 igb_set_rx_mode(netdev);
6166 /* turn on all-multi mode if wake on multicast is enabled */
6167 if (wufc & E1000_WUFC_MC) {
6168 rctl = rd32(E1000_RCTL);
6169 rctl |= E1000_RCTL_MPE;
6170 wr32(E1000_RCTL, rctl);
6173 ctrl = rd32(E1000_CTRL);
6174 /* advertise wake from D3Cold */
6175 #define E1000_CTRL_ADVD3WUC 0x00100000
6176 /* phy power management enable */
6177 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6178 ctrl |= E1000_CTRL_ADVD3WUC;
6179 wr32(E1000_CTRL, ctrl);
6181 /* Allow time for pending master requests to run */
6182 igb_disable_pcie_master(hw);
6184 wr32(E1000_WUC, E1000_WUC_PME_EN);
6185 wr32(E1000_WUFC, wufc);
6188 wr32(E1000_WUFC, 0);
6191 *enable_wake = wufc || adapter->en_mng_pt;
6193 igb_power_down_link(adapter);
6195 igb_power_up_link(adapter);
6197 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6198 * would have already happened in close and is redundant. */
6199 igb_release_hw_control(adapter);
6201 pci_disable_device(pdev);
6207 static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6212 retval = __igb_shutdown(pdev, &wake);
6217 pci_prepare_to_sleep(pdev);
6219 pci_wake_from_d3(pdev, false);
6220 pci_set_power_state(pdev, PCI_D3hot);
6226 static int igb_resume(struct pci_dev *pdev)
6228 struct net_device *netdev = pci_get_drvdata(pdev);
6229 struct igb_adapter *adapter = netdev_priv(netdev);
6230 struct e1000_hw *hw = &adapter->hw;
6233 pci_set_power_state(pdev, PCI_D0);
6234 pci_restore_state(pdev);
6235 pci_save_state(pdev);
6237 err = pci_enable_device_mem(pdev);
6240 "igb: Cannot enable PCI device from suspend\n");
6243 pci_set_master(pdev);
6245 pci_enable_wake(pdev, PCI_D3hot, 0);
6246 pci_enable_wake(pdev, PCI_D3cold, 0);
6248 if (igb_init_interrupt_scheme(adapter)) {
6249 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6255 /* let the f/w know that the h/w is now under the control of the
6257 igb_get_hw_control(adapter);
6259 wr32(E1000_WUS, ~0);
6261 if (netif_running(netdev)) {
6262 err = igb_open(netdev);
6267 netif_device_attach(netdev);
6273 static void igb_shutdown(struct pci_dev *pdev)
6277 __igb_shutdown(pdev, &wake);
6279 if (system_state == SYSTEM_POWER_OFF) {
6280 pci_wake_from_d3(pdev, wake);
6281 pci_set_power_state(pdev, PCI_D3hot);
6285 #ifdef CONFIG_NET_POLL_CONTROLLER
6287 * Polling 'interrupt' - used by things like netconsole to send skbs
6288 * without having to re-enable interrupts. It's not called while
6289 * the interrupt routine is executing.
6291 static void igb_netpoll(struct net_device *netdev)
6293 struct igb_adapter *adapter = netdev_priv(netdev);
6294 struct e1000_hw *hw = &adapter->hw;
6297 if (!adapter->msix_entries) {
6298 struct igb_q_vector *q_vector = adapter->q_vector[0];
6299 igb_irq_disable(adapter);
6300 napi_schedule(&q_vector->napi);
6304 for (i = 0; i < adapter->num_q_vectors; i++) {
6305 struct igb_q_vector *q_vector = adapter->q_vector[i];
6306 wr32(E1000_EIMC, q_vector->eims_value);
6307 napi_schedule(&q_vector->napi);
6310 #endif /* CONFIG_NET_POLL_CONTROLLER */
6313 * igb_io_error_detected - called when PCI error is detected
6314 * @pdev: Pointer to PCI device
6315 * @state: The current pci connection state
6317 * This function is called after a PCI bus error affecting
6318 * this device has been detected.
6320 static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6321 pci_channel_state_t state)
6323 struct net_device *netdev = pci_get_drvdata(pdev);
6324 struct igb_adapter *adapter = netdev_priv(netdev);
6326 netif_device_detach(netdev);
6328 if (state == pci_channel_io_perm_failure)
6329 return PCI_ERS_RESULT_DISCONNECT;
6331 if (netif_running(netdev))
6333 pci_disable_device(pdev);
6335 /* Request a slot slot reset. */
6336 return PCI_ERS_RESULT_NEED_RESET;
6340 * igb_io_slot_reset - called after the pci bus has been reset.
6341 * @pdev: Pointer to PCI device
6343 * Restart the card from scratch, as if from a cold-boot. Implementation
6344 * resembles the first-half of the igb_resume routine.
6346 static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6348 struct net_device *netdev = pci_get_drvdata(pdev);
6349 struct igb_adapter *adapter = netdev_priv(netdev);
6350 struct e1000_hw *hw = &adapter->hw;
6351 pci_ers_result_t result;
6354 if (pci_enable_device_mem(pdev)) {
6356 "Cannot re-enable PCI device after reset.\n");
6357 result = PCI_ERS_RESULT_DISCONNECT;
6359 pci_set_master(pdev);
6360 pci_restore_state(pdev);
6361 pci_save_state(pdev);
6363 pci_enable_wake(pdev, PCI_D3hot, 0);
6364 pci_enable_wake(pdev, PCI_D3cold, 0);
6367 wr32(E1000_WUS, ~0);
6368 result = PCI_ERS_RESULT_RECOVERED;
6371 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6373 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6374 "failed 0x%0x\n", err);
6375 /* non-fatal, continue */
6382 * igb_io_resume - called when traffic can start flowing again.
6383 * @pdev: Pointer to PCI device
6385 * This callback is called when the error recovery driver tells us that
6386 * its OK to resume normal operation. Implementation resembles the
6387 * second-half of the igb_resume routine.
6389 static void igb_io_resume(struct pci_dev *pdev)
6391 struct net_device *netdev = pci_get_drvdata(pdev);
6392 struct igb_adapter *adapter = netdev_priv(netdev);
6394 if (netif_running(netdev)) {
6395 if (igb_up(adapter)) {
6396 dev_err(&pdev->dev, "igb_up failed after reset\n");
6401 netif_device_attach(netdev);
6403 /* let the f/w know that the h/w is now under the control of the
6405 igb_get_hw_control(adapter);
6408 static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6411 u32 rar_low, rar_high;
6412 struct e1000_hw *hw = &adapter->hw;
6414 /* HW expects these in little endian so we reverse the byte order
6415 * from network order (big endian) to little endian
6417 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6418 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6419 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6421 /* Indicate to hardware the Address is Valid. */
6422 rar_high |= E1000_RAH_AV;
6424 if (hw->mac.type == e1000_82575)
6425 rar_high |= E1000_RAH_POOL_1 * qsel;
6427 rar_high |= E1000_RAH_POOL_1 << qsel;
6429 wr32(E1000_RAL(index), rar_low);
6431 wr32(E1000_RAH(index), rar_high);
6435 static int igb_set_vf_mac(struct igb_adapter *adapter,
6436 int vf, unsigned char *mac_addr)
6438 struct e1000_hw *hw = &adapter->hw;
6439 /* VF MAC addresses start at end of receive addresses and moves
6440 * torwards the first, as a result a collision should not be possible */
6441 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6443 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6445 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6450 static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6452 struct igb_adapter *adapter = netdev_priv(netdev);
6453 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6455 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6456 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6457 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6458 " change effective.");
6459 if (test_bit(__IGB_DOWN, &adapter->state)) {
6460 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6461 " but the PF device is not up.\n");
6462 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6463 " attempting to use the VF device.\n");
6465 return igb_set_vf_mac(adapter, vf, mac);
6468 static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6473 static int igb_ndo_get_vf_config(struct net_device *netdev,
6474 int vf, struct ifla_vf_info *ivi)
6476 struct igb_adapter *adapter = netdev_priv(netdev);
6477 if (vf >= adapter->vfs_allocated_count)
6480 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6482 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6483 ivi->qos = adapter->vf_data[vf].pf_qos;
6487 static void igb_vmm_control(struct igb_adapter *adapter)
6489 struct e1000_hw *hw = &adapter->hw;
6492 switch (hw->mac.type) {
6495 /* replication is not supported for 82575 */
6498 /* notify HW that the MAC is adding vlan tags */
6499 reg = rd32(E1000_DTXCTL);
6500 reg |= E1000_DTXCTL_VLAN_ADDED;
6501 wr32(E1000_DTXCTL, reg);
6503 /* enable replication vlan tag stripping */
6504 reg = rd32(E1000_RPLOLR);
6505 reg |= E1000_RPLOLR_STRVLAN;
6506 wr32(E1000_RPLOLR, reg);
6508 /* none of the above registers are supported by i350 */
6512 if (adapter->vfs_allocated_count) {
6513 igb_vmdq_set_loopback_pf(hw, true);
6514 igb_vmdq_set_replication_pf(hw, true);
6516 igb_vmdq_set_loopback_pf(hw, false);
6517 igb_vmdq_set_replication_pf(hw, false);