e100/e1000*/igb*/ixgb*: Add missing read memory barrier

commit 2d0bb1c1f4524befe9f0fcf0d0cd3081a451223f upstream.

Based on patches from Sonny Rao and Milton Miller...

Combined the patches to fix up clean_tx_irq and clean_rx_irq.

The PowerPC architecture does not require loads to independent bytes
to be ordered without adding an explicit barrier.

In ixgbe_clean_rx_irq we load the status bit then load the packet data.
With packet split disabled if these loads go out of order we get a
stale packet, but we will notice the bad sequence numbers and drop it.

The problem occurs with packet split enabled where the TCP/IP header
and data are in different descriptors. If the reads go out of order
we may have data that doesn't match the TCP/IP header. Since we use
hardware checksumming this bad data is never verified and it makes it
all the way to the application.

This bug was found during stress testing and adding this barrier has
been shown to fix it.  The bug can manifest as a data integrity issue
(bad payload data) or as a BUG in skb_pull().

This was a nasty bug to hunt down, if people agree with the fix I think
it's a candidate for stable.

Previously Submitted to e1000-devel only for ixgbe

http://marc.info/?l=e1000-devel&m=126593062701537&w=3

We've now seen this problem hit with other device drivers (e1000e mostly)
So I'm resubmitting with fixes for other Intel Device Drivers with
similar issues.

CC: Milton Miller <miltonm@bga.com>
CC: Anton Blanchard <anton@samba.org>
CC: Sonny Rao <sonnyrao@us.ibm.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>

diff --git a/drivers/net/e100.c b/drivers/net/e100.c
index 791080303db100cf7f5f53d961253846e6e3049e..f6bb11ec506679488b95600cbba3c77f6f823481 100644 (file)
--- a/drivers/net/e100.c
+++ b/drivers/net/e100.c
@@ -1768,6 +1768,7 @@ static int e100_tx_clean(struct nic *nic)
        for (cb = nic->cb_to_clean;
            cb->status & cpu_to_le16(cb_complete);
            cb = nic->cb_to_clean = cb->next) {
+               rmb(); /* read skb after status */
                DPRINTK(TX_DONE, DEBUG, "cb[%d]->status = 0x%04X\n",
                        (int)(((void*)cb - (void*)nic->cbs)/sizeof(struct cb)),
                        cb->status);
@@ -1914,6 +1915,7 @@ static int e100_rx_indicate(struct nic *nic, struct rx *rx,
        rfd_status = le16_to_cpu(rfd->status);
 
        DPRINTK(RX_STATUS, DEBUG, "status=0x%04X\n", rfd_status);
+       rmb(); /* read size after status bit */
 
        /* If data isn't ready, nothing to indicate */
        if (unlikely(!(rfd_status & cb_complete))) {

diff --git a/drivers/net/e1000/e1000_main.c b/drivers/net/e1000/e1000_main.c
index b15ece26ed8469136df4d40eced1b98839e4acce..13e34d26e5d474751e151f4e2dd675c9d0b06ab4 100644 (file)
--- a/drivers/net/e1000/e1000_main.c
+++ b/drivers/net/e1000/e1000_main.c
@@ -3433,6 +3433,7 @@ static bool e1000_clean_tx_irq(struct e1000_adapter *adapter,
        while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
               (count < tx_ring->count)) {
                bool cleaned = false;
+               rmb();  /* read buffer_info after eop_desc */
                for ( ; !cleaned; count++) {
                        tx_desc = E1000_TX_DESC(*tx_ring, i);
                        buffer_info = &tx_ring->buffer_info[i];
@@ -3622,6 +3623,7 @@ static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
                if (*work_done >= work_to_do)
                        break;
                (*work_done)++;
+               rmb(); /* read descriptor and rx_buffer_info after status DD */
 
                status = rx_desc->status;
                skb = buffer_info->skb;
@@ -3803,6 +3805,7 @@ static bool e1000_clean_rx_irq(struct e1000_adapter *adapter,
                if (*work_done >= work_to_do)
                        break;
                (*work_done)++;
+               rmb(); /* read descriptor and rx_buffer_info after status DD */
 
                status = rx_desc->status;
                skb = buffer_info->skb;

diff --git a/drivers/net/e1000e/netdev.c b/drivers/net/e1000e/netdev.c
index 64a06ed549060c27545d7cf63eb1c19df7b1f89a..56c9e69dfeeb981d19c81df810c5a7e6d6366478 100644 (file)
--- a/drivers/net/e1000e/netdev.c
+++ b/drivers/net/e1000e/netdev.c
@@ -637,6 +637,7 @@ static bool e1000_clean_tx_irq(struct e1000_adapter *adapter)
        while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
               (count < tx_ring->count)) {
                bool cleaned = false;
+               rmb(); /* read buffer_info after eop_desc */
                for (; !cleaned; count++) {
                        tx_desc = E1000_TX_DESC(*tx_ring, i);
                        buffer_info = &tx_ring->buffer_info[i];
@@ -739,6 +740,7 @@ static bool e1000_clean_rx_irq_ps(struct e1000_adapter *adapter,
                        break;
                (*work_done)++;
                skb = buffer_info->skb;
+               rmb();  /* read descriptor and rx_buffer_info after status DD */
 
                /* in the packet split case this is header only */
                prefetch(skb->data - NET_IP_ALIGN);
@@ -938,6 +940,8 @@ static bool e1000_clean_jumbo_rx_irq(struct e1000_adapter *adapter,
                if (*work_done >= work_to_do)
                        break;
                (*work_done)++;
+               rmb();  /* read descriptor and rx_buffer_info after status DD */
+               rmb();  /* read descriptor and rx_buffer_info after status DD */
 
                status = rx_desc->status;
                skb = buffer_info->skb;

diff --git a/drivers/net/igb/igb_main.c b/drivers/net/igb/igb_main.c
index c5edd4107eee53ea5bf7ebe6419916f3c8540ba0..acc618ac13d994ddfb0b37975e5d972a98a6e528 100644 (file)
--- a/drivers/net/igb/igb_main.c
+++ b/drivers/net/igb/igb_main.c
@@ -5005,6 +5005,7 @@ static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
 
        while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
               (count < tx_ring->count)) {
+               rmb();  /* read buffer_info after eop_desc status */
                for (cleaned = false; !cleaned; count++) {
                        tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
                        buffer_info = &tx_ring->buffer_info[i];
@@ -5212,6 +5213,7 @@ static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
                if (*work_done >= budget)
                        break;
                (*work_done)++;
+               rmb(); /* read descriptor and rx_buffer_info after status DD */
 
                skb = buffer_info->skb;
                prefetch(skb->data - NET_IP_ALIGN);

diff --git a/drivers/net/igbvf/netdev.c b/drivers/net/igbvf/netdev.c
index 1b1edad1eb5e68de5a583a2a1ac14d68e6a2a7dd..709e83dcd46f2f80e0572d0ff5a9afcd01561b00 100644 (file)
--- a/drivers/net/igbvf/netdev.c
+++ b/drivers/net/igbvf/netdev.c
@@ -247,6 +247,7 @@ static bool igbvf_clean_rx_irq(struct igbvf_adapter *adapter,
                if (*work_done >= work_to_do)
                        break;
                (*work_done)++;
+               rmb(); /* read descriptor and rx_buffer_info after status DD */
 
                buffer_info = &rx_ring->buffer_info[i];
 
@@ -777,6 +778,7 @@ static bool igbvf_clean_tx_irq(struct igbvf_ring *tx_ring)
 
        while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
               (count < tx_ring->count)) {
+               rmb();  /* read buffer_info after eop_desc status */
                for (cleaned = false; !cleaned; count++) {
                        tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);
                        buffer_info = &tx_ring->buffer_info[i];

diff --git a/drivers/net/ixgb/ixgb_main.c b/drivers/net/ixgb/ixgb_main.c
index c9fef65cb98be5e17e02ad656d0e92d62d89cd5e..be88a1db9f8a69a20e883a251a622741401b5f82 100644 (file)
--- a/drivers/net/ixgb/ixgb_main.c
+++ b/drivers/net/ixgb/ixgb_main.c
@@ -1811,6 +1811,7 @@ ixgb_clean_tx_irq(struct ixgb_adapter *adapter)
 
        while (eop_desc->status & IXGB_TX_DESC_STATUS_DD) {
 
+               rmb(); /* read buffer_info after eop_desc */
                for (cleaned = false; !cleaned; ) {
                        tx_desc = IXGB_TX_DESC(*tx_ring, i);
                        buffer_info = &tx_ring->buffer_info[i];
@@ -1946,6 +1947,7 @@ ixgb_clean_rx_irq(struct ixgb_adapter *adapter, int *work_done, int work_to_do)
                        break;
 
                (*work_done)++;
+               rmb();  /* read descriptor and rx_buffer_info after status DD */
                status = rx_desc->status;
                skb = buffer_info->skb;
                buffer_info->skb = NULL;

diff --git a/drivers/net/ixgbe/ixgbe_main.c b/drivers/net/ixgbe/ixgbe_main.c
index 6c00ee493a3bb1c1bae50b59fdb2afb58688c22e..156dfcbc71248bf627382b50ba1288e0cd57fd15 100644 (file)
--- a/drivers/net/ixgbe/ixgbe_main.c
+++ b/drivers/net/ixgbe/ixgbe_main.c
@@ -407,6 +407,7 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
        while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) &&
               (count < tx_ring->work_limit)) {
                bool cleaned = false;
+               rmb(); /* read buffer_info after eop_desc */
                for ( ; !cleaned; count++) {
                        struct sk_buff *skb;
                        tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, i);

diff --git a/drivers/net/ixgbevf/ixgbevf_main.c b/drivers/net/ixgbevf/ixgbevf_main.c
index 0cd6202dfaccfc541c0aee86471997e5189f2cf5..75be3c9f2cb1ec97c44e0bd10369625bda5adbc2 100644 (file)
--- a/drivers/net/ixgbevf/ixgbevf_main.c
+++ b/drivers/net/ixgbevf/ixgbevf_main.c
@@ -231,6 +231,7 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_adapter *adapter,
        while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) &&
               (count < tx_ring->work_limit)) {
                bool cleaned = false;
+               rmb(); /* read buffer_info after eop_desc */
                for ( ; !cleaned; count++) {
                        struct sk_buff *skb;
                        tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, i);
@@ -518,6 +519,7 @@ static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
                        break;
                (*work_done)++;
 
+               rmb(); /* read descriptor and rx_buffer_info after status DD */
                if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
                        hdr_info = le16_to_cpu(ixgbevf_get_hdr_info(rx_desc));
                        len = (hdr_info & IXGBE_RXDADV_HDRBUFLEN_MASK) >>