staging/rdma/hfi1: Prevent host software lock up

If packets stop egressing the hardware link, software can lock up.

Implement a timeout for send context halt recovery.  This patch increases the
timeout for packet egress to 500 us and timer resets to zero if the packet
occupancy changes. Also we bounce the link on time out.

Reviewed-by: Dean Luick <dean.luick@intel.com>
Signed-off-by: Vennila Megavannan <vennila.megavannan@intel.com>
Signed-off-by: Ira Weiny <ira.weiny@intel.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/drivers/staging/rdma/hfi1/pio.c b/drivers/staging/rdma/hfi1/pio.c
index 67dd93a6888c705ea0b74a69256036f8f20d14b0..e5c32db4bc67c96f58729a269142190dccb94396 100644 (file)
--- a/drivers/staging/rdma/hfi1/pio.c
+++ b/drivers/staging/rdma/hfi1/pio.c
@@ -922,10 +922,12 @@ void sc_disable(struct send_context *sc)
 static void sc_wait_for_packet_egress(struct send_context *sc, int pause)
 {
        struct hfi1_devdata *dd = sc->dd;
-       u64 reg;
+       u64 reg = 0;
+       u64 reg_prev;
        u32 loop = 0;
 
        while (1) {
+               reg_prev = reg;
                reg = read_csr(dd, sc->hw_context * 8 +
                               SEND_EGRESS_CTXT_STATUS);
                /* done if egress is stopped */
@@ -934,11 +936,17 @@ static void sc_wait_for_packet_egress(struct send_context *sc, int pause)
                reg = packet_occupancy(reg);
                if (reg == 0)
                        break;
-               if (loop > 100) {
+               /* counter is reset if occupancy count changes */
+               if (reg != reg_prev)
+                       loop = 0;
+               if (loop > 500) {
+                       /* timed out - bounce the link */
                        dd_dev_err(dd,
-                               "%s: context %u(%u) timeout waiting for packets to egress, remaining count %u\n",
+                               "%s: context %u(%u) timeout waiting for packets to egress, remaining count %u, bouncing link\n",
                                __func__, sc->sw_index,
                                sc->hw_context, (u32)reg);
+                       queue_work(dd->pport->hfi1_wq,
+                               &dd->pport->link_bounce_work);
                        break;
                }
                loop++;

diff --git a/drivers/staging/rdma/hfi1/sdma.c b/drivers/staging/rdma/hfi1/sdma.c
index 16d93fffe5def8f718326caeda0baf66d9d63e90..f254ed30bd45456836263dcb1632a30e094f4f2a 100644 (file)
--- a/drivers/staging/rdma/hfi1/sdma.c
+++ b/drivers/staging/rdma/hfi1/sdma.c
@@ -303,17 +303,26 @@ static void sdma_wait_for_packet_egress(struct sdma_engine *sde,
        u64 off = 8 * sde->this_idx;
        struct hfi1_devdata *dd = sde->dd;
        int lcnt = 0;
+       u64 reg_prev;
+       u64 reg = 0;
 
        while (1) {
-               u64 reg = read_csr(dd, off + SEND_EGRESS_SEND_DMA_STATUS);
+               reg_prev = reg;
+               reg = read_csr(dd, off + SEND_EGRESS_SEND_DMA_STATUS);
 
                reg &= SDMA_EGRESS_PACKET_OCCUPANCY_SMASK;
                reg >>= SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT;
                if (reg == 0)
                        break;
-               if (lcnt++ > 100) {
-                       dd_dev_err(dd, "%s: engine %u timeout waiting for packets to egress, remaining count %u\n",
+               /* counter is reest if accupancy count changes */
+               if (reg != reg_prev)
+                       lcnt = 0;
+               if (lcnt++ > 500) {
+                       /* timed out - bounce the link */
+                       dd_dev_err(dd, "%s: engine %u timeout waiting for packets to egress, remaining count %u, bouncing link\n",
                                  __func__, sde->this_idx, (u32)reg);
+                       queue_work(dd->pport->hfi1_wq,
+                               &dd->pport->link_bounce_work);
                        break;
                }
                udelay(1);