ionic: add heartbeat check

Most of our firmware has a heartbeat feature that the driver
can watch for to see if the FW is still alive and likely to
answer a dev_cmd or AdminQ request.

Signed-off-by: Shannon Nelson <snelson@pensando.io>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.c b/drivers/net/ethernet/pensando/ionic/ionic_dev.c
index d168a643532256011a6bb56644ab20500500e152..544a9f799afc6c9563156c990fe5db063c88561c 100644 (file)
--- a/drivers/net/ethernet/pensando/ionic/ionic_dev.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.c
@@ -84,6 +84,49 @@ void ionic_dev_teardown(struct ionic *ionic)
 }
 
 /* Devcmd Interface */
+int ionic_heartbeat_check(struct ionic *ionic)
+{
+       struct ionic_dev *idev = &ionic->idev;
+       unsigned long hb_time;
+       u32 fw_status;
+       u32 hb;
+
+       /* wait a little more than one second before testing again */
+       hb_time = jiffies;
+       if (time_before(hb_time, (idev->last_hb_time + (HZ * 2))))
+               return 0;
+
+       /* firmware is useful only if fw_status is non-zero */
+       fw_status = ioread32(&idev->dev_info_regs->fw_status);
+       if (!fw_status)
+               return -ENXIO;
+
+       /* early FW has no heartbeat, else FW will return non-zero */
+       hb = ioread32(&idev->dev_info_regs->fw_heartbeat);
+       if (!hb)
+               return 0;
+
+       /* are we stalled? */
+       if (hb == idev->last_hb) {
+               /* only complain once for each stall seen */
+               if (idev->last_hb_time != 1) {
+                       dev_info(ionic->dev, "FW heartbeat stalled at %d\n",
+                                idev->last_hb);
+                       idev->last_hb_time = 1;
+               }
+
+               return -ENXIO;
+       }
+
+       if (idev->last_hb_time == 1)
+               dev_info(ionic->dev, "FW heartbeat restored at %d\n", hb);
+
+       idev->last_hb = hb;
+       idev->last_hb_time = hb_time;
+
+       return 0;
+}
+
 u8 ionic_dev_cmd_status(struct ionic_dev *idev)
 {
        return ioread8(&idev->dev_cmd_regs->comp.comp.status);

diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.h b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
index 9610aeb7d5f42ad0921ec70f31dfc4312df3c4f1..1ffb3e4dec5da5b8d937138933d5743376fb678c 100644 (file)
--- a/drivers/net/ethernet/pensando/ionic/ionic_dev.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
@@ -123,6 +123,9 @@ struct ionic_dev {
        union ionic_dev_info_regs __iomem *dev_info_regs;
        union ionic_dev_cmd_regs __iomem *dev_cmd_regs;
 
+       unsigned long last_hb_time;
+       u32 last_hb;
+
        u64 __iomem *db_pages;
        dma_addr_t phy_db_pages;
 
@@ -295,5 +298,6 @@ void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb,
 void ionic_q_rewind(struct ionic_queue *q, struct ionic_desc_info *start);
 void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info,
                     unsigned int stop_index);
+int ionic_heartbeat_check(struct ionic *ionic);
 
 #endif /* _IONIC_DEV_H_ */

diff --git a/drivers/net/ethernet/pensando/ionic/ionic_main.c b/drivers/net/ethernet/pensando/ionic/ionic_main.c
index 15e432386b35a2ad0733c0df28e4002588bb66ea..52eb303e903f7864bf6294f368ce1e021270c391 100644 (file)
--- a/drivers/net/ethernet/pensando/ionic/ionic_main.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_main.c
@@ -245,6 +245,10 @@ static int ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
                goto err_out;
        }
 
+       err = ionic_heartbeat_check(lif->ionic);
+       if (err)
+               goto err_out;
+
        memcpy(adminq->head->desc, &ctx->cmd, sizeof(ctx->cmd));
 
        dev_dbg(&lif->netdev->dev, "post admin queue command:\n");
@@ -305,6 +309,14 @@ int ionic_napi(struct napi_struct *napi, int budget, ionic_cq_cb cb,
        return work_done;
 }
 
+static void ionic_dev_cmd_clean(struct ionic *ionic)
+{
+       union ionic_dev_cmd_regs *regs = ionic->idev.dev_cmd_regs;
+
+       iowrite32(0, &regs->doorbell);
+       memset_io(&regs->cmd, 0, sizeof(regs->cmd));
+}
+
 int ionic_dev_cmd_wait(struct ionic *ionic, unsigned long max_seconds)
 {
        struct ionic_dev *idev = &ionic->idev;
@@ -314,6 +326,7 @@ int ionic_dev_cmd_wait(struct ionic *ionic, unsigned long max_seconds)
        int opcode;
        int done;
        int err;
+       int hb;
 
        WARN_ON(in_interrupt());
 
@@ -328,7 +341,8 @@ try_again:
                if (done)
                        break;
                msleep(20);
-       } while (!done && time_before(jiffies, max_wait));
+               hb = ionic_heartbeat_check(ionic);
+       } while (!done && !hb && time_before(jiffies, max_wait));
        duration = jiffies - start_time;
 
        opcode = idev->dev_cmd_regs->cmd.cmd.opcode;
@@ -336,7 +350,15 @@ try_again:
                ionic_opcode_to_str(opcode), opcode,
                done, duration / HZ, duration);
 
+       if (!done && hb) {
+               ionic_dev_cmd_clean(ionic);
+               dev_warn(ionic->dev, "DEVCMD %s (%d) failed - FW halted\n",
+                        ionic_opcode_to_str(opcode), opcode);
+               return -ENXIO;
+       }
+
        if (!done && !time_before(jiffies, max_wait)) {
+               ionic_dev_cmd_clean(ionic);
                dev_warn(ionic->dev, "DEVCMD %s (%d) timeout after %ld secs\n",
                         ionic_opcode_to_str(opcode), opcode, max_seconds);
                return -ETIMEDOUT;