drivers/net/ethernet/mellanox/mlx5/core/health.c

   1 /*
   2  * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/kernel.h>
  34 #include <linux/random.h>
  35 #include <linux/vmalloc.h>
  36 #include <linux/hardirq.h>
  37 #include <linux/mlx5/driver.h>
  38 #include <linux/kern_levels.h>
  39 #include "mlx5_core.h"
  40 #include "lib/eq.h"
  41 #include "lib/mlx5.h"
  42 #include "lib/events.h"
  43 #include "lib/pci_vsc.h"
  44 #include "lib/tout.h"
  45 #include "diag/fw_tracer.h"
  46 #include "diag/reporter_vnic.h"
  47
  48 enum {
  49         MAX_MISSES                      = 3,
  50 };
  51
  52 enum {
  53         MLX5_HEALTH_SYNDR_FW_ERR                = 0x1,
  54         MLX5_HEALTH_SYNDR_IRISC_ERR             = 0x7,
  55         MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR  = 0x8,
  56         MLX5_HEALTH_SYNDR_CRC_ERR               = 0x9,
  57         MLX5_HEALTH_SYNDR_FETCH_PCI_ERR         = 0xa,
  58         MLX5_HEALTH_SYNDR_HW_FTL_ERR            = 0xb,
  59         MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR  = 0xc,
  60         MLX5_HEALTH_SYNDR_EQ_ERR                = 0xd,
  61         MLX5_HEALTH_SYNDR_EQ_INV                = 0xe,
  62         MLX5_HEALTH_SYNDR_FFSER_ERR             = 0xf,
  63         MLX5_HEALTH_SYNDR_HIGH_TEMP             = 0x10
  64 };
  65
  66 enum {
  67         MLX5_DROP_HEALTH_WORK,
  68 };
  69
  70 enum  {
  71         MLX5_SENSOR_NO_ERR              = 0,
  72         MLX5_SENSOR_PCI_COMM_ERR        = 1,
  73         MLX5_SENSOR_PCI_ERR             = 2,
  74         MLX5_SENSOR_NIC_DISABLED        = 3,
  75         MLX5_SENSOR_NIC_SW_RESET        = 4,
  76         MLX5_SENSOR_FW_SYND_RFR         = 5,
  77 };
  78
  79 enum {
  80         MLX5_SEVERITY_MASK              = 0x7,
  81         MLX5_SEVERITY_VALID_MASK        = 0x8,
  82 };
  83
  84 u8 mlx5_get_nic_state(struct mlx5_core_dev *dev)
  85 {
  86         return (ioread32be(&dev->iseg->cmdq_addr_l_sz) >> 8) & 7;
  87 }
  88
  89 void mlx5_set_nic_state(struct mlx5_core_dev *dev, u8 state)
  90 {
  91         u32 cur_cmdq_addr_l_sz;
  92
  93         cur_cmdq_addr_l_sz = ioread32be(&dev->iseg->cmdq_addr_l_sz);
  94         iowrite32be((cur_cmdq_addr_l_sz & 0xFFFFF000) |
  95                     state << MLX5_NIC_IFC_OFFSET,
  96                     &dev->iseg->cmdq_addr_l_sz);
  97 }
  98
  99 static bool sensor_pci_not_working(struct mlx5_core_dev *dev)
 100 {
 101         struct mlx5_core_health *health = &dev->priv.health;
 102         struct health_buffer __iomem *h = health->health;
 103
 104         /* Offline PCI reads return 0xffffffff */
 105         return (ioread32be(&h->fw_ver) == 0xffffffff);
 106 }
 107
 108 static int mlx5_health_get_rfr(u8 rfr_severity)
 109 {
 110         return rfr_severity >> MLX5_RFR_BIT_OFFSET;
 111 }
 112
 113 static bool sensor_fw_synd_rfr(struct mlx5_core_dev *dev)
 114 {
 115         struct mlx5_core_health *health = &dev->priv.health;
 116         struct health_buffer __iomem *h = health->health;
 117         u8 synd = ioread8(&h->synd);
 118         u8 rfr;
 119
 120         rfr = mlx5_health_get_rfr(ioread8(&h->rfr_severity));
 121
 122         if (rfr && synd)
 123                 mlx5_core_dbg(dev, "FW requests reset, synd: %d\n", synd);
 124         return rfr && synd;
 125 }
 126
 127 u32 mlx5_health_check_fatal_sensors(struct mlx5_core_dev *dev)
 128 {
 129         if (sensor_pci_not_working(dev))
 130                 return MLX5_SENSOR_PCI_COMM_ERR;
 131         if (pci_channel_offline(dev->pdev))
 132                 return MLX5_SENSOR_PCI_ERR;
 133         if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)
 134                 return MLX5_SENSOR_NIC_DISABLED;
 135         if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_SW_RESET)
 136                 return MLX5_SENSOR_NIC_SW_RESET;
 137         if (sensor_fw_synd_rfr(dev))
 138                 return MLX5_SENSOR_FW_SYND_RFR;
 139
 140         return MLX5_SENSOR_NO_ERR;
 141 }
 142
 143 static int lock_sem_sw_reset(struct mlx5_core_dev *dev, bool lock)
 144 {
 145         enum mlx5_vsc_state state;
 146         int ret;
 147
 148         if (!mlx5_core_is_pf(dev))
 149                 return -EBUSY;
 150
 151         /* Try to lock GW access, this stage doesn't return
 152          * EBUSY because locked GW does not mean that other PF
 153          * already started the reset.
 154          */
 155         ret = mlx5_vsc_gw_lock(dev);
 156         if (ret == -EBUSY)
 157                 return -EINVAL;
 158         if (ret)
 159                 return ret;
 160
 161         state = lock ? MLX5_VSC_LOCK : MLX5_VSC_UNLOCK;
 162         /* At this stage, if the return status == EBUSY, then we know
 163          * for sure that another PF started the reset, so don't allow
 164          * another reset.
 165          */
 166         ret = mlx5_vsc_sem_set_space(dev, MLX5_SEMAPHORE_SW_RESET, state);
 167         if (ret)
 168                 mlx5_core_warn(dev, "Failed to lock SW reset semaphore\n");
 169
 170         /* Unlock GW access */
 171         mlx5_vsc_gw_unlock(dev);
 172
 173         return ret;
 174 }
 175
 176 static bool reset_fw_if_needed(struct mlx5_core_dev *dev)
 177 {
 178         bool supported = (ioread32be(&dev->iseg->initializing) >>
 179                           MLX5_FW_RESET_SUPPORTED_OFFSET) & 1;
 180         u32 fatal_error;
 181
 182         if (!supported)
 183                 return false;
 184
 185         /* The reset only needs to be issued by one PF. The health buffer is
 186          * shared between all functions, and will be cleared during a reset.
 187          * Check again to avoid a redundant 2nd reset. If the fatal errors was
 188          * PCI related a reset won't help.
 189          */
 190         fatal_error = mlx5_health_check_fatal_sensors(dev);
 191         if (fatal_error == MLX5_SENSOR_PCI_COMM_ERR ||
 192             fatal_error == MLX5_SENSOR_NIC_DISABLED ||
 193             fatal_error == MLX5_SENSOR_NIC_SW_RESET) {
 194                 mlx5_core_warn(dev, "Not issuing FW reset. Either it's already done or won't help.");
 195                 return false;
 196         }
 197
 198         mlx5_core_warn(dev, "Issuing FW Reset\n");
 199         /* Write the NIC interface field to initiate the reset, the command
 200          * interface address also resides here, don't overwrite it.
 201          */
 202         mlx5_set_nic_state(dev, MLX5_NIC_IFC_SW_RESET);
 203
 204         return true;
 205 }
 206
 207 static void enter_error_state(struct mlx5_core_dev *dev, bool force)
 208 {
 209         if (mlx5_health_check_fatal_sensors(dev) || force) { /* protected state setting */
 210                 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
 211                 mlx5_cmd_flush(dev);
 212         }
 213
 214         mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_SYS_ERROR, (void *)1);
 215 }
 216
 217 void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
 218 {
 219         bool err_detected = false;
 220
 221         /* Mark the device as fatal in order to abort FW commands */
 222         if ((mlx5_health_check_fatal_sensors(dev) || force) &&
 223             dev->state == MLX5_DEVICE_STATE_UP) {
 224                 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
 225                 err_detected = true;
 226         }
 227         mutex_lock(&dev->intf_state_mutex);
 228         if (!err_detected && dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
 229                 goto unlock;/* a previous error is still being handled */
 230
 231         enter_error_state(dev, force);
 232 unlock:
 233         mutex_unlock(&dev->intf_state_mutex);
 234 }
 235
 236 void mlx5_error_sw_reset(struct mlx5_core_dev *dev)
 237 {
 238         unsigned long end, delay_ms = mlx5_tout_ms(dev, PCI_TOGGLE);
 239         int lock = -EBUSY;
 240
 241         mutex_lock(&dev->intf_state_mutex);
 242         if (dev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR)
 243                 goto unlock;
 244
 245         mlx5_core_err(dev, "start\n");
 246
 247         if (mlx5_health_check_fatal_sensors(dev) == MLX5_SENSOR_FW_SYND_RFR) {
 248                 /* Get cr-dump and reset FW semaphore */
 249                 lock = lock_sem_sw_reset(dev, true);
 250
 251                 if (lock == -EBUSY) {
 252                         delay_ms = mlx5_tout_ms(dev, FULL_CRDUMP);
 253                         goto recover_from_sw_reset;
 254                 }
 255                 /* Execute SW reset */
 256                 reset_fw_if_needed(dev);
 257         }
 258
 259 recover_from_sw_reset:
 260         /* Recover from SW reset */
 261         end = jiffies + msecs_to_jiffies(delay_ms);
 262         do {
 263                 if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)
 264                         break;
 265
 266                 msleep(20);
 267         } while (!time_after(jiffies, end));
 268
 269         if (mlx5_get_nic_state(dev) != MLX5_NIC_IFC_DISABLED) {
 270                 dev_err(&dev->pdev->dev, "NIC IFC still %d after %lums.\n",
 271                         mlx5_get_nic_state(dev), delay_ms);
 272         }
 273
 274         /* Release FW semaphore if you are the lock owner */
 275         if (!lock)
 276                 lock_sem_sw_reset(dev, false);
 277
 278         mlx5_core_err(dev, "end\n");
 279
 280 unlock:
 281         mutex_unlock(&dev->intf_state_mutex);
 282 }
 283
 284 static void mlx5_handle_bad_state(struct mlx5_core_dev *dev)
 285 {
 286         u8 nic_interface = mlx5_get_nic_state(dev);
 287
 288         switch (nic_interface) {
 289         case MLX5_NIC_IFC_FULL:
 290                 mlx5_core_warn(dev, "Expected to see disabled NIC but it is full driver\n");
 291                 break;
 292
 293         case MLX5_NIC_IFC_DISABLED:
 294                 mlx5_core_warn(dev, "starting teardown\n");
 295                 break;
 296
 297         case MLX5_NIC_IFC_NO_DRAM_NIC:
 298                 mlx5_core_warn(dev, "Expected to see disabled NIC but it is no dram nic\n");
 299                 break;
 300
 301         case MLX5_NIC_IFC_SW_RESET:
 302                 /* The IFC mode field is 3 bits, so it will read 0x7 in 2 cases:
 303                  * 1. PCI has been disabled (ie. PCI-AER, PF driver unloaded
 304                  *    and this is a VF), this is not recoverable by SW reset.
 305                  *    Logging of this is handled elsewhere.
 306                  * 2. FW reset has been issued by another function, driver can
 307                  *    be reloaded to recover after the mode switches to
 308                  *    MLX5_NIC_IFC_DISABLED.
 309                  */
 310                 if (dev->priv.health.fatal_error != MLX5_SENSOR_PCI_COMM_ERR)
 311                         mlx5_core_warn(dev, "NIC SW reset in progress\n");
 312                 break;
 313
 314         default:
 315                 mlx5_core_warn(dev, "Expected to see disabled NIC but it is has invalid value %d\n",
 316                                nic_interface);
 317         }
 318
 319         mlx5_disable_device(dev);
 320 }
 321
 322 int mlx5_health_wait_pci_up(struct mlx5_core_dev *dev)
 323 {
 324         unsigned long end;
 325
 326         end = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, FW_RESET));
 327         while (sensor_pci_not_working(dev)) {
 328                 if (time_after(jiffies, end))
 329                         return -ETIMEDOUT;
 330                 if (test_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) {
 331                         mlx5_core_warn(dev, "device is being removed, stop waiting for PCI\n");
 332                         return -ENODEV;
 333                 }
 334                 msleep(100);
 335         }
 336         return 0;
 337 }
 338
 339 static int mlx5_health_try_recover(struct mlx5_core_dev *dev)
 340 {
 341         mlx5_core_warn(dev, "handling bad device here\n");
 342         mlx5_handle_bad_state(dev);
 343         if (mlx5_health_wait_pci_up(dev)) {
 344                 mlx5_core_err(dev, "health recovery flow aborted, PCI reads still not working\n");
 345                 return -EIO;
 346         }
 347         mlx5_core_err(dev, "starting health recovery flow\n");
 348         if (mlx5_recover_device(dev) || mlx5_health_check_fatal_sensors(dev)) {
 349                 mlx5_core_err(dev, "health recovery failed\n");
 350                 return -EIO;
 351         }
 352
 353         mlx5_core_info(dev, "health recovery succeeded\n");
 354         return 0;
 355 }
 356
 357 static const char *hsynd_str(u8 synd)
 358 {
 359         switch (synd) {
 360         case MLX5_HEALTH_SYNDR_FW_ERR:
 361                 return "firmware internal error";
 362         case MLX5_HEALTH_SYNDR_IRISC_ERR:
 363                 return "irisc not responding";
 364         case MLX5_HEALTH_SYNDR_HW_UNRECOVERABLE_ERR:
 365                 return "unrecoverable hardware error";
 366         case MLX5_HEALTH_SYNDR_CRC_ERR:
 367                 return "firmware CRC error";
 368         case MLX5_HEALTH_SYNDR_FETCH_PCI_ERR:
 369                 return "ICM fetch PCI error";
 370         case MLX5_HEALTH_SYNDR_HW_FTL_ERR:
 371                 return "HW fatal error\n";
 372         case MLX5_HEALTH_SYNDR_ASYNC_EQ_OVERRUN_ERR:
 373                 return "async EQ buffer overrun";
 374         case MLX5_HEALTH_SYNDR_EQ_ERR:
 375                 return "EQ error";
 376         case MLX5_HEALTH_SYNDR_EQ_INV:
 377                 return "Invalid EQ referenced";
 378         case MLX5_HEALTH_SYNDR_FFSER_ERR:
 379                 return "FFSER error";
 380         case MLX5_HEALTH_SYNDR_HIGH_TEMP:
 381                 return "High temperature";
 382         default:
 383                 return "unrecognized error";
 384         }
 385 }
 386
 387 static const char *mlx5_loglevel_str(int level)
 388 {
 389         switch (level) {
 390         case LOGLEVEL_EMERG:
 391                 return "EMERGENCY";
 392         case LOGLEVEL_ALERT:
 393                 return "ALERT";
 394         case LOGLEVEL_CRIT:
 395                 return "CRITICAL";
 396         case LOGLEVEL_ERR:
 397                 return "ERROR";
 398         case LOGLEVEL_WARNING:
 399                 return "WARNING";
 400         case LOGLEVEL_NOTICE:
 401                 return "NOTICE";
 402         case LOGLEVEL_INFO:
 403                 return "INFO";
 404         case LOGLEVEL_DEBUG:
 405                 return "DEBUG";
 406         }
 407         return "Unknown log level";
 408 }
 409
 410 static int mlx5_health_get_severity(u8 rfr_severity)
 411 {
 412         return rfr_severity & MLX5_SEVERITY_VALID_MASK ?
 413                rfr_severity & MLX5_SEVERITY_MASK : LOGLEVEL_ERR;
 414 }
 415
 416 static void print_health_info(struct mlx5_core_dev *dev)
 417 {
 418         struct mlx5_core_health *health = &dev->priv.health;
 419         struct health_buffer __iomem *h = health->health;
 420         u8 rfr_severity;
 421         int severity;
 422         int i;
 423
 424         /* If the syndrome is 0, the device is OK and no need to print buffer */
 425         if (!ioread8(&h->synd))
 426                 return;
 427
 428         if (ioread32be(&h->fw_ver) == 0xFFFFFFFF) {
 429                 mlx5_log(dev, LOGLEVEL_ERR, "PCI slot is unavailable\n");
 430                 return;
 431         }
 432
 433         rfr_severity = ioread8(&h->rfr_severity);
 434         severity  = mlx5_health_get_severity(rfr_severity);
 435         mlx5_log(dev, severity, "Health issue observed, %s, severity(%d) %s:\n",
 436                  hsynd_str(ioread8(&h->synd)), severity, mlx5_loglevel_str(severity));
 437
 438         for (i = 0; i < ARRAY_SIZE(h->assert_var); i++)
 439                 mlx5_log(dev, severity, "assert_var[%d] 0x%08x\n", i,
 440                          ioread32be(h->assert_var + i));
 441
 442         mlx5_log(dev, severity, "assert_exit_ptr 0x%08x\n", ioread32be(&h->assert_exit_ptr));
 443         mlx5_log(dev, severity, "assert_callra 0x%08x\n", ioread32be(&h->assert_callra));
 444         mlx5_log(dev, severity, "fw_ver %d.%d.%d", fw_rev_maj(dev), fw_rev_min(dev),
 445                  fw_rev_sub(dev));
 446         mlx5_log(dev, severity, "time %u\n", ioread32be(&h->time));
 447         mlx5_log(dev, severity, "hw_id 0x%08x\n", ioread32be(&h->hw_id));
 448         mlx5_log(dev, severity, "rfr %d\n", mlx5_health_get_rfr(rfr_severity));
 449         mlx5_log(dev, severity, "severity %d (%s)\n", severity, mlx5_loglevel_str(severity));
 450         mlx5_log(dev, severity, "irisc_index %d\n", ioread8(&h->irisc_index));
 451         mlx5_log(dev, severity, "synd 0x%x: %s\n", ioread8(&h->synd),
 452                  hsynd_str(ioread8(&h->synd)));
 453         mlx5_log(dev, severity, "ext_synd 0x%04x\n", ioread16be(&h->ext_synd));
 454         mlx5_log(dev, severity, "raw fw_ver 0x%08x\n", ioread32be(&h->fw_ver));
 455 }
 456
 457 static int
 458 mlx5_fw_reporter_diagnose(struct devlink_health_reporter *reporter,
 459                           struct devlink_fmsg *fmsg,
 460                           struct netlink_ext_ack *extack)
 461 {
 462         struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter);
 463         struct mlx5_core_health *health = &dev->priv.health;
 464         struct health_buffer __iomem *h = health->health;
 465         u8 synd;
 466         int err;
 467
 468         synd = ioread8(&h->synd);
 469         err = devlink_fmsg_u8_pair_put(fmsg, "Syndrome", synd);
 470         if (err || !synd)
 471                 return err;
 472         return devlink_fmsg_string_pair_put(fmsg, "Description", hsynd_str(synd));
 473 }
 474
 475 struct mlx5_fw_reporter_ctx {
 476         u8 err_synd;
 477         int miss_counter;
 478 };
 479
 480 static int
 481 mlx5_fw_reporter_ctx_pairs_put(struct devlink_fmsg *fmsg,
 482                                struct mlx5_fw_reporter_ctx *fw_reporter_ctx)
 483 {
 484         int err;
 485
 486         err = devlink_fmsg_u8_pair_put(fmsg, "syndrome",
 487                                        fw_reporter_ctx->err_synd);
 488         if (err)
 489                 return err;
 490         err = devlink_fmsg_u32_pair_put(fmsg, "fw_miss_counter",
 491                                         fw_reporter_ctx->miss_counter);
 492         if (err)
 493                 return err;
 494         return 0;
 495 }
 496
 497 static int
 498 mlx5_fw_reporter_heath_buffer_data_put(struct mlx5_core_dev *dev,
 499                                        struct devlink_fmsg *fmsg)
 500 {
 501         struct mlx5_core_health *health = &dev->priv.health;
 502         struct health_buffer __iomem *h = health->health;
 503         u8 rfr_severity;
 504         int err;
 505         int i;
 506
 507         if (!ioread8(&h->synd))
 508                 return 0;
 509
 510         err = devlink_fmsg_pair_nest_start(fmsg, "health buffer");
 511         if (err)
 512                 return err;
 513         err = devlink_fmsg_obj_nest_start(fmsg);
 514         if (err)
 515                 return err;
 516         err = devlink_fmsg_arr_pair_nest_start(fmsg, "assert_var");
 517         if (err)
 518                 return err;
 519
 520         for (i = 0; i < ARRAY_SIZE(h->assert_var); i++) {
 521                 err = devlink_fmsg_u32_put(fmsg, ioread32be(h->assert_var + i));
 522                 if (err)
 523                         return err;
 524         }
 525         err = devlink_fmsg_arr_pair_nest_end(fmsg);
 526         if (err)
 527                 return err;
 528         err = devlink_fmsg_u32_pair_put(fmsg, "assert_exit_ptr",
 529                                         ioread32be(&h->assert_exit_ptr));
 530         if (err)
 531                 return err;
 532         err = devlink_fmsg_u32_pair_put(fmsg, "assert_callra",
 533                                         ioread32be(&h->assert_callra));
 534         if (err)
 535                 return err;
 536         err = devlink_fmsg_u32_pair_put(fmsg, "time", ioread32be(&h->time));
 537         if (err)
 538                 return err;
 539         err = devlink_fmsg_u32_pair_put(fmsg, "hw_id", ioread32be(&h->hw_id));
 540         if (err)
 541                 return err;
 542         rfr_severity = ioread8(&h->rfr_severity);
 543         err = devlink_fmsg_u8_pair_put(fmsg, "rfr", mlx5_health_get_rfr(rfr_severity));
 544         if (err)
 545                 return err;
 546         err = devlink_fmsg_u8_pair_put(fmsg, "severity", mlx5_health_get_severity(rfr_severity));
 547         if (err)
 548                 return err;
 549         err = devlink_fmsg_u8_pair_put(fmsg, "irisc_index",
 550                                        ioread8(&h->irisc_index));
 551         if (err)
 552                 return err;
 553         err = devlink_fmsg_u8_pair_put(fmsg, "synd", ioread8(&h->synd));
 554         if (err)
 555                 return err;
 556         err = devlink_fmsg_u32_pair_put(fmsg, "ext_synd",
 557                                         ioread16be(&h->ext_synd));
 558         if (err)
 559                 return err;
 560         err = devlink_fmsg_u32_pair_put(fmsg, "raw_fw_ver",
 561                                         ioread32be(&h->fw_ver));
 562         if (err)
 563                 return err;
 564         err = devlink_fmsg_obj_nest_end(fmsg);
 565         if (err)
 566                 return err;
 567         return devlink_fmsg_pair_nest_end(fmsg);
 568 }
 569
 570 static int
 571 mlx5_fw_reporter_dump(struct devlink_health_reporter *reporter,
 572                       struct devlink_fmsg *fmsg, void *priv_ctx,
 573                       struct netlink_ext_ack *extack)
 574 {
 575         struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter);
 576         int err;
 577
 578         err = mlx5_fw_tracer_trigger_core_dump_general(dev);
 579         if (err)
 580                 return err;
 581
 582         if (priv_ctx) {
 583                 struct mlx5_fw_reporter_ctx *fw_reporter_ctx = priv_ctx;
 584
 585                 err = mlx5_fw_reporter_ctx_pairs_put(fmsg, fw_reporter_ctx);
 586                 if (err)
 587                         return err;
 588         }
 589
 590         err = mlx5_fw_reporter_heath_buffer_data_put(dev, fmsg);
 591         if (err)
 592                 return err;
 593         return mlx5_fw_tracer_get_saved_traces_objects(dev->tracer, fmsg);
 594 }
 595
 596 static void mlx5_fw_reporter_err_work(struct work_struct *work)
 597 {
 598         struct mlx5_fw_reporter_ctx fw_reporter_ctx;
 599         struct mlx5_core_health *health;
 600
 601         health = container_of(work, struct mlx5_core_health, report_work);
 602
 603         if (IS_ERR_OR_NULL(health->fw_reporter))
 604                 return;
 605
 606         fw_reporter_ctx.err_synd = health->synd;
 607         fw_reporter_ctx.miss_counter = health->miss_counter;
 608         if (fw_reporter_ctx.err_synd) {
 609                 devlink_health_report(health->fw_reporter,
 610                                       "FW syndrome reported", &fw_reporter_ctx);
 611                 return;
 612         }
 613         if (fw_reporter_ctx.miss_counter)
 614                 devlink_health_report(health->fw_reporter,
 615                                       "FW miss counter reported",
 616                                       &fw_reporter_ctx);
 617 }
 618
 619 static const struct devlink_health_reporter_ops mlx5_fw_reporter_ops = {
 620                 .name = "fw",
 621                 .diagnose = mlx5_fw_reporter_diagnose,
 622                 .dump = mlx5_fw_reporter_dump,
 623 };
 624
 625 static int
 626 mlx5_fw_fatal_reporter_recover(struct devlink_health_reporter *reporter,
 627                                void *priv_ctx,
 628                                struct netlink_ext_ack *extack)
 629 {
 630         struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter);
 631
 632         return mlx5_health_try_recover(dev);
 633 }
 634
 635 static int
 636 mlx5_fw_fatal_reporter_dump(struct devlink_health_reporter *reporter,
 637                             struct devlink_fmsg *fmsg, void *priv_ctx,
 638                             struct netlink_ext_ack *extack)
 639 {
 640         struct mlx5_core_dev *dev = devlink_health_reporter_priv(reporter);
 641         u32 crdump_size = dev->priv.health.crdump_size;
 642         u32 *cr_data;
 643         int err;
 644
 645         if (!mlx5_core_is_pf(dev))
 646                 return -EPERM;
 647
 648         cr_data = kvmalloc(crdump_size, GFP_KERNEL);
 649         if (!cr_data)
 650                 return -ENOMEM;
 651         err = mlx5_crdump_collect(dev, cr_data);
 652         if (err)
 653                 goto free_data;
 654
 655         if (priv_ctx) {
 656                 struct mlx5_fw_reporter_ctx *fw_reporter_ctx = priv_ctx;
 657
 658                 err = mlx5_fw_reporter_ctx_pairs_put(fmsg, fw_reporter_ctx);
 659                 if (err)
 660                         goto free_data;
 661         }
 662
 663         err = devlink_fmsg_binary_pair_put(fmsg, "crdump_data", cr_data, crdump_size);
 664
 665 free_data:
 666         kvfree(cr_data);
 667         return err;
 668 }
 669
 670 static void mlx5_fw_fatal_reporter_err_work(struct work_struct *work)
 671 {
 672         struct mlx5_fw_reporter_ctx fw_reporter_ctx;
 673         struct mlx5_core_health *health;
 674         struct mlx5_core_dev *dev;
 675         struct devlink *devlink;
 676         struct mlx5_priv *priv;
 677
 678         health = container_of(work, struct mlx5_core_health, fatal_report_work);
 679         priv = container_of(health, struct mlx5_priv, health);
 680         dev = container_of(priv, struct mlx5_core_dev, priv);
 681         devlink = priv_to_devlink(dev);
 682
 683         mutex_lock(&dev->intf_state_mutex);
 684         if (test_bit(MLX5_DROP_HEALTH_WORK, &health->flags)) {
 685                 mlx5_core_err(dev, "health works are not permitted at this stage\n");
 686                 mutex_unlock(&dev->intf_state_mutex);
 687                 return;
 688         }
 689         mutex_unlock(&dev->intf_state_mutex);
 690         enter_error_state(dev, false);
 691         if (IS_ERR_OR_NULL(health->fw_fatal_reporter)) {
 692                 devl_lock(devlink);
 693                 if (mlx5_health_try_recover(dev))
 694                         mlx5_core_err(dev, "health recovery failed\n");
 695                 devl_unlock(devlink);
 696                 return;
 697         }
 698         fw_reporter_ctx.err_synd = health->synd;
 699         fw_reporter_ctx.miss_counter = health->miss_counter;
 700         if (devlink_health_report(health->fw_fatal_reporter,
 701                                   "FW fatal error reported", &fw_reporter_ctx) == -ECANCELED) {
 702                 /* If recovery wasn't performed, due to grace period,
 703                  * unload the driver. This ensures that the driver
 704                  * closes all its resources and it is not subjected to
 705                  * requests from the kernel.
 706                  */
 707                 mlx5_core_err(dev, "Driver is in error state. Unloading\n");
 708                 mlx5_unload_one(dev, false);
 709         }
 710 }
 711
 712 static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ops = {
 713                 .name = "fw_fatal",
 714                 .recover = mlx5_fw_fatal_reporter_recover,
 715                 .dump = mlx5_fw_fatal_reporter_dump,
 716 };
 717
 718 #define MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD 180000
 719 #define MLX5_FW_REPORTER_PF_GRACEFUL_PERIOD 60000
 720 #define MLX5_FW_REPORTER_VF_GRACEFUL_PERIOD 30000
 721 #define MLX5_FW_REPORTER_DEFAULT_GRACEFUL_PERIOD MLX5_FW_REPORTER_VF_GRACEFUL_PERIOD
 722
 723 void mlx5_fw_reporters_create(struct mlx5_core_dev *dev)
 724 {
 725         struct mlx5_core_health *health = &dev->priv.health;
 726         struct devlink *devlink = priv_to_devlink(dev);
 727         u64 grace_period;
 728
 729         if (mlx5_core_is_ecpf(dev)) {
 730                 grace_period = MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD;
 731         } else if (mlx5_core_is_pf(dev)) {
 732                 grace_period = MLX5_FW_REPORTER_PF_GRACEFUL_PERIOD;
 733         } else {
 734                 /* VF or SF */
 735                 grace_period = MLX5_FW_REPORTER_DEFAULT_GRACEFUL_PERIOD;
 736         }
 737
 738         health->fw_reporter =
 739                 devl_health_reporter_create(devlink, &mlx5_fw_reporter_ops,
 740                                             0, dev);
 741         if (IS_ERR(health->fw_reporter))
 742                 mlx5_core_warn(dev, "Failed to create fw reporter, err = %ld\n",
 743                                PTR_ERR(health->fw_reporter));
 744
 745         health->fw_fatal_reporter =
 746                 devl_health_reporter_create(devlink,
 747                                             &mlx5_fw_fatal_reporter_ops,
 748                                             grace_period,
 749                                             dev);
 750         if (IS_ERR(health->fw_fatal_reporter))
 751                 mlx5_core_warn(dev, "Failed to create fw fatal reporter, err = %ld\n",
 752                                PTR_ERR(health->fw_fatal_reporter));
 753 }
 754
 755 static void mlx5_fw_reporters_destroy(struct mlx5_core_dev *dev)
 756 {
 757         struct mlx5_core_health *health = &dev->priv.health;
 758
 759         if (!IS_ERR_OR_NULL(health->fw_reporter))
 760                 devlink_health_reporter_destroy(health->fw_reporter);
 761
 762         if (!IS_ERR_OR_NULL(health->fw_fatal_reporter))
 763                 devlink_health_reporter_destroy(health->fw_fatal_reporter);
 764 }
 765
 766 static unsigned long get_next_poll_jiffies(struct mlx5_core_dev *dev)
 767 {
 768         unsigned long next;
 769
 770         get_random_bytes(&next, sizeof(next));
 771         next %= HZ;
 772         next += jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, HEALTH_POLL_INTERVAL));
 773
 774         return next;
 775 }
 776
 777 void mlx5_trigger_health_work(struct mlx5_core_dev *dev)
 778 {
 779         struct mlx5_core_health *health = &dev->priv.health;
 780
 781         if (!mlx5_dev_is_lightweight(dev))
 782                 queue_work(health->wq, &health->fatal_report_work);
 783 }
 784
 785 #define MLX5_MSEC_PER_HOUR (MSEC_PER_SEC * 60 * 60)
 786 static void mlx5_health_log_ts_update(struct work_struct *work)
 787 {
 788         struct delayed_work *dwork = to_delayed_work(work);
 789         u32 out[MLX5_ST_SZ_DW(mrtc_reg)] = {};
 790         u32 in[MLX5_ST_SZ_DW(mrtc_reg)] = {};
 791         struct mlx5_core_health *health;
 792         struct mlx5_core_dev *dev;
 793         struct mlx5_priv *priv;
 794         u64 now_us;
 795
 796         health = container_of(dwork, struct mlx5_core_health, update_fw_log_ts_work);
 797         priv = container_of(health, struct mlx5_priv, health);
 798         dev = container_of(priv, struct mlx5_core_dev, priv);
 799
 800         now_us =  ktime_to_us(ktime_get_real());
 801
 802         MLX5_SET(mrtc_reg, in, time_h, now_us >> 32);
 803         MLX5_SET(mrtc_reg, in, time_l, now_us & 0xFFFFFFFF);
 804         mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MRTC, 0, 1);
 805
 806         queue_delayed_work(health->wq, &health->update_fw_log_ts_work,
 807                            msecs_to_jiffies(MLX5_MSEC_PER_HOUR));
 808 }
 809
 810 static void poll_health(struct timer_list *t)
 811 {
 812         struct mlx5_core_dev *dev = from_timer(dev, t, priv.health.timer);
 813         struct mlx5_core_health *health = &dev->priv.health;
 814         struct health_buffer __iomem *h = health->health;
 815         u32 fatal_error;
 816         u8 prev_synd;
 817         u32 count;
 818
 819         if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
 820                 goto out;
 821
 822         fatal_error = mlx5_health_check_fatal_sensors(dev);
 823
 824         if (fatal_error && !health->fatal_error) {
 825                 mlx5_core_err(dev, "Fatal error %u detected\n", fatal_error);
 826                 dev->priv.health.fatal_error = fatal_error;
 827                 print_health_info(dev);
 828                 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
 829                 mlx5_trigger_health_work(dev);
 830                 return;
 831         }
 832
 833         count = ioread32be(health->health_counter);
 834         if (count == health->prev)
 835                 ++health->miss_counter;
 836         else
 837                 health->miss_counter = 0;
 838
 839         health->prev = count;
 840         if (health->miss_counter == MAX_MISSES) {
 841                 mlx5_core_err(dev, "device's health compromised - reached miss count\n");
 842                 print_health_info(dev);
 843                 queue_work(health->wq, &health->report_work);
 844         }
 845
 846         prev_synd = health->synd;
 847         health->synd = ioread8(&h->synd);
 848         if (health->synd && health->synd != prev_synd)
 849                 queue_work(health->wq, &health->report_work);
 850
 851 out:
 852         mod_timer(&health->timer, get_next_poll_jiffies(dev));
 853 }
 854
 855 void mlx5_start_health_poll(struct mlx5_core_dev *dev)
 856 {
 857         u64 poll_interval_ms =  mlx5_tout_ms(dev, HEALTH_POLL_INTERVAL);
 858         struct mlx5_core_health *health = &dev->priv.health;
 859
 860         timer_setup(&health->timer, poll_health, 0);
 861         health->fatal_error = MLX5_SENSOR_NO_ERR;
 862         clear_bit(MLX5_DROP_HEALTH_WORK, &health->flags);
 863         health->health = &dev->iseg->health;
 864         health->health_counter = &dev->iseg->health_counter;
 865
 866         health->timer.expires = jiffies + msecs_to_jiffies(poll_interval_ms);
 867         add_timer(&health->timer);
 868 }
 869
 870 void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health)
 871 {
 872         struct mlx5_core_health *health = &dev->priv.health;
 873
 874         if (disable_health)
 875                 set_bit(MLX5_DROP_HEALTH_WORK, &health->flags);
 876
 877         del_timer_sync(&health->timer);
 878 }
 879
 880 void mlx5_start_health_fw_log_up(struct mlx5_core_dev *dev)
 881 {
 882         struct mlx5_core_health *health = &dev->priv.health;
 883
 884         if (mlx5_core_is_pf(dev) && MLX5_CAP_MCAM_REG(dev, mrtc))
 885                 queue_delayed_work(health->wq, &health->update_fw_log_ts_work, 0);
 886 }
 887
 888 void mlx5_drain_health_wq(struct mlx5_core_dev *dev)
 889 {
 890         struct mlx5_core_health *health = &dev->priv.health;
 891
 892         set_bit(MLX5_DROP_HEALTH_WORK, &health->flags);
 893         cancel_delayed_work_sync(&health->update_fw_log_ts_work);
 894         cancel_work_sync(&health->report_work);
 895         cancel_work_sync(&health->fatal_report_work);
 896 }
 897
 898 void mlx5_health_cleanup(struct mlx5_core_dev *dev)
 899 {
 900         struct mlx5_core_health *health = &dev->priv.health;
 901
 902         cancel_delayed_work_sync(&health->update_fw_log_ts_work);
 903         destroy_workqueue(health->wq);
 904         mlx5_reporter_vnic_destroy(dev);
 905         mlx5_fw_reporters_destroy(dev);
 906 }
 907
 908 int mlx5_health_init(struct mlx5_core_dev *dev)
 909 {
 910         struct devlink *devlink = priv_to_devlink(dev);
 911         struct mlx5_core_health *health;
 912         char *name;
 913
 914         if (!mlx5_dev_is_lightweight(dev)) {
 915                 devl_lock(devlink);
 916                 mlx5_fw_reporters_create(dev);
 917                 devl_unlock(devlink);
 918         }
 919         mlx5_reporter_vnic_create(dev);
 920
 921         health = &dev->priv.health;
 922         name = kmalloc(64, GFP_KERNEL);
 923         if (!name)
 924                 goto out_err;
 925
 926         strcpy(name, "mlx5_health");
 927         strcat(name, dev_name(dev->device));
 928         health->wq = create_singlethread_workqueue(name);
 929         kfree(name);
 930         if (!health->wq)
 931                 goto out_err;
 932         INIT_WORK(&health->fatal_report_work, mlx5_fw_fatal_reporter_err_work);
 933         INIT_WORK(&health->report_work, mlx5_fw_reporter_err_work);
 934         INIT_DELAYED_WORK(&health->update_fw_log_ts_work, mlx5_health_log_ts_update);
 935
 936         return 0;
 937
 938 out_err:
 939         mlx5_reporter_vnic_destroy(dev);
 940         mlx5_fw_reporters_destroy(dev);
 941         return -ENOMEM;
 942 }