drivers/edac/thunderx_edac.c

   1 /*
   2  * Cavium ThunderX memory controller kernel module
   3  *
   4  * This file is subject to the terms and conditions of the GNU General Public
   5  * License.  See the file "COPYING" in the main directory of this archive
   6  * for more details.
   7  *
   8  * Copyright Cavium, Inc. (C) 2015-2017. All rights reserved.
   9  *
  10  */
  11
  12 #include <linux/module.h>
  13 #include <linux/pci.h>
  14 #include <linux/edac.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/string.h>
  17 #include <linux/stop_machine.h>
  18 #include <linux/delay.h>
  19 #include <linux/sizes.h>
  20 #include <linux/atomic.h>
  21 #include <linux/bitfield.h>
  22 #include <linux/circ_buf.h>
  23
  24 #include <asm/page.h>
  25
  26 #include "edac_module.h"
  27
  28 #define phys_to_pfn(phys)       (PFN_DOWN(phys))
  29
  30 #define THUNDERX_NODE           GENMASK(45, 44)
  31
  32 enum {
  33         ERR_CORRECTED   = 1,
  34         ERR_UNCORRECTED = 2,
  35         ERR_UNKNOWN     = 3,
  36 };
  37
  38 struct error_descr {
  39         int     type;
  40         u64     mask;
  41         char    *descr;
  42 };
  43
  44 static void decode_register(char *str, size_t size,
  45                            const struct error_descr *descr,
  46                            const uint64_t reg)
  47 {
  48         int ret = 0;
  49
  50         while (descr->type && descr->mask && descr->descr) {
  51                 if (reg & descr->mask) {
  52                         ret = snprintf(str, size, "\n\t%s, %s",
  53                                        descr->type == ERR_CORRECTED ?
  54                                          "Corrected" : "Uncorrected",
  55                                        descr->descr);
  56                         str += ret;
  57                         size -= ret;
  58                 }
  59                 descr++;
  60         }
  61 }
  62
  63 static unsigned long get_bits(unsigned long data, int pos, int width)
  64 {
  65         return (data >> pos) & ((1 << width) - 1);
  66 }
  67
  68 #define L2C_CTL                 0x87E080800000
  69 #define L2C_CTL_DISIDXALIAS     BIT(0)
  70
  71 #define PCI_DEVICE_ID_THUNDER_LMC 0xa022
  72
  73 #define LMC_FADR                0x20
  74 #define LMC_FADR_FDIMM(x)       ((x >> 37) & 0x1)
  75 #define LMC_FADR_FBUNK(x)       ((x >> 36) & 0x1)
  76 #define LMC_FADR_FBANK(x)       ((x >> 32) & 0xf)
  77 #define LMC_FADR_FROW(x)        ((x >> 14) & 0xffff)
  78 #define LMC_FADR_FCOL(x)        ((x >> 0) & 0x1fff)
  79
  80 #define LMC_NXM_FADR            0x28
  81 #define LMC_ECC_SYND            0x38
  82
  83 #define LMC_ECC_PARITY_TEST     0x108
  84
  85 #define LMC_INT_W1S             0x150
  86
  87 #define LMC_INT_ENA_W1C         0x158
  88 #define LMC_INT_ENA_W1S         0x160
  89
  90 #define LMC_CONFIG              0x188
  91
  92 #define LMC_CONFIG_BG2          BIT(62)
  93 #define LMC_CONFIG_RANK_ENA     BIT(42)
  94 #define LMC_CONFIG_PBANK_LSB(x) (((x) >> 5) & 0xF)
  95 #define LMC_CONFIG_ROW_LSB(x)   (((x) >> 2) & 0x7)
  96
  97 #define LMC_CONTROL             0x190
  98 #define LMC_CONTROL_XOR_BANK    BIT(16)
  99
 100 #define LMC_INT                 0x1F0
 101
 102 #define LMC_INT_DDR_ERR         BIT(11)
 103 #define LMC_INT_DED_ERR         (0xFUL << 5)
 104 #define LMC_INT_SEC_ERR         (0xFUL << 1)
 105 #define LMC_INT_NXM_WR_MASK     BIT(0)
 106
 107 #define LMC_DDR_PLL_CTL         0x258
 108 #define LMC_DDR_PLL_CTL_DDR4    BIT(29)
 109
 110 #define LMC_FADR_SCRAMBLED      0x330
 111
 112 #define LMC_INT_UE              (LMC_INT_DDR_ERR | LMC_INT_DED_ERR | \
 113                                  LMC_INT_NXM_WR_MASK)
 114
 115 #define LMC_INT_CE              (LMC_INT_SEC_ERR)
 116
 117 static const struct error_descr lmc_errors[] = {
 118         {
 119                 .type  = ERR_CORRECTED,
 120                 .mask  = LMC_INT_SEC_ERR,
 121                 .descr = "Single-bit ECC error",
 122         },
 123         {
 124                 .type  = ERR_UNCORRECTED,
 125                 .mask  = LMC_INT_DDR_ERR,
 126                 .descr = "DDR chip error",
 127         },
 128         {
 129                 .type  = ERR_UNCORRECTED,
 130                 .mask  = LMC_INT_DED_ERR,
 131                 .descr = "Double-bit ECC error",
 132         },
 133         {
 134                 .type = ERR_UNCORRECTED,
 135                 .mask = LMC_INT_NXM_WR_MASK,
 136                 .descr = "Non-existent memory write",
 137         },
 138         {0, 0, NULL},
 139 };
 140
 141 #define LMC_INT_EN_DDR_ERROR_ALERT_ENA  BIT(5)
 142 #define LMC_INT_EN_DLCRAM_DED_ERR       BIT(4)
 143 #define LMC_INT_EN_DLCRAM_SEC_ERR       BIT(3)
 144 #define LMC_INT_INTR_DED_ENA            BIT(2)
 145 #define LMC_INT_INTR_SEC_ENA            BIT(1)
 146 #define LMC_INT_INTR_NXM_WR_ENA         BIT(0)
 147
 148 #define LMC_INT_ENA_ALL                 GENMASK(5, 0)
 149
 150 #define LMC_DDR_PLL_CTL         0x258
 151 #define LMC_DDR_PLL_CTL_DDR4    BIT(29)
 152
 153 #define LMC_CONTROL             0x190
 154 #define LMC_CONTROL_RDIMM       BIT(0)
 155
 156 #define LMC_SCRAM_FADR          0x330
 157
 158 #define LMC_CHAR_MASK0          0x228
 159 #define LMC_CHAR_MASK2          0x238
 160
 161 #define RING_ENTRIES    8
 162
 163 struct debugfs_entry {
 164         const char *name;
 165         umode_t mode;
 166         const struct file_operations fops;
 167 };
 168
 169 struct lmc_err_ctx {
 170         u64 reg_int;
 171         u64 reg_fadr;
 172         u64 reg_nxm_fadr;
 173         u64 reg_scram_fadr;
 174         u64 reg_ecc_synd;
 175 };
 176
 177 struct thunderx_lmc {
 178         void __iomem *regs;
 179         struct pci_dev *pdev;
 180         struct msix_entry msix_ent;
 181
 182         atomic_t ecc_int;
 183
 184         u64 mask0;
 185         u64 mask2;
 186         u64 parity_test;
 187         u64 node;
 188
 189         int xbits;
 190         int bank_width;
 191         int pbank_lsb;
 192         int dimm_lsb;
 193         int rank_lsb;
 194         int bank_lsb;
 195         int row_lsb;
 196         int col_hi_lsb;
 197
 198         int xor_bank;
 199         int l2c_alias;
 200
 201         struct page *mem;
 202
 203         struct lmc_err_ctx err_ctx[RING_ENTRIES];
 204         unsigned long ring_head;
 205         unsigned long ring_tail;
 206 };
 207
 208 #define ring_pos(pos, size) ((pos) & (size - 1))
 209
 210 #define DEBUGFS_STRUCT(_name, _mode, _write, _read)                         \
 211 static struct debugfs_entry debugfs_##_name = {                             \
 212         .name = __stringify(_name),                                         \
 213         .mode = VERIFY_OCTAL_PERMISSIONS(_mode),                            \
 214         .fops = {                                                           \
 215                 .open = simple_open,                                        \
 216                 .write = _write,                                            \
 217                 .read  = _read,                                             \
 218                 .llseek = generic_file_llseek,                              \
 219         },                                                                  \
 220 }
 221
 222 #define DEBUGFS_FIELD_ATTR(_type, _field)                                   \
 223 static ssize_t thunderx_##_type##_##_field##_read(struct file *file,        \
 224                                             char __user *data,              \
 225                                             size_t count, loff_t *ppos)     \
 226 {                                                                           \
 227         struct thunderx_##_type *pdata = file->private_data;                \
 228         char buf[20];                                                       \
 229                                                                             \
 230         snprintf(buf, count, "0x%016llx", pdata->_field);                   \
 231         return simple_read_from_buffer(data, count, ppos,                   \
 232                                        buf, sizeof(buf));                   \
 233 }                                                                           \
 234                                                                             \
 235 static ssize_t thunderx_##_type##_##_field##_write(struct file *file,       \
 236                                              const char __user *data,       \
 237                                              size_t count, loff_t *ppos)    \
 238 {                                                                           \
 239         struct thunderx_##_type *pdata = file->private_data;                \
 240         int res;                                                            \
 241                                                                             \
 242         res = kstrtoull_from_user(data, count, 0, &pdata->_field);          \
 243                                                                             \
 244         return res ? res : count;                                           \
 245 }                                                                           \
 246                                                                             \
 247 DEBUGFS_STRUCT(_field, 0600,                                                \
 248                    thunderx_##_type##_##_field##_write,                     \
 249                    thunderx_##_type##_##_field##_read)                      \
 250
 251 #define DEBUGFS_REG_ATTR(_type, _name, _reg)                                \
 252 static ssize_t thunderx_##_type##_##_name##_read(struct file *file,         \
 253                                            char __user *data,               \
 254                                            size_t count, loff_t *ppos)      \
 255 {                                                                           \
 256         struct thunderx_##_type *pdata = file->private_data;                \
 257         char buf[20];                                                       \
 258                                                                             \
 259         sprintf(buf, "0x%016llx", readq(pdata->regs + _reg));               \
 260         return simple_read_from_buffer(data, count, ppos,                   \
 261                                        buf, sizeof(buf));                   \
 262 }                                                                           \
 263                                                                             \
 264 static ssize_t thunderx_##_type##_##_name##_write(struct file *file,        \
 265                                             const char __user *data,        \
 266                                             size_t count, loff_t *ppos)     \
 267 {                                                                           \
 268         struct thunderx_##_type *pdata = file->private_data;                \
 269         u64 val;                                                            \
 270         int res;                                                            \
 271                                                                             \
 272         res = kstrtoull_from_user(data, count, 0, &val);                    \
 273                                                                             \
 274         if (!res) {                                                         \
 275                 writeq(val, pdata->regs + _reg);                            \
 276                 res = count;                                                \
 277         }                                                                   \
 278                                                                             \
 279         return res;                                                         \
 280 }                                                                           \
 281                                                                             \
 282 DEBUGFS_STRUCT(_name, 0600,                                                 \
 283                thunderx_##_type##_##_name##_write,                          \
 284                thunderx_##_type##_##_name##_read)
 285
 286 #define LMC_DEBUGFS_ENT(_field) DEBUGFS_FIELD_ATTR(lmc, _field)
 287
 288 /*
 289  * To get an ECC error injected, the following steps are needed:
 290  * - Setup the ECC injection by writing the appropriate parameters:
 291  *      echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask0
 292  *      echo <bit mask value> > /sys/kernel/debug/<device number>/ecc_mask2
 293  *      echo 0x802 > /sys/kernel/debug/<device number>/ecc_parity_test
 294  * - Do the actual injection:
 295  *      echo 1 > /sys/kernel/debug/<device number>/inject_ecc
 296  */
 297 static ssize_t thunderx_lmc_inject_int_write(struct file *file,
 298                                              const char __user *data,
 299                                              size_t count, loff_t *ppos)
 300 {
 301         struct thunderx_lmc *lmc = file->private_data;
 302         u64 val;
 303         int res;
 304
 305         res = kstrtoull_from_user(data, count, 0, &val);
 306
 307         if (!res) {
 308                 /* Trigger the interrupt */
 309                 writeq(val, lmc->regs + LMC_INT_W1S);
 310                 res = count;
 311         }
 312
 313         return res;
 314 }
 315
 316 static ssize_t thunderx_lmc_int_read(struct file *file,
 317                                      char __user *data,
 318                                      size_t count, loff_t *ppos)
 319 {
 320         struct thunderx_lmc *lmc = file->private_data;
 321         char buf[20];
 322         u64 lmc_int = readq(lmc->regs + LMC_INT);
 323
 324         snprintf(buf, sizeof(buf), "0x%016llx", lmc_int);
 325         return simple_read_from_buffer(data, count, ppos, buf, sizeof(buf));
 326 }
 327
 328 #define TEST_PATTERN 0xa5
 329
 330 static int inject_ecc_fn(void *arg)
 331 {
 332         struct thunderx_lmc *lmc = arg;
 333         uintptr_t addr, phys;
 334         unsigned int cline_size = cache_line_size();
 335         const unsigned int lines = PAGE_SIZE / cline_size;
 336         unsigned int i, cl_idx;
 337
 338         addr = (uintptr_t)page_address(lmc->mem);
 339         phys = (uintptr_t)page_to_phys(lmc->mem);
 340
 341         cl_idx = (phys & 0x7f) >> 4;
 342         lmc->parity_test &= ~(7ULL << 8);
 343         lmc->parity_test |= (cl_idx << 8);
 344
 345         writeq(lmc->mask0, lmc->regs + LMC_CHAR_MASK0);
 346         writeq(lmc->mask2, lmc->regs + LMC_CHAR_MASK2);
 347         writeq(lmc->parity_test, lmc->regs + LMC_ECC_PARITY_TEST);
 348
 349         readq(lmc->regs + LMC_CHAR_MASK0);
 350         readq(lmc->regs + LMC_CHAR_MASK2);
 351         readq(lmc->regs + LMC_ECC_PARITY_TEST);
 352
 353         for (i = 0; i < lines; i++) {
 354                 memset((void *)addr, TEST_PATTERN, cline_size);
 355                 barrier();
 356
 357                 /*
 358                  * Flush L1 cachelines to the PoC (L2).
 359                  * This will cause cacheline eviction to the L2.
 360                  */
 361                 asm volatile("dc civac, %0\n"
 362                              "dsb sy\n"
 363                              : : "r"(addr + i * cline_size));
 364         }
 365
 366         for (i = 0; i < lines; i++) {
 367                 /*
 368                  * Flush L2 cachelines to the DRAM.
 369                  * This will cause cacheline eviction to the DRAM
 370                  * and ECC corruption according to the masks set.
 371                  */
 372                 __asm__ volatile("sys #0,c11,C1,#2, %0\n"
 373                                  : : "r"(phys + i * cline_size));
 374         }
 375
 376         for (i = 0; i < lines; i++) {
 377                 /*
 378                  * Invalidate L2 cachelines.
 379                  * The subsequent load will cause cacheline fetch
 380                  * from the DRAM and an error interrupt
 381                  */
 382                 __asm__ volatile("sys #0,c11,C1,#1, %0"
 383                                  : : "r"(phys + i * cline_size));
 384         }
 385
 386         for (i = 0; i < lines; i++) {
 387                 /*
 388                  * Invalidate L1 cachelines.
 389                  * The subsequent load will cause cacheline fetch
 390                  * from the L2 and/or DRAM
 391                  */
 392                 asm volatile("dc ivac, %0\n"
 393                              "dsb sy\n"
 394                              : : "r"(addr + i * cline_size));
 395         }
 396
 397         return 0;
 398 }
 399
 400 static ssize_t thunderx_lmc_inject_ecc_write(struct file *file,
 401                                              const char __user *data,
 402                                              size_t count, loff_t *ppos)
 403 {
 404         struct thunderx_lmc *lmc = file->private_data;
 405         unsigned int cline_size = cache_line_size();
 406         u8 *tmp;
 407         void __iomem *addr;
 408         unsigned int offs, timeout = 100000;
 409
 410         atomic_set(&lmc->ecc_int, 0);
 411
 412         lmc->mem = alloc_pages_node(lmc->node, GFP_KERNEL, 0);
 413         if (!lmc->mem)
 414                 return -ENOMEM;
 415
 416         tmp = kmalloc(cline_size, GFP_KERNEL);
 417         if (!tmp) {
 418                 __free_pages(lmc->mem, 0);
 419                 return -ENOMEM;
 420         }
 421
 422         addr = page_address(lmc->mem);
 423
 424         while (!atomic_read(&lmc->ecc_int) && timeout--) {
 425                 stop_machine(inject_ecc_fn, lmc, NULL);
 426
 427                 for (offs = 0; offs < PAGE_SIZE; offs += cline_size) {
 428                         /*
 429                          * Do a load from the previously rigged location
 430                          * This should generate an error interrupt.
 431                          */
 432                         memcpy(tmp, addr + offs, cline_size);
 433                         asm volatile("dsb ld\n");
 434                 }
 435         }
 436
 437         kfree(tmp);
 438         __free_pages(lmc->mem, 0);
 439
 440         return count;
 441 }
 442
 443 LMC_DEBUGFS_ENT(mask0);
 444 LMC_DEBUGFS_ENT(mask2);
 445 LMC_DEBUGFS_ENT(parity_test);
 446
 447 DEBUGFS_STRUCT(inject_int, 0200, thunderx_lmc_inject_int_write, NULL);
 448 DEBUGFS_STRUCT(inject_ecc, 0200, thunderx_lmc_inject_ecc_write, NULL);
 449 DEBUGFS_STRUCT(int_w1c, 0400, NULL, thunderx_lmc_int_read);
 450
 451 static struct debugfs_entry *lmc_dfs_ents[] = {
 452         &debugfs_mask0,
 453         &debugfs_mask2,
 454         &debugfs_parity_test,
 455         &debugfs_inject_ecc,
 456         &debugfs_inject_int,
 457         &debugfs_int_w1c,
 458 };
 459
 460 static int thunderx_create_debugfs_nodes(struct dentry *parent,
 461                                           struct debugfs_entry *attrs[],
 462                                           void *data,
 463                                           size_t num)
 464 {
 465         int i;
 466         struct dentry *ent;
 467
 468         if (!IS_ENABLED(CONFIG_EDAC_DEBUG))
 469                 return 0;
 470
 471         if (!parent)
 472                 return -ENOENT;
 473
 474         for (i = 0; i < num; i++) {
 475                 ent = edac_debugfs_create_file(attrs[i]->name, attrs[i]->mode,
 476                                                parent, data, &attrs[i]->fops);
 477
 478                 if (IS_ERR(ent))
 479                         break;
 480         }
 481
 482         return i;
 483 }
 484
 485 static phys_addr_t thunderx_faddr_to_phys(u64 faddr, struct thunderx_lmc *lmc)
 486 {
 487         phys_addr_t addr = 0;
 488         int bank, xbits;
 489
 490         addr |= lmc->node << 40;
 491         addr |= LMC_FADR_FDIMM(faddr) << lmc->dimm_lsb;
 492         addr |= LMC_FADR_FBUNK(faddr) << lmc->rank_lsb;
 493         addr |= LMC_FADR_FROW(faddr) << lmc->row_lsb;
 494         addr |= (LMC_FADR_FCOL(faddr) >> 4) << lmc->col_hi_lsb;
 495
 496         bank = LMC_FADR_FBANK(faddr) << lmc->bank_lsb;
 497
 498         if (lmc->xor_bank)
 499                 bank ^= get_bits(addr, 12 + lmc->xbits, lmc->bank_width);
 500
 501         addr |= bank << lmc->bank_lsb;
 502
 503         xbits = PCI_FUNC(lmc->pdev->devfn);
 504
 505         if (lmc->l2c_alias)
 506                 xbits ^= get_bits(addr, 20, lmc->xbits) ^
 507                          get_bits(addr, 12, lmc->xbits);
 508
 509         addr |= xbits << 7;
 510
 511         return addr;
 512 }
 513
 514 static unsigned int thunderx_get_num_lmcs(unsigned int node)
 515 {
 516         unsigned int number = 0;
 517         struct pci_dev *pdev = NULL;
 518
 519         do {
 520                 pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
 521                                       PCI_DEVICE_ID_THUNDER_LMC,
 522                                       pdev);
 523                 if (pdev) {
 524 #ifdef CONFIG_NUMA
 525                         if (pdev->dev.numa_node == node)
 526                                 number++;
 527 #else
 528                         number++;
 529 #endif
 530                 }
 531         } while (pdev);
 532
 533         return number;
 534 }
 535
 536 #define LMC_MESSAGE_SIZE        120
 537 #define LMC_OTHER_SIZE          (50 * ARRAY_SIZE(lmc_errors))
 538
 539 static irqreturn_t thunderx_lmc_err_isr(int irq, void *dev_id)
 540 {
 541         struct mem_ctl_info *mci = dev_id;
 542         struct thunderx_lmc *lmc = mci->pvt_info;
 543
 544         unsigned long head = ring_pos(lmc->ring_head, ARRAY_SIZE(lmc->err_ctx));
 545         struct lmc_err_ctx *ctx = &lmc->err_ctx[head];
 546
 547         writeq(0, lmc->regs + LMC_CHAR_MASK0);
 548         writeq(0, lmc->regs + LMC_CHAR_MASK2);
 549         writeq(0x2, lmc->regs + LMC_ECC_PARITY_TEST);
 550
 551         ctx->reg_int = readq(lmc->regs + LMC_INT);
 552         ctx->reg_fadr = readq(lmc->regs + LMC_FADR);
 553         ctx->reg_nxm_fadr = readq(lmc->regs + LMC_NXM_FADR);
 554         ctx->reg_scram_fadr = readq(lmc->regs + LMC_SCRAM_FADR);
 555         ctx->reg_ecc_synd = readq(lmc->regs + LMC_ECC_SYND);
 556
 557         lmc->ring_head++;
 558
 559         atomic_set(&lmc->ecc_int, 1);
 560
 561         /* Clear the interrupt */
 562         writeq(ctx->reg_int, lmc->regs + LMC_INT);
 563
 564         return IRQ_WAKE_THREAD;
 565 }
 566
 567 static irqreturn_t thunderx_lmc_threaded_isr(int irq, void *dev_id)
 568 {
 569         struct mem_ctl_info *mci = dev_id;
 570         struct thunderx_lmc *lmc = mci->pvt_info;
 571         phys_addr_t phys_addr;
 572
 573         unsigned long tail;
 574         struct lmc_err_ctx *ctx;
 575
 576         irqreturn_t ret = IRQ_NONE;
 577
 578         char *msg;
 579         char *other;
 580
 581         msg = kmalloc(LMC_MESSAGE_SIZE, GFP_KERNEL);
 582         other =  kmalloc(LMC_OTHER_SIZE, GFP_KERNEL);
 583
 584         if (!msg || !other)
 585                 goto err_free;
 586
 587         while (CIRC_CNT(lmc->ring_head, lmc->ring_tail,
 588                 ARRAY_SIZE(lmc->err_ctx))) {
 589                 tail = ring_pos(lmc->ring_tail, ARRAY_SIZE(lmc->err_ctx));
 590
 591                 ctx = &lmc->err_ctx[tail];
 592
 593                 dev_dbg(&lmc->pdev->dev, "LMC_INT: %016llx\n",
 594                         ctx->reg_int);
 595                 dev_dbg(&lmc->pdev->dev, "LMC_FADR: %016llx\n",
 596                         ctx->reg_fadr);
 597                 dev_dbg(&lmc->pdev->dev, "LMC_NXM_FADR: %016llx\n",
 598                         ctx->reg_nxm_fadr);
 599                 dev_dbg(&lmc->pdev->dev, "LMC_SCRAM_FADR: %016llx\n",
 600                         ctx->reg_scram_fadr);
 601                 dev_dbg(&lmc->pdev->dev, "LMC_ECC_SYND: %016llx\n",
 602                         ctx->reg_ecc_synd);
 603
 604                 snprintf(msg, LMC_MESSAGE_SIZE,
 605                          "DIMM %lld rank %lld bank %lld row %lld col %lld",
 606                          LMC_FADR_FDIMM(ctx->reg_scram_fadr),
 607                          LMC_FADR_FBUNK(ctx->reg_scram_fadr),
 608                          LMC_FADR_FBANK(ctx->reg_scram_fadr),
 609                          LMC_FADR_FROW(ctx->reg_scram_fadr),
 610                          LMC_FADR_FCOL(ctx->reg_scram_fadr));
 611
 612                 decode_register(other, LMC_OTHER_SIZE, lmc_errors,
 613                                 ctx->reg_int);
 614
 615                 phys_addr = thunderx_faddr_to_phys(ctx->reg_fadr, lmc);
 616
 617                 if (ctx->reg_int & LMC_INT_UE)
 618                         edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
 619                                              phys_to_pfn(phys_addr),
 620                                              offset_in_page(phys_addr),
 621                                              0, -1, -1, -1, msg, other);
 622                 else if (ctx->reg_int & LMC_INT_CE)
 623                         edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1,
 624                                              phys_to_pfn(phys_addr),
 625                                              offset_in_page(phys_addr),
 626                                              0, -1, -1, -1, msg, other);
 627
 628                 lmc->ring_tail++;
 629         }
 630
 631         ret = IRQ_HANDLED;
 632
 633 err_free:
 634         kfree(msg);
 635         kfree(other);
 636
 637         return ret;
 638 }
 639
 640 static const struct pci_device_id thunderx_lmc_pci_tbl[] = {
 641         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_LMC) },
 642         { 0, },
 643 };
 644
 645 static inline int pci_dev_to_mc_idx(struct pci_dev *pdev)
 646 {
 647         int node = dev_to_node(&pdev->dev);
 648         int ret = PCI_FUNC(pdev->devfn);
 649
 650         ret += max(node, 0) << 3;
 651
 652         return ret;
 653 }
 654
 655 static int thunderx_lmc_probe(struct pci_dev *pdev,
 656                                 const struct pci_device_id *id)
 657 {
 658         struct thunderx_lmc *lmc;
 659         struct edac_mc_layer layer;
 660         struct mem_ctl_info *mci;
 661         u64 lmc_control, lmc_ddr_pll_ctl, lmc_config;
 662         int ret;
 663         u64 lmc_int;
 664         void *l2c_ioaddr;
 665
 666         layer.type = EDAC_MC_LAYER_SLOT;
 667         layer.size = 2;
 668         layer.is_virt_csrow = false;
 669
 670         ret = pcim_enable_device(pdev);
 671         if (ret) {
 672                 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
 673                 return ret;
 674         }
 675
 676         ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_lmc");
 677         if (ret) {
 678                 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
 679                 return ret;
 680         }
 681
 682         mci = edac_mc_alloc(pci_dev_to_mc_idx(pdev), 1, &layer,
 683                             sizeof(struct thunderx_lmc));
 684         if (!mci)
 685                 return -ENOMEM;
 686
 687         mci->pdev = &pdev->dev;
 688         lmc = mci->pvt_info;
 689
 690         pci_set_drvdata(pdev, mci);
 691
 692         lmc->regs = pcim_iomap_table(pdev)[0];
 693
 694         lmc_control = readq(lmc->regs + LMC_CONTROL);
 695         lmc_ddr_pll_ctl = readq(lmc->regs + LMC_DDR_PLL_CTL);
 696         lmc_config = readq(lmc->regs + LMC_CONFIG);
 697
 698         if (lmc_control & LMC_CONTROL_RDIMM) {
 699                 mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
 700                                            lmc_ddr_pll_ctl) ?
 701                                 MEM_RDDR4 : MEM_RDDR3;
 702         } else {
 703                 mci->mtype_cap = FIELD_GET(LMC_DDR_PLL_CTL_DDR4,
 704                                            lmc_ddr_pll_ctl) ?
 705                                 MEM_DDR4 : MEM_DDR3;
 706         }
 707
 708         mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
 709         mci->edac_cap = EDAC_FLAG_SECDED;
 710
 711         mci->mod_name = "thunderx-lmc";
 712         mci->ctl_name = "thunderx-lmc";
 713         mci->dev_name = dev_name(&pdev->dev);
 714         mci->scrub_mode = SCRUB_NONE;
 715
 716         lmc->pdev = pdev;
 717         lmc->msix_ent.entry = 0;
 718
 719         lmc->ring_head = 0;
 720         lmc->ring_tail = 0;
 721
 722         ret = pci_enable_msix_exact(pdev, &lmc->msix_ent, 1);
 723         if (ret) {
 724                 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
 725                 goto err_free;
 726         }
 727
 728         ret = devm_request_threaded_irq(&pdev->dev, lmc->msix_ent.vector,
 729                                         thunderx_lmc_err_isr,
 730                                         thunderx_lmc_threaded_isr, 0,
 731                                         "[EDAC] ThunderX LMC", mci);
 732         if (ret) {
 733                 dev_err(&pdev->dev, "Cannot set ISR: %d\n", ret);
 734                 goto err_free;
 735         }
 736
 737         lmc->node = FIELD_GET(THUNDERX_NODE, pci_resource_start(pdev, 0));
 738
 739         lmc->xbits = thunderx_get_num_lmcs(lmc->node) >> 1;
 740         lmc->bank_width = (FIELD_GET(LMC_DDR_PLL_CTL_DDR4, lmc_ddr_pll_ctl) &&
 741                            FIELD_GET(LMC_CONFIG_BG2, lmc_config)) ? 4 : 3;
 742
 743         lmc->pbank_lsb = (lmc_config >> 5) & 0xf;
 744         lmc->dimm_lsb  = 28 + lmc->pbank_lsb + lmc->xbits;
 745         lmc->rank_lsb = lmc->dimm_lsb;
 746         lmc->rank_lsb -= FIELD_GET(LMC_CONFIG_RANK_ENA, lmc_config) ? 1 : 0;
 747         lmc->bank_lsb = 7 + lmc->xbits;
 748         lmc->row_lsb = 14 + LMC_CONFIG_ROW_LSB(lmc_config) + lmc->xbits;
 749
 750         lmc->col_hi_lsb = lmc->bank_lsb + lmc->bank_width;
 751
 752         lmc->xor_bank = lmc_control & LMC_CONTROL_XOR_BANK;
 753
 754         l2c_ioaddr = ioremap(L2C_CTL | FIELD_PREP(THUNDERX_NODE, lmc->node), PAGE_SIZE);
 755         if (!l2c_ioaddr) {
 756                 dev_err(&pdev->dev, "Cannot map L2C_CTL\n");
 757                 ret = -ENOMEM;
 758                 goto err_free;
 759         }
 760
 761         lmc->l2c_alias = !(readq(l2c_ioaddr) & L2C_CTL_DISIDXALIAS);
 762
 763         iounmap(l2c_ioaddr);
 764
 765         ret = edac_mc_add_mc(mci);
 766         if (ret) {
 767                 dev_err(&pdev->dev, "Cannot add the MC: %d\n", ret);
 768                 goto err_free;
 769         }
 770
 771         lmc_int = readq(lmc->regs + LMC_INT);
 772         writeq(lmc_int, lmc->regs + LMC_INT);
 773
 774         writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1S);
 775
 776         if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
 777                 ret = thunderx_create_debugfs_nodes(mci->debugfs,
 778                                                     lmc_dfs_ents,
 779                                                     lmc,
 780                                                     ARRAY_SIZE(lmc_dfs_ents));
 781
 782                 if (ret != ARRAY_SIZE(lmc_dfs_ents)) {
 783                         dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
 784                                  ret, ret >= 0 ? " created" : "");
 785                 }
 786         }
 787
 788         return 0;
 789
 790 err_free:
 791         pci_set_drvdata(pdev, NULL);
 792         edac_mc_free(mci);
 793
 794         return ret;
 795 }
 796
 797 static void thunderx_lmc_remove(struct pci_dev *pdev)
 798 {
 799         struct mem_ctl_info *mci = pci_get_drvdata(pdev);
 800         struct thunderx_lmc *lmc = mci->pvt_info;
 801
 802         writeq(LMC_INT_ENA_ALL, lmc->regs + LMC_INT_ENA_W1C);
 803
 804         edac_mc_del_mc(&pdev->dev);
 805         edac_mc_free(mci);
 806 }
 807
 808 MODULE_DEVICE_TABLE(pci, thunderx_lmc_pci_tbl);
 809
 810 static struct pci_driver thunderx_lmc_driver = {
 811         .name     = "thunderx_lmc_edac",
 812         .probe    = thunderx_lmc_probe,
 813         .remove   = thunderx_lmc_remove,
 814         .id_table = thunderx_lmc_pci_tbl,
 815 };
 816
 817 /*---------------------- OCX driver ---------------------------------*/
 818
 819 #define PCI_DEVICE_ID_THUNDER_OCX 0xa013
 820
 821 #define OCX_LINK_INTS           3
 822 #define OCX_INTS                (OCX_LINK_INTS + 1)
 823 #define OCX_RX_LANES            24
 824 #define OCX_RX_LANE_STATS       15
 825
 826 #define OCX_COM_INT             0x100
 827 #define OCX_COM_INT_W1S         0x108
 828 #define OCX_COM_INT_ENA_W1S     0x110
 829 #define OCX_COM_INT_ENA_W1C     0x118
 830
 831 #define OCX_COM_IO_BADID                BIT(54)
 832 #define OCX_COM_MEM_BADID               BIT(53)
 833 #define OCX_COM_COPR_BADID              BIT(52)
 834 #define OCX_COM_WIN_REQ_BADID           BIT(51)
 835 #define OCX_COM_WIN_REQ_TOUT            BIT(50)
 836 #define OCX_COM_RX_LANE                 GENMASK(23, 0)
 837
 838 #define OCX_COM_INT_CE                  (OCX_COM_IO_BADID      | \
 839                                          OCX_COM_MEM_BADID     | \
 840                                          OCX_COM_COPR_BADID    | \
 841                                          OCX_COM_WIN_REQ_BADID | \
 842                                          OCX_COM_WIN_REQ_TOUT)
 843
 844 static const struct error_descr ocx_com_errors[] = {
 845         {
 846                 .type  = ERR_CORRECTED,
 847                 .mask  = OCX_COM_IO_BADID,
 848                 .descr = "Invalid IO transaction node ID",
 849         },
 850         {
 851                 .type  = ERR_CORRECTED,
 852                 .mask  = OCX_COM_MEM_BADID,
 853                 .descr = "Invalid memory transaction node ID",
 854         },
 855         {
 856                 .type  = ERR_CORRECTED,
 857                 .mask  = OCX_COM_COPR_BADID,
 858                 .descr = "Invalid coprocessor transaction node ID",
 859         },
 860         {
 861                 .type  = ERR_CORRECTED,
 862                 .mask  = OCX_COM_WIN_REQ_BADID,
 863                 .descr = "Invalid SLI transaction node ID",
 864         },
 865         {
 866                 .type  = ERR_CORRECTED,
 867                 .mask  = OCX_COM_WIN_REQ_TOUT,
 868                 .descr = "Window/core request timeout",
 869         },
 870         {0, 0, NULL},
 871 };
 872
 873 #define OCX_COM_LINKX_INT(x)            (0x120 + (x) * 8)
 874 #define OCX_COM_LINKX_INT_W1S(x)        (0x140 + (x) * 8)
 875 #define OCX_COM_LINKX_INT_ENA_W1S(x)    (0x160 + (x) * 8)
 876 #define OCX_COM_LINKX_INT_ENA_W1C(x)    (0x180 + (x) * 8)
 877
 878 #define OCX_COM_LINK_BAD_WORD                   BIT(13)
 879 #define OCX_COM_LINK_ALIGN_FAIL                 BIT(12)
 880 #define OCX_COM_LINK_ALIGN_DONE                 BIT(11)
 881 #define OCX_COM_LINK_UP                         BIT(10)
 882 #define OCX_COM_LINK_STOP                       BIT(9)
 883 #define OCX_COM_LINK_BLK_ERR                    BIT(8)
 884 #define OCX_COM_LINK_REINIT                     BIT(7)
 885 #define OCX_COM_LINK_LNK_DATA                   BIT(6)
 886 #define OCX_COM_LINK_RXFIFO_DBE                 BIT(5)
 887 #define OCX_COM_LINK_RXFIFO_SBE                 BIT(4)
 888 #define OCX_COM_LINK_TXFIFO_DBE                 BIT(3)
 889 #define OCX_COM_LINK_TXFIFO_SBE                 BIT(2)
 890 #define OCX_COM_LINK_REPLAY_DBE                 BIT(1)
 891 #define OCX_COM_LINK_REPLAY_SBE                 BIT(0)
 892
 893 static const struct error_descr ocx_com_link_errors[] = {
 894         {
 895                 .type  = ERR_CORRECTED,
 896                 .mask  = OCX_COM_LINK_REPLAY_SBE,
 897                 .descr = "Replay buffer single-bit error",
 898         },
 899         {
 900                 .type  = ERR_CORRECTED,
 901                 .mask  = OCX_COM_LINK_TXFIFO_SBE,
 902                 .descr = "TX FIFO single-bit error",
 903         },
 904         {
 905                 .type  = ERR_CORRECTED,
 906                 .mask  = OCX_COM_LINK_RXFIFO_SBE,
 907                 .descr = "RX FIFO single-bit error",
 908         },
 909         {
 910                 .type  = ERR_CORRECTED,
 911                 .mask  = OCX_COM_LINK_BLK_ERR,
 912                 .descr = "Block code error",
 913         },
 914         {
 915                 .type  = ERR_CORRECTED,
 916                 .mask  = OCX_COM_LINK_ALIGN_FAIL,
 917                 .descr = "Link alignment failure",
 918         },
 919         {
 920                 .type  = ERR_CORRECTED,
 921                 .mask  = OCX_COM_LINK_BAD_WORD,
 922                 .descr = "Bad code word",
 923         },
 924         {
 925                 .type  = ERR_UNCORRECTED,
 926                 .mask  = OCX_COM_LINK_REPLAY_DBE,
 927                 .descr = "Replay buffer double-bit error",
 928         },
 929         {
 930                 .type  = ERR_UNCORRECTED,
 931                 .mask  = OCX_COM_LINK_TXFIFO_DBE,
 932                 .descr = "TX FIFO double-bit error",
 933         },
 934         {
 935                 .type  = ERR_UNCORRECTED,
 936                 .mask  = OCX_COM_LINK_RXFIFO_DBE,
 937                 .descr = "RX FIFO double-bit error",
 938         },
 939         {
 940                 .type  = ERR_UNCORRECTED,
 941                 .mask  = OCX_COM_LINK_STOP,
 942                 .descr = "Link stopped",
 943         },
 944         {0, 0, NULL},
 945 };
 946
 947 #define OCX_COM_LINK_INT_UE       (OCX_COM_LINK_REPLAY_DBE | \
 948                                    OCX_COM_LINK_TXFIFO_DBE | \
 949                                    OCX_COM_LINK_RXFIFO_DBE | \
 950                                    OCX_COM_LINK_STOP)
 951
 952 #define OCX_COM_LINK_INT_CE       (OCX_COM_LINK_REPLAY_SBE | \
 953                                    OCX_COM_LINK_TXFIFO_SBE | \
 954                                    OCX_COM_LINK_RXFIFO_SBE | \
 955                                    OCX_COM_LINK_BLK_ERR    | \
 956                                    OCX_COM_LINK_ALIGN_FAIL | \
 957                                    OCX_COM_LINK_BAD_WORD)
 958
 959 #define OCX_LNE_INT(x)                  (0x8018 + (x) * 0x100)
 960 #define OCX_LNE_INT_EN(x)               (0x8020 + (x) * 0x100)
 961 #define OCX_LNE_BAD_CNT(x)              (0x8028 + (x) * 0x100)
 962 #define OCX_LNE_CFG(x)                  (0x8000 + (x) * 0x100)
 963 #define OCX_LNE_STAT(x, y)              (0x8040 + (x) * 0x100 + (y) * 8)
 964
 965 #define OCX_LNE_CFG_RX_BDRY_LOCK_DIS            BIT(8)
 966 #define OCX_LNE_CFG_RX_STAT_WRAP_DIS            BIT(2)
 967 #define OCX_LNE_CFG_RX_STAT_RDCLR               BIT(1)
 968 #define OCX_LNE_CFG_RX_STAT_ENA                 BIT(0)
 969
 970
 971 #define OCX_LANE_BAD_64B67B                     BIT(8)
 972 #define OCX_LANE_DSKEW_FIFO_OVFL                BIT(5)
 973 #define OCX_LANE_SCRM_SYNC_LOSS                 BIT(4)
 974 #define OCX_LANE_UKWN_CNTL_WORD                 BIT(3)
 975 #define OCX_LANE_CRC32_ERR                      BIT(2)
 976 #define OCX_LANE_BDRY_SYNC_LOSS                 BIT(1)
 977 #define OCX_LANE_SERDES_LOCK_LOSS               BIT(0)
 978
 979 #define OCX_COM_LANE_INT_UE       (0)
 980 #define OCX_COM_LANE_INT_CE       (OCX_LANE_SERDES_LOCK_LOSS | \
 981                                    OCX_LANE_BDRY_SYNC_LOSS   | \
 982                                    OCX_LANE_CRC32_ERR        | \
 983                                    OCX_LANE_UKWN_CNTL_WORD   | \
 984                                    OCX_LANE_SCRM_SYNC_LOSS   | \
 985                                    OCX_LANE_DSKEW_FIFO_OVFL  | \
 986                                    OCX_LANE_BAD_64B67B)
 987
 988 static const struct error_descr ocx_lane_errors[] = {
 989         {
 990                 .type  = ERR_CORRECTED,
 991                 .mask  = OCX_LANE_SERDES_LOCK_LOSS,
 992                 .descr = "RX SerDes lock lost",
 993         },
 994         {
 995                 .type  = ERR_CORRECTED,
 996                 .mask  = OCX_LANE_BDRY_SYNC_LOSS,
 997                 .descr = "RX word boundary lost",
 998         },
 999         {
1000                 .type  = ERR_CORRECTED,
1001                 .mask  = OCX_LANE_CRC32_ERR,
1002                 .descr = "CRC32 error",
1003         },
1004         {
1005                 .type  = ERR_CORRECTED,
1006                 .mask  = OCX_LANE_UKWN_CNTL_WORD,
1007                 .descr = "Unknown control word",
1008         },
1009         {
1010                 .type  = ERR_CORRECTED,
1011                 .mask  = OCX_LANE_SCRM_SYNC_LOSS,
1012                 .descr = "Scrambler synchronization lost",
1013         },
1014         {
1015                 .type  = ERR_CORRECTED,
1016                 .mask  = OCX_LANE_DSKEW_FIFO_OVFL,
1017                 .descr = "RX deskew FIFO overflow",
1018         },
1019         {
1020                 .type  = ERR_CORRECTED,
1021                 .mask  = OCX_LANE_BAD_64B67B,
1022                 .descr = "Bad 64B/67B codeword",
1023         },
1024         {0, 0, NULL},
1025 };
1026
1027 #define OCX_LNE_INT_ENA_ALL             (GENMASK(9, 8) | GENMASK(6, 0))
1028 #define OCX_COM_INT_ENA_ALL             (GENMASK(54, 50) | GENMASK(23, 0))
1029 #define OCX_COM_LINKX_INT_ENA_ALL       (GENMASK(13, 12) | \
1030                                          GENMASK(9, 7) | GENMASK(5, 0))
1031
1032 #define OCX_TLKX_ECC_CTL(x)             (0x10018 + (x) * 0x2000)
1033 #define OCX_RLKX_ECC_CTL(x)             (0x18018 + (x) * 0x2000)
1034
1035 struct ocx_com_err_ctx {
1036         u64 reg_com_int;
1037         u64 reg_lane_int[OCX_RX_LANES];
1038         u64 reg_lane_stat11[OCX_RX_LANES];
1039 };
1040
1041 struct ocx_link_err_ctx {
1042         u64 reg_com_link_int;
1043         int link;
1044 };
1045
1046 struct thunderx_ocx {
1047         void __iomem *regs;
1048         int com_link;
1049         struct pci_dev *pdev;
1050         struct edac_device_ctl_info *edac_dev;
1051
1052         struct dentry *debugfs;
1053         struct msix_entry msix_ent[OCX_INTS];
1054
1055         struct ocx_com_err_ctx com_err_ctx[RING_ENTRIES];
1056         struct ocx_link_err_ctx link_err_ctx[RING_ENTRIES];
1057
1058         unsigned long com_ring_head;
1059         unsigned long com_ring_tail;
1060
1061         unsigned long link_ring_head;
1062         unsigned long link_ring_tail;
1063 };
1064
1065 #define OCX_MESSAGE_SIZE        SZ_1K
1066 #define OCX_OTHER_SIZE          (50 * ARRAY_SIZE(ocx_com_link_errors))
1067
1068 /* This handler is threaded */
1069 static irqreturn_t thunderx_ocx_com_isr(int irq, void *irq_id)
1070 {
1071         struct msix_entry *msix = irq_id;
1072         struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1073                                                 msix_ent[msix->entry]);
1074
1075         int lane;
1076         unsigned long head = ring_pos(ocx->com_ring_head,
1077                                       ARRAY_SIZE(ocx->com_err_ctx));
1078         struct ocx_com_err_ctx *ctx = &ocx->com_err_ctx[head];
1079
1080         ctx->reg_com_int = readq(ocx->regs + OCX_COM_INT);
1081
1082         for (lane = 0; lane < OCX_RX_LANES; lane++) {
1083                 ctx->reg_lane_int[lane] =
1084                         readq(ocx->regs + OCX_LNE_INT(lane));
1085                 ctx->reg_lane_stat11[lane] =
1086                         readq(ocx->regs + OCX_LNE_STAT(lane, 11));
1087
1088                 writeq(ctx->reg_lane_int[lane], ocx->regs + OCX_LNE_INT(lane));
1089         }
1090
1091         writeq(ctx->reg_com_int, ocx->regs + OCX_COM_INT);
1092
1093         ocx->com_ring_head++;
1094
1095         return IRQ_WAKE_THREAD;
1096 }
1097
1098 static irqreturn_t thunderx_ocx_com_threaded_isr(int irq, void *irq_id)
1099 {
1100         struct msix_entry *msix = irq_id;
1101         struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1102                                                 msix_ent[msix->entry]);
1103
1104         irqreturn_t ret = IRQ_NONE;
1105
1106         unsigned long tail;
1107         struct ocx_com_err_ctx *ctx;
1108         int lane;
1109         char *msg;
1110         char *other;
1111
1112         msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1113         other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1114
1115         if (!msg || !other)
1116                 goto err_free;
1117
1118         while (CIRC_CNT(ocx->com_ring_head, ocx->com_ring_tail,
1119                         ARRAY_SIZE(ocx->com_err_ctx))) {
1120                 tail = ring_pos(ocx->com_ring_tail,
1121                                 ARRAY_SIZE(ocx->com_err_ctx));
1122                 ctx = &ocx->com_err_ctx[tail];
1123
1124                 snprintf(msg, OCX_MESSAGE_SIZE, "%s: OCX_COM_INT: %016llx",
1125                         ocx->edac_dev->ctl_name, ctx->reg_com_int);
1126
1127                 decode_register(other, OCX_OTHER_SIZE,
1128                                 ocx_com_errors, ctx->reg_com_int);
1129
1130                 strlcat(msg, other, OCX_MESSAGE_SIZE);
1131
1132                 for (lane = 0; lane < OCX_RX_LANES; lane++)
1133                         if (ctx->reg_com_int & BIT(lane)) {
1134                                 snprintf(other, OCX_OTHER_SIZE,
1135                                          "\n\tOCX_LNE_INT[%02d]: %016llx OCX_LNE_STAT11[%02d]: %016llx",
1136                                          lane, ctx->reg_lane_int[lane],
1137                                          lane, ctx->reg_lane_stat11[lane]);
1138
1139                                 strlcat(msg, other, OCX_MESSAGE_SIZE);
1140
1141                                 decode_register(other, OCX_OTHER_SIZE,
1142                                                 ocx_lane_errors,
1143                                                 ctx->reg_lane_int[lane]);
1144                                 strlcat(msg, other, OCX_MESSAGE_SIZE);
1145                         }
1146
1147                 if (ctx->reg_com_int & OCX_COM_INT_CE)
1148                         edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1149
1150                 ocx->com_ring_tail++;
1151         }
1152
1153         ret = IRQ_HANDLED;
1154
1155 err_free:
1156         kfree(other);
1157         kfree(msg);
1158
1159         return ret;
1160 }
1161
1162 static irqreturn_t thunderx_ocx_lnk_isr(int irq, void *irq_id)
1163 {
1164         struct msix_entry *msix = irq_id;
1165         struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1166                                                 msix_ent[msix->entry]);
1167         unsigned long head = ring_pos(ocx->link_ring_head,
1168                                       ARRAY_SIZE(ocx->link_err_ctx));
1169         struct ocx_link_err_ctx *ctx = &ocx->link_err_ctx[head];
1170
1171         ctx->link = msix->entry;
1172         ctx->reg_com_link_int = readq(ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1173
1174         writeq(ctx->reg_com_link_int, ocx->regs + OCX_COM_LINKX_INT(ctx->link));
1175
1176         ocx->link_ring_head++;
1177
1178         return IRQ_WAKE_THREAD;
1179 }
1180
1181 static irqreturn_t thunderx_ocx_lnk_threaded_isr(int irq, void *irq_id)
1182 {
1183         struct msix_entry *msix = irq_id;
1184         struct thunderx_ocx *ocx = container_of(msix, struct thunderx_ocx,
1185                                                 msix_ent[msix->entry]);
1186         irqreturn_t ret = IRQ_NONE;
1187         unsigned long tail;
1188         struct ocx_link_err_ctx *ctx;
1189
1190         char *msg;
1191         char *other;
1192
1193         msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1194         other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1195
1196         if (!msg || !other)
1197                 goto err_free;
1198
1199         while (CIRC_CNT(ocx->link_ring_head, ocx->link_ring_tail,
1200                         ARRAY_SIZE(ocx->link_err_ctx))) {
1201                 tail = ring_pos(ocx->link_ring_head,
1202                                 ARRAY_SIZE(ocx->link_err_ctx));
1203
1204                 ctx = &ocx->link_err_ctx[tail];
1205
1206                 snprintf(msg, OCX_MESSAGE_SIZE,
1207                          "%s: OCX_COM_LINK_INT[%d]: %016llx",
1208                          ocx->edac_dev->ctl_name,
1209                          ctx->link, ctx->reg_com_link_int);
1210
1211                 decode_register(other, OCX_OTHER_SIZE,
1212                                 ocx_com_link_errors, ctx->reg_com_link_int);
1213
1214                 strlcat(msg, other, OCX_MESSAGE_SIZE);
1215
1216                 if (ctx->reg_com_link_int & OCX_COM_LINK_INT_UE)
1217                         edac_device_handle_ue(ocx->edac_dev, 0, 0, msg);
1218                 else if (ctx->reg_com_link_int & OCX_COM_LINK_INT_CE)
1219                         edac_device_handle_ce(ocx->edac_dev, 0, 0, msg);
1220
1221                 ocx->link_ring_tail++;
1222         }
1223
1224         ret = IRQ_HANDLED;
1225 err_free:
1226         kfree(other);
1227         kfree(msg);
1228
1229         return ret;
1230 }
1231
1232 #define OCX_DEBUGFS_ATTR(_name, _reg)   DEBUGFS_REG_ATTR(ocx, _name, _reg)
1233
1234 OCX_DEBUGFS_ATTR(tlk0_ecc_ctl, OCX_TLKX_ECC_CTL(0));
1235 OCX_DEBUGFS_ATTR(tlk1_ecc_ctl, OCX_TLKX_ECC_CTL(1));
1236 OCX_DEBUGFS_ATTR(tlk2_ecc_ctl, OCX_TLKX_ECC_CTL(2));
1237
1238 OCX_DEBUGFS_ATTR(rlk0_ecc_ctl, OCX_RLKX_ECC_CTL(0));
1239 OCX_DEBUGFS_ATTR(rlk1_ecc_ctl, OCX_RLKX_ECC_CTL(1));
1240 OCX_DEBUGFS_ATTR(rlk2_ecc_ctl, OCX_RLKX_ECC_CTL(2));
1241
1242 OCX_DEBUGFS_ATTR(com_link0_int, OCX_COM_LINKX_INT_W1S(0));
1243 OCX_DEBUGFS_ATTR(com_link1_int, OCX_COM_LINKX_INT_W1S(1));
1244 OCX_DEBUGFS_ATTR(com_link2_int, OCX_COM_LINKX_INT_W1S(2));
1245
1246 OCX_DEBUGFS_ATTR(lne00_badcnt, OCX_LNE_BAD_CNT(0));
1247 OCX_DEBUGFS_ATTR(lne01_badcnt, OCX_LNE_BAD_CNT(1));
1248 OCX_DEBUGFS_ATTR(lne02_badcnt, OCX_LNE_BAD_CNT(2));
1249 OCX_DEBUGFS_ATTR(lne03_badcnt, OCX_LNE_BAD_CNT(3));
1250 OCX_DEBUGFS_ATTR(lne04_badcnt, OCX_LNE_BAD_CNT(4));
1251 OCX_DEBUGFS_ATTR(lne05_badcnt, OCX_LNE_BAD_CNT(5));
1252 OCX_DEBUGFS_ATTR(lne06_badcnt, OCX_LNE_BAD_CNT(6));
1253 OCX_DEBUGFS_ATTR(lne07_badcnt, OCX_LNE_BAD_CNT(7));
1254
1255 OCX_DEBUGFS_ATTR(lne08_badcnt, OCX_LNE_BAD_CNT(8));
1256 OCX_DEBUGFS_ATTR(lne09_badcnt, OCX_LNE_BAD_CNT(9));
1257 OCX_DEBUGFS_ATTR(lne10_badcnt, OCX_LNE_BAD_CNT(10));
1258 OCX_DEBUGFS_ATTR(lne11_badcnt, OCX_LNE_BAD_CNT(11));
1259 OCX_DEBUGFS_ATTR(lne12_badcnt, OCX_LNE_BAD_CNT(12));
1260 OCX_DEBUGFS_ATTR(lne13_badcnt, OCX_LNE_BAD_CNT(13));
1261 OCX_DEBUGFS_ATTR(lne14_badcnt, OCX_LNE_BAD_CNT(14));
1262 OCX_DEBUGFS_ATTR(lne15_badcnt, OCX_LNE_BAD_CNT(15));
1263
1264 OCX_DEBUGFS_ATTR(lne16_badcnt, OCX_LNE_BAD_CNT(16));
1265 OCX_DEBUGFS_ATTR(lne17_badcnt, OCX_LNE_BAD_CNT(17));
1266 OCX_DEBUGFS_ATTR(lne18_badcnt, OCX_LNE_BAD_CNT(18));
1267 OCX_DEBUGFS_ATTR(lne19_badcnt, OCX_LNE_BAD_CNT(19));
1268 OCX_DEBUGFS_ATTR(lne20_badcnt, OCX_LNE_BAD_CNT(20));
1269 OCX_DEBUGFS_ATTR(lne21_badcnt, OCX_LNE_BAD_CNT(21));
1270 OCX_DEBUGFS_ATTR(lne22_badcnt, OCX_LNE_BAD_CNT(22));
1271 OCX_DEBUGFS_ATTR(lne23_badcnt, OCX_LNE_BAD_CNT(23));
1272
1273 OCX_DEBUGFS_ATTR(com_int, OCX_COM_INT_W1S);
1274
1275 static struct debugfs_entry *ocx_dfs_ents[] = {
1276         &debugfs_tlk0_ecc_ctl,
1277         &debugfs_tlk1_ecc_ctl,
1278         &debugfs_tlk2_ecc_ctl,
1279
1280         &debugfs_rlk0_ecc_ctl,
1281         &debugfs_rlk1_ecc_ctl,
1282         &debugfs_rlk2_ecc_ctl,
1283
1284         &debugfs_com_link0_int,
1285         &debugfs_com_link1_int,
1286         &debugfs_com_link2_int,
1287
1288         &debugfs_lne00_badcnt,
1289         &debugfs_lne01_badcnt,
1290         &debugfs_lne02_badcnt,
1291         &debugfs_lne03_badcnt,
1292         &debugfs_lne04_badcnt,
1293         &debugfs_lne05_badcnt,
1294         &debugfs_lne06_badcnt,
1295         &debugfs_lne07_badcnt,
1296         &debugfs_lne08_badcnt,
1297         &debugfs_lne09_badcnt,
1298         &debugfs_lne10_badcnt,
1299         &debugfs_lne11_badcnt,
1300         &debugfs_lne12_badcnt,
1301         &debugfs_lne13_badcnt,
1302         &debugfs_lne14_badcnt,
1303         &debugfs_lne15_badcnt,
1304         &debugfs_lne16_badcnt,
1305         &debugfs_lne17_badcnt,
1306         &debugfs_lne18_badcnt,
1307         &debugfs_lne19_badcnt,
1308         &debugfs_lne20_badcnt,
1309         &debugfs_lne21_badcnt,
1310         &debugfs_lne22_badcnt,
1311         &debugfs_lne23_badcnt,
1312
1313         &debugfs_com_int,
1314 };
1315
1316 static const struct pci_device_id thunderx_ocx_pci_tbl[] = {
1317         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_OCX) },
1318         { 0, },
1319 };
1320
1321 static void thunderx_ocx_clearstats(struct thunderx_ocx *ocx)
1322 {
1323         int lane, stat, cfg;
1324
1325         for (lane = 0; lane < OCX_RX_LANES; lane++) {
1326                 cfg = readq(ocx->regs + OCX_LNE_CFG(lane));
1327                 cfg |= OCX_LNE_CFG_RX_STAT_RDCLR;
1328                 cfg &= ~OCX_LNE_CFG_RX_STAT_ENA;
1329                 writeq(cfg, ocx->regs + OCX_LNE_CFG(lane));
1330
1331                 for (stat = 0; stat < OCX_RX_LANE_STATS; stat++)
1332                         readq(ocx->regs + OCX_LNE_STAT(lane, stat));
1333         }
1334 }
1335
1336 static int thunderx_ocx_probe(struct pci_dev *pdev,
1337                               const struct pci_device_id *id)
1338 {
1339         struct thunderx_ocx *ocx;
1340         struct edac_device_ctl_info *edac_dev;
1341         char name[32];
1342         int idx;
1343         int i;
1344         int ret;
1345         u64 reg;
1346
1347         ret = pcim_enable_device(pdev);
1348         if (ret) {
1349                 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1350                 return ret;
1351         }
1352
1353         ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_ocx");
1354         if (ret) {
1355                 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1356                 return ret;
1357         }
1358
1359         idx = edac_device_alloc_index();
1360         snprintf(name, sizeof(name), "OCX%d", idx);
1361         edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_ocx),
1362                                               name, 1, "CCPI", 1, 0, idx);
1363         if (!edac_dev) {
1364                 dev_err(&pdev->dev, "Cannot allocate EDAC device\n");
1365                 return -ENOMEM;
1366         }
1367         ocx = edac_dev->pvt_info;
1368         ocx->edac_dev = edac_dev;
1369         ocx->com_ring_head = 0;
1370         ocx->com_ring_tail = 0;
1371         ocx->link_ring_head = 0;
1372         ocx->link_ring_tail = 0;
1373
1374         ocx->regs = pcim_iomap_table(pdev)[0];
1375         if (!ocx->regs) {
1376                 dev_err(&pdev->dev, "Cannot map PCI resources\n");
1377                 ret = -ENODEV;
1378                 goto err_free;
1379         }
1380
1381         ocx->pdev = pdev;
1382
1383         for (i = 0; i < OCX_INTS; i++) {
1384                 ocx->msix_ent[i].entry = i;
1385                 ocx->msix_ent[i].vector = 0;
1386         }
1387
1388         ret = pci_enable_msix_exact(pdev, ocx->msix_ent, OCX_INTS);
1389         if (ret) {
1390                 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
1391                 goto err_free;
1392         }
1393
1394         for (i = 0; i < OCX_INTS; i++) {
1395                 ret = devm_request_threaded_irq(&pdev->dev,
1396                                                 ocx->msix_ent[i].vector,
1397                                                 (i == 3) ?
1398                                                  thunderx_ocx_com_isr :
1399                                                  thunderx_ocx_lnk_isr,
1400                                                 (i == 3) ?
1401                                                  thunderx_ocx_com_threaded_isr :
1402                                                  thunderx_ocx_lnk_threaded_isr,
1403                                                 0, "[EDAC] ThunderX OCX",
1404                                                 &ocx->msix_ent[i]);
1405                 if (ret)
1406                         goto err_free;
1407         }
1408
1409         edac_dev->dev = &pdev->dev;
1410         edac_dev->dev_name = dev_name(&pdev->dev);
1411         edac_dev->mod_name = "thunderx-ocx";
1412         edac_dev->ctl_name = "thunderx-ocx";
1413
1414         ret = edac_device_add_device(edac_dev);
1415         if (ret) {
1416                 dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
1417                 goto err_free;
1418         }
1419
1420         if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
1421                 ocx->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
1422
1423                 ret = thunderx_create_debugfs_nodes(ocx->debugfs,
1424                                                     ocx_dfs_ents,
1425                                                     ocx,
1426                                                     ARRAY_SIZE(ocx_dfs_ents));
1427                 if (ret != ARRAY_SIZE(ocx_dfs_ents)) {
1428                         dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
1429                                  ret, ret >= 0 ? " created" : "");
1430                 }
1431         }
1432
1433         pci_set_drvdata(pdev, edac_dev);
1434
1435         thunderx_ocx_clearstats(ocx);
1436
1437         for (i = 0; i < OCX_RX_LANES; i++) {
1438                 writeq(OCX_LNE_INT_ENA_ALL,
1439                        ocx->regs + OCX_LNE_INT_EN(i));
1440
1441                 reg = readq(ocx->regs + OCX_LNE_INT(i));
1442                 writeq(reg, ocx->regs + OCX_LNE_INT(i));
1443
1444         }
1445
1446         for (i = 0; i < OCX_LINK_INTS; i++) {
1447                 reg = readq(ocx->regs + OCX_COM_LINKX_INT(i));
1448                 writeq(reg, ocx->regs + OCX_COM_LINKX_INT(i));
1449
1450                 writeq(OCX_COM_LINKX_INT_ENA_ALL,
1451                        ocx->regs + OCX_COM_LINKX_INT_ENA_W1S(i));
1452         }
1453
1454         reg = readq(ocx->regs + OCX_COM_INT);
1455         writeq(reg, ocx->regs + OCX_COM_INT);
1456
1457         writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1S);
1458
1459         return 0;
1460 err_free:
1461         edac_device_free_ctl_info(edac_dev);
1462
1463         return ret;
1464 }
1465
1466 static void thunderx_ocx_remove(struct pci_dev *pdev)
1467 {
1468         struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
1469         struct thunderx_ocx *ocx = edac_dev->pvt_info;
1470         int i;
1471
1472         writeq(OCX_COM_INT_ENA_ALL, ocx->regs + OCX_COM_INT_ENA_W1C);
1473
1474         for (i = 0; i < OCX_INTS; i++) {
1475                 writeq(OCX_COM_LINKX_INT_ENA_ALL,
1476                        ocx->regs + OCX_COM_LINKX_INT_ENA_W1C(i));
1477         }
1478
1479         edac_debugfs_remove_recursive(ocx->debugfs);
1480
1481         edac_device_del_device(&pdev->dev);
1482         edac_device_free_ctl_info(edac_dev);
1483 }
1484
1485 MODULE_DEVICE_TABLE(pci, thunderx_ocx_pci_tbl);
1486
1487 static struct pci_driver thunderx_ocx_driver = {
1488         .name     = "thunderx_ocx_edac",
1489         .probe    = thunderx_ocx_probe,
1490         .remove   = thunderx_ocx_remove,
1491         .id_table = thunderx_ocx_pci_tbl,
1492 };
1493
1494 /*---------------------- L2C driver ---------------------------------*/
1495
1496 #define PCI_DEVICE_ID_THUNDER_L2C_TAD 0xa02e
1497 #define PCI_DEVICE_ID_THUNDER_L2C_CBC 0xa02f
1498 #define PCI_DEVICE_ID_THUNDER_L2C_MCI 0xa030
1499
1500 #define L2C_TAD_INT_W1C         0x40000
1501 #define L2C_TAD_INT_W1S         0x40008
1502
1503 #define L2C_TAD_INT_ENA_W1C     0x40020
1504 #define L2C_TAD_INT_ENA_W1S     0x40028
1505
1506
1507 #define L2C_TAD_INT_L2DDBE       BIT(1)
1508 #define L2C_TAD_INT_SBFSBE       BIT(2)
1509 #define L2C_TAD_INT_SBFDBE       BIT(3)
1510 #define L2C_TAD_INT_FBFSBE       BIT(4)
1511 #define L2C_TAD_INT_FBFDBE       BIT(5)
1512 #define L2C_TAD_INT_TAGDBE       BIT(9)
1513 #define L2C_TAD_INT_RDDISLMC     BIT(15)
1514 #define L2C_TAD_INT_WRDISLMC     BIT(16)
1515 #define L2C_TAD_INT_LFBTO        BIT(17)
1516 #define L2C_TAD_INT_GSYNCTO      BIT(18)
1517 #define L2C_TAD_INT_RTGSBE       BIT(32)
1518 #define L2C_TAD_INT_RTGDBE       BIT(33)
1519 #define L2C_TAD_INT_RDDISOCI     BIT(34)
1520 #define L2C_TAD_INT_WRDISOCI     BIT(35)
1521
1522 #define L2C_TAD_INT_ECC         (L2C_TAD_INT_L2DDBE | \
1523                                  L2C_TAD_INT_SBFSBE | L2C_TAD_INT_SBFDBE | \
1524                                  L2C_TAD_INT_FBFSBE | L2C_TAD_INT_FBFDBE)
1525
1526 #define L2C_TAD_INT_CE          (L2C_TAD_INT_SBFSBE | \
1527                                  L2C_TAD_INT_FBFSBE)
1528
1529 #define L2C_TAD_INT_UE          (L2C_TAD_INT_L2DDBE | \
1530                                  L2C_TAD_INT_SBFDBE | \
1531                                  L2C_TAD_INT_FBFDBE | \
1532                                  L2C_TAD_INT_TAGDBE | \
1533                                  L2C_TAD_INT_RTGDBE | \
1534                                  L2C_TAD_INT_WRDISOCI | \
1535                                  L2C_TAD_INT_RDDISOCI | \
1536                                  L2C_TAD_INT_WRDISLMC | \
1537                                  L2C_TAD_INT_RDDISLMC | \
1538                                  L2C_TAD_INT_LFBTO    | \
1539                                  L2C_TAD_INT_GSYNCTO)
1540
1541 static const struct error_descr l2_tad_errors[] = {
1542         {
1543                 .type  = ERR_CORRECTED,
1544                 .mask  = L2C_TAD_INT_SBFSBE,
1545                 .descr = "SBF single-bit error",
1546         },
1547         {
1548                 .type  = ERR_CORRECTED,
1549                 .mask  = L2C_TAD_INT_FBFSBE,
1550                 .descr = "FBF single-bit error",
1551         },
1552         {
1553                 .type  = ERR_UNCORRECTED,
1554                 .mask  = L2C_TAD_INT_L2DDBE,
1555                 .descr = "L2D double-bit error",
1556         },
1557         {
1558                 .type  = ERR_UNCORRECTED,
1559                 .mask  = L2C_TAD_INT_SBFDBE,
1560                 .descr = "SBF double-bit error",
1561         },
1562         {
1563                 .type  = ERR_UNCORRECTED,
1564                 .mask  = L2C_TAD_INT_FBFDBE,
1565                 .descr = "FBF double-bit error",
1566         },
1567         {
1568                 .type  = ERR_UNCORRECTED,
1569                 .mask  = L2C_TAD_INT_TAGDBE,
1570                 .descr = "TAG double-bit error",
1571         },
1572         {
1573                 .type  = ERR_UNCORRECTED,
1574                 .mask  = L2C_TAD_INT_RTGDBE,
1575                 .descr = "RTG double-bit error",
1576         },
1577         {
1578                 .type  = ERR_UNCORRECTED,
1579                 .mask  = L2C_TAD_INT_WRDISOCI,
1580                 .descr = "Write to a disabled CCPI",
1581         },
1582         {
1583                 .type  = ERR_UNCORRECTED,
1584                 .mask  = L2C_TAD_INT_RDDISOCI,
1585                 .descr = "Read from a disabled CCPI",
1586         },
1587         {
1588                 .type  = ERR_UNCORRECTED,
1589                 .mask  = L2C_TAD_INT_WRDISLMC,
1590                 .descr = "Write to a disabled LMC",
1591         },
1592         {
1593                 .type  = ERR_UNCORRECTED,
1594                 .mask  = L2C_TAD_INT_RDDISLMC,
1595                 .descr = "Read from a disabled LMC",
1596         },
1597         {
1598                 .type  = ERR_UNCORRECTED,
1599                 .mask  = L2C_TAD_INT_LFBTO,
1600                 .descr = "LFB entry timeout",
1601         },
1602         {
1603                 .type  = ERR_UNCORRECTED,
1604                 .mask  = L2C_TAD_INT_GSYNCTO,
1605                 .descr = "Global sync CCPI timeout",
1606         },
1607         {0, 0, NULL},
1608 };
1609
1610 #define L2C_TAD_INT_TAG         (L2C_TAD_INT_TAGDBE)
1611
1612 #define L2C_TAD_INT_RTG         (L2C_TAD_INT_RTGDBE)
1613
1614 #define L2C_TAD_INT_DISLMC      (L2C_TAD_INT_WRDISLMC | L2C_TAD_INT_RDDISLMC)
1615
1616 #define L2C_TAD_INT_DISOCI      (L2C_TAD_INT_WRDISOCI | L2C_TAD_INT_RDDISOCI)
1617
1618 #define L2C_TAD_INT_ENA_ALL     (L2C_TAD_INT_ECC | L2C_TAD_INT_TAG | \
1619                                  L2C_TAD_INT_RTG | \
1620                                  L2C_TAD_INT_DISLMC | L2C_TAD_INT_DISOCI | \
1621                                  L2C_TAD_INT_LFBTO)
1622
1623 #define L2C_TAD_TIMETWO         0x50000
1624 #define L2C_TAD_TIMEOUT         0x50100
1625 #define L2C_TAD_ERR             0x60000
1626 #define L2C_TAD_TQD_ERR         0x60100
1627 #define L2C_TAD_TTG_ERR         0x60200
1628
1629
1630 #define L2C_CBC_INT_W1C         0x60000
1631
1632 #define L2C_CBC_INT_RSDSBE       BIT(0)
1633 #define L2C_CBC_INT_RSDDBE       BIT(1)
1634
1635 #define L2C_CBC_INT_RSD          (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_RSDDBE)
1636
1637 #define L2C_CBC_INT_MIBSBE       BIT(4)
1638 #define L2C_CBC_INT_MIBDBE       BIT(5)
1639
1640 #define L2C_CBC_INT_MIB          (L2C_CBC_INT_MIBSBE | L2C_CBC_INT_MIBDBE)
1641
1642 #define L2C_CBC_INT_IORDDISOCI   BIT(6)
1643 #define L2C_CBC_INT_IOWRDISOCI   BIT(7)
1644
1645 #define L2C_CBC_INT_IODISOCI     (L2C_CBC_INT_IORDDISOCI | \
1646                                   L2C_CBC_INT_IOWRDISOCI)
1647
1648 #define L2C_CBC_INT_CE           (L2C_CBC_INT_RSDSBE | L2C_CBC_INT_MIBSBE)
1649 #define L2C_CBC_INT_UE           (L2C_CBC_INT_RSDDBE | L2C_CBC_INT_MIBDBE)
1650
1651
1652 static const struct error_descr l2_cbc_errors[] = {
1653         {
1654                 .type  = ERR_CORRECTED,
1655                 .mask  = L2C_CBC_INT_RSDSBE,
1656                 .descr = "RSD single-bit error",
1657         },
1658         {
1659                 .type  = ERR_CORRECTED,
1660                 .mask  = L2C_CBC_INT_MIBSBE,
1661                 .descr = "MIB single-bit error",
1662         },
1663         {
1664                 .type  = ERR_UNCORRECTED,
1665                 .mask  = L2C_CBC_INT_RSDDBE,
1666                 .descr = "RSD double-bit error",
1667         },
1668         {
1669                 .type  = ERR_UNCORRECTED,
1670                 .mask  = L2C_CBC_INT_MIBDBE,
1671                 .descr = "MIB double-bit error",
1672         },
1673         {
1674                 .type  = ERR_UNCORRECTED,
1675                 .mask  = L2C_CBC_INT_IORDDISOCI,
1676                 .descr = "Read from a disabled CCPI",
1677         },
1678         {
1679                 .type  = ERR_UNCORRECTED,
1680                 .mask  = L2C_CBC_INT_IOWRDISOCI,
1681                 .descr = "Write to a disabled CCPI",
1682         },
1683         {0, 0, NULL},
1684 };
1685
1686 #define L2C_CBC_INT_W1S         0x60008
1687 #define L2C_CBC_INT_ENA_W1C     0x60020
1688
1689 #define L2C_CBC_INT_ENA_ALL      (L2C_CBC_INT_RSD | L2C_CBC_INT_MIB | \
1690                                   L2C_CBC_INT_IODISOCI)
1691
1692 #define L2C_CBC_INT_ENA_W1S     0x60028
1693
1694 #define L2C_CBC_IODISOCIERR     0x80008
1695 #define L2C_CBC_IOCERR          0x80010
1696 #define L2C_CBC_RSDERR          0x80018
1697 #define L2C_CBC_MIBERR          0x80020
1698
1699
1700 #define L2C_MCI_INT_W1C         0x0
1701
1702 #define L2C_MCI_INT_VBFSBE       BIT(0)
1703 #define L2C_MCI_INT_VBFDBE       BIT(1)
1704
1705 static const struct error_descr l2_mci_errors[] = {
1706         {
1707                 .type  = ERR_CORRECTED,
1708                 .mask  = L2C_MCI_INT_VBFSBE,
1709                 .descr = "VBF single-bit error",
1710         },
1711         {
1712                 .type  = ERR_UNCORRECTED,
1713                 .mask  = L2C_MCI_INT_VBFDBE,
1714                 .descr = "VBF double-bit error",
1715         },
1716         {0, 0, NULL},
1717 };
1718
1719 #define L2C_MCI_INT_W1S         0x8
1720 #define L2C_MCI_INT_ENA_W1C     0x20
1721
1722 #define L2C_MCI_INT_ENA_ALL      (L2C_MCI_INT_VBFSBE | L2C_MCI_INT_VBFDBE)
1723
1724 #define L2C_MCI_INT_ENA_W1S     0x28
1725
1726 #define L2C_MCI_ERR             0x10000
1727
1728 #define L2C_MESSAGE_SIZE        SZ_1K
1729 #define L2C_OTHER_SIZE          (50 * ARRAY_SIZE(l2_tad_errors))
1730
1731 struct l2c_err_ctx {
1732         char *reg_ext_name;
1733         u64  reg_int;
1734         u64  reg_ext;
1735 };
1736
1737 struct thunderx_l2c {
1738         void __iomem *regs;
1739         struct pci_dev *pdev;
1740         struct edac_device_ctl_info *edac_dev;
1741
1742         struct dentry *debugfs;
1743
1744         int index;
1745
1746         struct msix_entry msix_ent;
1747
1748         struct l2c_err_ctx err_ctx[RING_ENTRIES];
1749         unsigned long ring_head;
1750         unsigned long ring_tail;
1751 };
1752
1753 static irqreturn_t thunderx_l2c_tad_isr(int irq, void *irq_id)
1754 {
1755         struct msix_entry *msix = irq_id;
1756         struct thunderx_l2c *tad = container_of(msix, struct thunderx_l2c,
1757                                                 msix_ent);
1758
1759         unsigned long head = ring_pos(tad->ring_head, ARRAY_SIZE(tad->err_ctx));
1760         struct l2c_err_ctx *ctx = &tad->err_ctx[head];
1761
1762         ctx->reg_int = readq(tad->regs + L2C_TAD_INT_W1C);
1763
1764         if (ctx->reg_int & L2C_TAD_INT_ECC) {
1765                 ctx->reg_ext_name = "TQD_ERR";
1766                 ctx->reg_ext = readq(tad->regs + L2C_TAD_TQD_ERR);
1767         } else if (ctx->reg_int & L2C_TAD_INT_TAG) {
1768                 ctx->reg_ext_name = "TTG_ERR";
1769                 ctx->reg_ext = readq(tad->regs + L2C_TAD_TTG_ERR);
1770         } else if (ctx->reg_int & L2C_TAD_INT_LFBTO) {
1771                 ctx->reg_ext_name = "TIMEOUT";
1772                 ctx->reg_ext = readq(tad->regs + L2C_TAD_TIMEOUT);
1773         } else if (ctx->reg_int & L2C_TAD_INT_DISOCI) {
1774                 ctx->reg_ext_name = "ERR";
1775                 ctx->reg_ext = readq(tad->regs + L2C_TAD_ERR);
1776         }
1777
1778         writeq(ctx->reg_int, tad->regs + L2C_TAD_INT_W1C);
1779
1780         tad->ring_head++;
1781
1782         return IRQ_WAKE_THREAD;
1783 }
1784
1785 static irqreturn_t thunderx_l2c_cbc_isr(int irq, void *irq_id)
1786 {
1787         struct msix_entry *msix = irq_id;
1788         struct thunderx_l2c *cbc = container_of(msix, struct thunderx_l2c,
1789                                                 msix_ent);
1790
1791         unsigned long head = ring_pos(cbc->ring_head, ARRAY_SIZE(cbc->err_ctx));
1792         struct l2c_err_ctx *ctx = &cbc->err_ctx[head];
1793
1794         ctx->reg_int = readq(cbc->regs + L2C_CBC_INT_W1C);
1795
1796         if (ctx->reg_int & L2C_CBC_INT_RSD) {
1797                 ctx->reg_ext_name = "RSDERR";
1798                 ctx->reg_ext = readq(cbc->regs + L2C_CBC_RSDERR);
1799         } else if (ctx->reg_int & L2C_CBC_INT_MIB) {
1800                 ctx->reg_ext_name = "MIBERR";
1801                 ctx->reg_ext = readq(cbc->regs + L2C_CBC_MIBERR);
1802         } else if (ctx->reg_int & L2C_CBC_INT_IODISOCI) {
1803                 ctx->reg_ext_name = "IODISOCIERR";
1804                 ctx->reg_ext = readq(cbc->regs + L2C_CBC_IODISOCIERR);
1805         }
1806
1807         writeq(ctx->reg_int, cbc->regs + L2C_CBC_INT_W1C);
1808
1809         cbc->ring_head++;
1810
1811         return IRQ_WAKE_THREAD;
1812 }
1813
1814 static irqreturn_t thunderx_l2c_mci_isr(int irq, void *irq_id)
1815 {
1816         struct msix_entry *msix = irq_id;
1817         struct thunderx_l2c *mci = container_of(msix, struct thunderx_l2c,
1818                                                 msix_ent);
1819
1820         unsigned long head = ring_pos(mci->ring_head, ARRAY_SIZE(mci->err_ctx));
1821         struct l2c_err_ctx *ctx = &mci->err_ctx[head];
1822
1823         ctx->reg_int = readq(mci->regs + L2C_MCI_INT_W1C);
1824         ctx->reg_ext = readq(mci->regs + L2C_MCI_ERR);
1825
1826         writeq(ctx->reg_int, mci->regs + L2C_MCI_INT_W1C);
1827
1828         ctx->reg_ext_name = "ERR";
1829
1830         mci->ring_head++;
1831
1832         return IRQ_WAKE_THREAD;
1833 }
1834
1835 static irqreturn_t thunderx_l2c_threaded_isr(int irq, void *irq_id)
1836 {
1837         struct msix_entry *msix = irq_id;
1838         struct thunderx_l2c *l2c = container_of(msix, struct thunderx_l2c,
1839                                                 msix_ent);
1840
1841         unsigned long tail = ring_pos(l2c->ring_tail, ARRAY_SIZE(l2c->err_ctx));
1842         struct l2c_err_ctx *ctx = &l2c->err_ctx[tail];
1843         irqreturn_t ret = IRQ_NONE;
1844
1845         u64 mask_ue, mask_ce;
1846         const struct error_descr *l2_errors;
1847         char *reg_int_name;
1848
1849         char *msg;
1850         char *other;
1851
1852         msg = kmalloc(OCX_MESSAGE_SIZE, GFP_KERNEL);
1853         other = kmalloc(OCX_OTHER_SIZE, GFP_KERNEL);
1854
1855         if (!msg || !other)
1856                 goto err_free;
1857
1858         switch (l2c->pdev->device) {
1859         case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1860                 reg_int_name = "L2C_TAD_INT";
1861                 mask_ue = L2C_TAD_INT_UE;
1862                 mask_ce = L2C_TAD_INT_CE;
1863                 l2_errors = l2_tad_errors;
1864                 break;
1865         case PCI_DEVICE_ID_THUNDER_L2C_CBC:
1866                 reg_int_name = "L2C_CBC_INT";
1867                 mask_ue = L2C_CBC_INT_UE;
1868                 mask_ce = L2C_CBC_INT_CE;
1869                 l2_errors = l2_cbc_errors;
1870                 break;
1871         case PCI_DEVICE_ID_THUNDER_L2C_MCI:
1872                 reg_int_name = "L2C_MCI_INT";
1873                 mask_ue = L2C_MCI_INT_VBFDBE;
1874                 mask_ce = L2C_MCI_INT_VBFSBE;
1875                 l2_errors = l2_mci_errors;
1876                 break;
1877         default:
1878                 dev_err(&l2c->pdev->dev, "Unsupported device: %04x\n",
1879                         l2c->pdev->device);
1880                 goto err_free;
1881         }
1882
1883         while (CIRC_CNT(l2c->ring_head, l2c->ring_tail,
1884                         ARRAY_SIZE(l2c->err_ctx))) {
1885                 snprintf(msg, L2C_MESSAGE_SIZE,
1886                          "%s: %s: %016llx, %s: %016llx",
1887                          l2c->edac_dev->ctl_name, reg_int_name, ctx->reg_int,
1888                          ctx->reg_ext_name, ctx->reg_ext);
1889
1890                 decode_register(other, L2C_OTHER_SIZE, l2_errors, ctx->reg_int);
1891
1892                 strlcat(msg, other, L2C_MESSAGE_SIZE);
1893
1894                 if (ctx->reg_int & mask_ue)
1895                         edac_device_handle_ue(l2c->edac_dev, 0, 0, msg);
1896                 else if (ctx->reg_int & mask_ce)
1897                         edac_device_handle_ce(l2c->edac_dev, 0, 0, msg);
1898
1899                 l2c->ring_tail++;
1900         }
1901
1902         ret = IRQ_HANDLED;
1903
1904 err_free:
1905         kfree(other);
1906         kfree(msg);
1907
1908         return ret;
1909 }
1910
1911 #define L2C_DEBUGFS_ATTR(_name, _reg)   DEBUGFS_REG_ATTR(l2c, _name, _reg)
1912
1913 L2C_DEBUGFS_ATTR(tad_int, L2C_TAD_INT_W1S);
1914
1915 static struct debugfs_entry *l2c_tad_dfs_ents[] = {
1916         &debugfs_tad_int,
1917 };
1918
1919 L2C_DEBUGFS_ATTR(cbc_int, L2C_CBC_INT_W1S);
1920
1921 static struct debugfs_entry *l2c_cbc_dfs_ents[] = {
1922         &debugfs_cbc_int,
1923 };
1924
1925 L2C_DEBUGFS_ATTR(mci_int, L2C_MCI_INT_W1S);
1926
1927 static struct debugfs_entry *l2c_mci_dfs_ents[] = {
1928         &debugfs_mci_int,
1929 };
1930
1931 static const struct pci_device_id thunderx_l2c_pci_tbl[] = {
1932         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_TAD), },
1933         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_CBC), },
1934         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVICE_ID_THUNDER_L2C_MCI), },
1935         { 0, },
1936 };
1937
1938 static int thunderx_l2c_probe(struct pci_dev *pdev,
1939                               const struct pci_device_id *id)
1940 {
1941         struct thunderx_l2c *l2c;
1942         struct edac_device_ctl_info *edac_dev;
1943         struct debugfs_entry **l2c_devattr;
1944         size_t dfs_entries;
1945         irqreturn_t (*thunderx_l2c_isr)(int, void *) = NULL;
1946         char name[32];
1947         const char *fmt;
1948         u64 reg_en_offs, reg_en_mask;
1949         int idx;
1950         int ret;
1951
1952         ret = pcim_enable_device(pdev);
1953         if (ret) {
1954                 dev_err(&pdev->dev, "Cannot enable PCI device: %d\n", ret);
1955                 return ret;
1956         }
1957
1958         ret = pcim_iomap_regions(pdev, BIT(0), "thunderx_l2c");
1959         if (ret) {
1960                 dev_err(&pdev->dev, "Cannot map PCI resources: %d\n", ret);
1961                 return ret;
1962         }
1963
1964         switch (pdev->device) {
1965         case PCI_DEVICE_ID_THUNDER_L2C_TAD:
1966                 thunderx_l2c_isr = thunderx_l2c_tad_isr;
1967                 l2c_devattr = l2c_tad_dfs_ents;
1968                 dfs_entries = ARRAY_SIZE(l2c_tad_dfs_ents);
1969                 fmt = "L2C-TAD%d";
1970                 reg_en_offs = L2C_TAD_INT_ENA_W1S;
1971                 reg_en_mask = L2C_TAD_INT_ENA_ALL;
1972                 break;
1973         case PCI_DEVICE_ID_THUNDER_L2C_CBC:
1974                 thunderx_l2c_isr = thunderx_l2c_cbc_isr;
1975                 l2c_devattr = l2c_cbc_dfs_ents;
1976                 dfs_entries = ARRAY_SIZE(l2c_cbc_dfs_ents);
1977                 fmt = "L2C-CBC%d";
1978                 reg_en_offs = L2C_CBC_INT_ENA_W1S;
1979                 reg_en_mask = L2C_CBC_INT_ENA_ALL;
1980                 break;
1981         case PCI_DEVICE_ID_THUNDER_L2C_MCI:
1982                 thunderx_l2c_isr = thunderx_l2c_mci_isr;
1983                 l2c_devattr = l2c_mci_dfs_ents;
1984                 dfs_entries = ARRAY_SIZE(l2c_mci_dfs_ents);
1985                 fmt = "L2C-MCI%d";
1986                 reg_en_offs = L2C_MCI_INT_ENA_W1S;
1987                 reg_en_mask = L2C_MCI_INT_ENA_ALL;
1988                 break;
1989         default:
1990                 //Should never ever get here
1991                 dev_err(&pdev->dev, "Unsupported PCI device: %04x\n",
1992                         pdev->device);
1993                 return -EINVAL;
1994         }
1995
1996         idx = edac_device_alloc_index();
1997         snprintf(name, sizeof(name), fmt, idx);
1998
1999         edac_dev = edac_device_alloc_ctl_info(sizeof(struct thunderx_l2c),
2000                                               name, 1, "L2C", 1, 0, idx);
2001         if (!edac_dev) {
2002                 dev_err(&pdev->dev, "Cannot allocate EDAC device\n");
2003                 return -ENOMEM;
2004         }
2005
2006         l2c = edac_dev->pvt_info;
2007         l2c->edac_dev = edac_dev;
2008
2009         l2c->regs = pcim_iomap_table(pdev)[0];
2010         if (!l2c->regs) {
2011                 dev_err(&pdev->dev, "Cannot map PCI resources\n");
2012                 ret = -ENODEV;
2013                 goto err_free;
2014         }
2015
2016         l2c->pdev = pdev;
2017
2018         l2c->ring_head = 0;
2019         l2c->ring_tail = 0;
2020
2021         l2c->msix_ent.entry = 0;
2022         l2c->msix_ent.vector = 0;
2023
2024         ret = pci_enable_msix_exact(pdev, &l2c->msix_ent, 1);
2025         if (ret) {
2026                 dev_err(&pdev->dev, "Cannot enable interrupt: %d\n", ret);
2027                 goto err_free;
2028         }
2029
2030         ret = devm_request_threaded_irq(&pdev->dev, l2c->msix_ent.vector,
2031                                         thunderx_l2c_isr,
2032                                         thunderx_l2c_threaded_isr,
2033                                         0, "[EDAC] ThunderX L2C",
2034                                         &l2c->msix_ent);
2035         if (ret)
2036                 goto err_free;
2037
2038         edac_dev->dev = &pdev->dev;
2039         edac_dev->dev_name = dev_name(&pdev->dev);
2040         edac_dev->mod_name = "thunderx-l2c";
2041         edac_dev->ctl_name = "thunderx-l2c";
2042
2043         ret = edac_device_add_device(edac_dev);
2044         if (ret) {
2045                 dev_err(&pdev->dev, "Cannot add EDAC device: %d\n", ret);
2046                 goto err_free;
2047         }
2048
2049         if (IS_ENABLED(CONFIG_EDAC_DEBUG)) {
2050                 l2c->debugfs = edac_debugfs_create_dir(pdev->dev.kobj.name);
2051
2052                 ret = thunderx_create_debugfs_nodes(l2c->debugfs, l2c_devattr,
2053                                               l2c, dfs_entries);
2054
2055                 if (ret != dfs_entries) {
2056                         dev_warn(&pdev->dev, "Error creating debugfs entries: %d%s\n",
2057                                  ret, ret >= 0 ? " created" : "");
2058                 }
2059         }
2060
2061         pci_set_drvdata(pdev, edac_dev);
2062
2063         writeq(reg_en_mask, l2c->regs + reg_en_offs);
2064
2065         return 0;
2066
2067 err_free:
2068         edac_device_free_ctl_info(edac_dev);
2069
2070         return ret;
2071 }
2072
2073 static void thunderx_l2c_remove(struct pci_dev *pdev)
2074 {
2075         struct edac_device_ctl_info *edac_dev = pci_get_drvdata(pdev);
2076         struct thunderx_l2c *l2c = edac_dev->pvt_info;
2077
2078         switch (pdev->device) {
2079         case PCI_DEVICE_ID_THUNDER_L2C_TAD:
2080                 writeq(L2C_TAD_INT_ENA_ALL, l2c->regs + L2C_TAD_INT_ENA_W1C);
2081                 break;
2082         case PCI_DEVICE_ID_THUNDER_L2C_CBC:
2083                 writeq(L2C_CBC_INT_ENA_ALL, l2c->regs + L2C_CBC_INT_ENA_W1C);
2084                 break;
2085         case PCI_DEVICE_ID_THUNDER_L2C_MCI:
2086                 writeq(L2C_MCI_INT_ENA_ALL, l2c->regs + L2C_MCI_INT_ENA_W1C);
2087                 break;
2088         }
2089
2090         edac_debugfs_remove_recursive(l2c->debugfs);
2091
2092         edac_device_del_device(&pdev->dev);
2093         edac_device_free_ctl_info(edac_dev);
2094 }
2095
2096 MODULE_DEVICE_TABLE(pci, thunderx_l2c_pci_tbl);
2097
2098 static struct pci_driver thunderx_l2c_driver = {
2099         .name     = "thunderx_l2c_edac",
2100         .probe    = thunderx_l2c_probe,
2101         .remove   = thunderx_l2c_remove,
2102         .id_table = thunderx_l2c_pci_tbl,
2103 };
2104
2105 static int __init thunderx_edac_init(void)
2106 {
2107         int rc = 0;
2108
2109         if (ghes_get_devices())
2110                 return -EBUSY;
2111
2112         rc = pci_register_driver(&thunderx_lmc_driver);
2113         if (rc)
2114                 return rc;
2115
2116         rc = pci_register_driver(&thunderx_ocx_driver);
2117         if (rc)
2118                 goto err_lmc;
2119
2120         rc = pci_register_driver(&thunderx_l2c_driver);
2121         if (rc)
2122                 goto err_ocx;
2123
2124         return rc;
2125 err_ocx:
2126         pci_unregister_driver(&thunderx_ocx_driver);
2127 err_lmc:
2128         pci_unregister_driver(&thunderx_lmc_driver);
2129
2130         return rc;
2131 }
2132
2133 static void __exit thunderx_edac_exit(void)
2134 {
2135         pci_unregister_driver(&thunderx_l2c_driver);
2136         pci_unregister_driver(&thunderx_ocx_driver);
2137         pci_unregister_driver(&thunderx_lmc_driver);
2138
2139 }
2140
2141 module_init(thunderx_edac_init);
2142 module_exit(thunderx_edac_exit);
2143
2144 MODULE_LICENSE("GPL v2");
2145 MODULE_AUTHOR("Cavium, Inc.");
2146 MODULE_DESCRIPTION("EDAC Driver for Cavium ThunderX");