1 // SPDX-License-Identifier: GPL-2.0-only
2 #include "amd64_edac.h"
3 #include <asm/amd_nb.h>
5 static struct edac_pci_ctl_info *pci_ctl;
8 * Set by command line parameter. If BIOS has enabled the ECC, this override is
9 * cleared to prevent re-enabling the hardware by this driver.
11 static int ecc_enable_override;
12 module_param(ecc_enable_override, int, 0644);
14 static struct msr __percpu *msrs;
16 static inline u32 get_umc_reg(struct amd64_pvt *pvt, u32 reg)
18 if (!pvt->flags.zn_regs_v2)
22 case UMCCH_ADDR_CFG: return UMCCH_ADDR_CFG_DDR5;
23 case UMCCH_ADDR_MASK_SEC: return UMCCH_ADDR_MASK_SEC_DDR5;
24 case UMCCH_DIMM_CFG: return UMCCH_DIMM_CFG_DDR5;
27 WARN_ONCE(1, "%s: unknown register 0x%x", __func__, reg);
32 static struct ecc_settings **ecc_stngs;
34 /* Device for the PCI component */
35 static struct device *pci_ctl_dev;
38 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
39 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
42 *FIXME: Produce a better mapping/linearisation.
44 static const struct scrubrate {
45 u32 scrubval; /* bit pattern for scrub rate */
46 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
48 { 0x01, 1600000000UL},
70 { 0x00, 0UL}, /* scrubbing off */
73 int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
74 u32 *val, const char *func)
78 err = pci_read_config_dword(pdev, offset, val);
80 amd64_warn("%s: error reading F%dx%03x.\n",
81 func, PCI_FUNC(pdev->devfn), offset);
86 int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
87 u32 val, const char *func)
91 err = pci_write_config_dword(pdev, offset, val);
93 amd64_warn("%s: error writing to F%dx%03x.\n",
94 func, PCI_FUNC(pdev->devfn), offset);
100 * Select DCT to which PCI cfg accesses are routed
102 static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
106 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®);
107 reg &= (pvt->model == 0x30) ? ~3 : ~1;
109 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
114 * Depending on the family, F2 DCT reads need special handling:
116 * K8: has a single DCT only and no address offsets >= 0x100
118 * F10h: each DCT has its own set of regs
122 * F16h: has only 1 DCT
124 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
126 static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
127 int offset, u32 *val)
131 if (dct || offset >= 0x100)
138 * Note: If ganging is enabled, barring the regs
139 * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx
140 * return 0. (cf. Section 2.8.1 F10h BKDG)
142 if (dct_ganging_enabled(pvt))
151 * F15h: F2x1xx addresses do not map explicitly to DCT1.
152 * We should select which DCT we access using F1x10C[DctCfgSel]
154 dct = (dct && pvt->model == 0x30) ? 3 : dct;
155 f15h_select_dct(pvt, dct);
166 return amd64_read_pci_cfg(pvt->F2, offset, val);
170 * Memory scrubber control interface. For K8, memory scrubbing is handled by
171 * hardware and can involve L2 cache, dcache as well as the main memory. With
172 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
175 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
176 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
177 * bytes/sec for the setting.
179 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
180 * other archs, we might not have access to the caches directly.
184 * Scan the scrub rate mapping table for a close or matching bandwidth value to
185 * issue. If requested is too big, then use last maximum value found.
187 static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
193 * map the configured rate (new_bw) to a value specific to the AMD64
194 * memory controller and apply to register. Search for the first
195 * bandwidth entry that is greater or equal than the setting requested
196 * and program that. If at last entry, turn off DRAM scrubbing.
198 * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
199 * by falling back to the last element in scrubrates[].
201 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
203 * skip scrub rates which aren't recommended
204 * (see F10 BKDG, F3x58)
206 if (scrubrates[i].scrubval < min_rate)
209 if (scrubrates[i].bandwidth <= new_bw)
213 scrubval = scrubrates[i].scrubval;
215 if (pvt->fam == 0x15 && pvt->model == 0x60) {
216 f15h_select_dct(pvt, 0);
217 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
218 f15h_select_dct(pvt, 1);
219 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
221 pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
225 return scrubrates[i].bandwidth;
230 static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
232 struct amd64_pvt *pvt = mci->pvt_info;
233 u32 min_scrubrate = 0x5;
238 if (pvt->fam == 0x15) {
240 if (pvt->model < 0x10)
241 f15h_select_dct(pvt, 0);
243 if (pvt->model == 0x60)
246 return __set_scrub_rate(pvt, bw, min_scrubrate);
249 static int get_scrub_rate(struct mem_ctl_info *mci)
251 struct amd64_pvt *pvt = mci->pvt_info;
252 int i, retval = -EINVAL;
255 if (pvt->fam == 0x15) {
257 if (pvt->model < 0x10)
258 f15h_select_dct(pvt, 0);
260 if (pvt->model == 0x60)
261 amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
263 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
265 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
268 scrubval = scrubval & 0x001F;
270 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
271 if (scrubrates[i].scrubval == scrubval) {
272 retval = scrubrates[i].bandwidth;
280 * returns true if the SysAddr given by sys_addr matches the
281 * DRAM base/limit associated with node_id
283 static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
287 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
288 * all ones if the most significant implemented address bit is 1.
289 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
290 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
291 * Application Programming.
293 addr = sys_addr & 0x000000ffffffffffull;
295 return ((addr >= get_dram_base(pvt, nid)) &&
296 (addr <= get_dram_limit(pvt, nid)));
300 * Attempt to map a SysAddr to a node. On success, return a pointer to the
301 * mem_ctl_info structure for the node that the SysAddr maps to.
303 * On failure, return NULL.
305 static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
308 struct amd64_pvt *pvt;
313 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
314 * 3.4.4.2) registers to map the SysAddr to a node ID.
319 * The value of this field should be the same for all DRAM Base
320 * registers. Therefore we arbitrarily choose to read it from the
321 * register for node 0.
323 intlv_en = dram_intlv_en(pvt, 0);
326 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
327 if (base_limit_match(pvt, sys_addr, node_id))
333 if (unlikely((intlv_en != 0x01) &&
334 (intlv_en != 0x03) &&
335 (intlv_en != 0x07))) {
336 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
340 bits = (((u32) sys_addr) >> 12) & intlv_en;
342 for (node_id = 0; ; ) {
343 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
344 break; /* intlv_sel field matches */
346 if (++node_id >= DRAM_RANGES)
350 /* sanity test for sys_addr */
351 if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
352 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
353 "range for node %d with node interleaving enabled.\n",
354 __func__, sys_addr, node_id);
359 return edac_mc_find((int)node_id);
362 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
363 (unsigned long)sys_addr);
369 * compute the CS base address of the @csrow on the DRAM controller @dct.
370 * For details see F2x[5C:40] in the processor's BKDG
372 static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
373 u64 *base, u64 *mask)
375 u64 csbase, csmask, base_bits, mask_bits;
378 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
379 csbase = pvt->csels[dct].csbases[csrow];
380 csmask = pvt->csels[dct].csmasks[csrow];
381 base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
382 mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
386 * F16h and F15h, models 30h and later need two addr_shift values:
387 * 8 for high and 6 for low (cf. F16h BKDG).
389 } else if (pvt->fam == 0x16 ||
390 (pvt->fam == 0x15 && pvt->model >= 0x30)) {
391 csbase = pvt->csels[dct].csbases[csrow];
392 csmask = pvt->csels[dct].csmasks[csrow >> 1];
394 *base = (csbase & GENMASK_ULL(15, 5)) << 6;
395 *base |= (csbase & GENMASK_ULL(30, 19)) << 8;
398 /* poke holes for the csmask */
399 *mask &= ~((GENMASK_ULL(15, 5) << 6) |
400 (GENMASK_ULL(30, 19) << 8));
402 *mask |= (csmask & GENMASK_ULL(15, 5)) << 6;
403 *mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
407 csbase = pvt->csels[dct].csbases[csrow];
408 csmask = pvt->csels[dct].csmasks[csrow >> 1];
411 if (pvt->fam == 0x15)
412 base_bits = mask_bits =
413 GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
415 base_bits = mask_bits =
416 GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
419 *base = (csbase & base_bits) << addr_shift;
422 /* poke holes for the csmask */
423 *mask &= ~(mask_bits << addr_shift);
425 *mask |= (csmask & mask_bits) << addr_shift;
428 #define for_each_chip_select(i, dct, pvt) \
429 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
431 #define chip_select_base(i, dct, pvt) \
432 pvt->csels[dct].csbases[i]
434 #define for_each_chip_select_mask(i, dct, pvt) \
435 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
437 #define for_each_umc(i) \
438 for (i = 0; i < pvt->max_mcs; i++)
441 * @input_addr is an InputAddr associated with the node given by mci. Return the
442 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
444 static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
446 struct amd64_pvt *pvt;
452 for_each_chip_select(csrow, 0, pvt) {
453 if (!csrow_enabled(csrow, 0, pvt))
456 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
460 if ((input_addr & mask) == (base & mask)) {
461 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
462 (unsigned long)input_addr, csrow,
468 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
469 (unsigned long)input_addr, pvt->mc_node_id);
475 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
476 * for the node represented by mci. Info is passed back in *hole_base,
477 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
478 * info is invalid. Info may be invalid for either of the following reasons:
480 * - The revision of the node is not E or greater. In this case, the DRAM Hole
481 * Address Register does not exist.
483 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
484 * indicating that its contents are not valid.
486 * The values passed back in *hole_base, *hole_offset, and *hole_size are
487 * complete 32-bit values despite the fact that the bitfields in the DHAR
488 * only represent bits 31-24 of the base and offset values.
490 static int get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
491 u64 *hole_offset, u64 *hole_size)
493 struct amd64_pvt *pvt = mci->pvt_info;
495 /* only revE and later have the DRAM Hole Address Register */
496 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
497 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
498 pvt->ext_model, pvt->mc_node_id);
502 /* valid for Fam10h and above */
503 if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
504 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
508 if (!dhar_valid(pvt)) {
509 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
514 /* This node has Memory Hoisting */
516 /* +------------------+--------------------+--------------------+-----
517 * | memory | DRAM hole | relocated |
518 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
520 * | | | [0x100000000, |
521 * | | | (0x100000000+ |
522 * | | | (0xffffffff-x))] |
523 * +------------------+--------------------+--------------------+-----
525 * Above is a diagram of physical memory showing the DRAM hole and the
526 * relocated addresses from the DRAM hole. As shown, the DRAM hole
527 * starts at address x (the base address) and extends through address
528 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
529 * addresses in the hole so that they start at 0x100000000.
532 *hole_base = dhar_base(pvt);
533 *hole_size = (1ULL << 32) - *hole_base;
535 *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
536 : k8_dhar_offset(pvt);
538 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
539 pvt->mc_node_id, (unsigned long)*hole_base,
540 (unsigned long)*hole_offset, (unsigned long)*hole_size);
545 #ifdef CONFIG_EDAC_DEBUG
546 #define EDAC_DCT_ATTR_SHOW(reg) \
547 static ssize_t reg##_show(struct device *dev, \
548 struct device_attribute *mattr, char *data) \
550 struct mem_ctl_info *mci = to_mci(dev); \
551 struct amd64_pvt *pvt = mci->pvt_info; \
553 return sprintf(data, "0x%016llx\n", (u64)pvt->reg); \
556 EDAC_DCT_ATTR_SHOW(dhar);
557 EDAC_DCT_ATTR_SHOW(dbam0);
558 EDAC_DCT_ATTR_SHOW(top_mem);
559 EDAC_DCT_ATTR_SHOW(top_mem2);
561 static ssize_t dram_hole_show(struct device *dev, struct device_attribute *mattr,
564 struct mem_ctl_info *mci = to_mci(dev);
570 get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size);
572 return sprintf(data, "%llx %llx %llx\n", hole_base, hole_offset,
577 * update NUM_DBG_ATTRS in case you add new members
579 static DEVICE_ATTR(dhar, S_IRUGO, dhar_show, NULL);
580 static DEVICE_ATTR(dbam, S_IRUGO, dbam0_show, NULL);
581 static DEVICE_ATTR(topmem, S_IRUGO, top_mem_show, NULL);
582 static DEVICE_ATTR(topmem2, S_IRUGO, top_mem2_show, NULL);
583 static DEVICE_ATTR_RO(dram_hole);
585 static struct attribute *dbg_attrs[] = {
588 &dev_attr_topmem.attr,
589 &dev_attr_topmem2.attr,
590 &dev_attr_dram_hole.attr,
594 static const struct attribute_group dbg_group = {
598 static ssize_t inject_section_show(struct device *dev,
599 struct device_attribute *mattr, char *buf)
601 struct mem_ctl_info *mci = to_mci(dev);
602 struct amd64_pvt *pvt = mci->pvt_info;
603 return sprintf(buf, "0x%x\n", pvt->injection.section);
607 * store error injection section value which refers to one of 4 16-byte sections
608 * within a 64-byte cacheline
612 static ssize_t inject_section_store(struct device *dev,
613 struct device_attribute *mattr,
614 const char *data, size_t count)
616 struct mem_ctl_info *mci = to_mci(dev);
617 struct amd64_pvt *pvt = mci->pvt_info;
621 ret = kstrtoul(data, 10, &value);
626 amd64_warn("%s: invalid section 0x%lx\n", __func__, value);
630 pvt->injection.section = (u32) value;
634 static ssize_t inject_word_show(struct device *dev,
635 struct device_attribute *mattr, char *buf)
637 struct mem_ctl_info *mci = to_mci(dev);
638 struct amd64_pvt *pvt = mci->pvt_info;
639 return sprintf(buf, "0x%x\n", pvt->injection.word);
643 * store error injection word value which refers to one of 9 16-bit word of the
644 * 16-byte (128-bit + ECC bits) section
648 static ssize_t inject_word_store(struct device *dev,
649 struct device_attribute *mattr,
650 const char *data, size_t count)
652 struct mem_ctl_info *mci = to_mci(dev);
653 struct amd64_pvt *pvt = mci->pvt_info;
657 ret = kstrtoul(data, 10, &value);
662 amd64_warn("%s: invalid word 0x%lx\n", __func__, value);
666 pvt->injection.word = (u32) value;
670 static ssize_t inject_ecc_vector_show(struct device *dev,
671 struct device_attribute *mattr,
674 struct mem_ctl_info *mci = to_mci(dev);
675 struct amd64_pvt *pvt = mci->pvt_info;
676 return sprintf(buf, "0x%x\n", pvt->injection.bit_map);
680 * store 16 bit error injection vector which enables injecting errors to the
681 * corresponding bit within the error injection word above. When used during a
682 * DRAM ECC read, it holds the contents of the of the DRAM ECC bits.
684 static ssize_t inject_ecc_vector_store(struct device *dev,
685 struct device_attribute *mattr,
686 const char *data, size_t count)
688 struct mem_ctl_info *mci = to_mci(dev);
689 struct amd64_pvt *pvt = mci->pvt_info;
693 ret = kstrtoul(data, 16, &value);
697 if (value & 0xFFFF0000) {
698 amd64_warn("%s: invalid EccVector: 0x%lx\n", __func__, value);
702 pvt->injection.bit_map = (u32) value;
707 * Do a DRAM ECC read. Assemble staged values in the pvt area, format into
708 * fields needed by the injection registers and read the NB Array Data Port.
710 static ssize_t inject_read_store(struct device *dev,
711 struct device_attribute *mattr,
712 const char *data, size_t count)
714 struct mem_ctl_info *mci = to_mci(dev);
715 struct amd64_pvt *pvt = mci->pvt_info;
717 u32 section, word_bits;
720 ret = kstrtoul(data, 10, &value);
724 /* Form value to choose 16-byte section of cacheline */
725 section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section);
727 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section);
729 word_bits = SET_NB_DRAM_INJECTION_READ(pvt->injection);
731 /* Issue 'word' and 'bit' along with the READ request */
732 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits);
734 edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits);
740 * Do a DRAM ECC write. Assemble staged values in the pvt area and format into
741 * fields needed by the injection registers.
743 static ssize_t inject_write_store(struct device *dev,
744 struct device_attribute *mattr,
745 const char *data, size_t count)
747 struct mem_ctl_info *mci = to_mci(dev);
748 struct amd64_pvt *pvt = mci->pvt_info;
749 u32 section, word_bits, tmp;
753 ret = kstrtoul(data, 10, &value);
757 /* Form value to choose 16-byte section of cacheline */
758 section = F10_NB_ARRAY_DRAM | SET_NB_ARRAY_ADDR(pvt->injection.section);
760 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_ADDR, section);
762 word_bits = SET_NB_DRAM_INJECTION_WRITE(pvt->injection);
764 pr_notice_once("Don't forget to decrease MCE polling interval in\n"
765 "/sys/bus/machinecheck/devices/machinecheck<CPUNUM>/check_interval\n"
766 "so that you can get the error report faster.\n");
768 on_each_cpu(disable_caches, NULL, 1);
770 /* Issue 'word' and 'bit' along with the READ request */
771 amd64_write_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, word_bits);
774 /* wait until injection happens */
775 amd64_read_pci_cfg(pvt->F3, F10_NB_ARRAY_DATA, &tmp);
776 if (tmp & F10_NB_ARR_ECC_WR_REQ) {
781 on_each_cpu(enable_caches, NULL, 1);
783 edac_dbg(0, "section=0x%x word_bits=0x%x\n", section, word_bits);
789 * update NUM_INJ_ATTRS in case you add new members
792 static DEVICE_ATTR_RW(inject_section);
793 static DEVICE_ATTR_RW(inject_word);
794 static DEVICE_ATTR_RW(inject_ecc_vector);
795 static DEVICE_ATTR_WO(inject_write);
796 static DEVICE_ATTR_WO(inject_read);
798 static struct attribute *inj_attrs[] = {
799 &dev_attr_inject_section.attr,
800 &dev_attr_inject_word.attr,
801 &dev_attr_inject_ecc_vector.attr,
802 &dev_attr_inject_write.attr,
803 &dev_attr_inject_read.attr,
807 static umode_t inj_is_visible(struct kobject *kobj, struct attribute *attr, int idx)
809 struct device *dev = kobj_to_dev(kobj);
810 struct mem_ctl_info *mci = container_of(dev, struct mem_ctl_info, dev);
811 struct amd64_pvt *pvt = mci->pvt_info;
813 /* Families which have that injection hw */
814 if (pvt->fam >= 0x10 && pvt->fam <= 0x16)
820 static const struct attribute_group inj_group = {
822 .is_visible = inj_is_visible,
824 #endif /* CONFIG_EDAC_DEBUG */
827 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
828 * assumed that sys_addr maps to the node given by mci.
830 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
831 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
832 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
833 * then it is also involved in translating a SysAddr to a DramAddr. Sections
834 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
835 * These parts of the documentation are unclear. I interpret them as follows:
837 * When node n receives a SysAddr, it processes the SysAddr as follows:
839 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
840 * Limit registers for node n. If the SysAddr is not within the range
841 * specified by the base and limit values, then node n ignores the Sysaddr
842 * (since it does not map to node n). Otherwise continue to step 2 below.
844 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
845 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
846 * the range of relocated addresses (starting at 0x100000000) from the DRAM
847 * hole. If not, skip to step 3 below. Else get the value of the
848 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
849 * offset defined by this value from the SysAddr.
851 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
852 * Base register for node n. To obtain the DramAddr, subtract the base
853 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
855 static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
857 struct amd64_pvt *pvt = mci->pvt_info;
858 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
861 dram_base = get_dram_base(pvt, pvt->mc_node_id);
863 ret = get_dram_hole_info(mci, &hole_base, &hole_offset, &hole_size);
865 if ((sys_addr >= (1ULL << 32)) &&
866 (sys_addr < ((1ULL << 32) + hole_size))) {
867 /* use DHAR to translate SysAddr to DramAddr */
868 dram_addr = sys_addr - hole_offset;
870 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
871 (unsigned long)sys_addr,
872 (unsigned long)dram_addr);
879 * Translate the SysAddr to a DramAddr as shown near the start of
880 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
881 * only deals with 40-bit values. Therefore we discard bits 63-40 of
882 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
883 * discard are all 1s. Otherwise the bits we discard are all 0s. See
884 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
885 * Programmer's Manual Volume 1 Application Programming.
887 dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
889 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
890 (unsigned long)sys_addr, (unsigned long)dram_addr);
895 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
896 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
897 * for node interleaving.
899 static int num_node_interleave_bits(unsigned intlv_en)
901 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
904 BUG_ON(intlv_en > 7);
905 n = intlv_shift_table[intlv_en];
909 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
910 static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
912 struct amd64_pvt *pvt;
919 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
920 * concerning translating a DramAddr to an InputAddr.
922 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
923 input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
926 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
927 intlv_shift, (unsigned long)dram_addr,
928 (unsigned long)input_addr);
934 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
935 * assumed that @sys_addr maps to the node given by mci.
937 static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
942 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
944 edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n",
945 (unsigned long)sys_addr, (unsigned long)input_addr);
950 /* Map the Error address to a PAGE and PAGE OFFSET. */
951 static inline void error_address_to_page_and_offset(u64 error_address,
952 struct err_info *err)
954 err->page = (u32) (error_address >> PAGE_SHIFT);
955 err->offset = ((u32) error_address) & ~PAGE_MASK;
959 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
960 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
961 * of a node that detected an ECC memory error. mci represents the node that
962 * the error address maps to (possibly different from the node that detected
963 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
966 static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
970 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
973 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
974 "address 0x%lx\n", (unsigned long)sys_addr);
979 * See AMD PPR DF::LclNodeTypeMap
981 * This register gives information for nodes of the same type within a system.
983 * Reading this register from a GPU node will tell how many GPU nodes are in the
984 * system and what the lowest AMD Node ID value is for the GPU nodes. Use this
985 * info to fixup the Linux logical "Node ID" value set in the AMD NB code and EDAC.
987 static struct local_node_map {
992 #define PCI_DEVICE_ID_AMD_MI200_DF_F1 0x14d1
993 #define REG_LOCAL_NODE_TYPE_MAP 0x144
995 /* Local Node Type Map (LNTM) fields */
996 #define LNTM_NODE_COUNT GENMASK(27, 16)
997 #define LNTM_BASE_NODE_ID GENMASK(11, 0)
999 static int gpu_get_node_map(struct amd64_pvt *pvt)
1001 struct pci_dev *pdev;
1006 * Mapping of nodes from hardware-provided AMD Node ID to a
1007 * Linux logical one is applicable for MI200 models. Therefore,
1008 * return early for other heterogeneous systems.
1010 if (pvt->F3->device != PCI_DEVICE_ID_AMD_MI200_DF_F3)
1014 * Node ID 0 is reserved for CPUs. Therefore, a non-zero Node ID
1015 * means the values have been already cached.
1017 if (gpu_node_map.base_node_id)
1020 pdev = pci_get_device(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_MI200_DF_F1, NULL);
1026 ret = pci_read_config_dword(pdev, REG_LOCAL_NODE_TYPE_MAP, &tmp);
1030 gpu_node_map.node_count = FIELD_GET(LNTM_NODE_COUNT, tmp);
1031 gpu_node_map.base_node_id = FIELD_GET(LNTM_BASE_NODE_ID, tmp);
1038 static int fixup_node_id(int node_id, struct mce *m)
1040 /* MCA_IPID[InstanceIdHi] give the AMD Node ID for the bank. */
1041 u8 nid = (m->ipid >> 44) & 0xF;
1043 if (smca_get_bank_type(m->extcpu, m->bank) != SMCA_UMC_V2)
1046 /* Nodes below the GPU base node are CPU nodes and don't need a fixup. */
1047 if (nid < gpu_node_map.base_node_id)
1050 /* Convert the hardware-provided AMD Node ID to a Linux logical one. */
1051 return nid - gpu_node_map.base_node_id + 1;
1054 /* Protect the PCI config register pairs used for DF indirect access. */
1055 static DEFINE_MUTEX(df_indirect_mutex);
1058 * Data Fabric Indirect Access uses FICAA/FICAD.
1060 * Fabric Indirect Configuration Access Address (FICAA): Constructed based
1061 * on the device's Instance Id and the PCI function and register offset of
1062 * the desired register.
1064 * Fabric Indirect Configuration Access Data (FICAD): There are FICAD LO
1065 * and FICAD HI registers but so far we only need the LO register.
1067 * Use Instance Id 0xFF to indicate a broadcast read.
1069 #define DF_BROADCAST 0xFF
1070 static int __df_indirect_read(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo)
1076 if (node >= amd_nb_num())
1079 F4 = node_to_amd_nb(node)->link;
1083 ficaa = (instance_id == DF_BROADCAST) ? 0 : 1;
1084 ficaa |= reg & 0x3FC;
1085 ficaa |= (func & 0x7) << 11;
1086 ficaa |= instance_id << 16;
1088 mutex_lock(&df_indirect_mutex);
1090 err = pci_write_config_dword(F4, 0x5C, ficaa);
1092 pr_warn("Error writing DF Indirect FICAA, FICAA=0x%x\n", ficaa);
1096 err = pci_read_config_dword(F4, 0x98, lo);
1098 pr_warn("Error reading DF Indirect FICAD LO, FICAA=0x%x.\n", ficaa);
1101 mutex_unlock(&df_indirect_mutex);
1107 static int df_indirect_read_instance(u16 node, u8 func, u16 reg, u8 instance_id, u32 *lo)
1109 return __df_indirect_read(node, func, reg, instance_id, lo);
1112 static int df_indirect_read_broadcast(u16 node, u8 func, u16 reg, u32 *lo)
1114 return __df_indirect_read(node, func, reg, DF_BROADCAST, lo);
1124 static int umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr)
1126 u64 dram_base_addr, dram_limit_addr, dram_hole_base;
1128 u8 die_id_shift, die_id_mask, socket_id_shift, socket_id_mask;
1129 u8 intlv_num_dies, intlv_num_chan, intlv_num_sockets;
1130 u8 intlv_addr_sel, intlv_addr_bit;
1131 u8 num_intlv_bits, hashed_bit;
1132 u8 lgcy_mmio_hole_en, base = 0;
1133 u8 cs_mask, cs_id = 0;
1134 bool hash_enabled = false;
1136 struct addr_ctx ctx;
1138 memset(&ctx, 0, sizeof(ctx));
1140 /* Start from the normalized address */
1141 ctx.ret_addr = norm_addr;
1146 /* Read D18F0x1B4 (DramOffset), check if base 1 is used. */
1147 if (df_indirect_read_instance(nid, 0, 0x1B4, umc, &ctx.tmp))
1150 /* Remove HiAddrOffset from normalized address, if enabled: */
1151 if (ctx.tmp & BIT(0)) {
1152 u64 hi_addr_offset = (ctx.tmp & GENMASK_ULL(31, 20)) << 8;
1154 if (norm_addr >= hi_addr_offset) {
1155 ctx.ret_addr -= hi_addr_offset;
1160 /* Read D18F0x110 (DramBaseAddress). */
1161 if (df_indirect_read_instance(nid, 0, 0x110 + (8 * base), umc, &ctx.tmp))
1164 /* Check if address range is valid. */
1165 if (!(ctx.tmp & BIT(0))) {
1166 pr_err("%s: Invalid DramBaseAddress range: 0x%x.\n",
1171 lgcy_mmio_hole_en = ctx.tmp & BIT(1);
1172 intlv_num_chan = (ctx.tmp >> 4) & 0xF;
1173 intlv_addr_sel = (ctx.tmp >> 8) & 0x7;
1174 dram_base_addr = (ctx.tmp & GENMASK_ULL(31, 12)) << 16;
1176 /* {0, 1, 2, 3} map to address bits {8, 9, 10, 11} respectively */
1177 if (intlv_addr_sel > 3) {
1178 pr_err("%s: Invalid interleave address select %d.\n",
1179 __func__, intlv_addr_sel);
1183 /* Read D18F0x114 (DramLimitAddress). */
1184 if (df_indirect_read_instance(nid, 0, 0x114 + (8 * base), umc, &ctx.tmp))
1187 intlv_num_sockets = (ctx.tmp >> 8) & 0x1;
1188 intlv_num_dies = (ctx.tmp >> 10) & 0x3;
1189 dram_limit_addr = ((ctx.tmp & GENMASK_ULL(31, 12)) << 16) | GENMASK_ULL(27, 0);
1191 intlv_addr_bit = intlv_addr_sel + 8;
1193 /* Re-use intlv_num_chan by setting it equal to log2(#channels) */
1194 switch (intlv_num_chan) {
1195 case 0: intlv_num_chan = 0; break;
1196 case 1: intlv_num_chan = 1; break;
1197 case 3: intlv_num_chan = 2; break;
1198 case 5: intlv_num_chan = 3; break;
1199 case 7: intlv_num_chan = 4; break;
1201 case 8: intlv_num_chan = 1;
1202 hash_enabled = true;
1205 pr_err("%s: Invalid number of interleaved channels %d.\n",
1206 __func__, intlv_num_chan);
1210 num_intlv_bits = intlv_num_chan;
1212 if (intlv_num_dies > 2) {
1213 pr_err("%s: Invalid number of interleaved nodes/dies %d.\n",
1214 __func__, intlv_num_dies);
1218 num_intlv_bits += intlv_num_dies;
1220 /* Add a bit if sockets are interleaved. */
1221 num_intlv_bits += intlv_num_sockets;
1223 /* Assert num_intlv_bits <= 4 */
1224 if (num_intlv_bits > 4) {
1225 pr_err("%s: Invalid interleave bits %d.\n",
1226 __func__, num_intlv_bits);
1230 if (num_intlv_bits > 0) {
1231 u64 temp_addr_x, temp_addr_i, temp_addr_y;
1232 u8 die_id_bit, sock_id_bit, cs_fabric_id;
1235 * Read FabricBlockInstanceInformation3_CS[BlockFabricID].
1236 * This is the fabric id for this coherent slave. Use
1237 * umc/channel# as instance id of the coherent slave
1240 if (df_indirect_read_instance(nid, 0, 0x50, umc, &ctx.tmp))
1243 cs_fabric_id = (ctx.tmp >> 8) & 0xFF;
1246 /* If interleaved over more than 1 channel: */
1247 if (intlv_num_chan) {
1248 die_id_bit = intlv_num_chan;
1249 cs_mask = (1 << die_id_bit) - 1;
1250 cs_id = cs_fabric_id & cs_mask;
1253 sock_id_bit = die_id_bit;
1255 /* Read D18F1x208 (SystemFabricIdMask). */
1256 if (intlv_num_dies || intlv_num_sockets)
1257 if (df_indirect_read_broadcast(nid, 1, 0x208, &ctx.tmp))
1260 /* If interleaved over more than 1 die. */
1261 if (intlv_num_dies) {
1262 sock_id_bit = die_id_bit + intlv_num_dies;
1263 die_id_shift = (ctx.tmp >> 24) & 0xF;
1264 die_id_mask = (ctx.tmp >> 8) & 0xFF;
1266 cs_id |= ((cs_fabric_id & die_id_mask) >> die_id_shift) << die_id_bit;
1269 /* If interleaved over more than 1 socket. */
1270 if (intlv_num_sockets) {
1271 socket_id_shift = (ctx.tmp >> 28) & 0xF;
1272 socket_id_mask = (ctx.tmp >> 16) & 0xFF;
1274 cs_id |= ((cs_fabric_id & socket_id_mask) >> socket_id_shift) << sock_id_bit;
1278 * The pre-interleaved address consists of XXXXXXIIIYYYYY
1279 * where III is the ID for this CS, and XXXXXXYYYYY are the
1280 * address bits from the post-interleaved address.
1281 * "num_intlv_bits" has been calculated to tell us how many "I"
1282 * bits there are. "intlv_addr_bit" tells us how many "Y" bits
1283 * there are (where "I" starts).
1285 temp_addr_y = ctx.ret_addr & GENMASK_ULL(intlv_addr_bit - 1, 0);
1286 temp_addr_i = (cs_id << intlv_addr_bit);
1287 temp_addr_x = (ctx.ret_addr & GENMASK_ULL(63, intlv_addr_bit)) << num_intlv_bits;
1288 ctx.ret_addr = temp_addr_x | temp_addr_i | temp_addr_y;
1291 /* Add dram base address */
1292 ctx.ret_addr += dram_base_addr;
1294 /* If legacy MMIO hole enabled */
1295 if (lgcy_mmio_hole_en) {
1296 if (df_indirect_read_broadcast(nid, 0, 0x104, &ctx.tmp))
1299 dram_hole_base = ctx.tmp & GENMASK(31, 24);
1300 if (ctx.ret_addr >= dram_hole_base)
1301 ctx.ret_addr += (BIT_ULL(32) - dram_hole_base);
1305 /* Save some parentheses and grab ls-bit at the end. */
1306 hashed_bit = (ctx.ret_addr >> 12) ^
1307 (ctx.ret_addr >> 18) ^
1308 (ctx.ret_addr >> 21) ^
1309 (ctx.ret_addr >> 30) ^
1312 hashed_bit &= BIT(0);
1314 if (hashed_bit != ((ctx.ret_addr >> intlv_addr_bit) & BIT(0)))
1315 ctx.ret_addr ^= BIT(intlv_addr_bit);
1318 /* Is calculated system address is above DRAM limit address? */
1319 if (ctx.ret_addr > dram_limit_addr)
1322 *sys_addr = ctx.ret_addr;
1329 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
1332 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
1335 static unsigned long dct_determine_edac_cap(struct amd64_pvt *pvt)
1337 unsigned long edac_cap = EDAC_FLAG_NONE;
1340 bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
1344 if (pvt->dclr0 & BIT(bit))
1345 edac_cap = EDAC_FLAG_SECDED;
1350 static unsigned long umc_determine_edac_cap(struct amd64_pvt *pvt)
1352 u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
1353 unsigned long edac_cap = EDAC_FLAG_NONE;
1356 if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
1359 umc_en_mask |= BIT(i);
1361 /* UMC Configuration bit 12 (DimmEccEn) */
1362 if (pvt->umc[i].umc_cfg & BIT(12))
1363 dimm_ecc_en_mask |= BIT(i);
1366 if (umc_en_mask == dimm_ecc_en_mask)
1367 edac_cap = EDAC_FLAG_SECDED;
1373 * debug routine to display the memory sizes of all logical DIMMs and its
1376 static void dct_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
1378 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1379 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
1380 int dimm, size0, size1;
1382 if (pvt->fam == 0xf) {
1383 /* K8 families < revF not supported yet */
1384 if (pvt->ext_model < K8_REV_F)
1390 if (pvt->fam == 0x10) {
1391 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
1393 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
1394 pvt->csels[1].csbases :
1395 pvt->csels[0].csbases;
1398 dcsb = pvt->csels[1].csbases;
1400 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
1403 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1405 /* Dump memory sizes for DIMM and its CSROWs */
1406 for (dimm = 0; dimm < 4; dimm++) {
1408 if (dcsb[dimm * 2] & DCSB_CS_ENABLE)
1410 * For F15m60h, we need multiplier for LRDIMM cs_size
1411 * calculation. We pass dimm value to the dbam_to_cs
1412 * mapper so we can find the multiplier from the
1413 * corresponding DCSM.
1415 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
1416 DBAM_DIMM(dimm, dbam),
1420 if (dcsb[dimm * 2 + 1] & DCSB_CS_ENABLE)
1421 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
1422 DBAM_DIMM(dimm, dbam),
1425 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1427 dimm * 2 + 1, size1);
1432 static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
1434 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
1436 if (pvt->dram_type == MEM_LRDDR3) {
1437 u32 dcsm = pvt->csels[chan].csmasks[0];
1439 * It's assumed all LRDIMMs in a DCT are going to be of
1440 * same 'type' until proven otherwise. So, use a cs
1441 * value of '0' here to get dcsm value.
1443 edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
1446 edac_dbg(1, "All DIMMs support ECC:%s\n",
1447 (dclr & BIT(19)) ? "yes" : "no");
1450 edac_dbg(1, " PAR/ERR parity: %s\n",
1451 (dclr & BIT(8)) ? "enabled" : "disabled");
1453 if (pvt->fam == 0x10)
1454 edac_dbg(1, " DCT 128bit mode width: %s\n",
1455 (dclr & BIT(11)) ? "128b" : "64b");
1457 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
1458 (dclr & BIT(12)) ? "yes" : "no",
1459 (dclr & BIT(13)) ? "yes" : "no",
1460 (dclr & BIT(14)) ? "yes" : "no",
1461 (dclr & BIT(15)) ? "yes" : "no");
1464 #define CS_EVEN_PRIMARY BIT(0)
1465 #define CS_ODD_PRIMARY BIT(1)
1466 #define CS_EVEN_SECONDARY BIT(2)
1467 #define CS_ODD_SECONDARY BIT(3)
1468 #define CS_3R_INTERLEAVE BIT(4)
1470 #define CS_EVEN (CS_EVEN_PRIMARY | CS_EVEN_SECONDARY)
1471 #define CS_ODD (CS_ODD_PRIMARY | CS_ODD_SECONDARY)
1473 static int umc_get_cs_mode(int dimm, u8 ctrl, struct amd64_pvt *pvt)
1478 if (csrow_enabled(2 * dimm, ctrl, pvt))
1479 cs_mode |= CS_EVEN_PRIMARY;
1481 if (csrow_enabled(2 * dimm + 1, ctrl, pvt))
1482 cs_mode |= CS_ODD_PRIMARY;
1484 /* Asymmetric dual-rank DIMM support. */
1485 if (csrow_sec_enabled(2 * dimm + 1, ctrl, pvt))
1486 cs_mode |= CS_ODD_SECONDARY;
1489 * 3 Rank inteleaving support.
1490 * There should be only three bases enabled and their two masks should
1493 for_each_chip_select(base, ctrl, pvt)
1494 count += csrow_enabled(base, ctrl, pvt);
1497 pvt->csels[ctrl].csmasks[0] == pvt->csels[ctrl].csmasks[1]) {
1498 edac_dbg(1, "3R interleaving in use.\n");
1499 cs_mode |= CS_3R_INTERLEAVE;
1505 static int __addr_mask_to_cs_size(u32 addr_mask_orig, unsigned int cs_mode,
1506 int csrow_nr, int dimm)
1508 u32 msb, weight, num_zero_bits;
1509 u32 addr_mask_deinterleaved;
1513 * The number of zero bits in the mask is equal to the number of bits
1514 * in a full mask minus the number of bits in the current mask.
1516 * The MSB is the number of bits in the full mask because BIT[0] is
1519 * In the special 3 Rank interleaving case, a single bit is flipped
1520 * without swapping with the most significant bit. This can be handled
1521 * by keeping the MSB where it is and ignoring the single zero bit.
1523 msb = fls(addr_mask_orig) - 1;
1524 weight = hweight_long(addr_mask_orig);
1525 num_zero_bits = msb - weight - !!(cs_mode & CS_3R_INTERLEAVE);
1527 /* Take the number of zero bits off from the top of the mask. */
1528 addr_mask_deinterleaved = GENMASK_ULL(msb - num_zero_bits, 1);
1530 edac_dbg(1, "CS%d DIMM%d AddrMasks:\n", csrow_nr, dimm);
1531 edac_dbg(1, " Original AddrMask: 0x%x\n", addr_mask_orig);
1532 edac_dbg(1, " Deinterleaved AddrMask: 0x%x\n", addr_mask_deinterleaved);
1534 /* Register [31:1] = Address [39:9]. Size is in kBs here. */
1535 size = (addr_mask_deinterleaved >> 2) + 1;
1537 /* Return size in MBs. */
1541 static int umc_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc,
1542 unsigned int cs_mode, int csrow_nr)
1544 int cs_mask_nr = csrow_nr;
1548 /* No Chip Selects are enabled. */
1552 /* Requested size of an even CS but none are enabled. */
1553 if (!(cs_mode & CS_EVEN) && !(csrow_nr & 1))
1556 /* Requested size of an odd CS but none are enabled. */
1557 if (!(cs_mode & CS_ODD) && (csrow_nr & 1))
1561 * Family 17h introduced systems with one mask per DIMM,
1562 * and two Chip Selects per DIMM.
1564 * CS0 and CS1 -> MASK0 / DIMM0
1565 * CS2 and CS3 -> MASK1 / DIMM1
1567 * Family 19h Model 10h introduced systems with one mask per Chip Select,
1568 * and two Chip Selects per DIMM.
1570 * CS0 -> MASK0 -> DIMM0
1571 * CS1 -> MASK1 -> DIMM0
1572 * CS2 -> MASK2 -> DIMM1
1573 * CS3 -> MASK3 -> DIMM1
1575 * Keep the mask number equal to the Chip Select number for newer systems,
1576 * and shift the mask number for older systems.
1578 dimm = csrow_nr >> 1;
1580 if (!pvt->flags.zn_regs_v2)
1583 /* Asymmetric dual-rank DIMM support. */
1584 if ((csrow_nr & 1) && (cs_mode & CS_ODD_SECONDARY))
1585 addr_mask_orig = pvt->csels[umc].csmasks_sec[cs_mask_nr];
1587 addr_mask_orig = pvt->csels[umc].csmasks[cs_mask_nr];
1589 return __addr_mask_to_cs_size(addr_mask_orig, cs_mode, csrow_nr, dimm);
1592 static void umc_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
1594 int dimm, size0, size1, cs0, cs1, cs_mode;
1596 edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
1598 for (dimm = 0; dimm < 2; dimm++) {
1602 cs_mode = umc_get_cs_mode(dimm, ctrl, pvt);
1604 size0 = umc_addr_mask_to_cs_size(pvt, ctrl, cs_mode, cs0);
1605 size1 = umc_addr_mask_to_cs_size(pvt, ctrl, cs_mode, cs1);
1607 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1613 static void umc_dump_misc_regs(struct amd64_pvt *pvt)
1615 struct amd64_umc *umc;
1616 u32 i, tmp, umc_base;
1619 umc_base = get_umc_base(i);
1622 edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg);
1623 edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
1624 edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
1625 edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
1627 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp);
1628 edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp);
1630 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp);
1631 edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp);
1632 edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi);
1634 edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n",
1635 i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no",
1636 (umc->umc_cap_hi & BIT(31)) ? "yes" : "no");
1637 edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n",
1638 i, (umc->umc_cfg & BIT(12)) ? "yes" : "no");
1639 edac_dbg(1, "UMC%d x4 DIMMs present: %s\n",
1640 i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no");
1641 edac_dbg(1, "UMC%d x16 DIMMs present: %s\n",
1642 i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no");
1644 if (umc->dram_type == MEM_LRDDR4 || umc->dram_type == MEM_LRDDR5) {
1645 amd_smn_read(pvt->mc_node_id,
1646 umc_base + get_umc_reg(pvt, UMCCH_ADDR_CFG),
1648 edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
1649 i, 1 << ((tmp >> 4) & 0x3));
1652 umc_debug_display_dimm_sizes(pvt, i);
1656 static void dct_dump_misc_regs(struct amd64_pvt *pvt)
1658 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
1660 edac_dbg(1, " NB two channel DRAM capable: %s\n",
1661 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
1663 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
1664 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
1665 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
1667 debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
1669 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
1671 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
1672 pvt->dhar, dhar_base(pvt),
1673 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
1674 : f10_dhar_offset(pvt));
1676 dct_debug_display_dimm_sizes(pvt, 0);
1678 /* everything below this point is Fam10h and above */
1679 if (pvt->fam == 0xf)
1682 dct_debug_display_dimm_sizes(pvt, 1);
1684 /* Only if NOT ganged does dclr1 have valid info */
1685 if (!dct_ganging_enabled(pvt))
1686 debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
1688 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
1690 amd64_info("using x%u syndromes.\n", pvt->ecc_sym_sz);
1694 * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
1696 static void dct_prep_chip_selects(struct amd64_pvt *pvt)
1698 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
1699 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
1700 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
1701 } else if (pvt->fam == 0x15 && pvt->model == 0x30) {
1702 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
1703 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
1705 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
1706 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
1710 static void umc_prep_chip_selects(struct amd64_pvt *pvt)
1715 pvt->csels[umc].b_cnt = 4;
1716 pvt->csels[umc].m_cnt = pvt->flags.zn_regs_v2 ? 4 : 2;
1720 static void umc_read_base_mask(struct amd64_pvt *pvt)
1722 u32 umc_base_reg, umc_base_reg_sec;
1723 u32 umc_mask_reg, umc_mask_reg_sec;
1724 u32 base_reg, base_reg_sec;
1725 u32 mask_reg, mask_reg_sec;
1726 u32 *base, *base_sec;
1727 u32 *mask, *mask_sec;
1731 umc_base_reg = get_umc_base(umc) + UMCCH_BASE_ADDR;
1732 umc_base_reg_sec = get_umc_base(umc) + UMCCH_BASE_ADDR_SEC;
1734 for_each_chip_select(cs, umc, pvt) {
1735 base = &pvt->csels[umc].csbases[cs];
1736 base_sec = &pvt->csels[umc].csbases_sec[cs];
1738 base_reg = umc_base_reg + (cs * 4);
1739 base_reg_sec = umc_base_reg_sec + (cs * 4);
1741 if (!amd_smn_read(pvt->mc_node_id, base_reg, base))
1742 edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n",
1743 umc, cs, *base, base_reg);
1745 if (!amd_smn_read(pvt->mc_node_id, base_reg_sec, base_sec))
1746 edac_dbg(0, " DCSB_SEC%d[%d]=0x%08x reg: 0x%x\n",
1747 umc, cs, *base_sec, base_reg_sec);
1750 umc_mask_reg = get_umc_base(umc) + UMCCH_ADDR_MASK;
1751 umc_mask_reg_sec = get_umc_base(umc) + get_umc_reg(pvt, UMCCH_ADDR_MASK_SEC);
1753 for_each_chip_select_mask(cs, umc, pvt) {
1754 mask = &pvt->csels[umc].csmasks[cs];
1755 mask_sec = &pvt->csels[umc].csmasks_sec[cs];
1757 mask_reg = umc_mask_reg + (cs * 4);
1758 mask_reg_sec = umc_mask_reg_sec + (cs * 4);
1760 if (!amd_smn_read(pvt->mc_node_id, mask_reg, mask))
1761 edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n",
1762 umc, cs, *mask, mask_reg);
1764 if (!amd_smn_read(pvt->mc_node_id, mask_reg_sec, mask_sec))
1765 edac_dbg(0, " DCSM_SEC%d[%d]=0x%08x reg: 0x%x\n",
1766 umc, cs, *mask_sec, mask_reg_sec);
1772 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
1774 static void dct_read_base_mask(struct amd64_pvt *pvt)
1778 for_each_chip_select(cs, 0, pvt) {
1779 int reg0 = DCSB0 + (cs * 4);
1780 int reg1 = DCSB1 + (cs * 4);
1781 u32 *base0 = &pvt->csels[0].csbases[cs];
1782 u32 *base1 = &pvt->csels[1].csbases[cs];
1784 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
1785 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
1788 if (pvt->fam == 0xf)
1791 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
1792 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
1793 cs, *base1, (pvt->fam == 0x10) ? reg1
1797 for_each_chip_select_mask(cs, 0, pvt) {
1798 int reg0 = DCSM0 + (cs * 4);
1799 int reg1 = DCSM1 + (cs * 4);
1800 u32 *mask0 = &pvt->csels[0].csmasks[cs];
1801 u32 *mask1 = &pvt->csels[1].csmasks[cs];
1803 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
1804 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
1807 if (pvt->fam == 0xf)
1810 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
1811 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
1812 cs, *mask1, (pvt->fam == 0x10) ? reg1
1817 static void umc_determine_memory_type(struct amd64_pvt *pvt)
1819 struct amd64_umc *umc;
1825 if (!(umc->sdp_ctrl & UMC_SDP_INIT)) {
1826 umc->dram_type = MEM_EMPTY;
1831 * Check if the system supports the "DDR Type" field in UMC Config
1832 * and has DDR5 DIMMs in use.
1834 if (pvt->flags.zn_regs_v2 && ((umc->umc_cfg & GENMASK(2, 0)) == 0x1)) {
1835 if (umc->dimm_cfg & BIT(5))
1836 umc->dram_type = MEM_LRDDR5;
1837 else if (umc->dimm_cfg & BIT(4))
1838 umc->dram_type = MEM_RDDR5;
1840 umc->dram_type = MEM_DDR5;
1842 if (umc->dimm_cfg & BIT(5))
1843 umc->dram_type = MEM_LRDDR4;
1844 else if (umc->dimm_cfg & BIT(4))
1845 umc->dram_type = MEM_RDDR4;
1847 umc->dram_type = MEM_DDR4;
1850 edac_dbg(1, " UMC%d DIMM type: %s\n", i, edac_mem_types[umc->dram_type]);
1854 static void dct_determine_memory_type(struct amd64_pvt *pvt)
1856 u32 dram_ctrl, dcsm;
1860 if (pvt->ext_model >= K8_REV_F)
1863 pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
1867 if (pvt->dchr0 & DDR3_MODE)
1870 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
1874 if (pvt->model < 0x60)
1878 * Model 0x60h needs special handling:
1880 * We use a Chip Select value of '0' to obtain dcsm.
1881 * Theoretically, it is possible to populate LRDIMMs of different
1882 * 'Rank' value on a DCT. But this is not the common case. So,
1883 * it's reasonable to assume all DIMMs are going to be of same
1884 * 'type' until proven otherwise.
1886 amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
1887 dcsm = pvt->csels[0].csmasks[0];
1889 if (((dram_ctrl >> 8) & 0x7) == 0x2)
1890 pvt->dram_type = MEM_DDR4;
1891 else if (pvt->dclr0 & BIT(16))
1892 pvt->dram_type = MEM_DDR3;
1893 else if (dcsm & 0x3)
1894 pvt->dram_type = MEM_LRDDR3;
1896 pvt->dram_type = MEM_RDDR3;
1904 WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
1905 pvt->dram_type = MEM_EMPTY;
1908 edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
1912 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
1915 /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
1916 static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
1918 u16 mce_nid = topology_die_id(m->extcpu);
1919 struct mem_ctl_info *mci;
1924 mci = edac_mc_find(mce_nid);
1928 pvt = mci->pvt_info;
1930 if (pvt->fam == 0xf) {
1935 addr = m->addr & GENMASK_ULL(end_bit, start_bit);
1938 * Erratum 637 workaround
1940 if (pvt->fam == 0x15) {
1941 u64 cc6_base, tmp_addr;
1945 if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
1949 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
1950 intlv_en = tmp >> 21 & 0x7;
1952 /* add [47:27] + 3 trailing bits */
1953 cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3;
1955 /* reverse and add DramIntlvEn */
1956 cc6_base |= intlv_en ^ 0x7;
1958 /* pin at [47:24] */
1962 return cc6_base | (addr & GENMASK_ULL(23, 0));
1964 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
1967 tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
1969 /* OR DramIntlvSel into bits [14:12] */
1970 tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
1972 /* add remaining [11:0] bits from original MC4_ADDR */
1973 tmp_addr |= addr & GENMASK_ULL(11, 0);
1975 return cc6_base | tmp_addr;
1981 static struct pci_dev *pci_get_related_function(unsigned int vendor,
1982 unsigned int device,
1983 struct pci_dev *related)
1985 struct pci_dev *dev = NULL;
1987 while ((dev = pci_get_device(vendor, device, dev))) {
1988 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
1989 (dev->bus->number == related->bus->number) &&
1990 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1997 static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
1999 struct amd_northbridge *nb;
2000 struct pci_dev *f1 = NULL;
2001 unsigned int pci_func;
2002 int off = range << 3;
2005 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
2006 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
2008 if (pvt->fam == 0xf)
2011 if (!dram_rw(pvt, range))
2014 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
2015 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
2017 /* F15h: factor in CC6 save area by reading dst node's limit reg */
2018 if (pvt->fam != 0x15)
2021 nb = node_to_amd_nb(dram_dst_node(pvt, range));
2025 if (pvt->model == 0x60)
2026 pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
2027 else if (pvt->model == 0x30)
2028 pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
2030 pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
2032 f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
2036 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
2038 pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
2040 /* {[39:27],111b} */
2041 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
2043 pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
2046 pvt->ranges[range].lim.hi |= llim >> 13;
2051 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
2052 struct err_info *err)
2054 struct amd64_pvt *pvt = mci->pvt_info;
2056 error_address_to_page_and_offset(sys_addr, err);
2059 * Find out which node the error address belongs to. This may be
2060 * different from the node that detected the error.
2062 err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
2063 if (!err->src_mci) {
2064 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
2065 (unsigned long)sys_addr);
2066 err->err_code = ERR_NODE;
2070 /* Now map the sys_addr to a CSROW */
2071 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
2072 if (err->csrow < 0) {
2073 err->err_code = ERR_CSROW;
2077 /* CHIPKILL enabled */
2078 if (pvt->nbcfg & NBCFG_CHIPKILL) {
2079 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
2080 if (err->channel < 0) {
2082 * Syndrome didn't map, so we don't know which of the
2083 * 2 DIMMs is in error. So we need to ID 'both' of them
2086 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
2087 "possible error reporting race\n",
2089 err->err_code = ERR_CHANNEL;
2094 * non-chipkill ecc mode
2096 * The k8 documentation is unclear about how to determine the
2097 * channel number when using non-chipkill memory. This method
2098 * was obtained from email communication with someone at AMD.
2099 * (Wish the email was placed in this comment - norsk)
2101 err->channel = ((sys_addr & BIT(3)) != 0);
2105 static int ddr2_cs_size(unsigned i, bool dct_width)
2111 else if (!(i & 0x1))
2114 shift = (i + 1) >> 1;
2116 return 128 << (shift + !!dct_width);
2119 static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
2120 unsigned cs_mode, int cs_mask_nr)
2122 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
2124 if (pvt->ext_model >= K8_REV_F) {
2125 WARN_ON(cs_mode > 11);
2126 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
2128 else if (pvt->ext_model >= K8_REV_D) {
2130 WARN_ON(cs_mode > 10);
2133 * the below calculation, besides trying to win an obfuscated C
2134 * contest, maps cs_mode values to DIMM chip select sizes. The
2137 * cs_mode CS size (mb)
2138 * ======= ============
2151 * Basically, it calculates a value with which to shift the
2152 * smallest CS size of 32MB.
2154 * ddr[23]_cs_size have a similar purpose.
2156 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
2158 return 32 << (cs_mode - diff);
2161 WARN_ON(cs_mode > 6);
2162 return 32 << cs_mode;
2166 static int ddr3_cs_size(unsigned i, bool dct_width)
2171 if (i == 0 || i == 3 || i == 4)
2177 else if (!(i & 0x1))
2180 shift = (i + 1) >> 1;
2183 cs_size = (128 * (1 << !!dct_width)) << shift;
2188 static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
2193 if (i < 4 || i == 6)
2197 else if (!(i & 0x1))
2200 shift = (i + 1) >> 1;
2203 cs_size = rank_multiply * (128 << shift);
2208 static int ddr4_cs_size(unsigned i)
2217 /* Min cs_size = 1G */
2218 cs_size = 1024 * (1 << (i >> 1));
2223 static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
2224 unsigned cs_mode, int cs_mask_nr)
2226 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
2228 WARN_ON(cs_mode > 11);
2230 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
2231 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
2233 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
2237 * F15h supports only 64bit DCT interfaces
2239 static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
2240 unsigned cs_mode, int cs_mask_nr)
2242 WARN_ON(cs_mode > 12);
2244 return ddr3_cs_size(cs_mode, false);
2247 /* F15h M60h supports DDR4 mapping as well.. */
2248 static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
2249 unsigned cs_mode, int cs_mask_nr)
2252 u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
2254 WARN_ON(cs_mode > 12);
2256 if (pvt->dram_type == MEM_DDR4) {
2260 cs_size = ddr4_cs_size(cs_mode);
2261 } else if (pvt->dram_type == MEM_LRDDR3) {
2262 unsigned rank_multiply = dcsm & 0xf;
2264 if (rank_multiply == 3)
2266 cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
2268 /* Minimum cs size is 512mb for F15hM60h*/
2272 cs_size = ddr3_cs_size(cs_mode, false);
2279 * F16h and F15h model 30h have only limited cs_modes.
2281 static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
2282 unsigned cs_mode, int cs_mask_nr)
2284 WARN_ON(cs_mode > 12);
2286 if (cs_mode == 6 || cs_mode == 8 ||
2287 cs_mode == 9 || cs_mode == 12)
2290 return ddr3_cs_size(cs_mode, false);
2293 static void read_dram_ctl_register(struct amd64_pvt *pvt)
2296 if (pvt->fam == 0xf)
2299 if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
2300 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
2301 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
2303 edac_dbg(0, " DCTs operate in %s mode\n",
2304 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
2306 if (!dct_ganging_enabled(pvt))
2307 edac_dbg(0, " Address range split per DCT: %s\n",
2308 (dct_high_range_enabled(pvt) ? "yes" : "no"));
2310 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
2311 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
2312 (dct_memory_cleared(pvt) ? "yes" : "no"));
2314 edac_dbg(0, " channel interleave: %s, "
2315 "interleave bits selector: 0x%x\n",
2316 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
2317 dct_sel_interleave_addr(pvt));
2320 amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
2324 * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG,
2325 * 2.10.12 Memory Interleaving Modes).
2327 static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
2328 u8 intlv_en, int num_dcts_intlv,
2335 return (u8)(dct_sel);
2337 if (num_dcts_intlv == 2) {
2338 select = (sys_addr >> 8) & 0x3;
2339 channel = select ? 0x3 : 0;
2340 } else if (num_dcts_intlv == 4) {
2341 u8 intlv_addr = dct_sel_interleave_addr(pvt);
2342 switch (intlv_addr) {
2344 channel = (sys_addr >> 8) & 0x3;
2347 channel = (sys_addr >> 9) & 0x3;
2355 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
2356 * Interleaving Modes.
2358 static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
2359 bool hi_range_sel, u8 intlv_en)
2361 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
2363 if (dct_ganging_enabled(pvt))
2367 return dct_sel_high;
2370 * see F2x110[DctSelIntLvAddr] - channel interleave mode
2372 if (dct_interleave_enabled(pvt)) {
2373 u8 intlv_addr = dct_sel_interleave_addr(pvt);
2375 /* return DCT select function: 0=DCT0, 1=DCT1 */
2377 return sys_addr >> 6 & 1;
2379 if (intlv_addr & 0x2) {
2380 u8 shift = intlv_addr & 0x1 ? 9 : 6;
2381 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;
2383 return ((sys_addr >> shift) & 1) ^ temp;
2386 if (intlv_addr & 0x4) {
2387 u8 shift = intlv_addr & 0x1 ? 9 : 8;
2389 return (sys_addr >> shift) & 1;
2392 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
2395 if (dct_high_range_enabled(pvt))
2396 return ~dct_sel_high & 1;
2401 /* Convert the sys_addr to the normalized DCT address */
2402 static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
2403 u64 sys_addr, bool hi_rng,
2404 u32 dct_sel_base_addr)
2407 u64 dram_base = get_dram_base(pvt, range);
2408 u64 hole_off = f10_dhar_offset(pvt);
2409 u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16;
2414 * base address of high range is below 4Gb
2415 * (bits [47:27] at [31:11])
2416 * DRAM address space on this DCT is hoisted above 4Gb &&
2419 * remove hole offset from sys_addr
2421 * remove high range offset from sys_addr
2423 if ((!(dct_sel_base_addr >> 16) ||
2424 dct_sel_base_addr < dhar_base(pvt)) &&
2426 (sys_addr >= BIT_64(32)))
2427 chan_off = hole_off;
2429 chan_off = dct_sel_base_off;
2433 * we have a valid hole &&
2438 * remove dram base to normalize to DCT address
2440 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
2441 chan_off = hole_off;
2443 chan_off = dram_base;
2446 return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
2450 * checks if the csrow passed in is marked as SPARED, if so returns the new
2453 static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
2457 if (online_spare_swap_done(pvt, dct) &&
2458 csrow == online_spare_bad_dramcs(pvt, dct)) {
2460 for_each_chip_select(tmp_cs, dct, pvt) {
2461 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
2471 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
2472 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
2475 * -EINVAL: NOT FOUND
2476 * 0..csrow = Chip-Select Row
2478 static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
2480 struct mem_ctl_info *mci;
2481 struct amd64_pvt *pvt;
2482 u64 cs_base, cs_mask;
2483 int cs_found = -EINVAL;
2486 mci = edac_mc_find(nid);
2490 pvt = mci->pvt_info;
2492 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
2494 for_each_chip_select(csrow, dct, pvt) {
2495 if (!csrow_enabled(csrow, dct, pvt))
2498 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
2500 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
2501 csrow, cs_base, cs_mask);
2505 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
2506 (in_addr & cs_mask), (cs_base & cs_mask));
2508 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
2509 if (pvt->fam == 0x15 && pvt->model >= 0x30) {
2513 cs_found = f10_process_possible_spare(pvt, dct, csrow);
2515 edac_dbg(1, " MATCH csrow=%d\n", cs_found);
2523 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
2524 * swapped with a region located at the bottom of memory so that the GPU can use
2525 * the interleaved region and thus two channels.
2527 static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
2529 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
2531 if (pvt->fam == 0x10) {
2532 /* only revC3 and revE have that feature */
2533 if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
2537 amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
2539 if (!(swap_reg & 0x1))
2542 swap_base = (swap_reg >> 3) & 0x7f;
2543 swap_limit = (swap_reg >> 11) & 0x7f;
2544 rgn_size = (swap_reg >> 20) & 0x7f;
2545 tmp_addr = sys_addr >> 27;
2547 if (!(sys_addr >> 34) &&
2548 (((tmp_addr >= swap_base) &&
2549 (tmp_addr <= swap_limit)) ||
2550 (tmp_addr < rgn_size)))
2551 return sys_addr ^ (u64)swap_base << 27;
2556 /* For a given @dram_range, check if @sys_addr falls within it. */
2557 static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
2558 u64 sys_addr, int *chan_sel)
2560 int cs_found = -EINVAL;
2564 bool high_range = false;
2566 u8 node_id = dram_dst_node(pvt, range);
2567 u8 intlv_en = dram_intlv_en(pvt, range);
2568 u32 intlv_sel = dram_intlv_sel(pvt, range);
2570 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2571 range, sys_addr, get_dram_limit(pvt, range));
2573 if (dhar_valid(pvt) &&
2574 dhar_base(pvt) <= sys_addr &&
2575 sys_addr < BIT_64(32)) {
2576 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2581 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
2584 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
2586 dct_sel_base = dct_sel_baseaddr(pvt);
2589 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
2590 * select between DCT0 and DCT1.
2592 if (dct_high_range_enabled(pvt) &&
2593 !dct_ganging_enabled(pvt) &&
2594 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
2597 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
2599 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
2600 high_range, dct_sel_base);
2602 /* Remove node interleaving, see F1x120 */
2604 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
2605 (chan_addr & 0xfff);
2607 /* remove channel interleave */
2608 if (dct_interleave_enabled(pvt) &&
2609 !dct_high_range_enabled(pvt) &&
2610 !dct_ganging_enabled(pvt)) {
2612 if (dct_sel_interleave_addr(pvt) != 1) {
2613 if (dct_sel_interleave_addr(pvt) == 0x3)
2615 chan_addr = ((chan_addr >> 10) << 9) |
2616 (chan_addr & 0x1ff);
2618 /* A[6] or hash 6 */
2619 chan_addr = ((chan_addr >> 7) << 6) |
2623 chan_addr = ((chan_addr >> 13) << 12) |
2624 (chan_addr & 0xfff);
2627 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
2629 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
2632 *chan_sel = channel;
2637 static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
2638 u64 sys_addr, int *chan_sel)
2640 int cs_found = -EINVAL;
2641 int num_dcts_intlv = 0;
2642 u64 chan_addr, chan_offset;
2643 u64 dct_base, dct_limit;
2644 u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
2645 u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
2647 u64 dhar_offset = f10_dhar_offset(pvt);
2648 u8 intlv_addr = dct_sel_interleave_addr(pvt);
2649 u8 node_id = dram_dst_node(pvt, range);
2650 u8 intlv_en = dram_intlv_en(pvt, range);
2652 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
2653 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
2655 dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
2656 dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
2658 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
2659 range, sys_addr, get_dram_limit(pvt, range));
2661 if (!(get_dram_base(pvt, range) <= sys_addr) &&
2662 !(get_dram_limit(pvt, range) >= sys_addr))
2665 if (dhar_valid(pvt) &&
2666 dhar_base(pvt) <= sys_addr &&
2667 sys_addr < BIT_64(32)) {
2668 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
2673 /* Verify sys_addr is within DCT Range. */
2674 dct_base = (u64) dct_sel_baseaddr(pvt);
2675 dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
2677 if (!(dct_cont_base_reg & BIT(0)) &&
2678 !(dct_base <= (sys_addr >> 27) &&
2679 dct_limit >= (sys_addr >> 27)))
2682 /* Verify number of dct's that participate in channel interleaving. */
2683 num_dcts_intlv = (int) hweight8(intlv_en);
2685 if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
2688 if (pvt->model >= 0x60)
2689 channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en);
2691 channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
2692 num_dcts_intlv, dct_sel);
2694 /* Verify we stay within the MAX number of channels allowed */
2698 leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
2700 /* Get normalized DCT addr */
2701 if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
2702 chan_offset = dhar_offset;
2704 chan_offset = dct_base << 27;
2706 chan_addr = sys_addr - chan_offset;
2708 /* remove channel interleave */
2709 if (num_dcts_intlv == 2) {
2710 if (intlv_addr == 0x4)
2711 chan_addr = ((chan_addr >> 9) << 8) |
2713 else if (intlv_addr == 0x5)
2714 chan_addr = ((chan_addr >> 10) << 9) |
2715 (chan_addr & 0x1ff);
2719 } else if (num_dcts_intlv == 4) {
2720 if (intlv_addr == 0x4)
2721 chan_addr = ((chan_addr >> 10) << 8) |
2723 else if (intlv_addr == 0x5)
2724 chan_addr = ((chan_addr >> 11) << 9) |
2725 (chan_addr & 0x1ff);
2730 if (dct_offset_en) {
2731 amd64_read_pci_cfg(pvt->F1,
2732 DRAM_CONT_HIGH_OFF + (int) channel * 4,
2734 chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27;
2737 f15h_select_dct(pvt, channel);
2739 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
2743 * if channel = 3, then alias it to 1. This is because, in F15 M30h,
2744 * there is support for 4 DCT's, but only 2 are currently functional.
2745 * They are DCT0 and DCT3. But we have read all registers of DCT3 into
2746 * pvt->csels[1]. So we need to use '1' here to get correct info.
2747 * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications.
2749 alias_channel = (channel == 3) ? 1 : channel;
2751 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
2754 *chan_sel = alias_channel;
2759 static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
2763 int cs_found = -EINVAL;
2766 for (range = 0; range < DRAM_RANGES; range++) {
2767 if (!dram_rw(pvt, range))
2770 if (pvt->fam == 0x15 && pvt->model >= 0x30)
2771 cs_found = f15_m30h_match_to_this_node(pvt, range,
2775 else if ((get_dram_base(pvt, range) <= sys_addr) &&
2776 (get_dram_limit(pvt, range) >= sys_addr)) {
2777 cs_found = f1x_match_to_this_node(pvt, range,
2778 sys_addr, chan_sel);
2787 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
2788 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
2790 * The @sys_addr is usually an error address received from the hardware
2793 static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
2794 struct err_info *err)
2796 struct amd64_pvt *pvt = mci->pvt_info;
2798 error_address_to_page_and_offset(sys_addr, err);
2800 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
2801 if (err->csrow < 0) {
2802 err->err_code = ERR_CSROW;
2807 * We need the syndromes for channel detection only when we're
2808 * ganged. Otherwise @chan should already contain the channel at
2811 if (dct_ganging_enabled(pvt))
2812 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
2816 * These are tables of eigenvectors (one per line) which can be used for the
2817 * construction of the syndrome tables. The modified syndrome search algorithm
2818 * uses those to find the symbol in error and thus the DIMM.
2820 * Algorithm courtesy of Ross LaFetra from AMD.
2822 static const u16 x4_vectors[] = {
2823 0x2f57, 0x1afe, 0x66cc, 0xdd88,
2824 0x11eb, 0x3396, 0x7f4c, 0xeac8,
2825 0x0001, 0x0002, 0x0004, 0x0008,
2826 0x1013, 0x3032, 0x4044, 0x8088,
2827 0x106b, 0x30d6, 0x70fc, 0xe0a8,
2828 0x4857, 0xc4fe, 0x13cc, 0x3288,
2829 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
2830 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
2831 0x15c1, 0x2a42, 0x89ac, 0x4758,
2832 0x2b03, 0x1602, 0x4f0c, 0xca08,
2833 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
2834 0x8ba7, 0x465e, 0x244c, 0x1cc8,
2835 0x2b87, 0x164e, 0x642c, 0xdc18,
2836 0x40b9, 0x80de, 0x1094, 0x20e8,
2837 0x27db, 0x1eb6, 0x9dac, 0x7b58,
2838 0x11c1, 0x2242, 0x84ac, 0x4c58,
2839 0x1be5, 0x2d7a, 0x5e34, 0xa718,
2840 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
2841 0x4c97, 0xc87e, 0x11fc, 0x33a8,
2842 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2843 0x16b3, 0x3d62, 0x4f34, 0x8518,
2844 0x1e2f, 0x391a, 0x5cac, 0xf858,
2845 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2846 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2847 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2848 0x4397, 0xc27e, 0x17fc, 0x3ea8,
2849 0x1617, 0x3d3e, 0x6464, 0xb8b8,
2850 0x23ff, 0x12aa, 0xab6c, 0x56d8,
2851 0x2dfb, 0x1ba6, 0x913c, 0x7328,
2852 0x185d, 0x2ca6, 0x7914, 0x9e28,
2853 0x171b, 0x3e36, 0x7d7c, 0xebe8,
2854 0x4199, 0x82ee, 0x19f4, 0x2e58,
2855 0x4807, 0xc40e, 0x130c, 0x3208,
2856 0x1905, 0x2e0a, 0x5804, 0xac08,
2857 0x213f, 0x132a, 0xadfc, 0x5ba8,
2858 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
2861 static const u16 x8_vectors[] = {
2862 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2863 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2864 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
2865 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
2866 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
2867 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
2868 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
2869 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
2870 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
2871 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
2872 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
2873 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
2874 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
2875 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
2876 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
2877 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
2878 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
2879 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
2880 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
2883 static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
2886 unsigned int i, err_sym;
2888 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
2890 unsigned v_idx = err_sym * v_dim;
2891 unsigned v_end = (err_sym + 1) * v_dim;
2893 /* walk over all 16 bits of the syndrome */
2894 for (i = 1; i < (1U << 16); i <<= 1) {
2896 /* if bit is set in that eigenvector... */
2897 if (v_idx < v_end && vectors[v_idx] & i) {
2898 u16 ev_comp = vectors[v_idx++];
2900 /* ... and bit set in the modified syndrome, */
2910 /* can't get to zero, move to next symbol */
2915 edac_dbg(0, "syndrome(%x) not found\n", syndrome);
2919 static int map_err_sym_to_channel(int err_sym, int sym_size)
2930 return err_sym >> 4;
2935 /* imaginary bits not in a DIMM */
2937 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
2945 return err_sym >> 3;
2950 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
2952 struct amd64_pvt *pvt = mci->pvt_info;
2955 if (pvt->ecc_sym_sz == 8)
2956 err_sym = decode_syndrome(syndrome, x8_vectors,
2957 ARRAY_SIZE(x8_vectors),
2959 else if (pvt->ecc_sym_sz == 4)
2960 err_sym = decode_syndrome(syndrome, x4_vectors,
2961 ARRAY_SIZE(x4_vectors),
2964 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
2968 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
2971 static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err,
2974 enum hw_event_mc_err_type err_type;
2978 err_type = HW_EVENT_ERR_CORRECTED;
2979 else if (ecc_type == 1)
2980 err_type = HW_EVENT_ERR_UNCORRECTED;
2981 else if (ecc_type == 3)
2982 err_type = HW_EVENT_ERR_DEFERRED;
2984 WARN(1, "Something is rotten in the state of Denmark.\n");
2988 switch (err->err_code) {
2993 string = "Failed to map error addr to a node";
2996 string = "Failed to map error addr to a csrow";
2999 string = "Unknown syndrome - possible error reporting race";
3002 string = "MCA_SYND not valid - unknown syndrome and csrow";
3005 string = "Cannot decode normalized address";
3008 string = "WTF error";
3012 edac_mc_handle_error(err_type, mci, 1,
3013 err->page, err->offset, err->syndrome,
3014 err->csrow, err->channel, -1,
3018 static inline void decode_bus_error(int node_id, struct mce *m)
3020 struct mem_ctl_info *mci;
3021 struct amd64_pvt *pvt;
3022 u8 ecc_type = (m->status >> 45) & 0x3;
3023 u8 xec = XEC(m->status, 0x1f);
3024 u16 ec = EC(m->status);
3026 struct err_info err;
3028 mci = edac_mc_find(node_id);
3032 pvt = mci->pvt_info;
3034 /* Bail out early if this was an 'observed' error */
3035 if (PP(ec) == NBSL_PP_OBS)
3038 /* Do only ECC errors */
3039 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
3042 memset(&err, 0, sizeof(err));
3044 sys_addr = get_error_address(pvt, m);
3047 err.syndrome = extract_syndrome(m->status);
3049 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
3051 __log_ecc_error(mci, &err, ecc_type);
3055 * To find the UMC channel represented by this bank we need to match on its
3056 * instance_id. The instance_id of a bank is held in the lower 32 bits of its
3059 * Currently, we can derive the channel number by looking at the 6th nibble in
3060 * the instance_id. For example, instance_id=0xYXXXXX where Y is the channel
3063 * For DRAM ECC errors, the Chip Select number is given in bits [2:0] of
3064 * the MCA_SYND[ErrorInformation] field.
3066 static void umc_get_err_info(struct mce *m, struct err_info *err)
3068 err->channel = (m->ipid & GENMASK(31, 0)) >> 20;
3069 err->csrow = m->synd & 0x7;
3072 static void decode_umc_error(int node_id, struct mce *m)
3074 u8 ecc_type = (m->status >> 45) & 0x3;
3075 struct mem_ctl_info *mci;
3076 struct amd64_pvt *pvt;
3077 struct err_info err;
3080 node_id = fixup_node_id(node_id, m);
3082 mci = edac_mc_find(node_id);
3086 pvt = mci->pvt_info;
3088 memset(&err, 0, sizeof(err));
3090 if (m->status & MCI_STATUS_DEFERRED)
3093 if (!(m->status & MCI_STATUS_SYNDV)) {
3094 err.err_code = ERR_SYND;
3098 if (ecc_type == 2) {
3099 u8 length = (m->synd >> 18) & 0x3f;
3102 err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0);
3104 err.err_code = ERR_CHANNEL;
3107 pvt->ops->get_err_info(m, &err);
3109 if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
3110 err.err_code = ERR_NORM_ADDR;
3114 error_address_to_page_and_offset(sys_addr, &err);
3117 __log_ecc_error(mci, &err, ecc_type);
3121 * Use pvt->F3 which contains the F3 CPU PCI device to get the related
3122 * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error.
3125 reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
3127 /* Reserve the ADDRESS MAP Device */
3128 pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
3130 edac_dbg(1, "F1 not found: device 0x%x\n", pci_id1);
3134 /* Reserve the DCT Device */
3135 pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
3137 pci_dev_put(pvt->F1);
3140 edac_dbg(1, "F2 not found: device 0x%x\n", pci_id2);
3145 pci_ctl_dev = &pvt->F2->dev;
3147 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
3148 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
3149 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
3154 static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
3156 pvt->ecc_sym_sz = 4;
3158 if (pvt->fam >= 0x10) {
3161 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
3162 /* F16h has only DCT0, so no need to read dbam1. */
3163 if (pvt->fam != 0x16)
3164 amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
3166 /* F10h, revD and later can do x8 ECC too. */
3167 if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
3168 pvt->ecc_sym_sz = 8;
3173 * Retrieve the hardware registers of the memory controller.
3175 static void umc_read_mc_regs(struct amd64_pvt *pvt)
3177 u8 nid = pvt->mc_node_id;
3178 struct amd64_umc *umc;
3181 /* Read registers from each UMC */
3184 umc_base = get_umc_base(i);
3187 amd_smn_read(nid, umc_base + get_umc_reg(pvt, UMCCH_DIMM_CFG), &umc->dimm_cfg);
3188 amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
3189 amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
3190 amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
3191 amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi);
3196 * Retrieve the hardware registers of the memory controller (this includes the
3197 * 'Address Map' and 'Misc' device regs)
3199 static void dct_read_mc_regs(struct amd64_pvt *pvt)
3205 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
3206 * those are Read-As-Zero.
3208 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
3209 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
3211 /* Check first whether TOP_MEM2 is enabled: */
3212 rdmsrl(MSR_AMD64_SYSCFG, msr_val);
3213 if (msr_val & BIT(21)) {
3214 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
3215 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
3217 edac_dbg(0, " TOP_MEM2 disabled\n");
3220 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
3222 read_dram_ctl_register(pvt);
3224 for (range = 0; range < DRAM_RANGES; range++) {
3227 /* read settings for this DRAM range */
3228 read_dram_base_limit_regs(pvt, range);
3230 rw = dram_rw(pvt, range);
3234 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
3236 get_dram_base(pvt, range),
3237 get_dram_limit(pvt, range));
3239 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
3240 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
3241 (rw & 0x1) ? "R" : "-",
3242 (rw & 0x2) ? "W" : "-",
3243 dram_intlv_sel(pvt, range),
3244 dram_dst_node(pvt, range));
3247 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
3248 amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
3250 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
3252 amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
3253 amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
3255 if (!dct_ganging_enabled(pvt)) {
3256 amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
3257 amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
3260 determine_ecc_sym_sz(pvt);
3264 * NOTE: CPU Revision Dependent code
3267 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
3268 * k8 private pointer to -->
3269 * DRAM Bank Address mapping register
3271 * DCL register where dual_channel_active is
3273 * The DBAM register consists of 4 sets of 4 bits each definitions:
3276 * 0-3 CSROWs 0 and 1
3277 * 4-7 CSROWs 2 and 3
3278 * 8-11 CSROWs 4 and 5
3279 * 12-15 CSROWs 6 and 7
3281 * Values range from: 0 to 15
3282 * The meaning of the values depends on CPU revision and dual-channel state,
3283 * see relevant BKDG more info.
3285 * The memory controller provides for total of only 8 CSROWs in its current
3286 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
3287 * single channel or two (2) DIMMs in dual channel mode.
3289 * The following code logic collapses the various tables for CSROW based on CPU
3293 * The number of PAGE_SIZE pages on the specified CSROW number it
3297 static u32 dct_get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
3299 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
3300 u32 cs_mode, nr_pages;
3303 cs_mode = DBAM_DIMM(csrow_nr, dbam);
3305 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr);
3306 nr_pages <<= 20 - PAGE_SHIFT;
3308 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
3309 csrow_nr, dct, cs_mode);
3310 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
3315 static u32 umc_get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig)
3317 int csrow_nr = csrow_nr_orig;
3318 u32 cs_mode, nr_pages;
3320 cs_mode = umc_get_cs_mode(csrow_nr >> 1, dct, pvt);
3322 nr_pages = umc_addr_mask_to_cs_size(pvt, dct, cs_mode, csrow_nr);
3323 nr_pages <<= 20 - PAGE_SHIFT;
3325 edac_dbg(0, "csrow: %d, channel: %d, cs_mode %d\n",
3326 csrow_nr_orig, dct, cs_mode);
3327 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
3332 static void umc_init_csrows(struct mem_ctl_info *mci)
3334 struct amd64_pvt *pvt = mci->pvt_info;
3335 enum edac_type edac_mode = EDAC_NONE;
3336 enum dev_type dev_type = DEV_UNKNOWN;
3337 struct dimm_info *dimm;
3340 if (mci->edac_ctl_cap & EDAC_FLAG_S16ECD16ED) {
3341 edac_mode = EDAC_S16ECD16ED;
3343 } else if (mci->edac_ctl_cap & EDAC_FLAG_S8ECD8ED) {
3344 edac_mode = EDAC_S8ECD8ED;
3346 } else if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED) {
3347 edac_mode = EDAC_S4ECD4ED;
3349 } else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED) {
3350 edac_mode = EDAC_SECDED;
3354 for_each_chip_select(cs, umc, pvt) {
3355 if (!csrow_enabled(cs, umc, pvt))
3358 dimm = mci->csrows[cs]->channels[umc]->dimm;
3360 edac_dbg(1, "MC node: %d, csrow: %d\n",
3361 pvt->mc_node_id, cs);
3363 dimm->nr_pages = umc_get_csrow_nr_pages(pvt, umc, cs);
3364 dimm->mtype = pvt->umc[umc].dram_type;
3365 dimm->edac_mode = edac_mode;
3366 dimm->dtype = dev_type;
3373 * Initialize the array of csrow attribute instances, based on the values
3374 * from pci config hardware registers.
3376 static void dct_init_csrows(struct mem_ctl_info *mci)
3378 struct amd64_pvt *pvt = mci->pvt_info;
3379 enum edac_type edac_mode = EDAC_NONE;
3380 struct csrow_info *csrow;
3381 struct dimm_info *dimm;
3386 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
3390 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
3391 pvt->mc_node_id, val,
3392 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
3395 * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
3397 for_each_chip_select(i, 0, pvt) {
3398 bool row_dct0 = !!csrow_enabled(i, 0, pvt);
3399 bool row_dct1 = false;
3401 if (pvt->fam != 0xf)
3402 row_dct1 = !!csrow_enabled(i, 1, pvt);
3404 if (!row_dct0 && !row_dct1)
3407 csrow = mci->csrows[i];
3409 edac_dbg(1, "MC node: %d, csrow: %d\n",
3410 pvt->mc_node_id, i);
3413 nr_pages = dct_get_csrow_nr_pages(pvt, 0, i);
3414 csrow->channels[0]->dimm->nr_pages = nr_pages;
3417 /* K8 has only one DCT */
3418 if (pvt->fam != 0xf && row_dct1) {
3419 int row_dct1_pages = dct_get_csrow_nr_pages(pvt, 1, i);
3421 csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
3422 nr_pages += row_dct1_pages;
3425 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
3427 /* Determine DIMM ECC mode: */
3428 if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
3429 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL)
3434 for (j = 0; j < pvt->max_mcs; j++) {
3435 dimm = csrow->channels[j]->dimm;
3436 dimm->mtype = pvt->dram_type;
3437 dimm->edac_mode = edac_mode;
3443 /* get all cores on this DCT */
3444 static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
3448 for_each_online_cpu(cpu)
3449 if (topology_die_id(cpu) == nid)
3450 cpumask_set_cpu(cpu, mask);
3453 /* check MCG_CTL on all the cpus on this node */
3454 static bool nb_mce_bank_enabled_on_node(u16 nid)
3460 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
3461 amd64_warn("%s: Error allocating mask\n", __func__);
3465 get_cpus_on_this_dct_cpumask(mask, nid);
3467 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
3469 for_each_cpu(cpu, mask) {
3470 struct msr *reg = per_cpu_ptr(msrs, cpu);
3471 nbe = reg->l & MSR_MCGCTL_NBE;
3473 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
3475 (nbe ? "enabled" : "disabled"));
3483 free_cpumask_var(mask);
3487 static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
3489 cpumask_var_t cmask;
3492 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
3493 amd64_warn("%s: error allocating mask\n", __func__);
3497 get_cpus_on_this_dct_cpumask(cmask, nid);
3499 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
3501 for_each_cpu(cpu, cmask) {
3503 struct msr *reg = per_cpu_ptr(msrs, cpu);
3506 if (reg->l & MSR_MCGCTL_NBE)
3507 s->flags.nb_mce_enable = 1;
3509 reg->l |= MSR_MCGCTL_NBE;
3512 * Turn off NB MCE reporting only when it was off before
3514 if (!s->flags.nb_mce_enable)
3515 reg->l &= ~MSR_MCGCTL_NBE;
3518 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
3520 free_cpumask_var(cmask);
3525 static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
3529 u32 value, mask = 0x3; /* UECC/CECC enable */
3531 if (toggle_ecc_err_reporting(s, nid, ON)) {
3532 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
3536 amd64_read_pci_cfg(F3, NBCTL, &value);
3538 s->old_nbctl = value & mask;
3539 s->nbctl_valid = true;
3542 amd64_write_pci_cfg(F3, NBCTL, value);
3544 amd64_read_pci_cfg(F3, NBCFG, &value);
3546 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3547 nid, value, !!(value & NBCFG_ECC_ENABLE));
3549 if (!(value & NBCFG_ECC_ENABLE)) {
3550 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
3552 s->flags.nb_ecc_prev = 0;
3554 /* Attempt to turn on DRAM ECC Enable */
3555 value |= NBCFG_ECC_ENABLE;
3556 amd64_write_pci_cfg(F3, NBCFG, value);
3558 amd64_read_pci_cfg(F3, NBCFG, &value);
3560 if (!(value & NBCFG_ECC_ENABLE)) {
3561 amd64_warn("Hardware rejected DRAM ECC enable,"
3562 "check memory DIMM configuration.\n");
3565 amd64_info("Hardware accepted DRAM ECC Enable\n");
3568 s->flags.nb_ecc_prev = 1;
3571 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3572 nid, value, !!(value & NBCFG_ECC_ENABLE));
3577 static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
3580 u32 value, mask = 0x3; /* UECC/CECC enable */
3582 if (!s->nbctl_valid)
3585 amd64_read_pci_cfg(F3, NBCTL, &value);
3587 value |= s->old_nbctl;
3589 amd64_write_pci_cfg(F3, NBCTL, value);
3591 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
3592 if (!s->flags.nb_ecc_prev) {
3593 amd64_read_pci_cfg(F3, NBCFG, &value);
3594 value &= ~NBCFG_ECC_ENABLE;
3595 amd64_write_pci_cfg(F3, NBCFG, value);
3598 /* restore the NB Enable MCGCTL bit */
3599 if (toggle_ecc_err_reporting(s, nid, OFF))
3600 amd64_warn("Error restoring NB MCGCTL settings!\n");
3603 static bool dct_ecc_enabled(struct amd64_pvt *pvt)
3605 u16 nid = pvt->mc_node_id;
3606 bool nb_mce_en = false;
3610 amd64_read_pci_cfg(pvt->F3, NBCFG, &value);
3612 ecc_en = !!(value & NBCFG_ECC_ENABLE);
3614 nb_mce_en = nb_mce_bank_enabled_on_node(nid);
3616 edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
3617 MSR_IA32_MCG_CTL, nid);
3619 edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid, (ecc_en ? "enabled" : "disabled"));
3621 if (!ecc_en || !nb_mce_en)
3627 static bool umc_ecc_enabled(struct amd64_pvt *pvt)
3629 u8 umc_en_mask = 0, ecc_en_mask = 0;
3630 u16 nid = pvt->mc_node_id;
3631 struct amd64_umc *umc;
3637 /* Only check enabled UMCs. */
3638 if (!(umc->sdp_ctrl & UMC_SDP_INIT))
3641 umc_en_mask |= BIT(i);
3643 if (umc->umc_cap_hi & UMC_ECC_ENABLED)
3644 ecc_en_mask |= BIT(i);
3647 /* Check whether at least one UMC is enabled: */
3649 ecc_en = umc_en_mask == ecc_en_mask;
3651 edac_dbg(0, "Node %d: No enabled UMCs.\n", nid);
3653 edac_dbg(3, "Node %d: DRAM ECC %s.\n", nid, (ecc_en ? "enabled" : "disabled"));
3662 umc_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
3664 u8 i, ecc_en = 1, cpk_en = 1, dev_x4 = 1, dev_x16 = 1;
3667 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3668 ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED);
3669 cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP);
3671 dev_x4 &= !!(pvt->umc[i].dimm_cfg & BIT(6));
3672 dev_x16 &= !!(pvt->umc[i].dimm_cfg & BIT(7));
3676 /* Set chipkill only if ECC is enabled: */
3678 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3684 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3686 mci->edac_ctl_cap |= EDAC_FLAG_S16ECD16ED;
3688 mci->edac_ctl_cap |= EDAC_FLAG_S8ECD8ED;
3692 static void dct_setup_mci_misc_attrs(struct mem_ctl_info *mci)
3694 struct amd64_pvt *pvt = mci->pvt_info;
3696 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
3697 mci->edac_ctl_cap = EDAC_FLAG_NONE;
3699 if (pvt->nbcap & NBCAP_SECDED)
3700 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3702 if (pvt->nbcap & NBCAP_CHIPKILL)
3703 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3705 mci->edac_cap = dct_determine_edac_cap(pvt);
3706 mci->mod_name = EDAC_MOD_STR;
3707 mci->ctl_name = pvt->ctl_name;
3708 mci->dev_name = pci_name(pvt->F3);
3709 mci->ctl_page_to_phys = NULL;
3711 /* memory scrubber interface */
3712 mci->set_sdram_scrub_rate = set_scrub_rate;
3713 mci->get_sdram_scrub_rate = get_scrub_rate;
3715 dct_init_csrows(mci);
3718 static void umc_setup_mci_misc_attrs(struct mem_ctl_info *mci)
3720 struct amd64_pvt *pvt = mci->pvt_info;
3722 mci->mtype_cap = MEM_FLAG_DDR4 | MEM_FLAG_RDDR4;
3723 mci->edac_ctl_cap = EDAC_FLAG_NONE;
3725 umc_determine_edac_ctl_cap(mci, pvt);
3727 mci->edac_cap = umc_determine_edac_cap(pvt);
3728 mci->mod_name = EDAC_MOD_STR;
3729 mci->ctl_name = pvt->ctl_name;
3730 mci->dev_name = pci_name(pvt->F3);
3731 mci->ctl_page_to_phys = NULL;
3733 umc_init_csrows(mci);
3736 static int dct_hw_info_get(struct amd64_pvt *pvt)
3738 int ret = reserve_mc_sibling_devs(pvt, pvt->f1_id, pvt->f2_id);
3743 dct_prep_chip_selects(pvt);
3744 dct_read_base_mask(pvt);
3745 dct_read_mc_regs(pvt);
3746 dct_determine_memory_type(pvt);
3751 static int umc_hw_info_get(struct amd64_pvt *pvt)
3753 pvt->umc = kcalloc(pvt->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL);
3757 umc_prep_chip_selects(pvt);
3758 umc_read_base_mask(pvt);
3759 umc_read_mc_regs(pvt);
3760 umc_determine_memory_type(pvt);
3766 * The CPUs have one channel per UMC, so UMC number is equivalent to a
3767 * channel number. The GPUs have 8 channels per UMC, so the UMC number no
3768 * longer works as a channel number.
3770 * The channel number within a GPU UMC is given in MCA_IPID[15:12].
3771 * However, the IDs are split such that two UMC values go to one UMC, and
3772 * the channel numbers are split in two groups of four.
3774 * Refer to comment on gpu_get_umc_base().
3777 * UMC0 CH[3:0] = 0x0005[3:0]000
3778 * UMC0 CH[7:4] = 0x0015[3:0]000
3779 * UMC1 CH[3:0] = 0x0025[3:0]000
3780 * UMC1 CH[7:4] = 0x0035[3:0]000
3782 static void gpu_get_err_info(struct mce *m, struct err_info *err)
3784 u8 ch = (m->ipid & GENMASK(31, 0)) >> 20;
3785 u8 phy = ((m->ipid >> 12) & 0xf);
3787 err->channel = ch % 2 ? phy + 4 : phy;
3791 static int gpu_addr_mask_to_cs_size(struct amd64_pvt *pvt, u8 umc,
3792 unsigned int cs_mode, int csrow_nr)
3794 u32 addr_mask_orig = pvt->csels[umc].csmasks[csrow_nr];
3796 return __addr_mask_to_cs_size(addr_mask_orig, cs_mode, csrow_nr, csrow_nr >> 1);
3799 static void gpu_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
3801 int size, cs_mode, cs = 0;
3803 edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
3805 cs_mode = CS_EVEN_PRIMARY | CS_ODD_PRIMARY;
3807 for_each_chip_select(cs, ctrl, pvt) {
3808 size = gpu_addr_mask_to_cs_size(pvt, ctrl, cs_mode, cs);
3809 amd64_info(EDAC_MC ": %d: %5dMB\n", cs, size);
3813 static void gpu_dump_misc_regs(struct amd64_pvt *pvt)
3815 struct amd64_umc *umc;
3821 edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
3822 edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
3823 edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
3824 edac_dbg(1, "UMC%d All HBMs support ECC: yes\n", i);
3826 gpu_debug_display_dimm_sizes(pvt, i);
3830 static u32 gpu_get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
3833 int cs_mode = CS_EVEN_PRIMARY | CS_ODD_PRIMARY;
3835 nr_pages = gpu_addr_mask_to_cs_size(pvt, dct, cs_mode, csrow_nr);
3836 nr_pages <<= 20 - PAGE_SHIFT;
3838 edac_dbg(0, "csrow: %d, channel: %d\n", csrow_nr, dct);
3839 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
3844 static void gpu_init_csrows(struct mem_ctl_info *mci)
3846 struct amd64_pvt *pvt = mci->pvt_info;
3847 struct dimm_info *dimm;
3851 for_each_chip_select(cs, umc, pvt) {
3852 if (!csrow_enabled(cs, umc, pvt))
3855 dimm = mci->csrows[umc]->channels[cs]->dimm;
3857 edac_dbg(1, "MC node: %d, csrow: %d\n",
3858 pvt->mc_node_id, cs);
3860 dimm->nr_pages = gpu_get_csrow_nr_pages(pvt, umc, cs);
3861 dimm->edac_mode = EDAC_SECDED;
3862 dimm->mtype = pvt->dram_type;
3863 dimm->dtype = DEV_X16;
3869 static void gpu_setup_mci_misc_attrs(struct mem_ctl_info *mci)
3871 struct amd64_pvt *pvt = mci->pvt_info;
3873 mci->mtype_cap = MEM_FLAG_HBM2;
3874 mci->edac_ctl_cap = EDAC_FLAG_SECDED;
3876 mci->edac_cap = EDAC_FLAG_EC;
3877 mci->mod_name = EDAC_MOD_STR;
3878 mci->ctl_name = pvt->ctl_name;
3879 mci->dev_name = pci_name(pvt->F3);
3880 mci->ctl_page_to_phys = NULL;
3882 gpu_init_csrows(mci);
3885 /* ECC is enabled by default on GPU nodes */
3886 static bool gpu_ecc_enabled(struct amd64_pvt *pvt)
3891 static inline u32 gpu_get_umc_base(struct amd64_pvt *pvt, u8 umc, u8 channel)
3894 * On CPUs, there is one channel per UMC, so UMC numbering equals
3895 * channel numbering. On GPUs, there are eight channels per UMC,
3896 * so the channel numbering is different from UMC numbering.
3898 * On CPU nodes channels are selected in 6th nibble
3899 * UMC chY[3:0]= [(chY*2 + 1) : (chY*2)]50000;
3901 * On GPU nodes channels are selected in 3rd nibble
3902 * HBM chX[3:0]= [Y ]5X[3:0]000;
3903 * HBM chX[7:4]= [Y+1]5X[3:0]000
3905 * On MI300 APU nodes, same as GPU nodes but channels are selected
3906 * in the base address of 0x90000
3913 return pvt->gpu_umc_base + (umc << 20) + ((channel % 4) << 12);
3916 static void gpu_read_mc_regs(struct amd64_pvt *pvt)
3918 u8 nid = pvt->mc_node_id;
3919 struct amd64_umc *umc;
3922 /* Read registers from each UMC */
3924 umc_base = gpu_get_umc_base(pvt, i, 0);
3927 amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
3928 amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
3929 amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
3933 static void gpu_read_base_mask(struct amd64_pvt *pvt)
3935 u32 base_reg, mask_reg;
3940 for_each_chip_select(cs, umc, pvt) {
3941 base_reg = gpu_get_umc_base(pvt, umc, cs) + UMCCH_BASE_ADDR;
3942 base = &pvt->csels[umc].csbases[cs];
3944 if (!amd_smn_read(pvt->mc_node_id, base_reg, base)) {
3945 edac_dbg(0, " DCSB%d[%d]=0x%08x reg: 0x%x\n",
3946 umc, cs, *base, base_reg);
3949 mask_reg = gpu_get_umc_base(pvt, umc, cs) + UMCCH_ADDR_MASK;
3950 mask = &pvt->csels[umc].csmasks[cs];
3952 if (!amd_smn_read(pvt->mc_node_id, mask_reg, mask)) {
3953 edac_dbg(0, " DCSM%d[%d]=0x%08x reg: 0x%x\n",
3954 umc, cs, *mask, mask_reg);
3960 static void gpu_prep_chip_selects(struct amd64_pvt *pvt)
3965 pvt->csels[umc].b_cnt = 8;
3966 pvt->csels[umc].m_cnt = 8;
3970 static int gpu_hw_info_get(struct amd64_pvt *pvt)
3974 ret = gpu_get_node_map(pvt);
3978 pvt->umc = kcalloc(pvt->max_mcs, sizeof(struct amd64_umc), GFP_KERNEL);
3982 gpu_prep_chip_selects(pvt);
3983 gpu_read_base_mask(pvt);
3984 gpu_read_mc_regs(pvt);
3989 static void hw_info_put(struct amd64_pvt *pvt)
3991 pci_dev_put(pvt->F1);
3992 pci_dev_put(pvt->F2);
3996 static struct low_ops umc_ops = {
3997 .hw_info_get = umc_hw_info_get,
3998 .ecc_enabled = umc_ecc_enabled,
3999 .setup_mci_misc_attrs = umc_setup_mci_misc_attrs,
4000 .dump_misc_regs = umc_dump_misc_regs,
4001 .get_err_info = umc_get_err_info,
4004 static struct low_ops gpu_ops = {
4005 .hw_info_get = gpu_hw_info_get,
4006 .ecc_enabled = gpu_ecc_enabled,
4007 .setup_mci_misc_attrs = gpu_setup_mci_misc_attrs,
4008 .dump_misc_regs = gpu_dump_misc_regs,
4009 .get_err_info = gpu_get_err_info,
4012 /* Use Family 16h versions for defaults and adjust as needed below. */
4013 static struct low_ops dct_ops = {
4014 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
4015 .dbam_to_cs = f16_dbam_to_chip_select,
4016 .hw_info_get = dct_hw_info_get,
4017 .ecc_enabled = dct_ecc_enabled,
4018 .setup_mci_misc_attrs = dct_setup_mci_misc_attrs,
4019 .dump_misc_regs = dct_dump_misc_regs,
4022 static int per_family_init(struct amd64_pvt *pvt)
4024 pvt->ext_model = boot_cpu_data.x86_model >> 4;
4025 pvt->stepping = boot_cpu_data.x86_stepping;
4026 pvt->model = boot_cpu_data.x86_model;
4027 pvt->fam = boot_cpu_data.x86;
4031 * Decide on which ops group to use here and do any family/model
4034 if (pvt->fam >= 0x17)
4035 pvt->ops = &umc_ops;
4037 pvt->ops = &dct_ops;
4041 pvt->ctl_name = (pvt->ext_model >= K8_REV_F) ?
4042 "K8 revF or later" : "K8 revE or earlier";
4043 pvt->f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP;
4044 pvt->f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL;
4045 pvt->ops->map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow;
4046 pvt->ops->dbam_to_cs = k8_dbam_to_chip_select;
4050 pvt->ctl_name = "F10h";
4051 pvt->f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP;
4052 pvt->f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM;
4053 pvt->ops->dbam_to_cs = f10_dbam_to_chip_select;
4057 switch (pvt->model) {
4059 pvt->ctl_name = "F15h_M30h";
4060 pvt->f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
4061 pvt->f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2;
4064 pvt->ctl_name = "F15h_M60h";
4065 pvt->f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
4066 pvt->f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2;
4067 pvt->ops->dbam_to_cs = f15_m60h_dbam_to_chip_select;
4070 /* Richland is only client */
4073 pvt->ctl_name = "F15h";
4074 pvt->f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1;
4075 pvt->f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2;
4076 pvt->ops->dbam_to_cs = f15_dbam_to_chip_select;
4082 switch (pvt->model) {
4084 pvt->ctl_name = "F16h_M30h";
4085 pvt->f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1;
4086 pvt->f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2;
4089 pvt->ctl_name = "F16h";
4090 pvt->f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1;
4091 pvt->f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2;
4097 switch (pvt->model) {
4099 pvt->ctl_name = "F17h_M10h";
4102 pvt->ctl_name = "F17h_M30h";
4106 pvt->ctl_name = "F17h_M60h";
4109 pvt->ctl_name = "F17h_M70h";
4112 pvt->ctl_name = "F17h";
4118 pvt->ctl_name = "F18h";
4122 switch (pvt->model) {
4124 pvt->ctl_name = "F19h";
4128 pvt->ctl_name = "F19h_M10h";
4130 pvt->flags.zn_regs_v2 = 1;
4133 pvt->ctl_name = "F19h_M20h";
4136 if (pvt->F3->device == PCI_DEVICE_ID_AMD_MI200_DF_F3) {
4137 pvt->ctl_name = "MI200";
4139 pvt->dram_type = MEM_HBM2;
4140 pvt->gpu_umc_base = 0x50000;
4141 pvt->ops = &gpu_ops;
4143 pvt->ctl_name = "F19h_M30h";
4148 pvt->ctl_name = "F19h_M50h";
4151 pvt->ctl_name = "F19h_M60h";
4152 pvt->flags.zn_regs_v2 = 1;
4155 pvt->ctl_name = "F19h_M70h";
4156 pvt->flags.zn_regs_v2 = 1;
4159 pvt->ctl_name = "F19h_M90h";
4161 pvt->dram_type = MEM_HBM3;
4162 pvt->gpu_umc_base = 0x90000;
4163 pvt->ops = &gpu_ops;
4166 pvt->ctl_name = "F19h_MA0h";
4168 pvt->flags.zn_regs_v2 = 1;
4174 switch (pvt->model) {
4176 pvt->ctl_name = "F1Ah";
4178 pvt->flags.zn_regs_v2 = 1;
4181 pvt->ctl_name = "F1Ah_M40h";
4182 pvt->flags.zn_regs_v2 = 1;
4188 amd64_err("Unsupported family!\n");
4195 static const struct attribute_group *amd64_edac_attr_groups[] = {
4196 #ifdef CONFIG_EDAC_DEBUG
4204 * For heterogeneous and APU models EDAC CHIP_SELECT and CHANNEL layers
4205 * should be swapped to fit into the layers.
4207 static unsigned int get_layer_size(struct amd64_pvt *pvt, u8 layer)
4209 bool is_gpu = (pvt->ops == &gpu_ops);
4212 return is_gpu ? pvt->max_mcs
4213 : pvt->csels[0].b_cnt;
4215 return is_gpu ? pvt->csels[0].b_cnt
4219 static int init_one_instance(struct amd64_pvt *pvt)
4221 struct mem_ctl_info *mci = NULL;
4222 struct edac_mc_layer layers[2];
4225 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
4226 layers[0].size = get_layer_size(pvt, 0);
4227 layers[0].is_virt_csrow = true;
4228 layers[1].type = EDAC_MC_LAYER_CHANNEL;
4229 layers[1].size = get_layer_size(pvt, 1);
4230 layers[1].is_virt_csrow = false;
4232 mci = edac_mc_alloc(pvt->mc_node_id, ARRAY_SIZE(layers), layers, 0);
4236 mci->pvt_info = pvt;
4237 mci->pdev = &pvt->F3->dev;
4239 pvt->ops->setup_mci_misc_attrs(mci);
4242 if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
4243 edac_dbg(1, "failed edac_mc_add_mc()\n");
4251 static bool instance_has_memory(struct amd64_pvt *pvt)
4253 bool cs_enabled = false;
4254 int cs = 0, dct = 0;
4256 for (dct = 0; dct < pvt->max_mcs; dct++) {
4257 for_each_chip_select(cs, dct, pvt)
4258 cs_enabled |= csrow_enabled(cs, dct, pvt);
4264 static int probe_one_instance(unsigned int nid)
4266 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
4267 struct amd64_pvt *pvt = NULL;
4268 struct ecc_settings *s;
4272 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
4278 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
4282 pvt->mc_node_id = nid;
4285 ret = per_family_init(pvt);
4289 ret = pvt->ops->hw_info_get(pvt);
4294 if (!instance_has_memory(pvt)) {
4295 amd64_info("Node %d: No DIMMs detected.\n", nid);
4299 if (!pvt->ops->ecc_enabled(pvt)) {
4302 if (!ecc_enable_override)
4305 if (boot_cpu_data.x86 >= 0x17) {
4306 amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS.");
4309 amd64_warn("Forcing ECC on!\n");
4311 if (!enable_ecc_error_reporting(s, nid, F3))
4315 ret = init_one_instance(pvt);
4317 amd64_err("Error probing instance: %d\n", nid);
4319 if (boot_cpu_data.x86 < 0x17)
4320 restore_ecc_error_reporting(s, nid, F3);
4325 amd64_info("%s detected (node %d).\n", pvt->ctl_name, pvt->mc_node_id);
4327 /* Display and decode various registers for debug purposes. */
4328 pvt->ops->dump_misc_regs(pvt);
4338 ecc_stngs[nid] = NULL;
4344 static void remove_one_instance(unsigned int nid)
4346 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
4347 struct ecc_settings *s = ecc_stngs[nid];
4348 struct mem_ctl_info *mci;
4349 struct amd64_pvt *pvt;
4351 /* Remove from EDAC CORE tracking list */
4352 mci = edac_mc_del_mc(&F3->dev);
4356 pvt = mci->pvt_info;
4358 restore_ecc_error_reporting(s, nid, F3);
4360 kfree(ecc_stngs[nid]);
4361 ecc_stngs[nid] = NULL;
4363 /* Free the EDAC CORE resources */
4364 mci->pvt_info = NULL;
4371 static void setup_pci_device(void)
4376 pci_ctl = edac_pci_create_generic_ctl(pci_ctl_dev, EDAC_MOD_STR);
4378 pr_warn("%s(): Unable to create PCI control\n", __func__);
4379 pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
4383 static const struct x86_cpu_id amd64_cpuids[] = {
4384 X86_MATCH_VENDOR_FAM(AMD, 0x0F, NULL),
4385 X86_MATCH_VENDOR_FAM(AMD, 0x10, NULL),
4386 X86_MATCH_VENDOR_FAM(AMD, 0x15, NULL),
4387 X86_MATCH_VENDOR_FAM(AMD, 0x16, NULL),
4388 X86_MATCH_VENDOR_FAM(AMD, 0x17, NULL),
4389 X86_MATCH_VENDOR_FAM(HYGON, 0x18, NULL),
4390 X86_MATCH_VENDOR_FAM(AMD, 0x19, NULL),
4391 X86_MATCH_VENDOR_FAM(AMD, 0x1A, NULL),
4394 MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
4396 static int __init amd64_edac_init(void)
4402 if (ghes_get_devices())
4405 owner = edac_get_owner();
4406 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
4409 if (!x86_match_cpu(amd64_cpuids))
4418 ecc_stngs = kcalloc(amd_nb_num(), sizeof(ecc_stngs[0]), GFP_KERNEL);
4422 msrs = msrs_alloc();
4426 for (i = 0; i < amd_nb_num(); i++) {
4427 err = probe_one_instance(i);
4429 /* unwind properly */
4431 remove_one_instance(i);
4437 if (!edac_has_mcs()) {
4442 /* register stuff with EDAC MCE */
4443 if (boot_cpu_data.x86 >= 0x17) {
4444 amd_register_ecc_decoder(decode_umc_error);
4446 amd_register_ecc_decoder(decode_bus_error);
4450 #ifdef CONFIG_X86_32
4451 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
4469 static void __exit amd64_edac_exit(void)
4474 edac_pci_release_generic_ctl(pci_ctl);
4476 /* unregister from EDAC MCE */
4477 if (boot_cpu_data.x86 >= 0x17)
4478 amd_unregister_ecc_decoder(decode_umc_error);
4480 amd_unregister_ecc_decoder(decode_bus_error);
4482 for (i = 0; i < amd_nb_num(); i++)
4483 remove_one_instance(i);
4494 module_init(amd64_edac_init);
4495 module_exit(amd64_edac_exit);
4497 MODULE_LICENSE("GPL");
4498 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, Dave Peterson, Thayne Harbaugh; AMD");
4499 MODULE_DESCRIPTION("MC support for AMD64 memory controllers");
4501 module_param(edac_op_state, int, 0444);
4502 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");