1 #include "amd64_edac.h"
2 #include <asm/amd_nb.h>
4 static struct edac_pci_ctl_info *pci_ctl;
6 static int report_gart_errors;
7 module_param(report_gart_errors, int, 0644);
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
13 static int ecc_enable_override;
14 module_param(ecc_enable_override, int, 0644);
16 static struct msr __percpu *msrs;
19 static struct ecc_settings **ecc_stngs;
21 /* Number of Unified Memory Controllers */
25 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
26 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
29 *FIXME: Produce a better mapping/linearisation.
31 static const struct scrubrate {
32 u32 scrubval; /* bit pattern for scrub rate */
33 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
35 { 0x01, 1600000000UL},
57 { 0x00, 0UL}, /* scrubbing off */
60 int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
61 u32 *val, const char *func)
65 err = pci_read_config_dword(pdev, offset, val);
67 amd64_warn("%s: error reading F%dx%03x.\n",
68 func, PCI_FUNC(pdev->devfn), offset);
73 int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
74 u32 val, const char *func)
78 err = pci_write_config_dword(pdev, offset, val);
80 amd64_warn("%s: error writing to F%dx%03x.\n",
81 func, PCI_FUNC(pdev->devfn), offset);
87 * Select DCT to which PCI cfg accesses are routed
89 static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
93 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, ®);
94 reg &= (pvt->model == 0x30) ? ~3 : ~1;
96 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
101 * Depending on the family, F2 DCT reads need special handling:
103 * K8: has a single DCT only and no address offsets >= 0x100
105 * F10h: each DCT has its own set of regs
109 * F16h: has only 1 DCT
111 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
113 static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
114 int offset, u32 *val)
118 if (dct || offset >= 0x100)
125 * Note: If ganging is enabled, barring the regs
126 * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx
127 * return 0. (cf. Section 2.8.1 F10h BKDG)
129 if (dct_ganging_enabled(pvt))
138 * F15h: F2x1xx addresses do not map explicitly to DCT1.
139 * We should select which DCT we access using F1x10C[DctCfgSel]
141 dct = (dct && pvt->model == 0x30) ? 3 : dct;
142 f15h_select_dct(pvt, dct);
153 return amd64_read_pci_cfg(pvt->F2, offset, val);
157 * Memory scrubber control interface. For K8, memory scrubbing is handled by
158 * hardware and can involve L2 cache, dcache as well as the main memory. With
159 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
162 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
163 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
164 * bytes/sec for the setting.
166 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
167 * other archs, we might not have access to the caches directly.
170 static inline void __f17h_set_scrubval(struct amd64_pvt *pvt, u32 scrubval)
173 * Fam17h supports scrub values between 0x5 and 0x14. Also, the values
174 * are shifted down by 0x5, so scrubval 0x5 is written to the register
175 * as 0x0, scrubval 0x6 as 0x1, etc.
177 if (scrubval >= 0x5 && scrubval <= 0x14) {
179 pci_write_bits32(pvt->F6, F17H_SCR_LIMIT_ADDR, scrubval, 0xF);
180 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 1, 0x1);
182 pci_write_bits32(pvt->F6, F17H_SCR_BASE_ADDR, 0, 0x1);
186 * Scan the scrub rate mapping table for a close or matching bandwidth value to
187 * issue. If requested is too big, then use last maximum value found.
189 static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
195 * map the configured rate (new_bw) to a value specific to the AMD64
196 * memory controller and apply to register. Search for the first
197 * bandwidth entry that is greater or equal than the setting requested
198 * and program that. If at last entry, turn off DRAM scrubbing.
200 * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
201 * by falling back to the last element in scrubrates[].
203 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
205 * skip scrub rates which aren't recommended
206 * (see F10 BKDG, F3x58)
208 if (scrubrates[i].scrubval < min_rate)
211 if (scrubrates[i].bandwidth <= new_bw)
215 scrubval = scrubrates[i].scrubval;
217 if (pvt->fam == 0x17 || pvt->fam == 0x18) {
218 __f17h_set_scrubval(pvt, scrubval);
219 } else if (pvt->fam == 0x15 && pvt->model == 0x60) {
220 f15h_select_dct(pvt, 0);
221 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
222 f15h_select_dct(pvt, 1);
223 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
225 pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
229 return scrubrates[i].bandwidth;
234 static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
236 struct amd64_pvt *pvt = mci->pvt_info;
237 u32 min_scrubrate = 0x5;
242 if (pvt->fam == 0x15) {
244 if (pvt->model < 0x10)
245 f15h_select_dct(pvt, 0);
247 if (pvt->model == 0x60)
250 return __set_scrub_rate(pvt, bw, min_scrubrate);
253 static int get_scrub_rate(struct mem_ctl_info *mci)
255 struct amd64_pvt *pvt = mci->pvt_info;
256 int i, retval = -EINVAL;
262 if (pvt->model < 0x10)
263 f15h_select_dct(pvt, 0);
265 if (pvt->model == 0x60)
266 amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
271 amd64_read_pci_cfg(pvt->F6, F17H_SCR_BASE_ADDR, &scrubval);
272 if (scrubval & BIT(0)) {
273 amd64_read_pci_cfg(pvt->F6, F17H_SCR_LIMIT_ADDR, &scrubval);
282 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
286 scrubval = scrubval & 0x001F;
288 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
289 if (scrubrates[i].scrubval == scrubval) {
290 retval = scrubrates[i].bandwidth;
298 * returns true if the SysAddr given by sys_addr matches the
299 * DRAM base/limit associated with node_id
301 static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
305 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
306 * all ones if the most significant implemented address bit is 1.
307 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
308 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
309 * Application Programming.
311 addr = sys_addr & 0x000000ffffffffffull;
313 return ((addr >= get_dram_base(pvt, nid)) &&
314 (addr <= get_dram_limit(pvt, nid)));
318 * Attempt to map a SysAddr to a node. On success, return a pointer to the
319 * mem_ctl_info structure for the node that the SysAddr maps to.
321 * On failure, return NULL.
323 static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
326 struct amd64_pvt *pvt;
331 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
332 * 3.4.4.2) registers to map the SysAddr to a node ID.
337 * The value of this field should be the same for all DRAM Base
338 * registers. Therefore we arbitrarily choose to read it from the
339 * register for node 0.
341 intlv_en = dram_intlv_en(pvt, 0);
344 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
345 if (base_limit_match(pvt, sys_addr, node_id))
351 if (unlikely((intlv_en != 0x01) &&
352 (intlv_en != 0x03) &&
353 (intlv_en != 0x07))) {
354 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
358 bits = (((u32) sys_addr) >> 12) & intlv_en;
360 for (node_id = 0; ; ) {
361 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
362 break; /* intlv_sel field matches */
364 if (++node_id >= DRAM_RANGES)
368 /* sanity test for sys_addr */
369 if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
370 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
371 "range for node %d with node interleaving enabled.\n",
372 __func__, sys_addr, node_id);
377 return edac_mc_find((int)node_id);
380 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
381 (unsigned long)sys_addr);
387 * compute the CS base address of the @csrow on the DRAM controller @dct.
388 * For details see F2x[5C:40] in the processor's BKDG
390 static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
391 u64 *base, u64 *mask)
393 u64 csbase, csmask, base_bits, mask_bits;
396 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
397 csbase = pvt->csels[dct].csbases[csrow];
398 csmask = pvt->csels[dct].csmasks[csrow];
399 base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
400 mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
404 * F16h and F15h, models 30h and later need two addr_shift values:
405 * 8 for high and 6 for low (cf. F16h BKDG).
407 } else if (pvt->fam == 0x16 ||
408 (pvt->fam == 0x15 && pvt->model >= 0x30)) {
409 csbase = pvt->csels[dct].csbases[csrow];
410 csmask = pvt->csels[dct].csmasks[csrow >> 1];
412 *base = (csbase & GENMASK_ULL(15, 5)) << 6;
413 *base |= (csbase & GENMASK_ULL(30, 19)) << 8;
416 /* poke holes for the csmask */
417 *mask &= ~((GENMASK_ULL(15, 5) << 6) |
418 (GENMASK_ULL(30, 19) << 8));
420 *mask |= (csmask & GENMASK_ULL(15, 5)) << 6;
421 *mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
425 csbase = pvt->csels[dct].csbases[csrow];
426 csmask = pvt->csels[dct].csmasks[csrow >> 1];
429 if (pvt->fam == 0x15)
430 base_bits = mask_bits =
431 GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
433 base_bits = mask_bits =
434 GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
437 *base = (csbase & base_bits) << addr_shift;
440 /* poke holes for the csmask */
441 *mask &= ~(mask_bits << addr_shift);
443 *mask |= (csmask & mask_bits) << addr_shift;
446 #define for_each_chip_select(i, dct, pvt) \
447 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
449 #define chip_select_base(i, dct, pvt) \
450 pvt->csels[dct].csbases[i]
452 #define for_each_chip_select_mask(i, dct, pvt) \
453 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
455 #define for_each_umc(i) \
456 for (i = 0; i < num_umcs; i++)
459 * @input_addr is an InputAddr associated with the node given by mci. Return the
460 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
462 static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
464 struct amd64_pvt *pvt;
470 for_each_chip_select(csrow, 0, pvt) {
471 if (!csrow_enabled(csrow, 0, pvt))
474 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
478 if ((input_addr & mask) == (base & mask)) {
479 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
480 (unsigned long)input_addr, csrow,
486 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
487 (unsigned long)input_addr, pvt->mc_node_id);
493 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
494 * for the node represented by mci. Info is passed back in *hole_base,
495 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
496 * info is invalid. Info may be invalid for either of the following reasons:
498 * - The revision of the node is not E or greater. In this case, the DRAM Hole
499 * Address Register does not exist.
501 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
502 * indicating that its contents are not valid.
504 * The values passed back in *hole_base, *hole_offset, and *hole_size are
505 * complete 32-bit values despite the fact that the bitfields in the DHAR
506 * only represent bits 31-24 of the base and offset values.
508 int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
509 u64 *hole_offset, u64 *hole_size)
511 struct amd64_pvt *pvt = mci->pvt_info;
513 /* only revE and later have the DRAM Hole Address Register */
514 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
515 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
516 pvt->ext_model, pvt->mc_node_id);
520 /* valid for Fam10h and above */
521 if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
522 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
526 if (!dhar_valid(pvt)) {
527 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
532 /* This node has Memory Hoisting */
534 /* +------------------+--------------------+--------------------+-----
535 * | memory | DRAM hole | relocated |
536 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
538 * | | | [0x100000000, |
539 * | | | (0x100000000+ |
540 * | | | (0xffffffff-x))] |
541 * +------------------+--------------------+--------------------+-----
543 * Above is a diagram of physical memory showing the DRAM hole and the
544 * relocated addresses from the DRAM hole. As shown, the DRAM hole
545 * starts at address x (the base address) and extends through address
546 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
547 * addresses in the hole so that they start at 0x100000000.
550 *hole_base = dhar_base(pvt);
551 *hole_size = (1ULL << 32) - *hole_base;
553 *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
554 : k8_dhar_offset(pvt);
556 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
557 pvt->mc_node_id, (unsigned long)*hole_base,
558 (unsigned long)*hole_offset, (unsigned long)*hole_size);
562 EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
565 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
566 * assumed that sys_addr maps to the node given by mci.
568 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
569 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
570 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
571 * then it is also involved in translating a SysAddr to a DramAddr. Sections
572 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
573 * These parts of the documentation are unclear. I interpret them as follows:
575 * When node n receives a SysAddr, it processes the SysAddr as follows:
577 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
578 * Limit registers for node n. If the SysAddr is not within the range
579 * specified by the base and limit values, then node n ignores the Sysaddr
580 * (since it does not map to node n). Otherwise continue to step 2 below.
582 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
583 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
584 * the range of relocated addresses (starting at 0x100000000) from the DRAM
585 * hole. If not, skip to step 3 below. Else get the value of the
586 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
587 * offset defined by this value from the SysAddr.
589 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
590 * Base register for node n. To obtain the DramAddr, subtract the base
591 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
593 static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
595 struct amd64_pvt *pvt = mci->pvt_info;
596 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
599 dram_base = get_dram_base(pvt, pvt->mc_node_id);
601 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
604 if ((sys_addr >= (1ULL << 32)) &&
605 (sys_addr < ((1ULL << 32) + hole_size))) {
606 /* use DHAR to translate SysAddr to DramAddr */
607 dram_addr = sys_addr - hole_offset;
609 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
610 (unsigned long)sys_addr,
611 (unsigned long)dram_addr);
618 * Translate the SysAddr to a DramAddr as shown near the start of
619 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
620 * only deals with 40-bit values. Therefore we discard bits 63-40 of
621 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
622 * discard are all 1s. Otherwise the bits we discard are all 0s. See
623 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
624 * Programmer's Manual Volume 1 Application Programming.
626 dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
628 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
629 (unsigned long)sys_addr, (unsigned long)dram_addr);
634 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
635 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
636 * for node interleaving.
638 static int num_node_interleave_bits(unsigned intlv_en)
640 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
643 BUG_ON(intlv_en > 7);
644 n = intlv_shift_table[intlv_en];
648 /* Translate the DramAddr given by @dram_addr to an InputAddr. */
649 static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
651 struct amd64_pvt *pvt;
658 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
659 * concerning translating a DramAddr to an InputAddr.
661 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
662 input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
665 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
666 intlv_shift, (unsigned long)dram_addr,
667 (unsigned long)input_addr);
673 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
674 * assumed that @sys_addr maps to the node given by mci.
676 static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
681 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
683 edac_dbg(2, "SysAddr 0x%lx translates to InputAddr 0x%lx\n",
684 (unsigned long)sys_addr, (unsigned long)input_addr);
689 /* Map the Error address to a PAGE and PAGE OFFSET. */
690 static inline void error_address_to_page_and_offset(u64 error_address,
691 struct err_info *err)
693 err->page = (u32) (error_address >> PAGE_SHIFT);
694 err->offset = ((u32) error_address) & ~PAGE_MASK;
698 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
699 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
700 * of a node that detected an ECC memory error. mci represents the node that
701 * the error address maps to (possibly different from the node that detected
702 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
705 static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
709 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
712 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
713 "address 0x%lx\n", (unsigned long)sys_addr);
717 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
720 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
723 static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
725 unsigned long edac_cap = EDAC_FLAG_NONE;
729 u8 i, umc_en_mask = 0, dimm_ecc_en_mask = 0;
732 if (!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT))
735 umc_en_mask |= BIT(i);
737 /* UMC Configuration bit 12 (DimmEccEn) */
738 if (pvt->umc[i].umc_cfg & BIT(12))
739 dimm_ecc_en_mask |= BIT(i);
742 if (umc_en_mask == dimm_ecc_en_mask)
743 edac_cap = EDAC_FLAG_SECDED;
745 bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
749 if (pvt->dclr0 & BIT(bit))
750 edac_cap = EDAC_FLAG_SECDED;
756 static void debug_display_dimm_sizes(struct amd64_pvt *, u8);
758 static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
760 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
762 if (pvt->dram_type == MEM_LRDDR3) {
763 u32 dcsm = pvt->csels[chan].csmasks[0];
765 * It's assumed all LRDIMMs in a DCT are going to be of
766 * same 'type' until proven otherwise. So, use a cs
767 * value of '0' here to get dcsm value.
769 edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
772 edac_dbg(1, "All DIMMs support ECC:%s\n",
773 (dclr & BIT(19)) ? "yes" : "no");
776 edac_dbg(1, " PAR/ERR parity: %s\n",
777 (dclr & BIT(8)) ? "enabled" : "disabled");
779 if (pvt->fam == 0x10)
780 edac_dbg(1, " DCT 128bit mode width: %s\n",
781 (dclr & BIT(11)) ? "128b" : "64b");
783 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
784 (dclr & BIT(12)) ? "yes" : "no",
785 (dclr & BIT(13)) ? "yes" : "no",
786 (dclr & BIT(14)) ? "yes" : "no",
787 (dclr & BIT(15)) ? "yes" : "no");
791 * The Address Mask should be a contiguous set of bits in the non-interleaved
792 * case. So to check for CS interleaving, find the most- and least-significant
793 * bits of the mask, generate a contiguous bitmask, and compare the two.
795 static bool f17_cs_interleaved(struct amd64_pvt *pvt, u8 ctrl, int cs)
797 u32 mask = pvt->csels[ctrl].csmasks[cs >> 1];
798 u32 msb = fls(mask) - 1, lsb = ffs(mask) - 1;
799 u32 test_mask = GENMASK(msb, lsb);
801 edac_dbg(1, "mask=0x%08x test_mask=0x%08x\n", mask, test_mask);
803 return mask ^ test_mask;
806 static void debug_display_dimm_sizes_df(struct amd64_pvt *pvt, u8 ctrl)
808 int dimm, size0, size1, cs0, cs1;
810 edac_printk(KERN_DEBUG, EDAC_MC, "UMC%d chip selects:\n", ctrl);
812 for (dimm = 0; dimm < 4; dimm++) {
816 if (csrow_enabled(cs0, ctrl, pvt))
817 size0 = pvt->ops->dbam_to_cs(pvt, ctrl, 0, cs0);
822 if (csrow_enabled(cs1, ctrl, pvt)) {
824 * CS interleaving is only supported if both CSes have
825 * the same amount of memory. Because they are
826 * interleaved, it will look like both CSes have the
827 * full amount of memory. Save the size for both as
828 * half the amount we found on CS0, if interleaved.
830 if (f17_cs_interleaved(pvt, ctrl, cs1))
831 size1 = size0 = (size0 >> 1);
833 size1 = pvt->ops->dbam_to_cs(pvt, ctrl, 0, cs1);
836 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
842 static void __dump_misc_regs_df(struct amd64_pvt *pvt)
844 struct amd64_umc *umc;
845 u32 i, tmp, umc_base;
848 umc_base = get_umc_base(i);
851 edac_dbg(1, "UMC%d DIMM cfg: 0x%x\n", i, umc->dimm_cfg);
852 edac_dbg(1, "UMC%d UMC cfg: 0x%x\n", i, umc->umc_cfg);
853 edac_dbg(1, "UMC%d SDP ctrl: 0x%x\n", i, umc->sdp_ctrl);
854 edac_dbg(1, "UMC%d ECC ctrl: 0x%x\n", i, umc->ecc_ctrl);
856 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ECC_BAD_SYMBOL, &tmp);
857 edac_dbg(1, "UMC%d ECC bad symbol: 0x%x\n", i, tmp);
859 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_UMC_CAP, &tmp);
860 edac_dbg(1, "UMC%d UMC cap: 0x%x\n", i, tmp);
861 edac_dbg(1, "UMC%d UMC cap high: 0x%x\n", i, umc->umc_cap_hi);
863 edac_dbg(1, "UMC%d ECC capable: %s, ChipKill ECC capable: %s\n",
864 i, (umc->umc_cap_hi & BIT(30)) ? "yes" : "no",
865 (umc->umc_cap_hi & BIT(31)) ? "yes" : "no");
866 edac_dbg(1, "UMC%d All DIMMs support ECC: %s\n",
867 i, (umc->umc_cfg & BIT(12)) ? "yes" : "no");
868 edac_dbg(1, "UMC%d x4 DIMMs present: %s\n",
869 i, (umc->dimm_cfg & BIT(6)) ? "yes" : "no");
870 edac_dbg(1, "UMC%d x16 DIMMs present: %s\n",
871 i, (umc->dimm_cfg & BIT(7)) ? "yes" : "no");
873 if (pvt->dram_type == MEM_LRDDR4) {
874 amd_smn_read(pvt->mc_node_id, umc_base + UMCCH_ADDR_CFG, &tmp);
875 edac_dbg(1, "UMC%d LRDIMM %dx rank multiply\n",
876 i, 1 << ((tmp >> 4) & 0x3));
879 debug_display_dimm_sizes_df(pvt, i);
882 edac_dbg(1, "F0x104 (DRAM Hole Address): 0x%08x, base: 0x%08x\n",
883 pvt->dhar, dhar_base(pvt));
886 /* Display and decode various NB registers for debug purposes. */
887 static void __dump_misc_regs(struct amd64_pvt *pvt)
889 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
891 edac_dbg(1, " NB two channel DRAM capable: %s\n",
892 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
894 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
895 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
896 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
898 debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
900 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
902 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
903 pvt->dhar, dhar_base(pvt),
904 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
905 : f10_dhar_offset(pvt));
907 debug_display_dimm_sizes(pvt, 0);
909 /* everything below this point is Fam10h and above */
913 debug_display_dimm_sizes(pvt, 1);
915 /* Only if NOT ganged does dclr1 have valid info */
916 if (!dct_ganging_enabled(pvt))
917 debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
920 /* Display and decode various NB registers for debug purposes. */
921 static void dump_misc_regs(struct amd64_pvt *pvt)
924 __dump_misc_regs_df(pvt);
926 __dump_misc_regs(pvt);
928 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
930 amd64_info("using x%u syndromes.\n", pvt->ecc_sym_sz);
934 * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
936 static void prep_chip_selects(struct amd64_pvt *pvt)
938 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
939 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
940 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
941 } else if (pvt->fam == 0x15 && pvt->model == 0x30) {
942 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
943 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
945 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
946 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
951 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
953 static void read_dct_base_mask(struct amd64_pvt *pvt)
955 int base_reg0, base_reg1, mask_reg0, mask_reg1, cs;
957 prep_chip_selects(pvt);
960 base_reg0 = get_umc_base(0) + UMCCH_BASE_ADDR;
961 base_reg1 = get_umc_base(1) + UMCCH_BASE_ADDR;
962 mask_reg0 = get_umc_base(0) + UMCCH_ADDR_MASK;
963 mask_reg1 = get_umc_base(1) + UMCCH_ADDR_MASK;
971 for_each_chip_select(cs, 0, pvt) {
972 int reg0 = base_reg0 + (cs * 4);
973 int reg1 = base_reg1 + (cs * 4);
974 u32 *base0 = &pvt->csels[0].csbases[cs];
975 u32 *base1 = &pvt->csels[1].csbases[cs];
978 if (!amd_smn_read(pvt->mc_node_id, reg0, base0))
979 edac_dbg(0, " DCSB0[%d]=0x%08x reg: 0x%x\n",
982 if (!amd_smn_read(pvt->mc_node_id, reg1, base1))
983 edac_dbg(0, " DCSB1[%d]=0x%08x reg: 0x%x\n",
986 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
987 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
993 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
994 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
995 cs, *base1, (pvt->fam == 0x10) ? reg1
1000 for_each_chip_select_mask(cs, 0, pvt) {
1001 int reg0 = mask_reg0 + (cs * 4);
1002 int reg1 = mask_reg1 + (cs * 4);
1003 u32 *mask0 = &pvt->csels[0].csmasks[cs];
1004 u32 *mask1 = &pvt->csels[1].csmasks[cs];
1007 if (!amd_smn_read(pvt->mc_node_id, reg0, mask0))
1008 edac_dbg(0, " DCSM0[%d]=0x%08x reg: 0x%x\n",
1011 if (!amd_smn_read(pvt->mc_node_id, reg1, mask1))
1012 edac_dbg(0, " DCSM1[%d]=0x%08x reg: 0x%x\n",
1015 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
1016 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
1019 if (pvt->fam == 0xf)
1022 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
1023 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
1024 cs, *mask1, (pvt->fam == 0x10) ? reg1
1030 static void determine_memory_type(struct amd64_pvt *pvt)
1032 u32 dram_ctrl, dcsm;
1036 if (pvt->ext_model >= K8_REV_F)
1039 pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
1043 if (pvt->dchr0 & DDR3_MODE)
1046 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
1050 if (pvt->model < 0x60)
1054 * Model 0x60h needs special handling:
1056 * We use a Chip Select value of '0' to obtain dcsm.
1057 * Theoretically, it is possible to populate LRDIMMs of different
1058 * 'Rank' value on a DCT. But this is not the common case. So,
1059 * it's reasonable to assume all DIMMs are going to be of same
1060 * 'type' until proven otherwise.
1062 amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
1063 dcsm = pvt->csels[0].csmasks[0];
1065 if (((dram_ctrl >> 8) & 0x7) == 0x2)
1066 pvt->dram_type = MEM_DDR4;
1067 else if (pvt->dclr0 & BIT(16))
1068 pvt->dram_type = MEM_DDR3;
1069 else if (dcsm & 0x3)
1070 pvt->dram_type = MEM_LRDDR3;
1072 pvt->dram_type = MEM_RDDR3;
1081 if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(5))
1082 pvt->dram_type = MEM_LRDDR4;
1083 else if ((pvt->umc[0].dimm_cfg | pvt->umc[1].dimm_cfg) & BIT(4))
1084 pvt->dram_type = MEM_RDDR4;
1086 pvt->dram_type = MEM_DDR4;
1090 WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
1091 pvt->dram_type = MEM_EMPTY;
1096 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
1099 /* Get the number of DCT channels the memory controller is using. */
1100 static int k8_early_channel_count(struct amd64_pvt *pvt)
1104 if (pvt->ext_model >= K8_REV_F)
1105 /* RevF (NPT) and later */
1106 flag = pvt->dclr0 & WIDTH_128;
1108 /* RevE and earlier */
1109 flag = pvt->dclr0 & REVE_WIDTH_128;
1114 return (flag) ? 2 : 1;
1117 /* On F10h and later ErrAddr is MC4_ADDR[47:1] */
1118 static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
1120 u16 mce_nid = amd_get_nb_id(m->extcpu);
1121 struct mem_ctl_info *mci;
1126 mci = edac_mc_find(mce_nid);
1130 pvt = mci->pvt_info;
1132 if (pvt->fam == 0xf) {
1137 addr = m->addr & GENMASK_ULL(end_bit, start_bit);
1140 * Erratum 637 workaround
1142 if (pvt->fam == 0x15) {
1143 u64 cc6_base, tmp_addr;
1147 if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
1151 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
1152 intlv_en = tmp >> 21 & 0x7;
1154 /* add [47:27] + 3 trailing bits */
1155 cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3;
1157 /* reverse and add DramIntlvEn */
1158 cc6_base |= intlv_en ^ 0x7;
1160 /* pin at [47:24] */
1164 return cc6_base | (addr & GENMASK_ULL(23, 0));
1166 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
1169 tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
1171 /* OR DramIntlvSel into bits [14:12] */
1172 tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
1174 /* add remaining [11:0] bits from original MC4_ADDR */
1175 tmp_addr |= addr & GENMASK_ULL(11, 0);
1177 return cc6_base | tmp_addr;
1183 static struct pci_dev *pci_get_related_function(unsigned int vendor,
1184 unsigned int device,
1185 struct pci_dev *related)
1187 struct pci_dev *dev = NULL;
1189 while ((dev = pci_get_device(vendor, device, dev))) {
1190 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
1191 (dev->bus->number == related->bus->number) &&
1192 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1199 static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
1201 struct amd_northbridge *nb;
1202 struct pci_dev *f1 = NULL;
1203 unsigned int pci_func;
1204 int off = range << 3;
1207 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
1208 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
1210 if (pvt->fam == 0xf)
1213 if (!dram_rw(pvt, range))
1216 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1217 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
1219 /* F15h: factor in CC6 save area by reading dst node's limit reg */
1220 if (pvt->fam != 0x15)
1223 nb = node_to_amd_nb(dram_dst_node(pvt, range));
1227 if (pvt->model == 0x60)
1228 pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
1229 else if (pvt->model == 0x30)
1230 pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
1232 pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
1234 f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
1238 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1240 pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
1242 /* {[39:27],111b} */
1243 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1245 pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
1248 pvt->ranges[range].lim.hi |= llim >> 13;
1253 static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1254 struct err_info *err)
1256 struct amd64_pvt *pvt = mci->pvt_info;
1258 error_address_to_page_and_offset(sys_addr, err);
1261 * Find out which node the error address belongs to. This may be
1262 * different from the node that detected the error.
1264 err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
1265 if (!err->src_mci) {
1266 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1267 (unsigned long)sys_addr);
1268 err->err_code = ERR_NODE;
1272 /* Now map the sys_addr to a CSROW */
1273 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
1274 if (err->csrow < 0) {
1275 err->err_code = ERR_CSROW;
1279 /* CHIPKILL enabled */
1280 if (pvt->nbcfg & NBCFG_CHIPKILL) {
1281 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1282 if (err->channel < 0) {
1284 * Syndrome didn't map, so we don't know which of the
1285 * 2 DIMMs is in error. So we need to ID 'both' of them
1288 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
1289 "possible error reporting race\n",
1291 err->err_code = ERR_CHANNEL;
1296 * non-chipkill ecc mode
1298 * The k8 documentation is unclear about how to determine the
1299 * channel number when using non-chipkill memory. This method
1300 * was obtained from email communication with someone at AMD.
1301 * (Wish the email was placed in this comment - norsk)
1303 err->channel = ((sys_addr & BIT(3)) != 0);
1307 static int ddr2_cs_size(unsigned i, bool dct_width)
1313 else if (!(i & 0x1))
1316 shift = (i + 1) >> 1;
1318 return 128 << (shift + !!dct_width);
1321 static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1322 unsigned cs_mode, int cs_mask_nr)
1324 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1326 if (pvt->ext_model >= K8_REV_F) {
1327 WARN_ON(cs_mode > 11);
1328 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1330 else if (pvt->ext_model >= K8_REV_D) {
1332 WARN_ON(cs_mode > 10);
1335 * the below calculation, besides trying to win an obfuscated C
1336 * contest, maps cs_mode values to DIMM chip select sizes. The
1339 * cs_mode CS size (mb)
1340 * ======= ============
1353 * Basically, it calculates a value with which to shift the
1354 * smallest CS size of 32MB.
1356 * ddr[23]_cs_size have a similar purpose.
1358 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1360 return 32 << (cs_mode - diff);
1363 WARN_ON(cs_mode > 6);
1364 return 32 << cs_mode;
1369 * Get the number of DCT channels in use.
1372 * number of Memory Channels in operation
1374 * contents of the DCL0_LOW register
1376 static int f1x_early_channel_count(struct amd64_pvt *pvt)
1378 int i, j, channels = 0;
1380 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
1381 if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128))
1385 * Need to check if in unganged mode: In such, there are 2 channels,
1386 * but they are not in 128 bit mode and thus the above 'dclr0' status
1389 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1390 * their CSEnable bit on. If so, then SINGLE DIMM case.
1392 edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
1395 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1396 * is more than just one DIMM present in unganged mode. Need to check
1397 * both controllers since DIMMs can be placed in either one.
1399 for (i = 0; i < 2; i++) {
1400 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
1402 for (j = 0; j < 4; j++) {
1403 if (DBAM_DIMM(j, dbam) > 0) {
1413 amd64_info("MCT channel count: %d\n", channels);
1418 static int f17_early_channel_count(struct amd64_pvt *pvt)
1420 int i, channels = 0;
1422 /* SDP Control bit 31 (SdpInit) is clear for unused UMC channels */
1424 channels += !!(pvt->umc[i].sdp_ctrl & UMC_SDP_INIT);
1426 amd64_info("MCT channel count: %d\n", channels);
1431 static int ddr3_cs_size(unsigned i, bool dct_width)
1436 if (i == 0 || i == 3 || i == 4)
1442 else if (!(i & 0x1))
1445 shift = (i + 1) >> 1;
1448 cs_size = (128 * (1 << !!dct_width)) << shift;
1453 static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
1458 if (i < 4 || i == 6)
1462 else if (!(i & 0x1))
1465 shift = (i + 1) >> 1;
1468 cs_size = rank_multiply * (128 << shift);
1473 static int ddr4_cs_size(unsigned i)
1482 /* Min cs_size = 1G */
1483 cs_size = 1024 * (1 << (i >> 1));
1488 static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1489 unsigned cs_mode, int cs_mask_nr)
1491 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1493 WARN_ON(cs_mode > 11);
1495 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1496 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
1498 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1502 * F15h supports only 64bit DCT interfaces
1504 static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1505 unsigned cs_mode, int cs_mask_nr)
1507 WARN_ON(cs_mode > 12);
1509 return ddr3_cs_size(cs_mode, false);
1512 /* F15h M60h supports DDR4 mapping as well.. */
1513 static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1514 unsigned cs_mode, int cs_mask_nr)
1517 u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
1519 WARN_ON(cs_mode > 12);
1521 if (pvt->dram_type == MEM_DDR4) {
1525 cs_size = ddr4_cs_size(cs_mode);
1526 } else if (pvt->dram_type == MEM_LRDDR3) {
1527 unsigned rank_multiply = dcsm & 0xf;
1529 if (rank_multiply == 3)
1531 cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
1533 /* Minimum cs size is 512mb for F15hM60h*/
1537 cs_size = ddr3_cs_size(cs_mode, false);
1544 * F16h and F15h model 30h have only limited cs_modes.
1546 static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1547 unsigned cs_mode, int cs_mask_nr)
1549 WARN_ON(cs_mode > 12);
1551 if (cs_mode == 6 || cs_mode == 8 ||
1552 cs_mode == 9 || cs_mode == 12)
1555 return ddr3_cs_size(cs_mode, false);
1558 static int f17_base_addr_to_cs_size(struct amd64_pvt *pvt, u8 umc,
1559 unsigned int cs_mode, int csrow_nr)
1561 u32 base_addr = pvt->csels[umc].csbases[csrow_nr];
1563 /* Each mask is used for every two base addresses. */
1564 u32 addr_mask = pvt->csels[umc].csmasks[csrow_nr >> 1];
1566 /* Register [31:1] = Address [39:9]. Size is in kBs here. */
1567 u32 size = ((addr_mask >> 1) - (base_addr >> 1) + 1) >> 1;
1569 edac_dbg(1, "BaseAddr: 0x%x, AddrMask: 0x%x\n", base_addr, addr_mask);
1571 /* Return size in MBs. */
1575 static void read_dram_ctl_register(struct amd64_pvt *pvt)
1578 if (pvt->fam == 0xf)
1581 if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1582 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1583 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
1585 edac_dbg(0, " DCTs operate in %s mode\n",
1586 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
1588 if (!dct_ganging_enabled(pvt))
1589 edac_dbg(0, " Address range split per DCT: %s\n",
1590 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1592 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1593 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1594 (dct_memory_cleared(pvt) ? "yes" : "no"));
1596 edac_dbg(0, " channel interleave: %s, "
1597 "interleave bits selector: 0x%x\n",
1598 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1599 dct_sel_interleave_addr(pvt));
1602 amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
1606 * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG,
1607 * 2.10.12 Memory Interleaving Modes).
1609 static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1610 u8 intlv_en, int num_dcts_intlv,
1617 return (u8)(dct_sel);
1619 if (num_dcts_intlv == 2) {
1620 select = (sys_addr >> 8) & 0x3;
1621 channel = select ? 0x3 : 0;
1622 } else if (num_dcts_intlv == 4) {
1623 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1624 switch (intlv_addr) {
1626 channel = (sys_addr >> 8) & 0x3;
1629 channel = (sys_addr >> 9) & 0x3;
1637 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1638 * Interleaving Modes.
1640 static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1641 bool hi_range_sel, u8 intlv_en)
1643 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
1645 if (dct_ganging_enabled(pvt))
1649 return dct_sel_high;
1652 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1654 if (dct_interleave_enabled(pvt)) {
1655 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1657 /* return DCT select function: 0=DCT0, 1=DCT1 */
1659 return sys_addr >> 6 & 1;
1661 if (intlv_addr & 0x2) {
1662 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1663 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) & 1;
1665 return ((sys_addr >> shift) & 1) ^ temp;
1668 if (intlv_addr & 0x4) {
1669 u8 shift = intlv_addr & 0x1 ? 9 : 8;
1671 return (sys_addr >> shift) & 1;
1674 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1677 if (dct_high_range_enabled(pvt))
1678 return ~dct_sel_high & 1;
1683 /* Convert the sys_addr to the normalized DCT address */
1684 static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
1685 u64 sys_addr, bool hi_rng,
1686 u32 dct_sel_base_addr)
1689 u64 dram_base = get_dram_base(pvt, range);
1690 u64 hole_off = f10_dhar_offset(pvt);
1691 u64 dct_sel_base_off = (u64)(pvt->dct_sel_hi & 0xFFFFFC00) << 16;
1696 * base address of high range is below 4Gb
1697 * (bits [47:27] at [31:11])
1698 * DRAM address space on this DCT is hoisted above 4Gb &&
1701 * remove hole offset from sys_addr
1703 * remove high range offset from sys_addr
1705 if ((!(dct_sel_base_addr >> 16) ||
1706 dct_sel_base_addr < dhar_base(pvt)) &&
1708 (sys_addr >= BIT_64(32)))
1709 chan_off = hole_off;
1711 chan_off = dct_sel_base_off;
1715 * we have a valid hole &&
1720 * remove dram base to normalize to DCT address
1722 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
1723 chan_off = hole_off;
1725 chan_off = dram_base;
1728 return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
1732 * checks if the csrow passed in is marked as SPARED, if so returns the new
1735 static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
1739 if (online_spare_swap_done(pvt, dct) &&
1740 csrow == online_spare_bad_dramcs(pvt, dct)) {
1742 for_each_chip_select(tmp_cs, dct, pvt) {
1743 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1753 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1754 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1757 * -EINVAL: NOT FOUND
1758 * 0..csrow = Chip-Select Row
1760 static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
1762 struct mem_ctl_info *mci;
1763 struct amd64_pvt *pvt;
1764 u64 cs_base, cs_mask;
1765 int cs_found = -EINVAL;
1768 mci = edac_mc_find(nid);
1772 pvt = mci->pvt_info;
1774 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
1776 for_each_chip_select(csrow, dct, pvt) {
1777 if (!csrow_enabled(csrow, dct, pvt))
1780 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
1782 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1783 csrow, cs_base, cs_mask);
1787 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
1788 (in_addr & cs_mask), (cs_base & cs_mask));
1790 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1791 if (pvt->fam == 0x15 && pvt->model >= 0x30) {
1795 cs_found = f10_process_possible_spare(pvt, dct, csrow);
1797 edac_dbg(1, " MATCH csrow=%d\n", cs_found);
1805 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1806 * swapped with a region located at the bottom of memory so that the GPU can use
1807 * the interleaved region and thus two channels.
1809 static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
1811 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1813 if (pvt->fam == 0x10) {
1814 /* only revC3 and revE have that feature */
1815 if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
1819 amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
1821 if (!(swap_reg & 0x1))
1824 swap_base = (swap_reg >> 3) & 0x7f;
1825 swap_limit = (swap_reg >> 11) & 0x7f;
1826 rgn_size = (swap_reg >> 20) & 0x7f;
1827 tmp_addr = sys_addr >> 27;
1829 if (!(sys_addr >> 34) &&
1830 (((tmp_addr >= swap_base) &&
1831 (tmp_addr <= swap_limit)) ||
1832 (tmp_addr < rgn_size)))
1833 return sys_addr ^ (u64)swap_base << 27;
1838 /* For a given @dram_range, check if @sys_addr falls within it. */
1839 static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1840 u64 sys_addr, int *chan_sel)
1842 int cs_found = -EINVAL;
1846 bool high_range = false;
1848 u8 node_id = dram_dst_node(pvt, range);
1849 u8 intlv_en = dram_intlv_en(pvt, range);
1850 u32 intlv_sel = dram_intlv_sel(pvt, range);
1852 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1853 range, sys_addr, get_dram_limit(pvt, range));
1855 if (dhar_valid(pvt) &&
1856 dhar_base(pvt) <= sys_addr &&
1857 sys_addr < BIT_64(32)) {
1858 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1863 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1866 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
1868 dct_sel_base = dct_sel_baseaddr(pvt);
1871 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1872 * select between DCT0 and DCT1.
1874 if (dct_high_range_enabled(pvt) &&
1875 !dct_ganging_enabled(pvt) &&
1876 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1879 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
1881 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
1882 high_range, dct_sel_base);
1884 /* Remove node interleaving, see F1x120 */
1886 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1887 (chan_addr & 0xfff);
1889 /* remove channel interleave */
1890 if (dct_interleave_enabled(pvt) &&
1891 !dct_high_range_enabled(pvt) &&
1892 !dct_ganging_enabled(pvt)) {
1894 if (dct_sel_interleave_addr(pvt) != 1) {
1895 if (dct_sel_interleave_addr(pvt) == 0x3)
1897 chan_addr = ((chan_addr >> 10) << 9) |
1898 (chan_addr & 0x1ff);
1900 /* A[6] or hash 6 */
1901 chan_addr = ((chan_addr >> 7) << 6) |
1905 chan_addr = ((chan_addr >> 13) << 12) |
1906 (chan_addr & 0xfff);
1909 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
1911 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
1914 *chan_sel = channel;
1919 static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1920 u64 sys_addr, int *chan_sel)
1922 int cs_found = -EINVAL;
1923 int num_dcts_intlv = 0;
1924 u64 chan_addr, chan_offset;
1925 u64 dct_base, dct_limit;
1926 u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
1927 u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
1929 u64 dhar_offset = f10_dhar_offset(pvt);
1930 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1931 u8 node_id = dram_dst_node(pvt, range);
1932 u8 intlv_en = dram_intlv_en(pvt, range);
1934 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
1935 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
1937 dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
1938 dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
1940 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1941 range, sys_addr, get_dram_limit(pvt, range));
1943 if (!(get_dram_base(pvt, range) <= sys_addr) &&
1944 !(get_dram_limit(pvt, range) >= sys_addr))
1947 if (dhar_valid(pvt) &&
1948 dhar_base(pvt) <= sys_addr &&
1949 sys_addr < BIT_64(32)) {
1950 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1955 /* Verify sys_addr is within DCT Range. */
1956 dct_base = (u64) dct_sel_baseaddr(pvt);
1957 dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
1959 if (!(dct_cont_base_reg & BIT(0)) &&
1960 !(dct_base <= (sys_addr >> 27) &&
1961 dct_limit >= (sys_addr >> 27)))
1964 /* Verify number of dct's that participate in channel interleaving. */
1965 num_dcts_intlv = (int) hweight8(intlv_en);
1967 if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
1970 if (pvt->model >= 0x60)
1971 channel = f1x_determine_channel(pvt, sys_addr, false, intlv_en);
1973 channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
1974 num_dcts_intlv, dct_sel);
1976 /* Verify we stay within the MAX number of channels allowed */
1980 leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
1982 /* Get normalized DCT addr */
1983 if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
1984 chan_offset = dhar_offset;
1986 chan_offset = dct_base << 27;
1988 chan_addr = sys_addr - chan_offset;
1990 /* remove channel interleave */
1991 if (num_dcts_intlv == 2) {
1992 if (intlv_addr == 0x4)
1993 chan_addr = ((chan_addr >> 9) << 8) |
1995 else if (intlv_addr == 0x5)
1996 chan_addr = ((chan_addr >> 10) << 9) |
1997 (chan_addr & 0x1ff);
2001 } else if (num_dcts_intlv == 4) {
2002 if (intlv_addr == 0x4)
2003 chan_addr = ((chan_addr >> 10) << 8) |
2005 else if (intlv_addr == 0x5)
2006 chan_addr = ((chan_addr >> 11) << 9) |
2007 (chan_addr & 0x1ff);
2012 if (dct_offset_en) {
2013 amd64_read_pci_cfg(pvt->F1,
2014 DRAM_CONT_HIGH_OFF + (int) channel * 4,
2016 chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27;
2019 f15h_select_dct(pvt, channel);
2021 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
2025 * if channel = 3, then alias it to 1. This is because, in F15 M30h,
2026 * there is support for 4 DCT's, but only 2 are currently functional.
2027 * They are DCT0 and DCT3. But we have read all registers of DCT3 into
2028 * pvt->csels[1]. So we need to use '1' here to get correct info.
2029 * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications.
2031 alias_channel = (channel == 3) ? 1 : channel;
2033 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
2036 *chan_sel = alias_channel;
2041 static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
2045 int cs_found = -EINVAL;
2048 for (range = 0; range < DRAM_RANGES; range++) {
2049 if (!dram_rw(pvt, range))
2052 if (pvt->fam == 0x15 && pvt->model >= 0x30)
2053 cs_found = f15_m30h_match_to_this_node(pvt, range,
2057 else if ((get_dram_base(pvt, range) <= sys_addr) &&
2058 (get_dram_limit(pvt, range) >= sys_addr)) {
2059 cs_found = f1x_match_to_this_node(pvt, range,
2060 sys_addr, chan_sel);
2069 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
2070 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
2072 * The @sys_addr is usually an error address received from the hardware
2075 static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
2076 struct err_info *err)
2078 struct amd64_pvt *pvt = mci->pvt_info;
2080 error_address_to_page_and_offset(sys_addr, err);
2082 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
2083 if (err->csrow < 0) {
2084 err->err_code = ERR_CSROW;
2089 * We need the syndromes for channel detection only when we're
2090 * ganged. Otherwise @chan should already contain the channel at
2093 if (dct_ganging_enabled(pvt))
2094 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
2098 * debug routine to display the memory sizes of all logical DIMMs and its
2101 static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
2103 int dimm, size0, size1;
2104 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
2105 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
2107 if (pvt->fam == 0xf) {
2108 /* K8 families < revF not supported yet */
2109 if (pvt->ext_model < K8_REV_F)
2115 if (pvt->fam == 0x10) {
2116 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
2118 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
2119 pvt->csels[1].csbases :
2120 pvt->csels[0].csbases;
2123 dcsb = pvt->csels[1].csbases;
2125 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
2128 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
2130 /* Dump memory sizes for DIMM and its CSROWs */
2131 for (dimm = 0; dimm < 4; dimm++) {
2134 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
2136 * For F15m60h, we need multiplier for LRDIMM cs_size
2137 * calculation. We pass dimm value to the dbam_to_cs
2138 * mapper so we can find the multiplier from the
2139 * corresponding DCSM.
2141 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
2142 DBAM_DIMM(dimm, dbam),
2146 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
2147 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
2148 DBAM_DIMM(dimm, dbam),
2151 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
2153 dimm * 2 + 1, size1);
2157 static struct amd64_family_type family_types[] = {
2160 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
2161 .f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2163 .early_channel_count = k8_early_channel_count,
2164 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
2165 .dbam_to_cs = k8_dbam_to_chip_select,
2170 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
2171 .f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2173 .early_channel_count = f1x_early_channel_count,
2174 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2175 .dbam_to_cs = f10_dbam_to_chip_select,
2180 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
2181 .f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2,
2183 .early_channel_count = f1x_early_channel_count,
2184 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2185 .dbam_to_cs = f15_dbam_to_chip_select,
2189 .ctl_name = "F15h_M30h",
2190 .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
2191 .f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2,
2193 .early_channel_count = f1x_early_channel_count,
2194 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2195 .dbam_to_cs = f16_dbam_to_chip_select,
2199 .ctl_name = "F15h_M60h",
2200 .f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1,
2201 .f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2,
2203 .early_channel_count = f1x_early_channel_count,
2204 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2205 .dbam_to_cs = f15_m60h_dbam_to_chip_select,
2210 .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
2211 .f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2,
2213 .early_channel_count = f1x_early_channel_count,
2214 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2215 .dbam_to_cs = f16_dbam_to_chip_select,
2219 .ctl_name = "F16h_M30h",
2220 .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1,
2221 .f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2,
2223 .early_channel_count = f1x_early_channel_count,
2224 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
2225 .dbam_to_cs = f16_dbam_to_chip_select,
2230 .f0_id = PCI_DEVICE_ID_AMD_17H_DF_F0,
2231 .f6_id = PCI_DEVICE_ID_AMD_17H_DF_F6,
2233 .early_channel_count = f17_early_channel_count,
2234 .dbam_to_cs = f17_base_addr_to_cs_size,
2238 .ctl_name = "F17h_M10h",
2239 .f0_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F0,
2240 .f6_id = PCI_DEVICE_ID_AMD_17H_M10H_DF_F6,
2242 .early_channel_count = f17_early_channel_count,
2243 .dbam_to_cs = f17_base_addr_to_cs_size,
2247 .ctl_name = "F17h_M30h",
2248 .f0_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F0,
2249 .f6_id = PCI_DEVICE_ID_AMD_17H_M30H_DF_F6,
2251 .early_channel_count = f17_early_channel_count,
2252 .dbam_to_cs = f17_base_addr_to_cs_size,
2258 * These are tables of eigenvectors (one per line) which can be used for the
2259 * construction of the syndrome tables. The modified syndrome search algorithm
2260 * uses those to find the symbol in error and thus the DIMM.
2262 * Algorithm courtesy of Ross LaFetra from AMD.
2264 static const u16 x4_vectors[] = {
2265 0x2f57, 0x1afe, 0x66cc, 0xdd88,
2266 0x11eb, 0x3396, 0x7f4c, 0xeac8,
2267 0x0001, 0x0002, 0x0004, 0x0008,
2268 0x1013, 0x3032, 0x4044, 0x8088,
2269 0x106b, 0x30d6, 0x70fc, 0xe0a8,
2270 0x4857, 0xc4fe, 0x13cc, 0x3288,
2271 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
2272 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
2273 0x15c1, 0x2a42, 0x89ac, 0x4758,
2274 0x2b03, 0x1602, 0x4f0c, 0xca08,
2275 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
2276 0x8ba7, 0x465e, 0x244c, 0x1cc8,
2277 0x2b87, 0x164e, 0x642c, 0xdc18,
2278 0x40b9, 0x80de, 0x1094, 0x20e8,
2279 0x27db, 0x1eb6, 0x9dac, 0x7b58,
2280 0x11c1, 0x2242, 0x84ac, 0x4c58,
2281 0x1be5, 0x2d7a, 0x5e34, 0xa718,
2282 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
2283 0x4c97, 0xc87e, 0x11fc, 0x33a8,
2284 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2285 0x16b3, 0x3d62, 0x4f34, 0x8518,
2286 0x1e2f, 0x391a, 0x5cac, 0xf858,
2287 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2288 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2289 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2290 0x4397, 0xc27e, 0x17fc, 0x3ea8,
2291 0x1617, 0x3d3e, 0x6464, 0xb8b8,
2292 0x23ff, 0x12aa, 0xab6c, 0x56d8,
2293 0x2dfb, 0x1ba6, 0x913c, 0x7328,
2294 0x185d, 0x2ca6, 0x7914, 0x9e28,
2295 0x171b, 0x3e36, 0x7d7c, 0xebe8,
2296 0x4199, 0x82ee, 0x19f4, 0x2e58,
2297 0x4807, 0xc40e, 0x130c, 0x3208,
2298 0x1905, 0x2e0a, 0x5804, 0xac08,
2299 0x213f, 0x132a, 0xadfc, 0x5ba8,
2300 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
2303 static const u16 x8_vectors[] = {
2304 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2305 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2306 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
2307 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
2308 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
2309 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
2310 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
2311 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
2312 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
2313 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
2314 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
2315 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
2316 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
2317 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
2318 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
2319 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
2320 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
2321 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
2322 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
2325 static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
2328 unsigned int i, err_sym;
2330 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
2332 unsigned v_idx = err_sym * v_dim;
2333 unsigned v_end = (err_sym + 1) * v_dim;
2335 /* walk over all 16 bits of the syndrome */
2336 for (i = 1; i < (1U << 16); i <<= 1) {
2338 /* if bit is set in that eigenvector... */
2339 if (v_idx < v_end && vectors[v_idx] & i) {
2340 u16 ev_comp = vectors[v_idx++];
2342 /* ... and bit set in the modified syndrome, */
2352 /* can't get to zero, move to next symbol */
2357 edac_dbg(0, "syndrome(%x) not found\n", syndrome);
2361 static int map_err_sym_to_channel(int err_sym, int sym_size)
2374 return err_sym >> 4;
2380 /* imaginary bits not in a DIMM */
2382 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
2394 return err_sym >> 3;
2400 static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
2402 struct amd64_pvt *pvt = mci->pvt_info;
2405 if (pvt->ecc_sym_sz == 8)
2406 err_sym = decode_syndrome(syndrome, x8_vectors,
2407 ARRAY_SIZE(x8_vectors),
2409 else if (pvt->ecc_sym_sz == 4)
2410 err_sym = decode_syndrome(syndrome, x4_vectors,
2411 ARRAY_SIZE(x4_vectors),
2414 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
2418 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
2421 static void __log_ecc_error(struct mem_ctl_info *mci, struct err_info *err,
2424 enum hw_event_mc_err_type err_type;
2428 err_type = HW_EVENT_ERR_CORRECTED;
2429 else if (ecc_type == 1)
2430 err_type = HW_EVENT_ERR_UNCORRECTED;
2431 else if (ecc_type == 3)
2432 err_type = HW_EVENT_ERR_DEFERRED;
2434 WARN(1, "Something is rotten in the state of Denmark.\n");
2438 switch (err->err_code) {
2443 string = "Failed to map error addr to a node";
2446 string = "Failed to map error addr to a csrow";
2449 string = "Unknown syndrome - possible error reporting race";
2452 string = "MCA_SYND not valid - unknown syndrome and csrow";
2455 string = "Cannot decode normalized address";
2458 string = "WTF error";
2462 edac_mc_handle_error(err_type, mci, 1,
2463 err->page, err->offset, err->syndrome,
2464 err->csrow, err->channel, -1,
2468 static inline void decode_bus_error(int node_id, struct mce *m)
2470 struct mem_ctl_info *mci;
2471 struct amd64_pvt *pvt;
2472 u8 ecc_type = (m->status >> 45) & 0x3;
2473 u8 xec = XEC(m->status, 0x1f);
2474 u16 ec = EC(m->status);
2476 struct err_info err;
2478 mci = edac_mc_find(node_id);
2482 pvt = mci->pvt_info;
2484 /* Bail out early if this was an 'observed' error */
2485 if (PP(ec) == NBSL_PP_OBS)
2488 /* Do only ECC errors */
2489 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
2492 memset(&err, 0, sizeof(err));
2494 sys_addr = get_error_address(pvt, m);
2497 err.syndrome = extract_syndrome(m->status);
2499 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
2501 __log_ecc_error(mci, &err, ecc_type);
2505 * To find the UMC channel represented by this bank we need to match on its
2506 * instance_id. The instance_id of a bank is held in the lower 32 bits of its
2509 * Currently, we can derive the channel number by looking at the 6th nibble in
2510 * the instance_id. For example, instance_id=0xYXXXXX where Y is the channel
2513 static int find_umc_channel(struct mce *m)
2515 return (m->ipid & GENMASK(31, 0)) >> 20;
2518 static void decode_umc_error(int node_id, struct mce *m)
2520 u8 ecc_type = (m->status >> 45) & 0x3;
2521 struct mem_ctl_info *mci;
2522 struct amd64_pvt *pvt;
2523 struct err_info err;
2526 mci = edac_mc_find(node_id);
2530 pvt = mci->pvt_info;
2532 memset(&err, 0, sizeof(err));
2534 if (m->status & MCI_STATUS_DEFERRED)
2537 err.channel = find_umc_channel(m);
2539 if (umc_normaddr_to_sysaddr(m->addr, pvt->mc_node_id, err.channel, &sys_addr)) {
2540 err.err_code = ERR_NORM_ADDR;
2544 error_address_to_page_and_offset(sys_addr, &err);
2546 if (!(m->status & MCI_STATUS_SYNDV)) {
2547 err.err_code = ERR_SYND;
2551 if (ecc_type == 2) {
2552 u8 length = (m->synd >> 18) & 0x3f;
2555 err.syndrome = (m->synd >> 32) & GENMASK(length - 1, 0);
2557 err.err_code = ERR_CHANNEL;
2560 err.csrow = m->synd & 0x7;
2563 __log_ecc_error(mci, &err, ecc_type);
2567 * Use pvt->F3 which contains the F3 CPU PCI device to get the related
2568 * F1 (AddrMap) and F2 (Dct) devices. Return negative value on error.
2569 * Reserve F0 and F6 on systems with a UMC.
2572 reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 pci_id1, u16 pci_id2)
2575 pvt->F0 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2577 amd64_err("F0 not found, device 0x%x (broken BIOS?)\n", pci_id1);
2581 pvt->F6 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2583 pci_dev_put(pvt->F0);
2586 amd64_err("F6 not found: device 0x%x (broken BIOS?)\n", pci_id2);
2590 edac_dbg(1, "F0: %s\n", pci_name(pvt->F0));
2591 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2592 edac_dbg(1, "F6: %s\n", pci_name(pvt->F6));
2597 /* Reserve the ADDRESS MAP Device */
2598 pvt->F1 = pci_get_related_function(pvt->F3->vendor, pci_id1, pvt->F3);
2600 amd64_err("F1 not found: device 0x%x (broken BIOS?)\n", pci_id1);
2604 /* Reserve the DCT Device */
2605 pvt->F2 = pci_get_related_function(pvt->F3->vendor, pci_id2, pvt->F3);
2607 pci_dev_put(pvt->F1);
2610 amd64_err("F2 not found: device 0x%x (broken BIOS?)\n", pci_id2);
2614 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
2615 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
2616 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
2621 static void free_mc_sibling_devs(struct amd64_pvt *pvt)
2624 pci_dev_put(pvt->F0);
2625 pci_dev_put(pvt->F6);
2627 pci_dev_put(pvt->F1);
2628 pci_dev_put(pvt->F2);
2632 static void determine_ecc_sym_sz(struct amd64_pvt *pvt)
2634 pvt->ecc_sym_sz = 4;
2640 /* Check enabled channels only: */
2641 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
2642 if (pvt->umc[i].ecc_ctrl & BIT(9)) {
2643 pvt->ecc_sym_sz = 16;
2645 } else if (pvt->umc[i].ecc_ctrl & BIT(7)) {
2646 pvt->ecc_sym_sz = 8;
2651 } else if (pvt->fam >= 0x10) {
2654 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
2655 /* F16h has only DCT0, so no need to read dbam1. */
2656 if (pvt->fam != 0x16)
2657 amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
2659 /* F10h, revD and later can do x8 ECC too. */
2660 if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
2661 pvt->ecc_sym_sz = 8;
2666 * Retrieve the hardware registers of the memory controller.
2668 static void __read_mc_regs_df(struct amd64_pvt *pvt)
2670 u8 nid = pvt->mc_node_id;
2671 struct amd64_umc *umc;
2674 /* Read registers from each UMC */
2677 umc_base = get_umc_base(i);
2680 amd_smn_read(nid, umc_base + UMCCH_DIMM_CFG, &umc->dimm_cfg);
2681 amd_smn_read(nid, umc_base + UMCCH_UMC_CFG, &umc->umc_cfg);
2682 amd_smn_read(nid, umc_base + UMCCH_SDP_CTRL, &umc->sdp_ctrl);
2683 amd_smn_read(nid, umc_base + UMCCH_ECC_CTRL, &umc->ecc_ctrl);
2684 amd_smn_read(nid, umc_base + UMCCH_UMC_CAP_HI, &umc->umc_cap_hi);
2689 * Retrieve the hardware registers of the memory controller (this includes the
2690 * 'Address Map' and 'Misc' device regs)
2692 static void read_mc_regs(struct amd64_pvt *pvt)
2698 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2699 * those are Read-As-Zero.
2701 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
2702 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
2704 /* Check first whether TOP_MEM2 is enabled: */
2705 rdmsrl(MSR_K8_SYSCFG, msr_val);
2706 if (msr_val & BIT(21)) {
2707 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
2708 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
2710 edac_dbg(0, " TOP_MEM2 disabled\n");
2714 __read_mc_regs_df(pvt);
2715 amd64_read_pci_cfg(pvt->F0, DF_DHAR, &pvt->dhar);
2720 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
2722 read_dram_ctl_register(pvt);
2724 for (range = 0; range < DRAM_RANGES; range++) {
2727 /* read settings for this DRAM range */
2728 read_dram_base_limit_regs(pvt, range);
2730 rw = dram_rw(pvt, range);
2734 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
2736 get_dram_base(pvt, range),
2737 get_dram_limit(pvt, range));
2739 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2740 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
2741 (rw & 0x1) ? "R" : "-",
2742 (rw & 0x2) ? "W" : "-",
2743 dram_intlv_sel(pvt, range),
2744 dram_dst_node(pvt, range));
2747 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
2748 amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
2750 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
2752 amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
2753 amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
2755 if (!dct_ganging_enabled(pvt)) {
2756 amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
2757 amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
2761 read_dct_base_mask(pvt);
2763 determine_memory_type(pvt);
2764 edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
2766 determine_ecc_sym_sz(pvt);
2768 dump_misc_regs(pvt);
2772 * NOTE: CPU Revision Dependent code
2775 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
2776 * k8 private pointer to -->
2777 * DRAM Bank Address mapping register
2779 * DCL register where dual_channel_active is
2781 * The DBAM register consists of 4 sets of 4 bits each definitions:
2784 * 0-3 CSROWs 0 and 1
2785 * 4-7 CSROWs 2 and 3
2786 * 8-11 CSROWs 4 and 5
2787 * 12-15 CSROWs 6 and 7
2789 * Values range from: 0 to 15
2790 * The meaning of the values depends on CPU revision and dual-channel state,
2791 * see relevant BKDG more info.
2793 * The memory controller provides for total of only 8 CSROWs in its current
2794 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2795 * single channel or two (2) DIMMs in dual channel mode.
2797 * The following code logic collapses the various tables for CSROW based on CPU
2801 * The number of PAGE_SIZE pages on the specified CSROW number it
2805 static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr_orig)
2807 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
2808 int csrow_nr = csrow_nr_orig;
2809 u32 cs_mode, nr_pages;
2814 cs_mode = DBAM_DIMM(csrow_nr, dbam);
2816 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, csrow_nr);
2817 nr_pages <<= 20 - PAGE_SHIFT;
2819 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
2820 csrow_nr_orig, dct, cs_mode);
2821 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
2827 * Initialize the array of csrow attribute instances, based on the values
2828 * from pci config hardware registers.
2830 static int init_csrows(struct mem_ctl_info *mci)
2832 struct amd64_pvt *pvt = mci->pvt_info;
2833 enum edac_type edac_mode = EDAC_NONE;
2834 struct csrow_info *csrow;
2835 struct dimm_info *dimm;
2836 int i, j, empty = 1;
2841 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
2845 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2846 pvt->mc_node_id, val,
2847 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
2851 * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
2853 for_each_chip_select(i, 0, pvt) {
2854 bool row_dct0 = !!csrow_enabled(i, 0, pvt);
2855 bool row_dct1 = false;
2857 if (pvt->fam != 0xf)
2858 row_dct1 = !!csrow_enabled(i, 1, pvt);
2860 if (!row_dct0 && !row_dct1)
2863 csrow = mci->csrows[i];
2866 edac_dbg(1, "MC node: %d, csrow: %d\n",
2867 pvt->mc_node_id, i);
2870 nr_pages = get_csrow_nr_pages(pvt, 0, i);
2871 csrow->channels[0]->dimm->nr_pages = nr_pages;
2874 /* K8 has only one DCT */
2875 if (pvt->fam != 0xf && row_dct1) {
2876 int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i);
2878 csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
2879 nr_pages += row_dct1_pages;
2882 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
2884 /* Determine DIMM ECC mode: */
2886 if (mci->edac_ctl_cap & EDAC_FLAG_S4ECD4ED)
2887 edac_mode = EDAC_S4ECD4ED;
2888 else if (mci->edac_ctl_cap & EDAC_FLAG_SECDED)
2889 edac_mode = EDAC_SECDED;
2891 } else if (pvt->nbcfg & NBCFG_ECC_ENABLE) {
2892 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL)
2897 for (j = 0; j < pvt->channel_count; j++) {
2898 dimm = csrow->channels[j]->dimm;
2899 dimm->mtype = pvt->dram_type;
2900 dimm->edac_mode = edac_mode;
2907 /* get all cores on this DCT */
2908 static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
2912 for_each_online_cpu(cpu)
2913 if (amd_get_nb_id(cpu) == nid)
2914 cpumask_set_cpu(cpu, mask);
2917 /* check MCG_CTL on all the cpus on this node */
2918 static bool nb_mce_bank_enabled_on_node(u16 nid)
2924 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2925 amd64_warn("%s: Error allocating mask\n", __func__);
2929 get_cpus_on_this_dct_cpumask(mask, nid);
2931 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
2933 for_each_cpu(cpu, mask) {
2934 struct msr *reg = per_cpu_ptr(msrs, cpu);
2935 nbe = reg->l & MSR_MCGCTL_NBE;
2937 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2939 (nbe ? "enabled" : "disabled"));
2947 free_cpumask_var(mask);
2951 static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
2953 cpumask_var_t cmask;
2956 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2957 amd64_warn("%s: error allocating mask\n", __func__);
2961 get_cpus_on_this_dct_cpumask(cmask, nid);
2963 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2965 for_each_cpu(cpu, cmask) {
2967 struct msr *reg = per_cpu_ptr(msrs, cpu);
2970 if (reg->l & MSR_MCGCTL_NBE)
2971 s->flags.nb_mce_enable = 1;
2973 reg->l |= MSR_MCGCTL_NBE;
2976 * Turn off NB MCE reporting only when it was off before
2978 if (!s->flags.nb_mce_enable)
2979 reg->l &= ~MSR_MCGCTL_NBE;
2982 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2984 free_cpumask_var(cmask);
2989 static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
2993 u32 value, mask = 0x3; /* UECC/CECC enable */
2995 if (toggle_ecc_err_reporting(s, nid, ON)) {
2996 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
3000 amd64_read_pci_cfg(F3, NBCTL, &value);
3002 s->old_nbctl = value & mask;
3003 s->nbctl_valid = true;
3006 amd64_write_pci_cfg(F3, NBCTL, value);
3008 amd64_read_pci_cfg(F3, NBCFG, &value);
3010 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3011 nid, value, !!(value & NBCFG_ECC_ENABLE));
3013 if (!(value & NBCFG_ECC_ENABLE)) {
3014 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
3016 s->flags.nb_ecc_prev = 0;
3018 /* Attempt to turn on DRAM ECC Enable */
3019 value |= NBCFG_ECC_ENABLE;
3020 amd64_write_pci_cfg(F3, NBCFG, value);
3022 amd64_read_pci_cfg(F3, NBCFG, &value);
3024 if (!(value & NBCFG_ECC_ENABLE)) {
3025 amd64_warn("Hardware rejected DRAM ECC enable,"
3026 "check memory DIMM configuration.\n");
3029 amd64_info("Hardware accepted DRAM ECC Enable\n");
3032 s->flags.nb_ecc_prev = 1;
3035 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
3036 nid, value, !!(value & NBCFG_ECC_ENABLE));
3041 static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
3044 u32 value, mask = 0x3; /* UECC/CECC enable */
3046 if (!s->nbctl_valid)
3049 amd64_read_pci_cfg(F3, NBCTL, &value);
3051 value |= s->old_nbctl;
3053 amd64_write_pci_cfg(F3, NBCTL, value);
3055 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
3056 if (!s->flags.nb_ecc_prev) {
3057 amd64_read_pci_cfg(F3, NBCFG, &value);
3058 value &= ~NBCFG_ECC_ENABLE;
3059 amd64_write_pci_cfg(F3, NBCFG, value);
3062 /* restore the NB Enable MCGCTL bit */
3063 if (toggle_ecc_err_reporting(s, nid, OFF))
3064 amd64_warn("Error restoring NB MCGCTL settings!\n");
3068 * EDAC requires that the BIOS have ECC enabled before
3069 * taking over the processing of ECC errors. A command line
3070 * option allows to force-enable hardware ECC later in
3071 * enable_ecc_error_reporting().
3073 static const char *ecc_msg =
3074 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
3075 " Either enable ECC checking or force module loading by setting "
3076 "'ecc_enable_override'.\n"
3077 " (Note that use of the override may cause unknown side effects.)\n";
3079 static bool ecc_enabled(struct pci_dev *F3, u16 nid)
3081 bool nb_mce_en = false;
3085 if (boot_cpu_data.x86 >= 0x17) {
3086 u8 umc_en_mask = 0, ecc_en_mask = 0;
3089 u32 base = get_umc_base(i);
3091 /* Only check enabled UMCs. */
3092 if (amd_smn_read(nid, base + UMCCH_SDP_CTRL, &value))
3095 if (!(value & UMC_SDP_INIT))
3098 umc_en_mask |= BIT(i);
3100 if (amd_smn_read(nid, base + UMCCH_UMC_CAP_HI, &value))
3103 if (value & UMC_ECC_ENABLED)
3104 ecc_en_mask |= BIT(i);
3107 /* Check whether at least one UMC is enabled: */
3109 ecc_en = umc_en_mask == ecc_en_mask;
3111 edac_dbg(0, "Node %d: No enabled UMCs.\n", nid);
3113 /* Assume UMC MCA banks are enabled. */
3116 amd64_read_pci_cfg(F3, NBCFG, &value);
3118 ecc_en = !!(value & NBCFG_ECC_ENABLE);
3120 nb_mce_en = nb_mce_bank_enabled_on_node(nid);
3122 edac_dbg(0, "NB MCE bank disabled, set MSR 0x%08x[4] on node %d to enable.\n",
3123 MSR_IA32_MCG_CTL, nid);
3126 amd64_info("Node %d: DRAM ECC %s.\n",
3127 nid, (ecc_en ? "enabled" : "disabled"));
3129 if (!ecc_en || !nb_mce_en) {
3130 amd64_info("%s", ecc_msg);
3137 f17h_determine_edac_ctl_cap(struct mem_ctl_info *mci, struct amd64_pvt *pvt)
3139 u8 i, ecc_en = 1, cpk_en = 1;
3142 if (pvt->umc[i].sdp_ctrl & UMC_SDP_INIT) {
3143 ecc_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_ENABLED);
3144 cpk_en &= !!(pvt->umc[i].umc_cap_hi & UMC_ECC_CHIPKILL_CAP);
3148 /* Set chipkill only if ECC is enabled: */
3150 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3153 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3157 static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
3158 struct amd64_family_type *fam)
3160 struct amd64_pvt *pvt = mci->pvt_info;
3162 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
3163 mci->edac_ctl_cap = EDAC_FLAG_NONE;
3166 f17h_determine_edac_ctl_cap(mci, pvt);
3168 if (pvt->nbcap & NBCAP_SECDED)
3169 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
3171 if (pvt->nbcap & NBCAP_CHIPKILL)
3172 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
3175 mci->edac_cap = determine_edac_cap(pvt);
3176 mci->mod_name = EDAC_MOD_STR;
3177 mci->ctl_name = fam->ctl_name;
3178 mci->dev_name = pci_name(pvt->F3);
3179 mci->ctl_page_to_phys = NULL;
3181 /* memory scrubber interface */
3182 mci->set_sdram_scrub_rate = set_scrub_rate;
3183 mci->get_sdram_scrub_rate = get_scrub_rate;
3187 * returns a pointer to the family descriptor on success, NULL otherwise.
3189 static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
3191 struct amd64_family_type *fam_type = NULL;
3193 pvt->ext_model = boot_cpu_data.x86_model >> 4;
3194 pvt->stepping = boot_cpu_data.x86_stepping;
3195 pvt->model = boot_cpu_data.x86_model;
3196 pvt->fam = boot_cpu_data.x86;
3200 fam_type = &family_types[K8_CPUS];
3201 pvt->ops = &family_types[K8_CPUS].ops;
3205 fam_type = &family_types[F10_CPUS];
3206 pvt->ops = &family_types[F10_CPUS].ops;
3210 if (pvt->model == 0x30) {
3211 fam_type = &family_types[F15_M30H_CPUS];
3212 pvt->ops = &family_types[F15_M30H_CPUS].ops;
3214 } else if (pvt->model == 0x60) {
3215 fam_type = &family_types[F15_M60H_CPUS];
3216 pvt->ops = &family_types[F15_M60H_CPUS].ops;
3220 fam_type = &family_types[F15_CPUS];
3221 pvt->ops = &family_types[F15_CPUS].ops;
3225 if (pvt->model == 0x30) {
3226 fam_type = &family_types[F16_M30H_CPUS];
3227 pvt->ops = &family_types[F16_M30H_CPUS].ops;
3230 fam_type = &family_types[F16_CPUS];
3231 pvt->ops = &family_types[F16_CPUS].ops;
3235 if (pvt->model >= 0x10 && pvt->model <= 0x2f) {
3236 fam_type = &family_types[F17_M10H_CPUS];
3237 pvt->ops = &family_types[F17_M10H_CPUS].ops;
3239 } else if (pvt->model >= 0x30 && pvt->model <= 0x3f) {
3240 fam_type = &family_types[F17_M30H_CPUS];
3241 pvt->ops = &family_types[F17_M30H_CPUS].ops;
3246 fam_type = &family_types[F17_CPUS];
3247 pvt->ops = &family_types[F17_CPUS].ops;
3249 if (pvt->fam == 0x18)
3250 family_types[F17_CPUS].ctl_name = "F18h";
3254 amd64_err("Unsupported family!\n");
3258 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
3260 (pvt->ext_model >= K8_REV_F ? "revF or later "
3261 : "revE or earlier ")
3262 : ""), pvt->mc_node_id);
3266 static const struct attribute_group *amd64_edac_attr_groups[] = {
3267 #ifdef CONFIG_EDAC_DEBUG
3268 &amd64_edac_dbg_group,
3270 #ifdef CONFIG_EDAC_AMD64_ERROR_INJECTION
3271 &amd64_edac_inj_group,
3276 /* Set the number of Unified Memory Controllers in the system. */
3277 static void compute_num_umcs(void)
3279 u8 model = boot_cpu_data.x86_model;
3281 if (boot_cpu_data.x86 < 0x17)
3284 if (model >= 0x30 && model <= 0x3f)
3289 edac_dbg(1, "Number of UMCs: %x", num_umcs);
3292 static int init_one_instance(unsigned int nid)
3294 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3295 struct amd64_family_type *fam_type = NULL;
3296 struct mem_ctl_info *mci = NULL;
3297 struct edac_mc_layer layers[2];
3298 struct amd64_pvt *pvt = NULL;
3299 u16 pci_id1, pci_id2;
3303 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
3307 pvt->mc_node_id = nid;
3311 fam_type = per_family_init(pvt);
3315 if (pvt->fam >= 0x17) {
3316 pvt->umc = kcalloc(num_umcs, sizeof(struct amd64_umc), GFP_KERNEL);
3322 pci_id1 = fam_type->f0_id;
3323 pci_id2 = fam_type->f6_id;
3325 pci_id1 = fam_type->f1_id;
3326 pci_id2 = fam_type->f2_id;
3329 err = reserve_mc_sibling_devs(pvt, pci_id1, pci_id2);
3336 * We need to determine how many memory channels there are. Then use
3337 * that information for calculating the size of the dynamic instance
3338 * tables in the 'mci' structure.
3341 pvt->channel_count = pvt->ops->early_channel_count(pvt);
3342 if (pvt->channel_count < 0)
3346 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
3347 layers[0].size = pvt->csels[0].b_cnt;
3348 layers[0].is_virt_csrow = true;
3349 layers[1].type = EDAC_MC_LAYER_CHANNEL;
3352 * Always allocate two channels since we can have setups with DIMMs on
3353 * only one channel. Also, this simplifies handling later for the price
3354 * of a couple of KBs tops.
3356 * On Fam17h+, the number of controllers may be greater than two. So set
3357 * the size equal to the maximum number of UMCs.
3359 if (pvt->fam >= 0x17)
3360 layers[1].size = num_umcs;
3363 layers[1].is_virt_csrow = false;
3365 mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0);
3369 mci->pvt_info = pvt;
3370 mci->pdev = &pvt->F3->dev;
3372 setup_mci_misc_attrs(mci, fam_type);
3374 if (init_csrows(mci))
3375 mci->edac_cap = EDAC_FLAG_NONE;
3378 if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
3379 edac_dbg(1, "failed edac_mc_add_mc()\n");
3389 free_mc_sibling_devs(pvt);
3392 if (pvt->fam >= 0x17)
3402 static int probe_one_instance(unsigned int nid)
3404 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3405 struct ecc_settings *s;
3409 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
3415 if (!ecc_enabled(F3, nid)) {
3418 if (!ecc_enable_override)
3421 if (boot_cpu_data.x86 >= 0x17) {
3422 amd64_warn("Forcing ECC on is not recommended on newer systems. Please enable ECC in BIOS.");
3425 amd64_warn("Forcing ECC on!\n");
3427 if (!enable_ecc_error_reporting(s, nid, F3))
3431 ret = init_one_instance(nid);
3433 amd64_err("Error probing instance: %d\n", nid);
3435 if (boot_cpu_data.x86 < 0x17)
3436 restore_ecc_error_reporting(s, nid, F3);
3445 ecc_stngs[nid] = NULL;
3451 static void remove_one_instance(unsigned int nid)
3453 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
3454 struct ecc_settings *s = ecc_stngs[nid];
3455 struct mem_ctl_info *mci;
3456 struct amd64_pvt *pvt;
3458 mci = find_mci_by_dev(&F3->dev);
3461 /* Remove from EDAC CORE tracking list */
3462 mci = edac_mc_del_mc(&F3->dev);
3466 pvt = mci->pvt_info;
3468 restore_ecc_error_reporting(s, nid, F3);
3470 free_mc_sibling_devs(pvt);
3472 kfree(ecc_stngs[nid]);
3473 ecc_stngs[nid] = NULL;
3475 /* Free the EDAC CORE resources */
3476 mci->pvt_info = NULL;
3482 static void setup_pci_device(void)
3484 struct mem_ctl_info *mci;
3485 struct amd64_pvt *pvt;
3490 mci = edac_mc_find(0);
3494 pvt = mci->pvt_info;
3496 pci_ctl = edac_pci_create_generic_ctl(&pvt->F0->dev, EDAC_MOD_STR);
3498 pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
3500 pr_warn("%s(): Unable to create PCI control\n", __func__);
3501 pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
3505 static const struct x86_cpu_id amd64_cpuids[] = {
3506 { X86_VENDOR_AMD, 0xF, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3507 { X86_VENDOR_AMD, 0x10, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3508 { X86_VENDOR_AMD, 0x15, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3509 { X86_VENDOR_AMD, 0x16, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3510 { X86_VENDOR_AMD, 0x17, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3511 { X86_VENDOR_HYGON, 0x18, X86_MODEL_ANY, X86_FEATURE_ANY, 0 },
3514 MODULE_DEVICE_TABLE(x86cpu, amd64_cpuids);
3516 static int __init amd64_edac_init(void)
3522 owner = edac_get_owner();
3523 if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
3526 if (!x86_match_cpu(amd64_cpuids))
3529 if (amd_cache_northbridges() < 0)
3535 ecc_stngs = kcalloc(amd_nb_num(), sizeof(ecc_stngs[0]), GFP_KERNEL);
3539 msrs = msrs_alloc();
3545 for (i = 0; i < amd_nb_num(); i++) {
3546 err = probe_one_instance(i);
3548 /* unwind properly */
3550 remove_one_instance(i);
3556 if (!edac_has_mcs()) {
3561 /* register stuff with EDAC MCE */
3562 if (report_gart_errors)
3563 amd_report_gart_errors(true);
3565 if (boot_cpu_data.x86 >= 0x17)
3566 amd_register_ecc_decoder(decode_umc_error);
3568 amd_register_ecc_decoder(decode_bus_error);
3572 #ifdef CONFIG_X86_32
3573 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
3576 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
3591 static void __exit amd64_edac_exit(void)
3596 edac_pci_release_generic_ctl(pci_ctl);
3598 /* unregister from EDAC MCE */
3599 amd_report_gart_errors(false);
3601 if (boot_cpu_data.x86 >= 0x17)
3602 amd_unregister_ecc_decoder(decode_umc_error);
3604 amd_unregister_ecc_decoder(decode_bus_error);
3606 for (i = 0; i < amd_nb_num(); i++)
3607 remove_one_instance(i);
3616 module_init(amd64_edac_init);
3617 module_exit(amd64_edac_exit);
3619 MODULE_LICENSE("GPL");
3620 MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
3621 "Dave Peterson, Thayne Harbaugh");
3622 MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
3623 EDAC_AMD64_VERSION);
3625 module_param(edac_op_state, int, 0444);
3626 MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");