Commit | Line | Data |
---|---|---|
10590a9d QZ |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Driver for Intel client SoC with integrated memory controller using IBECC | |
4 | * | |
5 | * Copyright (C) 2020 Intel Corporation | |
6 | * | |
7 | * The In-Band ECC (IBECC) IP provides ECC protection to all or specific | |
8 | * regions of the physical memory space. It's used for memory controllers | |
9 | * that don't support the out-of-band ECC which often needs an additional | |
10 | * storage device to each channel for storing ECC data. | |
11 | */ | |
12 | ||
13 | #include <linux/module.h> | |
14 | #include <linux/init.h> | |
15 | #include <linux/pci.h> | |
16 | #include <linux/slab.h> | |
17 | #include <linux/irq_work.h> | |
18 | #include <linux/llist.h> | |
19 | #include <linux/genalloc.h> | |
20 | #include <linux/edac.h> | |
21 | #include <linux/bits.h> | |
22 | #include <linux/io.h> | |
23 | #include <asm/mach_traps.h> | |
24 | #include <asm/nmi.h> | |
0b7338b2 | 25 | #include <asm/mce.h> |
10590a9d QZ |
26 | |
27 | #include "edac_mc.h" | |
28 | #include "edac_module.h" | |
29 | ||
ad774bd5 | 30 | #define IGEN6_REVISION "v2.5" |
10590a9d QZ |
31 | |
32 | #define EDAC_MOD_STR "igen6_edac" | |
33 | #define IGEN6_NMI_NAME "igen6_ibecc" | |
34 | ||
35 | /* Debug macros */ | |
36 | #define igen6_printk(level, fmt, arg...) \ | |
37 | edac_printk(level, "igen6", fmt, ##arg) | |
38 | ||
39 | #define igen6_mc_printk(mci, level, fmt, arg...) \ | |
40 | edac_mc_chipset_printk(mci, level, "igen6", fmt, ##arg) | |
41 | ||
42 | #define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo)) | |
43 | ||
0b7338b2 | 44 | #define NUM_IMC 2 /* Max memory controllers */ |
10590a9d QZ |
45 | #define NUM_CHANNELS 2 /* Max channels */ |
46 | #define NUM_DIMMS 2 /* Max DIMMs per channel */ | |
47 | ||
48 | #define _4GB BIT_ULL(32) | |
49 | ||
50 | /* Size of physical memory */ | |
51 | #define TOM_OFFSET 0xa0 | |
52 | /* Top of low usable DRAM */ | |
53 | #define TOLUD_OFFSET 0xbc | |
54 | /* Capability register C */ | |
55 | #define CAPID_C_OFFSET 0xec | |
56 | #define CAPID_C_IBECC BIT(15) | |
57 | ||
0b7338b2 QZ |
58 | /* Capability register E */ |
59 | #define CAPID_E_OFFSET 0xf0 | |
60 | #define CAPID_E_IBECC BIT(12) | |
61 | ||
10590a9d QZ |
62 | /* Error Status */ |
63 | #define ERRSTS_OFFSET 0xc8 | |
64 | #define ERRSTS_CE BIT_ULL(6) | |
65 | #define ERRSTS_UE BIT_ULL(7) | |
66 | ||
67 | /* Error Command */ | |
68 | #define ERRCMD_OFFSET 0xca | |
69 | #define ERRCMD_CE BIT_ULL(6) | |
70 | #define ERRCMD_UE BIT_ULL(7) | |
71 | ||
72 | /* IBECC MMIO base address */ | |
73 | #define IBECC_BASE (res_cfg->ibecc_base) | |
74 | #define IBECC_ACTIVATE_OFFSET IBECC_BASE | |
75 | #define IBECC_ACTIVATE_EN BIT(0) | |
76 | ||
77 | /* IBECC error log */ | |
ad774bd5 | 78 | #define ECC_ERROR_LOG_OFFSET (IBECC_BASE + res_cfg->ibecc_error_log_offset) |
10590a9d QZ |
79 | #define ECC_ERROR_LOG_CE BIT_ULL(62) |
80 | #define ECC_ERROR_LOG_UE BIT_ULL(63) | |
81 | #define ECC_ERROR_LOG_ADDR_SHIFT 5 | |
82 | #define ECC_ERROR_LOG_ADDR(v) GET_BITFIELD(v, 5, 38) | |
83 | #define ECC_ERROR_LOG_SYND(v) GET_BITFIELD(v, 46, 61) | |
84 | ||
85 | /* Host MMIO base address */ | |
86 | #define MCHBAR_OFFSET 0x48 | |
87 | #define MCHBAR_EN BIT_ULL(0) | |
88 | #define MCHBAR_BASE(v) (GET_BITFIELD(v, 16, 38) << 16) | |
89 | #define MCHBAR_SIZE 0x10000 | |
90 | ||
91 | /* Parameters for the channel decode stage */ | |
ad774bd5 QZ |
92 | #define IMC_BASE (res_cfg->imc_base) |
93 | #define MAD_INTER_CHANNEL_OFFSET IMC_BASE | |
10590a9d QZ |
94 | #define MAD_INTER_CHANNEL_DDR_TYPE(v) GET_BITFIELD(v, 0, 2) |
95 | #define MAD_INTER_CHANNEL_ECHM(v) GET_BITFIELD(v, 3, 3) | |
96 | #define MAD_INTER_CHANNEL_CH_L_MAP(v) GET_BITFIELD(v, 4, 4) | |
97 | #define MAD_INTER_CHANNEL_CH_S_SIZE(v) ((u64)GET_BITFIELD(v, 12, 19) << 29) | |
98 | ||
99 | /* Parameters for DRAM decode stage */ | |
ad774bd5 | 100 | #define MAD_INTRA_CH0_OFFSET (IMC_BASE + 4) |
10590a9d QZ |
101 | #define MAD_INTRA_CH_DIMM_L_MAP(v) GET_BITFIELD(v, 0, 0) |
102 | ||
103 | /* DIMM characteristics */ | |
ad774bd5 | 104 | #define MAD_DIMM_CH0_OFFSET (IMC_BASE + 0xc) |
10590a9d QZ |
105 | #define MAD_DIMM_CH_DIMM_L_SIZE(v) ((u64)GET_BITFIELD(v, 0, 6) << 29) |
106 | #define MAD_DIMM_CH_DLW(v) GET_BITFIELD(v, 7, 8) | |
107 | #define MAD_DIMM_CH_DIMM_S_SIZE(v) ((u64)GET_BITFIELD(v, 16, 22) << 29) | |
108 | #define MAD_DIMM_CH_DSW(v) GET_BITFIELD(v, 24, 25) | |
109 | ||
ad774bd5 QZ |
110 | /* Hash for memory controller selection */ |
111 | #define MAD_MC_HASH_OFFSET (IMC_BASE + 0x1b8) | |
112 | #define MAC_MC_HASH_LSB(v) GET_BITFIELD(v, 1, 3) | |
113 | ||
10590a9d | 114 | /* Hash for channel selection */ |
ad774bd5 | 115 | #define CHANNEL_HASH_OFFSET (IMC_BASE + 0x24) |
10590a9d | 116 | /* Hash for enhanced channel selection */ |
ad774bd5 | 117 | #define CHANNEL_EHASH_OFFSET (IMC_BASE + 0x28) |
10590a9d QZ |
118 | #define CHANNEL_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6) |
119 | #define CHANNEL_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26) | |
120 | #define CHANNEL_HASH_MODE(v) GET_BITFIELD(v, 28, 28) | |
121 | ||
0b7338b2 QZ |
122 | /* Parameters for memory slice decode stage */ |
123 | #define MEM_SLICE_HASH_MASK(v) (GET_BITFIELD(v, 6, 19) << 6) | |
124 | #define MEM_SLICE_HASH_LSB_MASK_BIT(v) GET_BITFIELD(v, 24, 26) | |
125 | ||
10590a9d | 126 | static struct res_config { |
0b7338b2 | 127 | bool machine_check; |
10590a9d | 128 | int num_imc; |
ad774bd5 | 129 | u32 imc_base; |
0b7338b2 QZ |
130 | u32 cmf_base; |
131 | u32 cmf_size; | |
132 | u32 ms_hash_offset; | |
10590a9d | 133 | u32 ibecc_base; |
ad774bd5 | 134 | u32 ibecc_error_log_offset; |
10590a9d QZ |
135 | bool (*ibecc_available)(struct pci_dev *pdev); |
136 | /* Convert error address logged in IBECC to system physical address */ | |
0b7338b2 | 137 | u64 (*err_addr_to_sys_addr)(u64 eaddr, int mc); |
10590a9d | 138 | /* Convert error address logged in IBECC to integrated memory controller address */ |
ad774bd5 | 139 | u64 (*err_addr_to_imc_addr)(u64 eaddr, int mc); |
10590a9d QZ |
140 | } *res_cfg; |
141 | ||
142 | struct igen6_imc { | |
143 | int mc; | |
144 | struct mem_ctl_info *mci; | |
145 | struct pci_dev *pdev; | |
146 | struct device dev; | |
147 | void __iomem *window; | |
0b7338b2 | 148 | u64 size; |
10590a9d QZ |
149 | u64 ch_s_size; |
150 | int ch_l_map; | |
151 | u64 dimm_s_size[NUM_CHANNELS]; | |
152 | u64 dimm_l_size[NUM_CHANNELS]; | |
153 | int dimm_l_map[NUM_CHANNELS]; | |
154 | }; | |
155 | ||
156 | static struct igen6_pvt { | |
157 | struct igen6_imc imc[NUM_IMC]; | |
0b7338b2 QZ |
158 | u64 ms_hash; |
159 | u64 ms_s_size; | |
160 | int ms_l_map; | |
10590a9d QZ |
161 | } *igen6_pvt; |
162 | ||
163 | /* The top of low usable DRAM */ | |
164 | static u32 igen6_tolud; | |
165 | /* The size of physical memory */ | |
166 | static u64 igen6_tom; | |
167 | ||
168 | struct decoded_addr { | |
169 | int mc; | |
170 | u64 imc_addr; | |
171 | u64 sys_addr; | |
172 | int channel_idx; | |
173 | u64 channel_addr; | |
174 | int sub_channel_idx; | |
175 | u64 sub_channel_addr; | |
176 | }; | |
177 | ||
178 | struct ecclog_node { | |
179 | struct llist_node llnode; | |
180 | int mc; | |
181 | u64 ecclog; | |
182 | }; | |
183 | ||
184 | /* | |
185 | * In the NMI handler, the driver uses the lock-less memory allocator | |
186 | * to allocate memory to store the IBECC error logs and links the logs | |
187 | * to the lock-less list. Delay printk() and the work of error reporting | |
188 | * to EDAC core in a worker. | |
189 | */ | |
190 | #define ECCLOG_POOL_SIZE PAGE_SIZE | |
77429eeb | 191 | static LLIST_HEAD(ecclog_llist); |
10590a9d QZ |
192 | static struct gen_pool *ecclog_pool; |
193 | static char ecclog_buf[ECCLOG_POOL_SIZE]; | |
194 | static struct irq_work ecclog_irq_work; | |
195 | static struct work_struct ecclog_work; | |
196 | ||
197 | /* Compute die IDs for Elkhart Lake with IBECC */ | |
198 | #define DID_EHL_SKU5 0x4514 | |
199 | #define DID_EHL_SKU6 0x4528 | |
200 | #define DID_EHL_SKU7 0x452a | |
201 | #define DID_EHL_SKU8 0x4516 | |
202 | #define DID_EHL_SKU9 0x452c | |
203 | #define DID_EHL_SKU10 0x452e | |
204 | #define DID_EHL_SKU11 0x4532 | |
205 | #define DID_EHL_SKU12 0x4518 | |
206 | #define DID_EHL_SKU13 0x451a | |
207 | #define DID_EHL_SKU14 0x4534 | |
208 | #define DID_EHL_SKU15 0x4536 | |
209 | ||
4e591c05 QZ |
210 | /* Compute die IDs for ICL-NNPI with IBECC */ |
211 | #define DID_ICL_SKU8 0x4581 | |
212 | #define DID_ICL_SKU10 0x4585 | |
213 | #define DID_ICL_SKU11 0x4589 | |
214 | #define DID_ICL_SKU12 0x458d | |
215 | ||
0b7338b2 QZ |
216 | /* Compute die IDs for Tiger Lake with IBECC */ |
217 | #define DID_TGL_SKU 0x9a14 | |
218 | ||
ad774bd5 QZ |
219 | /* Compute die IDs for Alder Lake with IBECC */ |
220 | #define DID_ADL_SKU1 0x4601 | |
221 | #define DID_ADL_SKU2 0x4602 | |
222 | #define DID_ADL_SKU3 0x4621 | |
223 | #define DID_ADL_SKU4 0x4641 | |
224 | ||
10590a9d QZ |
225 | static bool ehl_ibecc_available(struct pci_dev *pdev) |
226 | { | |
227 | u32 v; | |
228 | ||
229 | if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v)) | |
230 | return false; | |
231 | ||
232 | return !!(CAPID_C_IBECC & v); | |
233 | } | |
234 | ||
0b7338b2 | 235 | static u64 ehl_err_addr_to_sys_addr(u64 eaddr, int mc) |
10590a9d QZ |
236 | { |
237 | return eaddr; | |
238 | } | |
239 | ||
ad774bd5 | 240 | static u64 ehl_err_addr_to_imc_addr(u64 eaddr, int mc) |
10590a9d QZ |
241 | { |
242 | if (eaddr < igen6_tolud) | |
243 | return eaddr; | |
244 | ||
245 | if (igen6_tom <= _4GB) | |
246 | return eaddr + igen6_tolud - _4GB; | |
247 | ||
248 | if (eaddr < _4GB) | |
249 | return eaddr + igen6_tolud - igen6_tom; | |
250 | ||
251 | return eaddr; | |
252 | } | |
253 | ||
4e591c05 QZ |
254 | static bool icl_ibecc_available(struct pci_dev *pdev) |
255 | { | |
256 | u32 v; | |
257 | ||
258 | if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v)) | |
259 | return false; | |
260 | ||
261 | return !(CAPID_C_IBECC & v) && | |
262 | (boot_cpu_data.x86_stepping >= 1); | |
263 | } | |
264 | ||
0b7338b2 QZ |
265 | static bool tgl_ibecc_available(struct pci_dev *pdev) |
266 | { | |
267 | u32 v; | |
268 | ||
269 | if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v)) | |
270 | return false; | |
271 | ||
272 | return !(CAPID_E_IBECC & v); | |
273 | } | |
274 | ||
275 | static u64 mem_addr_to_sys_addr(u64 maddr) | |
276 | { | |
277 | if (maddr < igen6_tolud) | |
278 | return maddr; | |
279 | ||
280 | if (igen6_tom <= _4GB) | |
281 | return maddr - igen6_tolud + _4GB; | |
282 | ||
283 | if (maddr < _4GB) | |
284 | return maddr - igen6_tolud + igen6_tom; | |
285 | ||
286 | return maddr; | |
287 | } | |
288 | ||
289 | static u64 mem_slice_hash(u64 addr, u64 mask, u64 hash_init, int intlv_bit) | |
290 | { | |
291 | u64 hash_addr = addr & mask, hash = hash_init; | |
292 | u64 intlv = (addr >> intlv_bit) & 1; | |
293 | int i; | |
294 | ||
295 | for (i = 6; i < 20; i++) | |
296 | hash ^= (hash_addr >> i) & 1; | |
297 | ||
298 | return hash ^ intlv; | |
299 | } | |
300 | ||
301 | static u64 tgl_err_addr_to_mem_addr(u64 eaddr, int mc) | |
302 | { | |
303 | u64 maddr, hash, mask, ms_s_size; | |
304 | int intlv_bit; | |
305 | u32 ms_hash; | |
306 | ||
307 | ms_s_size = igen6_pvt->ms_s_size; | |
308 | if (eaddr >= ms_s_size) | |
309 | return eaddr + ms_s_size; | |
310 | ||
311 | ms_hash = igen6_pvt->ms_hash; | |
312 | ||
313 | mask = MEM_SLICE_HASH_MASK(ms_hash); | |
314 | intlv_bit = MEM_SLICE_HASH_LSB_MASK_BIT(ms_hash) + 6; | |
315 | ||
316 | maddr = GET_BITFIELD(eaddr, intlv_bit, 63) << (intlv_bit + 1) | | |
317 | GET_BITFIELD(eaddr, 0, intlv_bit - 1); | |
318 | ||
319 | hash = mem_slice_hash(maddr, mask, mc, intlv_bit); | |
320 | ||
321 | return maddr | (hash << intlv_bit); | |
322 | } | |
323 | ||
324 | static u64 tgl_err_addr_to_sys_addr(u64 eaddr, int mc) | |
325 | { | |
326 | u64 maddr = tgl_err_addr_to_mem_addr(eaddr, mc); | |
327 | ||
328 | return mem_addr_to_sys_addr(maddr); | |
329 | } | |
330 | ||
ad774bd5 | 331 | static u64 tgl_err_addr_to_imc_addr(u64 eaddr, int mc) |
0b7338b2 QZ |
332 | { |
333 | return eaddr; | |
334 | } | |
335 | ||
ad774bd5 QZ |
336 | static u64 adl_err_addr_to_sys_addr(u64 eaddr, int mc) |
337 | { | |
338 | return mem_addr_to_sys_addr(eaddr); | |
339 | } | |
340 | ||
341 | static u64 adl_err_addr_to_imc_addr(u64 eaddr, int mc) | |
342 | { | |
343 | u64 imc_addr, ms_s_size = igen6_pvt->ms_s_size; | |
344 | struct igen6_imc *imc = &igen6_pvt->imc[mc]; | |
345 | int intlv_bit; | |
346 | u32 mc_hash; | |
347 | ||
348 | if (eaddr >= 2 * ms_s_size) | |
349 | return eaddr - ms_s_size; | |
350 | ||
351 | mc_hash = readl(imc->window + MAD_MC_HASH_OFFSET); | |
352 | ||
353 | intlv_bit = MAC_MC_HASH_LSB(mc_hash) + 6; | |
354 | ||
355 | imc_addr = GET_BITFIELD(eaddr, intlv_bit + 1, 63) << intlv_bit | | |
356 | GET_BITFIELD(eaddr, 0, intlv_bit - 1); | |
357 | ||
358 | return imc_addr; | |
359 | } | |
360 | ||
10590a9d | 361 | static struct res_config ehl_cfg = { |
0b7338b2 | 362 | .num_imc = 1, |
ad774bd5 | 363 | .imc_base = 0x5000, |
0b7338b2 QZ |
364 | .ibecc_base = 0xdc00, |
365 | .ibecc_available = ehl_ibecc_available, | |
ad774bd5 | 366 | .ibecc_error_log_offset = 0x170, |
0b7338b2 QZ |
367 | .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr, |
368 | .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr, | |
10590a9d QZ |
369 | }; |
370 | ||
4e591c05 | 371 | static struct res_config icl_cfg = { |
0b7338b2 | 372 | .num_imc = 1, |
ad774bd5 | 373 | .imc_base = 0x5000, |
0b7338b2 | 374 | .ibecc_base = 0xd800, |
ad774bd5 | 375 | .ibecc_error_log_offset = 0x170, |
0b7338b2 QZ |
376 | .ibecc_available = icl_ibecc_available, |
377 | .err_addr_to_sys_addr = ehl_err_addr_to_sys_addr, | |
378 | .err_addr_to_imc_addr = ehl_err_addr_to_imc_addr, | |
379 | }; | |
380 | ||
381 | static struct res_config tgl_cfg = { | |
382 | .machine_check = true, | |
383 | .num_imc = 2, | |
ad774bd5 | 384 | .imc_base = 0x5000, |
0b7338b2 QZ |
385 | .cmf_base = 0x11000, |
386 | .cmf_size = 0x800, | |
387 | .ms_hash_offset = 0xac, | |
388 | .ibecc_base = 0xd400, | |
ad774bd5 | 389 | .ibecc_error_log_offset = 0x170, |
0b7338b2 QZ |
390 | .ibecc_available = tgl_ibecc_available, |
391 | .err_addr_to_sys_addr = tgl_err_addr_to_sys_addr, | |
392 | .err_addr_to_imc_addr = tgl_err_addr_to_imc_addr, | |
4e591c05 QZ |
393 | }; |
394 | ||
ad774bd5 QZ |
395 | static struct res_config adl_cfg = { |
396 | .machine_check = true, | |
397 | .num_imc = 2, | |
398 | .imc_base = 0xd800, | |
399 | .ibecc_base = 0xd400, | |
400 | .ibecc_error_log_offset = 0x68, | |
401 | .ibecc_available = tgl_ibecc_available, | |
402 | .err_addr_to_sys_addr = adl_err_addr_to_sys_addr, | |
403 | .err_addr_to_imc_addr = adl_err_addr_to_imc_addr, | |
404 | }; | |
405 | ||
10590a9d QZ |
406 | static const struct pci_device_id igen6_pci_tbl[] = { |
407 | { PCI_VDEVICE(INTEL, DID_EHL_SKU5), (kernel_ulong_t)&ehl_cfg }, | |
408 | { PCI_VDEVICE(INTEL, DID_EHL_SKU6), (kernel_ulong_t)&ehl_cfg }, | |
409 | { PCI_VDEVICE(INTEL, DID_EHL_SKU7), (kernel_ulong_t)&ehl_cfg }, | |
410 | { PCI_VDEVICE(INTEL, DID_EHL_SKU8), (kernel_ulong_t)&ehl_cfg }, | |
411 | { PCI_VDEVICE(INTEL, DID_EHL_SKU9), (kernel_ulong_t)&ehl_cfg }, | |
412 | { PCI_VDEVICE(INTEL, DID_EHL_SKU10), (kernel_ulong_t)&ehl_cfg }, | |
413 | { PCI_VDEVICE(INTEL, DID_EHL_SKU11), (kernel_ulong_t)&ehl_cfg }, | |
414 | { PCI_VDEVICE(INTEL, DID_EHL_SKU12), (kernel_ulong_t)&ehl_cfg }, | |
415 | { PCI_VDEVICE(INTEL, DID_EHL_SKU13), (kernel_ulong_t)&ehl_cfg }, | |
416 | { PCI_VDEVICE(INTEL, DID_EHL_SKU14), (kernel_ulong_t)&ehl_cfg }, | |
417 | { PCI_VDEVICE(INTEL, DID_EHL_SKU15), (kernel_ulong_t)&ehl_cfg }, | |
4e591c05 QZ |
418 | { PCI_VDEVICE(INTEL, DID_ICL_SKU8), (kernel_ulong_t)&icl_cfg }, |
419 | { PCI_VDEVICE(INTEL, DID_ICL_SKU10), (kernel_ulong_t)&icl_cfg }, | |
420 | { PCI_VDEVICE(INTEL, DID_ICL_SKU11), (kernel_ulong_t)&icl_cfg }, | |
421 | { PCI_VDEVICE(INTEL, DID_ICL_SKU12), (kernel_ulong_t)&icl_cfg }, | |
0b7338b2 | 422 | { PCI_VDEVICE(INTEL, DID_TGL_SKU), (kernel_ulong_t)&tgl_cfg }, |
ad774bd5 QZ |
423 | { PCI_VDEVICE(INTEL, DID_ADL_SKU1), (kernel_ulong_t)&adl_cfg }, |
424 | { PCI_VDEVICE(INTEL, DID_ADL_SKU2), (kernel_ulong_t)&adl_cfg }, | |
425 | { PCI_VDEVICE(INTEL, DID_ADL_SKU3), (kernel_ulong_t)&adl_cfg }, | |
426 | { PCI_VDEVICE(INTEL, DID_ADL_SKU4), (kernel_ulong_t)&adl_cfg }, | |
10590a9d QZ |
427 | { }, |
428 | }; | |
429 | MODULE_DEVICE_TABLE(pci, igen6_pci_tbl); | |
430 | ||
431 | static enum dev_type get_width(int dimm_l, u32 mad_dimm) | |
432 | { | |
433 | u32 w = dimm_l ? MAD_DIMM_CH_DLW(mad_dimm) : | |
434 | MAD_DIMM_CH_DSW(mad_dimm); | |
435 | ||
436 | switch (w) { | |
437 | case 0: | |
438 | return DEV_X8; | |
439 | case 1: | |
440 | return DEV_X16; | |
441 | case 2: | |
442 | return DEV_X32; | |
443 | default: | |
444 | return DEV_UNKNOWN; | |
445 | } | |
446 | } | |
447 | ||
448 | static enum mem_type get_memory_type(u32 mad_inter) | |
449 | { | |
450 | u32 t = MAD_INTER_CHANNEL_DDR_TYPE(mad_inter); | |
451 | ||
452 | switch (t) { | |
453 | case 0: | |
454 | return MEM_DDR4; | |
455 | case 1: | |
456 | return MEM_DDR3; | |
457 | case 2: | |
458 | return MEM_LPDDR3; | |
459 | case 3: | |
460 | return MEM_LPDDR4; | |
461 | case 4: | |
462 | return MEM_WIO2; | |
463 | default: | |
464 | return MEM_UNKNOWN; | |
465 | } | |
466 | } | |
467 | ||
468 | static int decode_chan_idx(u64 addr, u64 mask, int intlv_bit) | |
469 | { | |
470 | u64 hash_addr = addr & mask, hash = 0; | |
471 | u64 intlv = (addr >> intlv_bit) & 1; | |
472 | int i; | |
473 | ||
474 | for (i = 6; i < 20; i++) | |
475 | hash ^= (hash_addr >> i) & 1; | |
476 | ||
477 | return (int)hash ^ intlv; | |
478 | } | |
479 | ||
480 | static u64 decode_channel_addr(u64 addr, int intlv_bit) | |
481 | { | |
482 | u64 channel_addr; | |
483 | ||
484 | /* Remove the interleave bit and shift upper part down to fill gap */ | |
485 | channel_addr = GET_BITFIELD(addr, intlv_bit + 1, 63) << intlv_bit; | |
486 | channel_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1); | |
487 | ||
488 | return channel_addr; | |
489 | } | |
490 | ||
491 | static void decode_addr(u64 addr, u32 hash, u64 s_size, int l_map, | |
492 | int *idx, u64 *sub_addr) | |
493 | { | |
494 | int intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6; | |
495 | ||
496 | if (addr > 2 * s_size) { | |
497 | *sub_addr = addr - s_size; | |
498 | *idx = l_map; | |
499 | return; | |
500 | } | |
501 | ||
502 | if (CHANNEL_HASH_MODE(hash)) { | |
503 | *sub_addr = decode_channel_addr(addr, intlv_bit); | |
504 | *idx = decode_chan_idx(addr, CHANNEL_HASH_MASK(hash), intlv_bit); | |
505 | } else { | |
506 | *sub_addr = decode_channel_addr(addr, 6); | |
507 | *idx = GET_BITFIELD(addr, 6, 6); | |
508 | } | |
509 | } | |
510 | ||
511 | static int igen6_decode(struct decoded_addr *res) | |
512 | { | |
513 | struct igen6_imc *imc = &igen6_pvt->imc[res->mc]; | |
514 | u64 addr = res->imc_addr, sub_addr, s_size; | |
515 | int idx, l_map; | |
516 | u32 hash; | |
517 | ||
518 | if (addr >= igen6_tom) { | |
519 | edac_dbg(0, "Address 0x%llx out of range\n", addr); | |
520 | return -EINVAL; | |
521 | } | |
522 | ||
523 | /* Decode channel */ | |
524 | hash = readl(imc->window + CHANNEL_HASH_OFFSET); | |
525 | s_size = imc->ch_s_size; | |
526 | l_map = imc->ch_l_map; | |
527 | decode_addr(addr, hash, s_size, l_map, &idx, &sub_addr); | |
528 | res->channel_idx = idx; | |
529 | res->channel_addr = sub_addr; | |
530 | ||
531 | /* Decode sub-channel/DIMM */ | |
532 | hash = readl(imc->window + CHANNEL_EHASH_OFFSET); | |
533 | s_size = imc->dimm_s_size[idx]; | |
534 | l_map = imc->dimm_l_map[idx]; | |
535 | decode_addr(res->channel_addr, hash, s_size, l_map, &idx, &sub_addr); | |
536 | res->sub_channel_idx = idx; | |
537 | res->sub_channel_addr = sub_addr; | |
538 | ||
539 | return 0; | |
540 | } | |
541 | ||
542 | static void igen6_output_error(struct decoded_addr *res, | |
543 | struct mem_ctl_info *mci, u64 ecclog) | |
544 | { | |
545 | enum hw_event_mc_err_type type = ecclog & ECC_ERROR_LOG_UE ? | |
546 | HW_EVENT_ERR_UNCORRECTED : | |
547 | HW_EVENT_ERR_CORRECTED; | |
548 | ||
549 | edac_mc_handle_error(type, mci, 1, | |
550 | res->sys_addr >> PAGE_SHIFT, | |
551 | res->sys_addr & ~PAGE_MASK, | |
552 | ECC_ERROR_LOG_SYND(ecclog), | |
553 | res->channel_idx, res->sub_channel_idx, | |
554 | -1, "", ""); | |
555 | } | |
556 | ||
557 | static struct gen_pool *ecclog_gen_pool_create(void) | |
558 | { | |
559 | struct gen_pool *pool; | |
560 | ||
561 | pool = gen_pool_create(ilog2(sizeof(struct ecclog_node)), -1); | |
562 | if (!pool) | |
563 | return NULL; | |
564 | ||
565 | if (gen_pool_add(pool, (unsigned long)ecclog_buf, ECCLOG_POOL_SIZE, -1)) { | |
566 | gen_pool_destroy(pool); | |
567 | return NULL; | |
568 | } | |
569 | ||
570 | return pool; | |
571 | } | |
572 | ||
573 | static int ecclog_gen_pool_add(int mc, u64 ecclog) | |
574 | { | |
575 | struct ecclog_node *node; | |
576 | ||
577 | node = (void *)gen_pool_alloc(ecclog_pool, sizeof(*node)); | |
578 | if (!node) | |
579 | return -ENOMEM; | |
580 | ||
581 | node->mc = mc; | |
582 | node->ecclog = ecclog; | |
583 | llist_add(&node->llnode, &ecclog_llist); | |
584 | ||
585 | return 0; | |
586 | } | |
587 | ||
588 | /* | |
589 | * Either the memory-mapped I/O status register ECC_ERROR_LOG or the PCI | |
590 | * configuration space status register ERRSTS can indicate whether a | |
591 | * correctable error or an uncorrectable error occurred. We only use the | |
592 | * ECC_ERROR_LOG register to check error type, but need to clear both | |
593 | * registers to enable future error events. | |
594 | */ | |
595 | static u64 ecclog_read_and_clear(struct igen6_imc *imc) | |
596 | { | |
597 | u64 ecclog = readq(imc->window + ECC_ERROR_LOG_OFFSET); | |
598 | ||
599 | if (ecclog & (ECC_ERROR_LOG_CE | ECC_ERROR_LOG_UE)) { | |
600 | /* Clear CE/UE bits by writing 1s */ | |
601 | writeq(ecclog, imc->window + ECC_ERROR_LOG_OFFSET); | |
602 | return ecclog; | |
603 | } | |
604 | ||
605 | return 0; | |
606 | } | |
607 | ||
608 | static void errsts_clear(struct igen6_imc *imc) | |
609 | { | |
610 | u16 errsts; | |
611 | ||
612 | if (pci_read_config_word(imc->pdev, ERRSTS_OFFSET, &errsts)) { | |
613 | igen6_printk(KERN_ERR, "Failed to read ERRSTS\n"); | |
614 | return; | |
615 | } | |
616 | ||
617 | /* Clear CE/UE bits by writing 1s */ | |
618 | if (errsts & (ERRSTS_CE | ERRSTS_UE)) | |
619 | pci_write_config_word(imc->pdev, ERRSTS_OFFSET, errsts); | |
620 | } | |
621 | ||
622 | static int errcmd_enable_error_reporting(bool enable) | |
623 | { | |
624 | struct igen6_imc *imc = &igen6_pvt->imc[0]; | |
625 | u16 errcmd; | |
626 | int rc; | |
627 | ||
628 | rc = pci_read_config_word(imc->pdev, ERRCMD_OFFSET, &errcmd); | |
629 | if (rc) | |
630 | return rc; | |
631 | ||
632 | if (enable) | |
633 | errcmd |= ERRCMD_CE | ERRSTS_UE; | |
634 | else | |
635 | errcmd &= ~(ERRCMD_CE | ERRSTS_UE); | |
636 | ||
637 | rc = pci_write_config_word(imc->pdev, ERRCMD_OFFSET, errcmd); | |
638 | if (rc) | |
639 | return rc; | |
640 | ||
641 | return 0; | |
642 | } | |
643 | ||
644 | static int ecclog_handler(void) | |
645 | { | |
646 | struct igen6_imc *imc; | |
647 | int i, n = 0; | |
648 | u64 ecclog; | |
649 | ||
650 | for (i = 0; i < res_cfg->num_imc; i++) { | |
651 | imc = &igen6_pvt->imc[i]; | |
652 | ||
653 | /* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */ | |
654 | ||
655 | ecclog = ecclog_read_and_clear(imc); | |
656 | if (!ecclog) | |
657 | continue; | |
658 | ||
659 | if (!ecclog_gen_pool_add(i, ecclog)) | |
660 | irq_work_queue(&ecclog_irq_work); | |
661 | ||
662 | n++; | |
663 | } | |
664 | ||
665 | return n; | |
666 | } | |
667 | ||
668 | static void ecclog_work_cb(struct work_struct *work) | |
669 | { | |
670 | struct ecclog_node *node, *tmp; | |
671 | struct mem_ctl_info *mci; | |
672 | struct llist_node *head; | |
673 | struct decoded_addr res; | |
674 | u64 eaddr; | |
675 | ||
676 | head = llist_del_all(&ecclog_llist); | |
677 | if (!head) | |
678 | return; | |
679 | ||
680 | llist_for_each_entry_safe(node, tmp, head, llnode) { | |
681 | memset(&res, 0, sizeof(res)); | |
682 | eaddr = ECC_ERROR_LOG_ADDR(node->ecclog) << | |
683 | ECC_ERROR_LOG_ADDR_SHIFT; | |
684 | res.mc = node->mc; | |
0b7338b2 | 685 | res.sys_addr = res_cfg->err_addr_to_sys_addr(eaddr, res.mc); |
ad774bd5 | 686 | res.imc_addr = res_cfg->err_addr_to_imc_addr(eaddr, res.mc); |
10590a9d QZ |
687 | |
688 | mci = igen6_pvt->imc[res.mc].mci; | |
689 | ||
690 | edac_dbg(2, "MC %d, ecclog = 0x%llx\n", node->mc, node->ecclog); | |
691 | igen6_mc_printk(mci, KERN_DEBUG, "HANDLING IBECC MEMORY ERROR\n"); | |
692 | igen6_mc_printk(mci, KERN_DEBUG, "ADDR 0x%llx ", res.sys_addr); | |
693 | ||
694 | if (!igen6_decode(&res)) | |
695 | igen6_output_error(&res, mci, node->ecclog); | |
696 | ||
697 | gen_pool_free(ecclog_pool, (unsigned long)node, sizeof(*node)); | |
698 | } | |
699 | } | |
700 | ||
701 | static void ecclog_irq_work_cb(struct irq_work *irq_work) | |
702 | { | |
703 | int i; | |
704 | ||
705 | for (i = 0; i < res_cfg->num_imc; i++) | |
706 | errsts_clear(&igen6_pvt->imc[i]); | |
707 | ||
708 | if (!llist_empty(&ecclog_llist)) | |
709 | schedule_work(&ecclog_work); | |
710 | } | |
711 | ||
712 | static int ecclog_nmi_handler(unsigned int cmd, struct pt_regs *regs) | |
713 | { | |
714 | unsigned char reason; | |
715 | ||
716 | if (!ecclog_handler()) | |
717 | return NMI_DONE; | |
718 | ||
719 | /* | |
720 | * Both In-Band ECC correctable error and uncorrectable error are | |
721 | * reported by SERR# NMI. The NMI generic code (see pci_serr_error()) | |
722 | * doesn't clear the bit NMI_REASON_CLEAR_SERR (in port 0x61) to | |
723 | * re-enable the SERR# NMI after NMI handling. So clear this bit here | |
724 | * to re-enable SERR# NMI for receiving future In-Band ECC errors. | |
725 | */ | |
726 | reason = x86_platform.get_nmi_reason() & NMI_REASON_CLEAR_MASK; | |
727 | reason |= NMI_REASON_CLEAR_SERR; | |
728 | outb(reason, NMI_REASON_PORT); | |
729 | reason &= ~NMI_REASON_CLEAR_SERR; | |
730 | outb(reason, NMI_REASON_PORT); | |
731 | ||
732 | return NMI_HANDLED; | |
733 | } | |
734 | ||
0b7338b2 QZ |
735 | static int ecclog_mce_handler(struct notifier_block *nb, unsigned long val, |
736 | void *data) | |
737 | { | |
738 | struct mce *mce = (struct mce *)data; | |
739 | char *type; | |
740 | ||
741 | if (mce->kflags & MCE_HANDLED_CEC) | |
742 | return NOTIFY_DONE; | |
743 | ||
744 | /* | |
745 | * Ignore unless this is a memory related error. | |
746 | * We don't check the bit MCI_STATUS_ADDRV of MCi_STATUS here, | |
747 | * since this bit isn't set on some CPU (e.g., Tiger Lake UP3). | |
748 | */ | |
749 | if ((mce->status & 0xefff) >> 7 != 1) | |
750 | return NOTIFY_DONE; | |
751 | ||
752 | if (mce->mcgstatus & MCG_STATUS_MCIP) | |
753 | type = "Exception"; | |
754 | else | |
755 | type = "Event"; | |
756 | ||
757 | edac_dbg(0, "CPU %d: Machine Check %s: 0x%llx Bank %d: 0x%llx\n", | |
758 | mce->extcpu, type, mce->mcgstatus, | |
759 | mce->bank, mce->status); | |
760 | edac_dbg(0, "TSC 0x%llx\n", mce->tsc); | |
761 | edac_dbg(0, "ADDR 0x%llx\n", mce->addr); | |
762 | edac_dbg(0, "MISC 0x%llx\n", mce->misc); | |
763 | edac_dbg(0, "PROCESSOR %u:0x%x TIME %llu SOCKET %u APIC 0x%x\n", | |
764 | mce->cpuvendor, mce->cpuid, mce->time, | |
765 | mce->socketid, mce->apicid); | |
766 | /* | |
767 | * We just use the Machine Check for the memory error notification. | |
768 | * Each memory controller is associated with an IBECC instance. | |
769 | * Directly read and clear the error information(error address and | |
770 | * error type) on all the IBECC instances so that we know on which | |
771 | * memory controller the memory error(s) occurred. | |
772 | */ | |
773 | if (!ecclog_handler()) | |
774 | return NOTIFY_DONE; | |
775 | ||
776 | mce->kflags |= MCE_HANDLED_EDAC; | |
777 | ||
778 | return NOTIFY_DONE; | |
779 | } | |
780 | ||
781 | static struct notifier_block ecclog_mce_dec = { | |
782 | .notifier_call = ecclog_mce_handler, | |
783 | .priority = MCE_PRIO_EDAC, | |
784 | }; | |
785 | ||
10590a9d QZ |
786 | static bool igen6_check_ecc(struct igen6_imc *imc) |
787 | { | |
788 | u32 activate = readl(imc->window + IBECC_ACTIVATE_OFFSET); | |
789 | ||
790 | return !!(activate & IBECC_ACTIVATE_EN); | |
791 | } | |
792 | ||
793 | static int igen6_get_dimm_config(struct mem_ctl_info *mci) | |
794 | { | |
795 | struct igen6_imc *imc = mci->pvt_info; | |
796 | u32 mad_inter, mad_intra, mad_dimm; | |
797 | int i, j, ndimms, mc = imc->mc; | |
798 | struct dimm_info *dimm; | |
799 | enum mem_type mtype; | |
800 | enum dev_type dtype; | |
801 | u64 dsize; | |
802 | bool ecc; | |
803 | ||
804 | edac_dbg(2, "\n"); | |
805 | ||
806 | mad_inter = readl(imc->window + MAD_INTER_CHANNEL_OFFSET); | |
807 | mtype = get_memory_type(mad_inter); | |
808 | ecc = igen6_check_ecc(imc); | |
809 | imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(mad_inter); | |
810 | imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(mad_inter); | |
811 | ||
812 | for (i = 0; i < NUM_CHANNELS; i++) { | |
813 | mad_intra = readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4); | |
814 | mad_dimm = readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4); | |
815 | ||
816 | imc->dimm_l_size[i] = MAD_DIMM_CH_DIMM_L_SIZE(mad_dimm); | |
817 | imc->dimm_s_size[i] = MAD_DIMM_CH_DIMM_S_SIZE(mad_dimm); | |
818 | imc->dimm_l_map[i] = MAD_INTRA_CH_DIMM_L_MAP(mad_intra); | |
0b7338b2 QZ |
819 | imc->size += imc->dimm_s_size[i]; |
820 | imc->size += imc->dimm_l_size[i]; | |
10590a9d QZ |
821 | ndimms = 0; |
822 | ||
823 | for (j = 0; j < NUM_DIMMS; j++) { | |
824 | dimm = edac_get_dimm(mci, i, j, 0); | |
825 | ||
826 | if (j ^ imc->dimm_l_map[i]) { | |
827 | dtype = get_width(0, mad_dimm); | |
828 | dsize = imc->dimm_s_size[i]; | |
829 | } else { | |
830 | dtype = get_width(1, mad_dimm); | |
831 | dsize = imc->dimm_l_size[i]; | |
832 | } | |
833 | ||
834 | if (!dsize) | |
835 | continue; | |
836 | ||
837 | dimm->grain = 64; | |
838 | dimm->mtype = mtype; | |
839 | dimm->dtype = dtype; | |
840 | dimm->nr_pages = MiB_TO_PAGES(dsize >> 20); | |
841 | dimm->edac_mode = EDAC_SECDED; | |
842 | snprintf(dimm->label, sizeof(dimm->label), | |
843 | "MC#%d_Chan#%d_DIMM#%d", mc, i, j); | |
844 | edac_dbg(0, "MC %d, Channel %d, DIMM %d, Size %llu MiB (%u pages)\n", | |
845 | mc, i, j, dsize >> 20, dimm->nr_pages); | |
846 | ||
847 | ndimms++; | |
848 | } | |
849 | ||
850 | if (ndimms && !ecc) { | |
851 | igen6_printk(KERN_ERR, "MC%d In-Band ECC is disabled\n", mc); | |
852 | return -ENODEV; | |
853 | } | |
854 | } | |
855 | ||
0b7338b2 QZ |
856 | edac_dbg(0, "MC %d, total size %llu MiB\n", mc, imc->size >> 20); |
857 | ||
10590a9d QZ |
858 | return 0; |
859 | } | |
860 | ||
861 | #ifdef CONFIG_EDAC_DEBUG | |
2223d8c7 QZ |
862 | /* Top of upper usable DRAM */ |
863 | static u64 igen6_touud; | |
864 | #define TOUUD_OFFSET 0xa8 | |
865 | ||
10590a9d QZ |
866 | static void igen6_reg_dump(struct igen6_imc *imc) |
867 | { | |
868 | int i; | |
869 | ||
870 | edac_dbg(2, "CHANNEL_HASH : 0x%x\n", | |
871 | readl(imc->window + CHANNEL_HASH_OFFSET)); | |
872 | edac_dbg(2, "CHANNEL_EHASH : 0x%x\n", | |
873 | readl(imc->window + CHANNEL_EHASH_OFFSET)); | |
874 | edac_dbg(2, "MAD_INTER_CHANNEL: 0x%x\n", | |
875 | readl(imc->window + MAD_INTER_CHANNEL_OFFSET)); | |
876 | edac_dbg(2, "ECC_ERROR_LOG : 0x%llx\n", | |
877 | readq(imc->window + ECC_ERROR_LOG_OFFSET)); | |
878 | ||
879 | for (i = 0; i < NUM_CHANNELS; i++) { | |
880 | edac_dbg(2, "MAD_INTRA_CH%d : 0x%x\n", i, | |
881 | readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4)); | |
882 | edac_dbg(2, "MAD_DIMM_CH%d : 0x%x\n", i, | |
883 | readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4)); | |
884 | } | |
885 | edac_dbg(2, "TOLUD : 0x%x", igen6_tolud); | |
2223d8c7 | 886 | edac_dbg(2, "TOUUD : 0x%llx", igen6_touud); |
10590a9d QZ |
887 | edac_dbg(2, "TOM : 0x%llx", igen6_tom); |
888 | } | |
2223d8c7 QZ |
889 | |
890 | static struct dentry *igen6_test; | |
891 | ||
892 | static int debugfs_u64_set(void *data, u64 val) | |
893 | { | |
894 | u64 ecclog; | |
895 | ||
896 | if ((val >= igen6_tolud && val < _4GB) || val >= igen6_touud) { | |
897 | edac_dbg(0, "Address 0x%llx out of range\n", val); | |
898 | return 0; | |
899 | } | |
900 | ||
901 | pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val); | |
902 | ||
903 | val >>= ECC_ERROR_LOG_ADDR_SHIFT; | |
904 | ecclog = (val << ECC_ERROR_LOG_ADDR_SHIFT) | ECC_ERROR_LOG_CE; | |
905 | ||
906 | if (!ecclog_gen_pool_add(0, ecclog)) | |
907 | irq_work_queue(&ecclog_irq_work); | |
908 | ||
909 | return 0; | |
910 | } | |
911 | DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n"); | |
912 | ||
913 | static void igen6_debug_setup(void) | |
914 | { | |
915 | igen6_test = edac_debugfs_create_dir("igen6_test"); | |
916 | if (!igen6_test) | |
917 | return; | |
918 | ||
919 | if (!edac_debugfs_create_file("addr", 0200, igen6_test, | |
920 | NULL, &fops_u64_wo)) { | |
921 | debugfs_remove(igen6_test); | |
922 | igen6_test = NULL; | |
923 | } | |
924 | } | |
925 | ||
926 | static void igen6_debug_teardown(void) | |
927 | { | |
928 | debugfs_remove_recursive(igen6_test); | |
929 | } | |
10590a9d QZ |
930 | #else |
931 | static void igen6_reg_dump(struct igen6_imc *imc) {} | |
2223d8c7 QZ |
932 | static void igen6_debug_setup(void) {} |
933 | static void igen6_debug_teardown(void) {} | |
10590a9d QZ |
934 | #endif |
935 | ||
936 | static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar) | |
937 | { | |
938 | union { | |
939 | u64 v; | |
940 | struct { | |
941 | u32 v_lo; | |
942 | u32 v_hi; | |
943 | }; | |
944 | } u; | |
945 | ||
946 | edac_dbg(2, "\n"); | |
947 | ||
948 | if (!res_cfg->ibecc_available(pdev)) { | |
949 | edac_dbg(2, "No In-Band ECC IP\n"); | |
950 | goto fail; | |
951 | } | |
952 | ||
953 | if (pci_read_config_dword(pdev, TOLUD_OFFSET, &igen6_tolud)) { | |
954 | igen6_printk(KERN_ERR, "Failed to read TOLUD\n"); | |
955 | goto fail; | |
956 | } | |
957 | ||
958 | igen6_tolud &= GENMASK(31, 20); | |
959 | ||
960 | if (pci_read_config_dword(pdev, TOM_OFFSET, &u.v_lo)) { | |
961 | igen6_printk(KERN_ERR, "Failed to read lower TOM\n"); | |
962 | goto fail; | |
963 | } | |
964 | ||
965 | if (pci_read_config_dword(pdev, TOM_OFFSET + 4, &u.v_hi)) { | |
966 | igen6_printk(KERN_ERR, "Failed to read upper TOM\n"); | |
967 | goto fail; | |
968 | } | |
969 | ||
970 | igen6_tom = u.v & GENMASK_ULL(38, 20); | |
971 | ||
972 | if (pci_read_config_dword(pdev, MCHBAR_OFFSET, &u.v_lo)) { | |
973 | igen6_printk(KERN_ERR, "Failed to read lower MCHBAR\n"); | |
974 | goto fail; | |
975 | } | |
976 | ||
977 | if (pci_read_config_dword(pdev, MCHBAR_OFFSET + 4, &u.v_hi)) { | |
978 | igen6_printk(KERN_ERR, "Failed to read upper MCHBAR\n"); | |
979 | goto fail; | |
980 | } | |
981 | ||
982 | if (!(u.v & MCHBAR_EN)) { | |
983 | igen6_printk(KERN_ERR, "MCHBAR is disabled\n"); | |
984 | goto fail; | |
985 | } | |
986 | ||
987 | *mchbar = MCHBAR_BASE(u.v); | |
988 | ||
2223d8c7 QZ |
989 | #ifdef CONFIG_EDAC_DEBUG |
990 | if (pci_read_config_dword(pdev, TOUUD_OFFSET, &u.v_lo)) | |
991 | edac_dbg(2, "Failed to read lower TOUUD\n"); | |
992 | else if (pci_read_config_dword(pdev, TOUUD_OFFSET + 4, &u.v_hi)) | |
993 | edac_dbg(2, "Failed to read upper TOUUD\n"); | |
994 | else | |
995 | igen6_touud = u.v & GENMASK_ULL(38, 20); | |
996 | #endif | |
997 | ||
10590a9d QZ |
998 | return 0; |
999 | fail: | |
1000 | return -ENODEV; | |
1001 | } | |
1002 | ||
1003 | static int igen6_register_mci(int mc, u64 mchbar, struct pci_dev *pdev) | |
1004 | { | |
1005 | struct edac_mc_layer layers[2]; | |
1006 | struct mem_ctl_info *mci; | |
1007 | struct igen6_imc *imc; | |
1008 | void __iomem *window; | |
1009 | int rc; | |
1010 | ||
1011 | edac_dbg(2, "\n"); | |
1012 | ||
1013 | mchbar += mc * MCHBAR_SIZE; | |
1014 | window = ioremap(mchbar, MCHBAR_SIZE); | |
1015 | if (!window) { | |
1016 | igen6_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", mchbar); | |
1017 | return -ENODEV; | |
1018 | } | |
1019 | ||
1020 | layers[0].type = EDAC_MC_LAYER_CHANNEL; | |
1021 | layers[0].size = NUM_CHANNELS; | |
1022 | layers[0].is_virt_csrow = false; | |
1023 | layers[1].type = EDAC_MC_LAYER_SLOT; | |
1024 | layers[1].size = NUM_DIMMS; | |
1025 | layers[1].is_virt_csrow = true; | |
1026 | ||
1027 | mci = edac_mc_alloc(mc, ARRAY_SIZE(layers), layers, 0); | |
1028 | if (!mci) { | |
1029 | rc = -ENOMEM; | |
1030 | goto fail; | |
1031 | } | |
1032 | ||
1033 | mci->ctl_name = kasprintf(GFP_KERNEL, "Intel_client_SoC MC#%d", mc); | |
1034 | if (!mci->ctl_name) { | |
1035 | rc = -ENOMEM; | |
1036 | goto fail2; | |
1037 | } | |
1038 | ||
1039 | mci->mtype_cap = MEM_FLAG_LPDDR4 | MEM_FLAG_DDR4; | |
1040 | mci->edac_ctl_cap = EDAC_FLAG_SECDED; | |
1041 | mci->edac_cap = EDAC_FLAG_SECDED; | |
1042 | mci->mod_name = EDAC_MOD_STR; | |
1043 | mci->dev_name = pci_name(pdev); | |
1044 | mci->pvt_info = &igen6_pvt->imc[mc]; | |
1045 | ||
1046 | imc = mci->pvt_info; | |
1047 | device_initialize(&imc->dev); | |
1048 | /* | |
1049 | * EDAC core uses mci->pdev(pointer of structure device) as | |
1050 | * memory controller ID. The client SoCs attach one or more | |
1051 | * memory controllers to single pci_dev (single pci_dev->dev | |
1052 | * can be for multiple memory controllers). | |
1053 | * | |
1054 | * To make mci->pdev unique, assign pci_dev->dev to mci->pdev | |
1055 | * for the first memory controller and assign a unique imc->dev | |
1056 | * to mci->pdev for each non-first memory controller. | |
1057 | */ | |
1058 | mci->pdev = mc ? &imc->dev : &pdev->dev; | |
1059 | imc->mc = mc; | |
1060 | imc->pdev = pdev; | |
1061 | imc->window = window; | |
1062 | ||
1063 | igen6_reg_dump(imc); | |
1064 | ||
1065 | rc = igen6_get_dimm_config(mci); | |
1066 | if (rc) | |
1067 | goto fail3; | |
1068 | ||
1069 | rc = edac_mc_add_mc(mci); | |
1070 | if (rc) { | |
1071 | igen6_printk(KERN_ERR, "Failed to register mci#%d\n", mc); | |
1072 | goto fail3; | |
1073 | } | |
1074 | ||
1075 | imc->mci = mci; | |
1076 | return 0; | |
1077 | fail3: | |
1078 | kfree(mci->ctl_name); | |
1079 | fail2: | |
1080 | edac_mc_free(mci); | |
1081 | fail: | |
1082 | iounmap(window); | |
1083 | return rc; | |
1084 | } | |
1085 | ||
1086 | static void igen6_unregister_mcis(void) | |
1087 | { | |
1088 | struct mem_ctl_info *mci; | |
1089 | struct igen6_imc *imc; | |
1090 | int i; | |
1091 | ||
1092 | edac_dbg(2, "\n"); | |
1093 | ||
1094 | for (i = 0; i < res_cfg->num_imc; i++) { | |
1095 | imc = &igen6_pvt->imc[i]; | |
1096 | mci = imc->mci; | |
1097 | if (!mci) | |
1098 | continue; | |
1099 | ||
1100 | edac_mc_del_mc(mci->pdev); | |
1101 | kfree(mci->ctl_name); | |
1102 | edac_mc_free(mci); | |
1103 | iounmap(imc->window); | |
1104 | } | |
1105 | } | |
1106 | ||
0b7338b2 QZ |
1107 | static int igen6_mem_slice_setup(u64 mchbar) |
1108 | { | |
1109 | struct igen6_imc *imc = &igen6_pvt->imc[0]; | |
1110 | u64 base = mchbar + res_cfg->cmf_base; | |
1111 | u32 offset = res_cfg->ms_hash_offset; | |
1112 | u32 size = res_cfg->cmf_size; | |
1113 | u64 ms_s_size, ms_hash; | |
1114 | void __iomem *cmf; | |
1115 | int ms_l_map; | |
1116 | ||
1117 | edac_dbg(2, "\n"); | |
1118 | ||
1119 | if (imc[0].size < imc[1].size) { | |
1120 | ms_s_size = imc[0].size; | |
1121 | ms_l_map = 1; | |
1122 | } else { | |
1123 | ms_s_size = imc[1].size; | |
1124 | ms_l_map = 0; | |
1125 | } | |
1126 | ||
1127 | igen6_pvt->ms_s_size = ms_s_size; | |
1128 | igen6_pvt->ms_l_map = ms_l_map; | |
1129 | ||
1130 | edac_dbg(0, "ms_s_size: %llu MiB, ms_l_map %d\n", | |
1131 | ms_s_size >> 20, ms_l_map); | |
1132 | ||
ad774bd5 QZ |
1133 | if (!size) |
1134 | return 0; | |
1135 | ||
0b7338b2 QZ |
1136 | cmf = ioremap(base, size); |
1137 | if (!cmf) { | |
1138 | igen6_printk(KERN_ERR, "Failed to ioremap cmf 0x%llx\n", base); | |
1139 | return -ENODEV; | |
1140 | } | |
1141 | ||
1142 | ms_hash = readq(cmf + offset); | |
1143 | igen6_pvt->ms_hash = ms_hash; | |
1144 | ||
1145 | edac_dbg(0, "MEM_SLICE_HASH: 0x%llx\n", ms_hash); | |
1146 | ||
1147 | iounmap(cmf); | |
1148 | ||
1149 | return 0; | |
1150 | } | |
1151 | ||
1152 | static int register_err_handler(void) | |
1153 | { | |
1154 | int rc; | |
1155 | ||
1156 | if (res_cfg->machine_check) { | |
1157 | mce_register_decode_chain(&ecclog_mce_dec); | |
1158 | return 0; | |
1159 | } | |
1160 | ||
1161 | rc = register_nmi_handler(NMI_SERR, ecclog_nmi_handler, | |
1162 | 0, IGEN6_NMI_NAME); | |
1163 | if (rc) { | |
1164 | igen6_printk(KERN_ERR, "Failed to register NMI handler\n"); | |
1165 | return rc; | |
1166 | } | |
1167 | ||
1168 | return 0; | |
1169 | } | |
1170 | ||
1171 | static void unregister_err_handler(void) | |
1172 | { | |
1173 | if (res_cfg->machine_check) { | |
1174 | mce_unregister_decode_chain(&ecclog_mce_dec); | |
1175 | return; | |
1176 | } | |
1177 | ||
1178 | unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); | |
1179 | } | |
1180 | ||
10590a9d QZ |
1181 | static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent) |
1182 | { | |
1183 | u64 mchbar; | |
1184 | int i, rc; | |
1185 | ||
1186 | edac_dbg(2, "\n"); | |
1187 | ||
1188 | igen6_pvt = kzalloc(sizeof(*igen6_pvt), GFP_KERNEL); | |
1189 | if (!igen6_pvt) | |
1190 | return -ENOMEM; | |
1191 | ||
1192 | res_cfg = (struct res_config *)ent->driver_data; | |
1193 | ||
1194 | rc = igen6_pci_setup(pdev, &mchbar); | |
1195 | if (rc) | |
1196 | goto fail; | |
1197 | ||
1198 | for (i = 0; i < res_cfg->num_imc; i++) { | |
1199 | rc = igen6_register_mci(i, mchbar, pdev); | |
1200 | if (rc) | |
1201 | goto fail2; | |
1202 | } | |
1203 | ||
0b7338b2 QZ |
1204 | if (res_cfg->num_imc > 1) { |
1205 | rc = igen6_mem_slice_setup(mchbar); | |
1206 | if (rc) | |
1207 | goto fail2; | |
1208 | } | |
1209 | ||
10590a9d QZ |
1210 | ecclog_pool = ecclog_gen_pool_create(); |
1211 | if (!ecclog_pool) { | |
1212 | rc = -ENOMEM; | |
1213 | goto fail2; | |
1214 | } | |
1215 | ||
1216 | INIT_WORK(&ecclog_work, ecclog_work_cb); | |
1217 | init_irq_work(&ecclog_irq_work, ecclog_irq_work_cb); | |
1218 | ||
1219 | /* Check if any pending errors before registering the NMI handler */ | |
1220 | ecclog_handler(); | |
1221 | ||
0b7338b2 QZ |
1222 | rc = register_err_handler(); |
1223 | if (rc) | |
10590a9d | 1224 | goto fail3; |
10590a9d QZ |
1225 | |
1226 | /* Enable error reporting */ | |
1227 | rc = errcmd_enable_error_reporting(true); | |
1228 | if (rc) { | |
1229 | igen6_printk(KERN_ERR, "Failed to enable error reporting\n"); | |
1230 | goto fail4; | |
1231 | } | |
1232 | ||
2223d8c7 | 1233 | igen6_debug_setup(); |
10590a9d QZ |
1234 | return 0; |
1235 | fail4: | |
1236 | unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME); | |
1237 | fail3: | |
1238 | gen_pool_destroy(ecclog_pool); | |
1239 | fail2: | |
1240 | igen6_unregister_mcis(); | |
1241 | fail: | |
1242 | kfree(igen6_pvt); | |
1243 | return rc; | |
1244 | } | |
1245 | ||
1246 | static void igen6_remove(struct pci_dev *pdev) | |
1247 | { | |
1248 | edac_dbg(2, "\n"); | |
1249 | ||
2223d8c7 | 1250 | igen6_debug_teardown(); |
10590a9d | 1251 | errcmd_enable_error_reporting(false); |
0b7338b2 | 1252 | unregister_err_handler(); |
10590a9d QZ |
1253 | irq_work_sync(&ecclog_irq_work); |
1254 | flush_work(&ecclog_work); | |
1255 | gen_pool_destroy(ecclog_pool); | |
1256 | igen6_unregister_mcis(); | |
1257 | kfree(igen6_pvt); | |
1258 | } | |
1259 | ||
1260 | static struct pci_driver igen6_driver = { | |
1261 | .name = EDAC_MOD_STR, | |
1262 | .probe = igen6_probe, | |
1263 | .remove = igen6_remove, | |
1264 | .id_table = igen6_pci_tbl, | |
1265 | }; | |
1266 | ||
1267 | static int __init igen6_init(void) | |
1268 | { | |
1269 | const char *owner; | |
1270 | int rc; | |
1271 | ||
1272 | edac_dbg(2, "\n"); | |
1273 | ||
1274 | owner = edac_get_owner(); | |
1275 | if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) | |
1276 | return -ENODEV; | |
1277 | ||
1278 | edac_op_state = EDAC_OPSTATE_NMI; | |
1279 | ||
1280 | rc = pci_register_driver(&igen6_driver); | |
1281 | if (rc) | |
1282 | return rc; | |
1283 | ||
1284 | igen6_printk(KERN_INFO, "%s\n", IGEN6_REVISION); | |
1285 | ||
1286 | return 0; | |
1287 | } | |
1288 | ||
1289 | static void __exit igen6_exit(void) | |
1290 | { | |
1291 | edac_dbg(2, "\n"); | |
1292 | ||
1293 | pci_unregister_driver(&igen6_driver); | |
1294 | } | |
1295 | ||
1296 | module_init(igen6_init); | |
1297 | module_exit(igen6_exit); | |
1298 | ||
1299 | MODULE_LICENSE("GPL v2"); | |
1300 | MODULE_AUTHOR("Qiuxu Zhuo"); | |
1301 | MODULE_DESCRIPTION("MC Driver for Intel client SoC using In-Band ECC"); |