Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
011d8261 BP |
2 | #include <linux/mm.h> |
3 | #include <linux/gfp.h> | |
4 | #include <linux/kernel.h> | |
5 | ||
6 | #include <asm/mce.h> | |
7 | ||
8 | #include "debugfs.h" | |
9 | ||
10 | /* | |
11 | * RAS Correctable Errors Collector | |
12 | * | |
13 | * This is a simple gadget which collects correctable errors and counts their | |
14 | * occurrence per physical page address. | |
15 | * | |
16 | * We've opted for possibly the simplest data structure to collect those - an | |
17 | * array of the size of a memory page. It stores 512 u64's with the following | |
18 | * structure: | |
19 | * | |
20 | * [63 ... PFN ... 12 | 11 ... generation ... 10 | 9 ... count ... 0] | |
21 | * | |
22 | * The generation in the two highest order bits is two bits which are set to 11b | |
23 | * on every insertion. During the course of each entry's existence, the | |
24 | * generation field gets decremented during spring cleaning to 10b, then 01b and | |
25 | * then 00b. | |
26 | * | |
27 | * This way we're employing the natural numeric ordering to make sure that newly | |
28 | * inserted/touched elements have higher 12-bit counts (which we've manufactured) | |
29 | * and thus iterating over the array initially won't kick out those elements | |
30 | * which were inserted last. | |
31 | * | |
32 | * Spring cleaning is what we do when we reach a certain number CLEAN_ELEMS of | |
33 | * elements entered into the array, during which, we're decaying all elements. | |
34 | * If, after decay, an element gets inserted again, its generation is set to 11b | |
35 | * to make sure it has higher numerical count than other, older elements and | |
36 | * thus emulate an an LRU-like behavior when deleting elements to free up space | |
37 | * in the page. | |
38 | * | |
39 | * When an element reaches it's max count of count_threshold, we try to poison | |
40 | * it by assuming that errors triggered count_threshold times in a single page | |
41 | * are excessive and that page shouldn't be used anymore. count_threshold is | |
42 | * initialized to COUNT_MASK which is the maximum. | |
43 | * | |
44 | * That error event entry causes cec_add_elem() to return !0 value and thus | |
45 | * signal to its callers to log the error. | |
46 | * | |
47 | * To the question why we've chosen a page and moving elements around with | |
48 | * memmove(), it is because it is a very simple structure to handle and max data | |
49 | * movement is 4K which on highly optimized modern CPUs is almost unnoticeable. | |
50 | * We wanted to avoid the pointer traversal of more complex structures like a | |
51 | * linked list or some sort of a balancing search tree. | |
52 | * | |
53 | * Deleting an element takes O(n) but since it is only a single page, it should | |
54 | * be fast enough and it shouldn't happen all too often depending on error | |
55 | * patterns. | |
56 | */ | |
57 | ||
58 | #undef pr_fmt | |
59 | #define pr_fmt(fmt) "RAS: " fmt | |
60 | ||
61 | /* | |
62 | * We use DECAY_BITS bits of PAGE_SHIFT bits for counting decay, i.e., how long | |
63 | * elements have stayed in the array without having been accessed again. | |
64 | */ | |
65 | #define DECAY_BITS 2 | |
66 | #define DECAY_MASK ((1ULL << DECAY_BITS) - 1) | |
67 | #define MAX_ELEMS (PAGE_SIZE / sizeof(u64)) | |
68 | ||
69 | /* | |
70 | * Threshold amount of inserted elements after which we start spring | |
71 | * cleaning. | |
72 | */ | |
73 | #define CLEAN_ELEMS (MAX_ELEMS >> DECAY_BITS) | |
74 | ||
75 | /* Bits which count the number of errors happened in this 4K page. */ | |
76 | #define COUNT_BITS (PAGE_SHIFT - DECAY_BITS) | |
77 | #define COUNT_MASK ((1ULL << COUNT_BITS) - 1) | |
78 | #define FULL_COUNT_MASK (PAGE_SIZE - 1) | |
79 | ||
80 | /* | |
81 | * u64: [ 63 ... 12 | DECAY_BITS | COUNT_BITS ] | |
82 | */ | |
83 | ||
84 | #define PFN(e) ((e) >> PAGE_SHIFT) | |
85 | #define DECAY(e) (((e) >> COUNT_BITS) & DECAY_MASK) | |
86 | #define COUNT(e) ((unsigned int)(e) & COUNT_MASK) | |
87 | #define FULL_COUNT(e) ((e) & (PAGE_SIZE - 1)) | |
88 | ||
89 | static struct ce_array { | |
90 | u64 *array; /* container page */ | |
91 | unsigned int n; /* number of elements in the array */ | |
92 | ||
93 | unsigned int decay_count; /* | |
94 | * number of element insertions/increments | |
95 | * since the last spring cleaning. | |
96 | */ | |
97 | ||
98 | u64 pfns_poisoned; /* | |
99 | * number of PFNs which got poisoned. | |
100 | */ | |
101 | ||
102 | u64 ces_entered; /* | |
103 | * The number of correctable errors | |
104 | * entered into the collector. | |
105 | */ | |
106 | ||
107 | u64 decays_done; /* | |
108 | * Times we did spring cleaning. | |
109 | */ | |
110 | ||
111 | union { | |
112 | struct { | |
113 | __u32 disabled : 1, /* cmdline disabled */ | |
114 | __resv : 31; | |
115 | }; | |
116 | __u32 flags; | |
117 | }; | |
118 | } ce_arr; | |
119 | ||
120 | static DEFINE_MUTEX(ce_mutex); | |
121 | static u64 dfs_pfn; | |
122 | ||
123 | /* Amount of errors after which we offline */ | |
124 | static unsigned int count_threshold = COUNT_MASK; | |
125 | ||
126 | /* | |
127 | * The timer "decays" element count each timer_interval which is 24hrs by | |
128 | * default. | |
129 | */ | |
130 | ||
131 | #define CEC_TIMER_DEFAULT_INTERVAL 24 * 60 * 60 /* 24 hrs */ | |
132 | #define CEC_TIMER_MIN_INTERVAL 1 * 60 * 60 /* 1h */ | |
133 | #define CEC_TIMER_MAX_INTERVAL 30 * 24 * 60 * 60 /* one month */ | |
134 | static struct timer_list cec_timer; | |
135 | static u64 timer_interval = CEC_TIMER_DEFAULT_INTERVAL; | |
136 | ||
137 | /* | |
138 | * Decrement decay value. We're using DECAY_BITS bits to denote decay of an | |
139 | * element in the array. On insertion and any access, it gets reset to max. | |
140 | */ | |
141 | static void do_spring_cleaning(struct ce_array *ca) | |
142 | { | |
143 | int i; | |
144 | ||
145 | for (i = 0; i < ca->n; i++) { | |
146 | u8 decay = DECAY(ca->array[i]); | |
147 | ||
148 | if (!decay) | |
149 | continue; | |
150 | ||
151 | decay--; | |
152 | ||
153 | ca->array[i] &= ~(DECAY_MASK << COUNT_BITS); | |
154 | ca->array[i] |= (decay << COUNT_BITS); | |
155 | } | |
156 | ca->decay_count = 0; | |
157 | ca->decays_done++; | |
158 | } | |
159 | ||
160 | /* | |
161 | * @interval in seconds | |
162 | */ | |
163 | static void cec_mod_timer(struct timer_list *t, unsigned long interval) | |
164 | { | |
165 | unsigned long iv; | |
166 | ||
167 | iv = interval * HZ + jiffies; | |
168 | ||
169 | mod_timer(t, round_jiffies(iv)); | |
170 | } | |
171 | ||
172 | static void cec_timer_fn(unsigned long data) | |
173 | { | |
174 | struct ce_array *ca = (struct ce_array *)data; | |
175 | ||
176 | do_spring_cleaning(ca); | |
177 | ||
178 | cec_mod_timer(&cec_timer, timer_interval); | |
179 | } | |
180 | ||
181 | /* | |
182 | * @to: index of the smallest element which is >= then @pfn. | |
183 | * | |
184 | * Return the index of the pfn if found, otherwise negative value. | |
185 | */ | |
186 | static int __find_elem(struct ce_array *ca, u64 pfn, unsigned int *to) | |
187 | { | |
188 | u64 this_pfn; | |
189 | int min = 0, max = ca->n; | |
190 | ||
191 | while (min < max) { | |
192 | int tmp = (max + min) >> 1; | |
193 | ||
194 | this_pfn = PFN(ca->array[tmp]); | |
195 | ||
196 | if (this_pfn < pfn) | |
197 | min = tmp + 1; | |
198 | else if (this_pfn > pfn) | |
199 | max = tmp; | |
200 | else { | |
201 | min = tmp; | |
202 | break; | |
203 | } | |
204 | } | |
205 | ||
206 | if (to) | |
207 | *to = min; | |
208 | ||
209 | this_pfn = PFN(ca->array[min]); | |
210 | ||
211 | if (this_pfn == pfn) | |
212 | return min; | |
213 | ||
214 | return -ENOKEY; | |
215 | } | |
216 | ||
217 | static int find_elem(struct ce_array *ca, u64 pfn, unsigned int *to) | |
218 | { | |
219 | WARN_ON(!to); | |
220 | ||
221 | if (!ca->n) { | |
222 | *to = 0; | |
223 | return -ENOKEY; | |
224 | } | |
225 | return __find_elem(ca, pfn, to); | |
226 | } | |
227 | ||
228 | static void del_elem(struct ce_array *ca, int idx) | |
229 | { | |
230 | /* Save us a function call when deleting the last element. */ | |
231 | if (ca->n - (idx + 1)) | |
232 | memmove((void *)&ca->array[idx], | |
233 | (void *)&ca->array[idx + 1], | |
234 | (ca->n - (idx + 1)) * sizeof(u64)); | |
235 | ||
236 | ca->n--; | |
237 | } | |
238 | ||
239 | static u64 del_lru_elem_unlocked(struct ce_array *ca) | |
240 | { | |
241 | unsigned int min = FULL_COUNT_MASK; | |
242 | int i, min_idx = 0; | |
243 | ||
244 | for (i = 0; i < ca->n; i++) { | |
245 | unsigned int this = FULL_COUNT(ca->array[i]); | |
246 | ||
247 | if (min > this) { | |
248 | min = this; | |
249 | min_idx = i; | |
250 | } | |
251 | } | |
252 | ||
253 | del_elem(ca, min_idx); | |
254 | ||
255 | return PFN(ca->array[min_idx]); | |
256 | } | |
257 | ||
258 | /* | |
259 | * We return the 0th pfn in the error case under the assumption that it cannot | |
260 | * be poisoned and excessive CEs in there are a serious deal anyway. | |
261 | */ | |
262 | static u64 __maybe_unused del_lru_elem(void) | |
263 | { | |
264 | struct ce_array *ca = &ce_arr; | |
265 | u64 pfn; | |
266 | ||
267 | if (!ca->n) | |
268 | return 0; | |
269 | ||
270 | mutex_lock(&ce_mutex); | |
271 | pfn = del_lru_elem_unlocked(ca); | |
272 | mutex_unlock(&ce_mutex); | |
273 | ||
274 | return pfn; | |
275 | } | |
276 | ||
277 | ||
278 | int cec_add_elem(u64 pfn) | |
279 | { | |
280 | struct ce_array *ca = &ce_arr; | |
281 | unsigned int to; | |
282 | int count, ret = 0; | |
283 | ||
284 | /* | |
285 | * We can be called very early on the identify_cpu() path where we are | |
286 | * not initialized yet. We ignore the error for simplicity. | |
287 | */ | |
288 | if (!ce_arr.array || ce_arr.disabled) | |
289 | return -ENODEV; | |
290 | ||
291 | ca->ces_entered++; | |
292 | ||
293 | mutex_lock(&ce_mutex); | |
294 | ||
295 | if (ca->n == MAX_ELEMS) | |
296 | WARN_ON(!del_lru_elem_unlocked(ca)); | |
297 | ||
298 | ret = find_elem(ca, pfn, &to); | |
299 | if (ret < 0) { | |
300 | /* | |
301 | * Shift range [to-end] to make room for one more element. | |
302 | */ | |
303 | memmove((void *)&ca->array[to + 1], | |
304 | (void *)&ca->array[to], | |
305 | (ca->n - to) * sizeof(u64)); | |
306 | ||
307 | ca->array[to] = (pfn << PAGE_SHIFT) | | |
308 | (DECAY_MASK << COUNT_BITS) | 1; | |
309 | ||
310 | ca->n++; | |
311 | ||
312 | ret = 0; | |
313 | ||
314 | goto decay; | |
315 | } | |
316 | ||
317 | count = COUNT(ca->array[to]); | |
318 | ||
319 | if (count < count_threshold) { | |
320 | ca->array[to] |= (DECAY_MASK << COUNT_BITS); | |
321 | ca->array[to]++; | |
322 | ||
323 | ret = 0; | |
324 | } else { | |
325 | u64 pfn = ca->array[to] >> PAGE_SHIFT; | |
326 | ||
327 | if (!pfn_valid(pfn)) { | |
328 | pr_warn("CEC: Invalid pfn: 0x%llx\n", pfn); | |
329 | } else { | |
330 | /* We have reached max count for this page, soft-offline it. */ | |
331 | pr_err("Soft-offlining pfn: 0x%llx\n", pfn); | |
332 | memory_failure_queue(pfn, 0, MF_SOFT_OFFLINE); | |
333 | ca->pfns_poisoned++; | |
334 | } | |
335 | ||
336 | del_elem(ca, to); | |
337 | ||
338 | /* | |
339 | * Return a >0 value to denote that we've reached the offlining | |
340 | * threshold. | |
341 | */ | |
342 | ret = 1; | |
343 | ||
344 | goto unlock; | |
345 | } | |
346 | ||
347 | decay: | |
348 | ca->decay_count++; | |
349 | ||
350 | if (ca->decay_count >= CLEAN_ELEMS) | |
351 | do_spring_cleaning(ca); | |
352 | ||
353 | unlock: | |
354 | mutex_unlock(&ce_mutex); | |
355 | ||
356 | return ret; | |
357 | } | |
358 | ||
359 | static int u64_get(void *data, u64 *val) | |
360 | { | |
361 | *val = *(u64 *)data; | |
362 | ||
363 | return 0; | |
364 | } | |
365 | ||
366 | static int pfn_set(void *data, u64 val) | |
367 | { | |
368 | *(u64 *)data = val; | |
369 | ||
370 | return cec_add_elem(val); | |
371 | } | |
372 | ||
373 | DEFINE_DEBUGFS_ATTRIBUTE(pfn_ops, u64_get, pfn_set, "0x%llx\n"); | |
374 | ||
375 | static int decay_interval_set(void *data, u64 val) | |
376 | { | |
377 | *(u64 *)data = val; | |
378 | ||
379 | if (val < CEC_TIMER_MIN_INTERVAL) | |
380 | return -EINVAL; | |
381 | ||
382 | if (val > CEC_TIMER_MAX_INTERVAL) | |
383 | return -EINVAL; | |
384 | ||
385 | timer_interval = val; | |
386 | ||
387 | cec_mod_timer(&cec_timer, timer_interval); | |
388 | return 0; | |
389 | } | |
390 | DEFINE_DEBUGFS_ATTRIBUTE(decay_interval_ops, u64_get, decay_interval_set, "%lld\n"); | |
391 | ||
392 | static int count_threshold_set(void *data, u64 val) | |
393 | { | |
394 | *(u64 *)data = val; | |
395 | ||
396 | if (val > COUNT_MASK) | |
397 | val = COUNT_MASK; | |
398 | ||
399 | count_threshold = val; | |
400 | ||
401 | return 0; | |
402 | } | |
403 | DEFINE_DEBUGFS_ATTRIBUTE(count_threshold_ops, u64_get, count_threshold_set, "%lld\n"); | |
404 | ||
405 | static int array_dump(struct seq_file *m, void *v) | |
406 | { | |
407 | struct ce_array *ca = &ce_arr; | |
408 | u64 prev = 0; | |
409 | int i; | |
410 | ||
411 | mutex_lock(&ce_mutex); | |
412 | ||
413 | seq_printf(m, "{ n: %d\n", ca->n); | |
414 | for (i = 0; i < ca->n; i++) { | |
415 | u64 this = PFN(ca->array[i]); | |
416 | ||
417 | seq_printf(m, " %03d: [%016llx|%03llx]\n", i, this, FULL_COUNT(ca->array[i])); | |
418 | ||
419 | WARN_ON(prev > this); | |
420 | ||
421 | prev = this; | |
422 | } | |
423 | ||
424 | seq_printf(m, "}\n"); | |
425 | ||
426 | seq_printf(m, "Stats:\nCEs: %llu\nofflined pages: %llu\n", | |
427 | ca->ces_entered, ca->pfns_poisoned); | |
428 | ||
429 | seq_printf(m, "Flags: 0x%x\n", ca->flags); | |
430 | ||
431 | seq_printf(m, "Timer interval: %lld seconds\n", timer_interval); | |
432 | seq_printf(m, "Decays: %lld\n", ca->decays_done); | |
433 | ||
434 | seq_printf(m, "Action threshold: %d\n", count_threshold); | |
435 | ||
436 | mutex_unlock(&ce_mutex); | |
437 | ||
438 | return 0; | |
439 | } | |
440 | ||
441 | static int array_open(struct inode *inode, struct file *filp) | |
442 | { | |
443 | return single_open(filp, array_dump, NULL); | |
444 | } | |
445 | ||
446 | static const struct file_operations array_ops = { | |
447 | .owner = THIS_MODULE, | |
448 | .open = array_open, | |
449 | .read = seq_read, | |
450 | .llseek = seq_lseek, | |
451 | .release = single_release, | |
452 | }; | |
453 | ||
454 | static int __init create_debugfs_nodes(void) | |
455 | { | |
456 | struct dentry *d, *pfn, *decay, *count, *array; | |
457 | ||
458 | d = debugfs_create_dir("cec", ras_debugfs_dir); | |
459 | if (!d) { | |
460 | pr_warn("Error creating cec debugfs node!\n"); | |
461 | return -1; | |
462 | } | |
463 | ||
464 | pfn = debugfs_create_file("pfn", S_IRUSR | S_IWUSR, d, &dfs_pfn, &pfn_ops); | |
465 | if (!pfn) { | |
466 | pr_warn("Error creating pfn debugfs node!\n"); | |
467 | goto err; | |
468 | } | |
469 | ||
470 | array = debugfs_create_file("array", S_IRUSR, d, NULL, &array_ops); | |
471 | if (!array) { | |
472 | pr_warn("Error creating array debugfs node!\n"); | |
473 | goto err; | |
474 | } | |
475 | ||
476 | decay = debugfs_create_file("decay_interval", S_IRUSR | S_IWUSR, d, | |
477 | &timer_interval, &decay_interval_ops); | |
478 | if (!decay) { | |
479 | pr_warn("Error creating decay_interval debugfs node!\n"); | |
480 | goto err; | |
481 | } | |
482 | ||
483 | count = debugfs_create_file("count_threshold", S_IRUSR | S_IWUSR, d, | |
484 | &count_threshold, &count_threshold_ops); | |
32288daf | 485 | if (!count) { |
011d8261 BP |
486 | pr_warn("Error creating count_threshold debugfs node!\n"); |
487 | goto err; | |
488 | } | |
489 | ||
490 | ||
491 | return 0; | |
492 | ||
493 | err: | |
494 | debugfs_remove_recursive(d); | |
495 | ||
496 | return 1; | |
497 | } | |
498 | ||
499 | void __init cec_init(void) | |
500 | { | |
501 | if (ce_arr.disabled) | |
502 | return; | |
503 | ||
504 | ce_arr.array = (void *)get_zeroed_page(GFP_KERNEL); | |
505 | if (!ce_arr.array) { | |
506 | pr_err("Error allocating CE array page!\n"); | |
507 | return; | |
508 | } | |
509 | ||
510 | if (create_debugfs_nodes()) | |
511 | return; | |
512 | ||
513 | setup_timer(&cec_timer, cec_timer_fn, (unsigned long)&ce_arr); | |
514 | cec_mod_timer(&cec_timer, CEC_TIMER_DEFAULT_INTERVAL); | |
515 | ||
516 | pr_info("Correctable Errors collector initialized.\n"); | |
517 | } | |
518 | ||
519 | int __init parse_cec_param(char *str) | |
520 | { | |
521 | if (!str) | |
522 | return 0; | |
523 | ||
524 | if (*str == '=') | |
525 | str++; | |
526 | ||
69a33000 | 527 | if (!strcmp(str, "cec_disable")) |
011d8261 BP |
528 | ce_arr.disabled = 1; |
529 | else | |
530 | return 0; | |
531 | ||
532 | return 1; | |
533 | } |