projects / linux-2.6-block.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Merge tag 'perf-tools-fixes-for-v6.17-2025-09-16' of git://git.kernel.org/pub/scm...
[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_ras.c
... / ...
CommitLineData
1/*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 *
23 */
24#include <linux/debugfs.h>
25#include <linux/list.h>
26#include <linux/module.h>
27#include <linux/uaccess.h>
28#include <linux/reboot.h>
29#include <linux/syscalls.h>
30#include <linux/pm_runtime.h>
31#include <linux/list_sort.h>
32
33#include "amdgpu.h"
34#include "amdgpu_ras.h"
35#include "amdgpu_atomfirmware.h"
36#include "amdgpu_xgmi.h"
37#include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
38#include "nbio_v4_3.h"
39#include "nbif_v6_3_1.h"
40#include "nbio_v7_9.h"
41#include "atom.h"
42#include "amdgpu_reset.h"
43#include "amdgpu_psp.h"
44
45#ifdef CONFIG_X86_MCE_AMD
46#include <asm/mce.h>
47
48static bool notifier_registered;
49#endif
50static const char *RAS_FS_NAME = "ras";
51
52const char *ras_error_string[] = {
53 "none",
54 "parity",
55 "single_correctable",
56 "multi_uncorrectable",
57 "poison",
58};
59
60const char *ras_block_string[] = {
61 "umc",
62 "sdma",
63 "gfx",
64 "mmhub",
65 "athub",
66 "pcie_bif",
67 "hdp",
68 "xgmi_wafl",
69 "df",
70 "smn",
71 "sem",
72 "mp0",
73 "mp1",
74 "fuse",
75 "mca",
76 "vcn",
77 "jpeg",
78 "ih",
79 "mpio",
80 "mmsch",
81};
82
83const char *ras_mca_block_string[] = {
84 "mca_mp0",
85 "mca_mp1",
86 "mca_mpio",
87 "mca_iohc",
88};
89
90struct amdgpu_ras_block_list {
91 /* ras block link */
92 struct list_head node;
93
94 struct amdgpu_ras_block_object *ras_obj;
95};
96
97const char *get_ras_block_str(struct ras_common_if *ras_block)
98{
99 if (!ras_block)
100 return "NULL";
101
102 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT ||
103 ras_block->block >= ARRAY_SIZE(ras_block_string))
104 return "OUT OF RANGE";
105
106 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA)
107 return ras_mca_block_string[ras_block->sub_block_index];
108
109 return ras_block_string[ras_block->block];
110}
111
112#define ras_block_str(_BLOCK_) \
113 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range")
114
115#define ras_err_str(i) (ras_error_string[ffs(i)])
116
117#define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
118
119/* inject address is 52 bits */
120#define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52)
121
122/* typical ECC bad page rate is 1 bad page per 100MB VRAM */
123#define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL)
124
125#define MAX_UMC_POISON_POLLING_TIME_ASYNC 300 //ms
126
127#define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms
128
129#define MAX_FLUSH_RETIRE_DWORK_TIMES 100
130
131enum amdgpu_ras_retire_page_reservation {
132 AMDGPU_RAS_RETIRE_PAGE_RESERVED,
133 AMDGPU_RAS_RETIRE_PAGE_PENDING,
134 AMDGPU_RAS_RETIRE_PAGE_FAULT,
135};
136
137atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0);
138
139static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
140 uint64_t addr);
141static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
142 uint64_t addr);
143#ifdef CONFIG_X86_MCE_AMD
144static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev);
145struct mce_notifier_adev_list {
146 struct amdgpu_device *devs[MAX_GPU_INSTANCE];
147 int num_gpu;
148};
149static struct mce_notifier_adev_list mce_adev_list;
150#endif
151
152void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
153{
154 if (adev && amdgpu_ras_get_context(adev))
155 amdgpu_ras_get_context(adev)->error_query_ready = ready;
156}
157
158static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev)
159{
160 if (adev && amdgpu_ras_get_context(adev))
161 return amdgpu_ras_get_context(adev)->error_query_ready;
162
163 return false;
164}
165
166static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address)
167{
168 struct ras_err_data err_data;
169 struct eeprom_table_record err_rec;
170 int ret;
171
172 if ((address >= adev->gmc.mc_vram_size) ||
173 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
174 dev_warn(adev->dev,
175 "RAS WARN: input address 0x%llx is invalid.\n",
176 address);
177 return -EINVAL;
178 }
179
180 if (amdgpu_ras_check_bad_page(adev, address)) {
181 dev_warn(adev->dev,
182 "RAS WARN: 0x%llx has already been marked as bad page!\n",
183 address);
184 return 0;
185 }
186
187 ret = amdgpu_ras_error_data_init(&err_data);
188 if (ret)
189 return ret;
190
191 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
192 err_data.err_addr = &err_rec;
193 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0);
194
195 if (amdgpu_bad_page_threshold != 0) {
196 amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
197 err_data.err_addr_cnt, false);
198 amdgpu_ras_save_bad_pages(adev, NULL);
199 }
200
201 amdgpu_ras_error_data_fini(&err_data);
202
203 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n");
204 dev_warn(adev->dev, "Clear EEPROM:\n");
205 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n");
206
207 return 0;
208}
209
210static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
211 size_t size, loff_t *pos)
212{
213 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
214 struct ras_query_if info = {
215 .head = obj->head,
216 };
217 ssize_t s;
218 char val[128];
219
220 if (amdgpu_ras_query_error_status(obj->adev, &info))
221 return -EINVAL;
222
223 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */
224 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
225 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
226 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
227 dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
228 }
229
230 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n",
231 "ue", info.ue_count,
232 "ce", info.ce_count);
233 if (*pos >= s)
234 return 0;
235
236 s -= *pos;
237 s = min_t(u64, s, size);
238
239
240 if (copy_to_user(buf, &val[*pos], s))
241 return -EINVAL;
242
243 *pos += s;
244
245 return s;
246}
247
248static const struct file_operations amdgpu_ras_debugfs_ops = {
249 .owner = THIS_MODULE,
250 .read = amdgpu_ras_debugfs_read,
251 .write = NULL,
252 .llseek = default_llseek
253};
254
255static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
256{
257 int i;
258
259 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
260 *block_id = i;
261 if (strcmp(name, ras_block_string[i]) == 0)
262 return 0;
263 }
264 return -EINVAL;
265}
266
267static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
268 const char __user *buf, size_t size,
269 loff_t *pos, struct ras_debug_if *data)
270{
271 ssize_t s = min_t(u64, 64, size);
272 char str[65];
273 char block_name[33];
274 char err[9] = "ue";
275 int op = -1;
276 int block_id;
277 uint32_t sub_block;
278 u64 address, value;
279 /* default value is 0 if the mask is not set by user */
280 u32 instance_mask = 0;
281
282 if (*pos)
283 return -EINVAL;
284 *pos = size;
285
286 memset(str, 0, sizeof(str));
287 memset(data, 0, sizeof(*data));
288
289 if (copy_from_user(str, buf, s))
290 return -EINVAL;
291
292 if (sscanf(str, "disable %32s", block_name) == 1)
293 op = 0;
294 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
295 op = 1;
296 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
297 op = 2;
298 else if (strstr(str, "retire_page") != NULL)
299 op = 3;
300 else if (str[0] && str[1] && str[2] && str[3])
301 /* ascii string, but commands are not matched. */
302 return -EINVAL;
303
304 if (op != -1) {
305 if (op == 3) {
306 if (sscanf(str, "%*s 0x%llx", &address) != 1 &&
307 sscanf(str, "%*s %llu", &address) != 1)
308 return -EINVAL;
309
310 data->op = op;
311 data->inject.address = address;
312
313 return 0;
314 }
315
316 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id))
317 return -EINVAL;
318
319 data->head.block = block_id;
320 /* only ue, ce and poison errors are supported */
321 if (!memcmp("ue", err, 2))
322 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
323 else if (!memcmp("ce", err, 2))
324 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
325 else if (!memcmp("poison", err, 6))
326 data->head.type = AMDGPU_RAS_ERROR__POISON;
327 else
328 return -EINVAL;
329
330 data->op = op;
331
332 if (op == 2) {
333 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x",
334 &sub_block, &address, &value, &instance_mask) != 4 &&
335 sscanf(str, "%*s %*s %*s %u %llu %llu %u",
336 &sub_block, &address, &value, &instance_mask) != 4 &&
337 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx",
338 &sub_block, &address, &value) != 3 &&
339 sscanf(str, "%*s %*s %*s %u %llu %llu",
340 &sub_block, &address, &value) != 3)
341 return -EINVAL;
342 data->head.sub_block_index = sub_block;
343 data->inject.address = address;
344 data->inject.value = value;
345 data->inject.instance_mask = instance_mask;
346 }
347 } else {
348 if (size < sizeof(*data))
349 return -EINVAL;
350
351 if (copy_from_user(data, buf, sizeof(*data)))
352 return -EINVAL;
353 }
354
355 return 0;
356}
357
358static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev,
359 struct ras_debug_if *data)
360{
361 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
362 uint32_t mask, inst_mask = data->inject.instance_mask;
363
364 /* no need to set instance mask if there is only one instance */
365 if (num_xcc <= 1 && inst_mask) {
366 data->inject.instance_mask = 0;
367 dev_dbg(adev->dev,
368 "RAS inject mask(0x%x) isn't supported and force it to 0.\n",
369 inst_mask);
370
371 return;
372 }
373
374 switch (data->head.block) {
375 case AMDGPU_RAS_BLOCK__GFX:
376 mask = GENMASK(num_xcc - 1, 0);
377 break;
378 case AMDGPU_RAS_BLOCK__SDMA:
379 mask = GENMASK(adev->sdma.num_instances - 1, 0);
380 break;
381 case AMDGPU_RAS_BLOCK__VCN:
382 case AMDGPU_RAS_BLOCK__JPEG:
383 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0);
384 break;
385 default:
386 mask = inst_mask;
387 break;
388 }
389
390 /* remove invalid bits in instance mask */
391 data->inject.instance_mask &= mask;
392 if (inst_mask != data->inject.instance_mask)
393 dev_dbg(adev->dev,
394 "Adjust RAS inject mask 0x%x to 0x%x\n",
395 inst_mask, data->inject.instance_mask);
396}
397
398/**
399 * DOC: AMDGPU RAS debugfs control interface
400 *
401 * The control interface accepts struct ras_debug_if which has two members.
402 *
403 * First member: ras_debug_if::head or ras_debug_if::inject.
404 *
405 * head is used to indicate which IP block will be under control.
406 *
407 * head has four members, they are block, type, sub_block_index, name.
408 * block: which IP will be under control.
409 * type: what kind of error will be enabled/disabled/injected.
410 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
411 * name: the name of IP.
412 *
413 * inject has three more members than head, they are address, value and mask.
414 * As their names indicate, inject operation will write the
415 * value to the address.
416 *
417 * The second member: struct ras_debug_if::op.
418 * It has three kinds of operations.
419 *
420 * - 0: disable RAS on the block. Take ::head as its data.
421 * - 1: enable RAS on the block. Take ::head as its data.
422 * - 2: inject errors on the block. Take ::inject as its data.
423 *
424 * How to use the interface?
425 *
426 * In a program
427 *
428 * Copy the struct ras_debug_if in your code and initialize it.
429 * Write the struct to the control interface.
430 *
431 * From shell
432 *
433 * .. code-block:: bash
434 *
435 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
436 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
437 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
438 *
439 * Where N, is the card which you want to affect.
440 *
441 * "disable" requires only the block.
442 * "enable" requires the block and error type.
443 * "inject" requires the block, error type, address, and value.
444 *
445 * The block is one of: umc, sdma, gfx, etc.
446 * see ras_block_string[] for details
447 *
448 * The error type is one of: ue, ce and poison where,
449 * ue is multi-uncorrectable
450 * ce is single-correctable
451 * poison is poison
452 *
453 * The sub-block is a the sub-block index, pass 0 if there is no sub-block.
454 * The address and value are hexadecimal numbers, leading 0x is optional.
455 * The mask means instance mask, is optional, default value is 0x1.
456 *
457 * For instance,
458 *
459 * .. code-block:: bash
460 *
461 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
462 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl
463 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
464 *
465 * How to check the result of the operation?
466 *
467 * To check disable/enable, see "ras" features at,
468 * /sys/class/drm/card[0/1/2...]/device/ras/features
469 *
470 * To check inject, see the corresponding error count at,
471 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count
472 *
473 * .. note::
474 * Operations are only allowed on blocks which are supported.
475 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask
476 * to see which blocks support RAS on a particular asic.
477 *
478 */
479static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f,
480 const char __user *buf,
481 size_t size, loff_t *pos)
482{
483 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
484 struct ras_debug_if data;
485 int ret = 0;
486
487 if (!amdgpu_ras_get_error_query_ready(adev)) {
488 dev_warn(adev->dev, "RAS WARN: error injection "
489 "currently inaccessible\n");
490 return size;
491 }
492
493 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
494 if (ret)
495 return ret;
496
497 if (data.op == 3) {
498 ret = amdgpu_reserve_page_direct(adev, data.inject.address);
499 if (!ret)
500 return size;
501 else
502 return ret;
503 }
504
505 if (!amdgpu_ras_is_supported(adev, data.head.block))
506 return -EINVAL;
507
508 switch (data.op) {
509 case 0:
510 ret = amdgpu_ras_feature_enable(adev, &data.head, 0);
511 break;
512 case 1:
513 ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
514 break;
515 case 2:
516 if ((data.inject.address >= adev->gmc.mc_vram_size &&
517 adev->gmc.mc_vram_size) ||
518 (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
519 dev_warn(adev->dev, "RAS WARN: input address "
520 "0x%llx is invalid.",
521 data.inject.address);
522 ret = -EINVAL;
523 break;
524 }
525
526 /* umc ce/ue error injection for a bad page is not allowed */
527 if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) &&
528 amdgpu_ras_check_bad_page(adev, data.inject.address)) {
529 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has "
530 "already been marked as bad!\n",
531 data.inject.address);
532 break;
533 }
534
535 amdgpu_ras_instance_mask_check(adev, &data);
536
537 /* data.inject.address is offset instead of absolute gpu address */
538 ret = amdgpu_ras_error_inject(adev, &data.inject);
539 break;
540 default:
541 ret = -EINVAL;
542 break;
543 }
544
545 if (ret)
546 return ret;
547
548 return size;
549}
550
551/**
552 * DOC: AMDGPU RAS debugfs EEPROM table reset interface
553 *
554 * Some boards contain an EEPROM which is used to persistently store a list of
555 * bad pages which experiences ECC errors in vram. This interface provides
556 * a way to reset the EEPROM, e.g., after testing error injection.
557 *
558 * Usage:
559 *
560 * .. code-block:: bash
561 *
562 * echo 1 > ../ras/ras_eeprom_reset
563 *
564 * will reset EEPROM table to 0 entries.
565 *
566 */
567static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f,
568 const char __user *buf,
569 size_t size, loff_t *pos)
570{
571 struct amdgpu_device *adev =
572 (struct amdgpu_device *)file_inode(f)->i_private;
573 int ret;
574
575 ret = amdgpu_ras_eeprom_reset_table(
576 &(amdgpu_ras_get_context(adev)->eeprom_control));
577
578 if (!ret) {
579 /* Something was written to EEPROM.
580 */
581 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS;
582 return size;
583 } else {
584 return ret;
585 }
586}
587
588static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
589 .owner = THIS_MODULE,
590 .read = NULL,
591 .write = amdgpu_ras_debugfs_ctrl_write,
592 .llseek = default_llseek
593};
594
595static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = {
596 .owner = THIS_MODULE,
597 .read = NULL,
598 .write = amdgpu_ras_debugfs_eeprom_write,
599 .llseek = default_llseek
600};
601
602/**
603 * DOC: AMDGPU RAS sysfs Error Count Interface
604 *
605 * It allows the user to read the error count for each IP block on the gpu through
606 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
607 *
608 * It outputs the multiple lines which report the uncorrected (ue) and corrected
609 * (ce) error counts.
610 *
611 * The format of one line is below,
612 *
613 * [ce|ue]: count
614 *
615 * Example:
616 *
617 * .. code-block:: bash
618 *
619 * ue: 0
620 * ce: 1
621 *
622 */
623static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
624 struct device_attribute *attr, char *buf)
625{
626 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
627 struct ras_query_if info = {
628 .head = obj->head,
629 };
630
631 if (!amdgpu_ras_get_error_query_ready(obj->adev))
632 return sysfs_emit(buf, "Query currently inaccessible\n");
633
634 if (amdgpu_ras_query_error_status(obj->adev, &info))
635 return -EINVAL;
636
637 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
638 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
639 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
640 dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
641 }
642
643 if (info.head.block == AMDGPU_RAS_BLOCK__UMC)
644 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
645 "ce", info.ce_count, "de", info.de_count);
646 else
647 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count,
648 "ce", info.ce_count);
649}
650
651/* obj begin */
652
653#define get_obj(obj) do { (obj)->use++; } while (0)
654#define alive_obj(obj) ((obj)->use)
655
656static inline void put_obj(struct ras_manager *obj)
657{
658 if (obj && (--obj->use == 0)) {
659 list_del(&obj->node);
660 amdgpu_ras_error_data_fini(&obj->err_data);
661 }
662
663 if (obj && (obj->use < 0))
664 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head));
665}
666
667/* make one obj and return it. */
668static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
669 struct ras_common_if *head)
670{
671 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
672 struct ras_manager *obj;
673
674 if (!adev->ras_enabled || !con)
675 return NULL;
676
677 if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
678 return NULL;
679
680 if (head->block == AMDGPU_RAS_BLOCK__MCA) {
681 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
682 return NULL;
683
684 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
685 } else
686 obj = &con->objs[head->block];
687
688 /* already exist. return obj? */
689 if (alive_obj(obj))
690 return NULL;
691
692 if (amdgpu_ras_error_data_init(&obj->err_data))
693 return NULL;
694
695 obj->head = *head;
696 obj->adev = adev;
697 list_add(&obj->node, &con->head);
698 get_obj(obj);
699
700 return obj;
701}
702
703/* return an obj equal to head, or the first when head is NULL */
704struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
705 struct ras_common_if *head)
706{
707 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
708 struct ras_manager *obj;
709 int i;
710
711 if (!adev->ras_enabled || !con)
712 return NULL;
713
714 if (head) {
715 if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
716 return NULL;
717
718 if (head->block == AMDGPU_RAS_BLOCK__MCA) {
719 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
720 return NULL;
721
722 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
723 } else
724 obj = &con->objs[head->block];
725
726 if (alive_obj(obj))
727 return obj;
728 } else {
729 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
730 obj = &con->objs[i];
731 if (alive_obj(obj))
732 return obj;
733 }
734 }
735
736 return NULL;
737}
738/* obj end */
739
740/* feature ctl begin */
741static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
742 struct ras_common_if *head)
743{
744 return adev->ras_hw_enabled & BIT(head->block);
745}
746
747static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
748 struct ras_common_if *head)
749{
750 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
751
752 return con->features & BIT(head->block);
753}
754
755/*
756 * if obj is not created, then create one.
757 * set feature enable flag.
758 */
759static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
760 struct ras_common_if *head, int enable)
761{
762 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
763 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
764
765 /* If hardware does not support ras, then do not create obj.
766 * But if hardware support ras, we can create the obj.
767 * Ras framework checks con->hw_supported to see if it need do
768 * corresponding initialization.
769 * IP checks con->support to see if it need disable ras.
770 */
771 if (!amdgpu_ras_is_feature_allowed(adev, head))
772 return 0;
773
774 if (enable) {
775 if (!obj) {
776 obj = amdgpu_ras_create_obj(adev, head);
777 if (!obj)
778 return -EINVAL;
779 } else {
780 /* In case we create obj somewhere else */
781 get_obj(obj);
782 }
783 con->features |= BIT(head->block);
784 } else {
785 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
786 con->features &= ~BIT(head->block);
787 put_obj(obj);
788 }
789 }
790
791 return 0;
792}
793
794/* wrapper of psp_ras_enable_features */
795int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
796 struct ras_common_if *head, bool enable)
797{
798 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
799 union ta_ras_cmd_input *info;
800 int ret;
801
802 if (!con)
803 return -EINVAL;
804
805 /* For non-gfx ip, do not enable ras feature if it is not allowed */
806 /* For gfx ip, regardless of feature support status, */
807 /* Force issue enable or disable ras feature commands */
808 if (head->block != AMDGPU_RAS_BLOCK__GFX &&
809 !amdgpu_ras_is_feature_allowed(adev, head))
810 return 0;
811
812 /* Only enable gfx ras feature from host side */
813 if (head->block == AMDGPU_RAS_BLOCK__GFX &&
814 !amdgpu_sriov_vf(adev) &&
815 !amdgpu_ras_intr_triggered()) {
816 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL);
817 if (!info)
818 return -ENOMEM;
819
820 if (!enable) {
821 info->disable_features = (struct ta_ras_disable_features_input) {
822 .block_id = amdgpu_ras_block_to_ta(head->block),
823 .error_type = amdgpu_ras_error_to_ta(head->type),
824 };
825 } else {
826 info->enable_features = (struct ta_ras_enable_features_input) {
827 .block_id = amdgpu_ras_block_to_ta(head->block),
828 .error_type = amdgpu_ras_error_to_ta(head->type),
829 };
830 }
831
832 ret = psp_ras_enable_features(&adev->psp, info, enable);
833 if (ret) {
834 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n",
835 enable ? "enable":"disable",
836 get_ras_block_str(head),
837 amdgpu_ras_is_poison_mode_supported(adev), ret);
838 kfree(info);
839 return ret;
840 }
841
842 kfree(info);
843 }
844
845 /* setup the obj */
846 __amdgpu_ras_feature_enable(adev, head, enable);
847
848 return 0;
849}
850
851/* Only used in device probe stage and called only once. */
852int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
853 struct ras_common_if *head, bool enable)
854{
855 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
856 int ret;
857
858 if (!con)
859 return -EINVAL;
860
861 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
862 if (enable) {
863 /* There is no harm to issue a ras TA cmd regardless of
864 * the currecnt ras state.
865 * If current state == target state, it will do nothing
866 * But sometimes it requests driver to reset and repost
867 * with error code -EAGAIN.
868 */
869 ret = amdgpu_ras_feature_enable(adev, head, 1);
870 /* With old ras TA, we might fail to enable ras.
871 * Log it and just setup the object.
872 * TODO need remove this WA in the future.
873 */
874 if (ret == -EINVAL) {
875 ret = __amdgpu_ras_feature_enable(adev, head, 1);
876 if (!ret)
877 dev_info(adev->dev,
878 "RAS INFO: %s setup object\n",
879 get_ras_block_str(head));
880 }
881 } else {
882 /* setup the object then issue a ras TA disable cmd.*/
883 ret = __amdgpu_ras_feature_enable(adev, head, 1);
884 if (ret)
885 return ret;
886
887 /* gfx block ras disable cmd must send to ras-ta */
888 if (head->block == AMDGPU_RAS_BLOCK__GFX)
889 con->features |= BIT(head->block);
890
891 ret = amdgpu_ras_feature_enable(adev, head, 0);
892
893 /* clean gfx block ras features flag */
894 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
895 con->features &= ~BIT(head->block);
896 }
897 } else
898 ret = amdgpu_ras_feature_enable(adev, head, enable);
899
900 return ret;
901}
902
903static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
904 bool bypass)
905{
906 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
907 struct ras_manager *obj, *tmp;
908
909 list_for_each_entry_safe(obj, tmp, &con->head, node) {
910 /* bypass psp.
911 * aka just release the obj and corresponding flags
912 */
913 if (bypass) {
914 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0))
915 break;
916 } else {
917 if (amdgpu_ras_feature_enable(adev, &obj->head, 0))
918 break;
919 }
920 }
921
922 return con->features;
923}
924
925static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
926 bool bypass)
927{
928 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
929 int i;
930 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE;
931
932 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
933 struct ras_common_if head = {
934 .block = i,
935 .type = default_ras_type,
936 .sub_block_index = 0,
937 };
938
939 if (i == AMDGPU_RAS_BLOCK__MCA)
940 continue;
941
942 if (bypass) {
943 /*
944 * bypass psp. vbios enable ras for us.
945 * so just create the obj
946 */
947 if (__amdgpu_ras_feature_enable(adev, &head, 1))
948 break;
949 } else {
950 if (amdgpu_ras_feature_enable(adev, &head, 1))
951 break;
952 }
953 }
954
955 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
956 struct ras_common_if head = {
957 .block = AMDGPU_RAS_BLOCK__MCA,
958 .type = default_ras_type,
959 .sub_block_index = i,
960 };
961
962 if (bypass) {
963 /*
964 * bypass psp. vbios enable ras for us.
965 * so just create the obj
966 */
967 if (__amdgpu_ras_feature_enable(adev, &head, 1))
968 break;
969 } else {
970 if (amdgpu_ras_feature_enable(adev, &head, 1))
971 break;
972 }
973 }
974
975 return con->features;
976}
977/* feature ctl end */
978
979static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj,
980 enum amdgpu_ras_block block)
981{
982 if (!block_obj)
983 return -EINVAL;
984
985 if (block_obj->ras_comm.block == block)
986 return 0;
987
988 return -EINVAL;
989}
990
991static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev,
992 enum amdgpu_ras_block block, uint32_t sub_block_index)
993{
994 struct amdgpu_ras_block_list *node, *tmp;
995 struct amdgpu_ras_block_object *obj;
996
997 if (block >= AMDGPU_RAS_BLOCK__LAST)
998 return NULL;
999
1000 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
1001 if (!node->ras_obj) {
1002 dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
1003 continue;
1004 }
1005
1006 obj = node->ras_obj;
1007 if (obj->ras_block_match) {
1008 if (obj->ras_block_match(obj, block, sub_block_index) == 0)
1009 return obj;
1010 } else {
1011 if (amdgpu_ras_block_match_default(obj, block) == 0)
1012 return obj;
1013 }
1014 }
1015
1016 return NULL;
1017}
1018
1019static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data)
1020{
1021 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
1022 int ret = 0;
1023
1024 /*
1025 * choosing right query method according to
1026 * whether smu support query error information
1027 */
1028 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc));
1029 if (ret == -EOPNOTSUPP) {
1030 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1031 adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
1032 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
1033
1034 /* umc query_ras_error_address is also responsible for clearing
1035 * error status
1036 */
1037 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1038 adev->umc.ras->ras_block.hw_ops->query_ras_error_address)
1039 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data);
1040 } else if (!ret) {
1041 if (adev->umc.ras &&
1042 adev->umc.ras->ecc_info_query_ras_error_count)
1043 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data);
1044
1045 if (adev->umc.ras &&
1046 adev->umc.ras->ecc_info_query_ras_error_address)
1047 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data);
1048 }
1049}
1050
1051static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev,
1052 struct ras_manager *ras_mgr,
1053 struct ras_err_data *err_data,
1054 struct ras_query_context *qctx,
1055 const char *blk_name,
1056 bool is_ue,
1057 bool is_de)
1058{
1059 struct amdgpu_smuio_mcm_config_info *mcm_info;
1060 struct ras_err_node *err_node;
1061 struct ras_err_info *err_info;
1062 u64 event_id = qctx->evid.event_id;
1063
1064 if (is_ue) {
1065 for_each_ras_error(err_node, err_data) {
1066 err_info = &err_node->err_info;
1067 mcm_info = &err_info->mcm_info;
1068 if (err_info->ue_count) {
1069 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1070 "%lld new uncorrectable hardware errors detected in %s block\n",
1071 mcm_info->socket_id,
1072 mcm_info->die_id,
1073 err_info->ue_count,
1074 blk_name);
1075 }
1076 }
1077
1078 for_each_ras_error(err_node, &ras_mgr->err_data) {
1079 err_info = &err_node->err_info;
1080 mcm_info = &err_info->mcm_info;
1081 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1082 "%lld uncorrectable hardware errors detected in total in %s block\n",
1083 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name);
1084 }
1085
1086 } else {
1087 if (is_de) {
1088 for_each_ras_error(err_node, err_data) {
1089 err_info = &err_node->err_info;
1090 mcm_info = &err_info->mcm_info;
1091 if (err_info->de_count) {
1092 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1093 "%lld new deferred hardware errors detected in %s block\n",
1094 mcm_info->socket_id,
1095 mcm_info->die_id,
1096 err_info->de_count,
1097 blk_name);
1098 }
1099 }
1100
1101 for_each_ras_error(err_node, &ras_mgr->err_data) {
1102 err_info = &err_node->err_info;
1103 mcm_info = &err_info->mcm_info;
1104 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1105 "%lld deferred hardware errors detected in total in %s block\n",
1106 mcm_info->socket_id, mcm_info->die_id,
1107 err_info->de_count, blk_name);
1108 }
1109 } else {
1110 if (adev->debug_disable_ce_logs)
1111 return;
1112
1113 for_each_ras_error(err_node, err_data) {
1114 err_info = &err_node->err_info;
1115 mcm_info = &err_info->mcm_info;
1116 if (err_info->ce_count) {
1117 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1118 "%lld new correctable hardware errors detected in %s block\n",
1119 mcm_info->socket_id,
1120 mcm_info->die_id,
1121 err_info->ce_count,
1122 blk_name);
1123 }
1124 }
1125
1126 for_each_ras_error(err_node, &ras_mgr->err_data) {
1127 err_info = &err_node->err_info;
1128 mcm_info = &err_info->mcm_info;
1129 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1130 "%lld correctable hardware errors detected in total in %s block\n",
1131 mcm_info->socket_id, mcm_info->die_id,
1132 err_info->ce_count, blk_name);
1133 }
1134 }
1135 }
1136}
1137
1138static inline bool err_data_has_source_info(struct ras_err_data *data)
1139{
1140 return !list_empty(&data->err_node_list);
1141}
1142
1143static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev,
1144 struct ras_query_if *query_if,
1145 struct ras_err_data *err_data,
1146 struct ras_query_context *qctx)
1147{
1148 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head);
1149 const char *blk_name = get_ras_block_str(&query_if->head);
1150 u64 event_id = qctx->evid.event_id;
1151
1152 if (err_data->ce_count) {
1153 if (err_data_has_source_info(err_data)) {
1154 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1155 blk_name, false, false);
1156 } else if (!adev->aid_mask &&
1157 adev->smuio.funcs &&
1158 adev->smuio.funcs->get_socket_id &&
1159 adev->smuio.funcs->get_die_id) {
1160 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1161 "%ld correctable hardware errors "
1162 "detected in %s block\n",
1163 adev->smuio.funcs->get_socket_id(adev),
1164 adev->smuio.funcs->get_die_id(adev),
1165 ras_mgr->err_data.ce_count,
1166 blk_name);
1167 } else {
1168 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors "
1169 "detected in %s block\n",
1170 ras_mgr->err_data.ce_count,
1171 blk_name);
1172 }
1173 }
1174
1175 if (err_data->ue_count) {
1176 if (err_data_has_source_info(err_data)) {
1177 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1178 blk_name, true, false);
1179 } else if (!adev->aid_mask &&
1180 adev->smuio.funcs &&
1181 adev->smuio.funcs->get_socket_id &&
1182 adev->smuio.funcs->get_die_id) {
1183 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1184 "%ld uncorrectable hardware errors "
1185 "detected in %s block\n",
1186 adev->smuio.funcs->get_socket_id(adev),
1187 adev->smuio.funcs->get_die_id(adev),
1188 ras_mgr->err_data.ue_count,
1189 blk_name);
1190 } else {
1191 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors "
1192 "detected in %s block\n",
1193 ras_mgr->err_data.ue_count,
1194 blk_name);
1195 }
1196 }
1197
1198 if (err_data->de_count) {
1199 if (err_data_has_source_info(err_data)) {
1200 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
1201 blk_name, false, true);
1202 } else if (!adev->aid_mask &&
1203 adev->smuio.funcs &&
1204 adev->smuio.funcs->get_socket_id &&
1205 adev->smuio.funcs->get_die_id) {
1206 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1207 "%ld deferred hardware errors "
1208 "detected in %s block\n",
1209 adev->smuio.funcs->get_socket_id(adev),
1210 adev->smuio.funcs->get_die_id(adev),
1211 ras_mgr->err_data.de_count,
1212 blk_name);
1213 } else {
1214 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors "
1215 "detected in %s block\n",
1216 ras_mgr->err_data.de_count,
1217 blk_name);
1218 }
1219 }
1220}
1221
1222static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev,
1223 struct ras_query_if *query_if,
1224 struct ras_err_data *err_data,
1225 struct ras_query_context *qctx)
1226{
1227 unsigned long new_ue, new_ce, new_de;
1228 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head);
1229 const char *blk_name = get_ras_block_str(&query_if->head);
1230 u64 event_id = qctx->evid.event_id;
1231
1232 new_ce = err_data->ce_count - obj->err_data.ce_count;
1233 new_ue = err_data->ue_count - obj->err_data.ue_count;
1234 new_de = err_data->de_count - obj->err_data.de_count;
1235
1236 if (new_ce) {
1237 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors "
1238 "detected in %s block\n",
1239 new_ce,
1240 blk_name);
1241 }
1242
1243 if (new_ue) {
1244 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors "
1245 "detected in %s block\n",
1246 new_ue,
1247 blk_name);
1248 }
1249
1250 if (new_de) {
1251 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors "
1252 "detected in %s block\n",
1253 new_de,
1254 blk_name);
1255 }
1256}
1257
1258static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data)
1259{
1260 struct ras_err_node *err_node;
1261 struct ras_err_info *err_info;
1262
1263 if (err_data_has_source_info(err_data)) {
1264 for_each_ras_error(err_node, err_data) {
1265 err_info = &err_node->err_info;
1266 amdgpu_ras_error_statistic_de_count(&obj->err_data,
1267 &err_info->mcm_info, err_info->de_count);
1268 amdgpu_ras_error_statistic_ce_count(&obj->err_data,
1269 &err_info->mcm_info, err_info->ce_count);
1270 amdgpu_ras_error_statistic_ue_count(&obj->err_data,
1271 &err_info->mcm_info, err_info->ue_count);
1272 }
1273 } else {
1274 /* for legacy asic path which doesn't has error source info */
1275 obj->err_data.ue_count += err_data->ue_count;
1276 obj->err_data.ce_count += err_data->ce_count;
1277 obj->err_data.de_count += err_data->de_count;
1278 }
1279}
1280
1281static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj,
1282 struct ras_err_data *err_data)
1283{
1284 /* Host reports absolute counts */
1285 obj->err_data.ue_count = err_data->ue_count;
1286 obj->err_data.ce_count = err_data->ce_count;
1287 obj->err_data.de_count = err_data->de_count;
1288}
1289
1290static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
1291{
1292 struct ras_common_if head;
1293
1294 memset(&head, 0, sizeof(head));
1295 head.block = blk;
1296
1297 return amdgpu_ras_find_obj(adev, &head);
1298}
1299
1300int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
1301 const struct aca_info *aca_info, void *data)
1302{
1303 struct ras_manager *obj;
1304
1305 /* in resume phase, no need to create aca fs node */
1306 if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
1307 return 0;
1308
1309 obj = get_ras_manager(adev, blk);
1310 if (!obj)
1311 return -EINVAL;
1312
1313 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data);
1314}
1315
1316int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
1317{
1318 struct ras_manager *obj;
1319
1320 obj = get_ras_manager(adev, blk);
1321 if (!obj)
1322 return -EINVAL;
1323
1324 amdgpu_aca_remove_handle(&obj->aca_handle);
1325
1326 return 0;
1327}
1328
1329static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
1330 enum aca_error_type type, struct ras_err_data *err_data,
1331 struct ras_query_context *qctx)
1332{
1333 struct ras_manager *obj;
1334
1335 obj = get_ras_manager(adev, blk);
1336 if (!obj)
1337 return -EINVAL;
1338
1339 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx);
1340}
1341
1342ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr,
1343 struct aca_handle *handle, char *buf, void *data)
1344{
1345 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle);
1346 struct ras_query_if info = {
1347 .head = obj->head,
1348 };
1349
1350 if (!amdgpu_ras_get_error_query_ready(obj->adev))
1351 return sysfs_emit(buf, "Query currently inaccessible\n");
1352
1353 if (amdgpu_ras_query_error_status(obj->adev, &info))
1354 return -EINVAL;
1355
1356 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
1357 "ce", info.ce_count, "de", info.de_count);
1358}
1359
1360static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev,
1361 struct ras_query_if *info,
1362 struct ras_err_data *err_data,
1363 struct ras_query_context *qctx,
1364 unsigned int error_query_mode)
1365{
1366 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT;
1367 struct amdgpu_ras_block_object *block_obj = NULL;
1368 int ret;
1369
1370 if (blk == AMDGPU_RAS_BLOCK_COUNT)
1371 return -EINVAL;
1372
1373 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY)
1374 return -EINVAL;
1375
1376 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
1377 return amdgpu_virt_req_ras_err_count(adev, blk, err_data);
1378 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) {
1379 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) {
1380 amdgpu_ras_get_ecc_info(adev, err_data);
1381 } else {
1382 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0);
1383 if (!block_obj || !block_obj->hw_ops) {
1384 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1385 get_ras_block_str(&info->head));
1386 return -EINVAL;
1387 }
1388
1389 if (block_obj->hw_ops->query_ras_error_count)
1390 block_obj->hw_ops->query_ras_error_count(adev, err_data);
1391
1392 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) ||
1393 (info->head.block == AMDGPU_RAS_BLOCK__GFX) ||
1394 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) {
1395 if (block_obj->hw_ops->query_ras_error_status)
1396 block_obj->hw_ops->query_ras_error_status(adev);
1397 }
1398 }
1399 } else {
1400 if (amdgpu_aca_is_enabled(adev)) {
1401 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx);
1402 if (ret)
1403 return ret;
1404
1405 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx);
1406 if (ret)
1407 return ret;
1408
1409 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx);
1410 if (ret)
1411 return ret;
1412 } else {
1413 /* FIXME: add code to check return value later */
1414 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx);
1415 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx);
1416 }
1417 }
1418
1419 return 0;
1420}
1421
1422/* query/inject/cure begin */
1423static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev,
1424 struct ras_query_if *info,
1425 enum ras_event_type type)
1426{
1427 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1428 struct ras_err_data err_data;
1429 struct ras_query_context qctx;
1430 unsigned int error_query_mode;
1431 int ret;
1432
1433 if (!obj)
1434 return -EINVAL;
1435
1436 ret = amdgpu_ras_error_data_init(&err_data);
1437 if (ret)
1438 return ret;
1439
1440 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode))
1441 return -EINVAL;
1442
1443 memset(&qctx, 0, sizeof(qctx));
1444 qctx.evid.type = type;
1445 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type);
1446
1447 if (!down_read_trylock(&adev->reset_domain->sem)) {
1448 ret = -EIO;
1449 goto out_fini_err_data;
1450 }
1451
1452 ret = amdgpu_ras_query_error_status_helper(adev, info,
1453 &err_data,
1454 &qctx,
1455 error_query_mode);
1456 up_read(&adev->reset_domain->sem);
1457 if (ret)
1458 goto out_fini_err_data;
1459
1460 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
1461 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data);
1462 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx);
1463 } else {
1464 /* Host provides absolute error counts. First generate the report
1465 * using the previous VF internal count against new host count.
1466 * Then Update VF internal count.
1467 */
1468 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx);
1469 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data);
1470 }
1471
1472 info->ue_count = obj->err_data.ue_count;
1473 info->ce_count = obj->err_data.ce_count;
1474 info->de_count = obj->err_data.de_count;
1475
1476out_fini_err_data:
1477 amdgpu_ras_error_data_fini(&err_data);
1478
1479 return ret;
1480}
1481
1482int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info)
1483{
1484 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID);
1485}
1486
1487int amdgpu_ras_reset_error_count(struct amdgpu_device *adev,
1488 enum amdgpu_ras_block block)
1489{
1490 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1491 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
1492 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
1493
1494 if (!block_obj || !block_obj->hw_ops) {
1495 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1496 ras_block_str(block));
1497 return -EOPNOTSUPP;
1498 }
1499
1500 if (!amdgpu_ras_is_supported(adev, block) ||
1501 !amdgpu_ras_get_aca_debug_mode(adev))
1502 return -EOPNOTSUPP;
1503
1504 if (amdgpu_sriov_vf(adev))
1505 return -EOPNOTSUPP;
1506
1507 /* skip ras error reset in gpu reset */
1508 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) &&
1509 ((smu_funcs && smu_funcs->set_debug_mode) ||
1510 (mca_funcs && mca_funcs->mca_set_debug_mode)))
1511 return -EOPNOTSUPP;
1512
1513 if (block_obj->hw_ops->reset_ras_error_count)
1514 block_obj->hw_ops->reset_ras_error_count(adev);
1515
1516 return 0;
1517}
1518
1519int amdgpu_ras_reset_error_status(struct amdgpu_device *adev,
1520 enum amdgpu_ras_block block)
1521{
1522 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1523
1524 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP)
1525 return 0;
1526
1527 if ((block == AMDGPU_RAS_BLOCK__GFX) ||
1528 (block == AMDGPU_RAS_BLOCK__MMHUB)) {
1529 if (block_obj->hw_ops->reset_ras_error_status)
1530 block_obj->hw_ops->reset_ras_error_status(adev);
1531 }
1532
1533 return 0;
1534}
1535
1536/* wrapper of psp_ras_trigger_error */
1537int amdgpu_ras_error_inject(struct amdgpu_device *adev,
1538 struct ras_inject_if *info)
1539{
1540 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1541 struct ta_ras_trigger_error_input block_info = {
1542 .block_id = amdgpu_ras_block_to_ta(info->head.block),
1543 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type),
1544 .sub_block_index = info->head.sub_block_index,
1545 .address = info->address,
1546 .value = info->value,
1547 };
1548 int ret = -EINVAL;
1549 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev,
1550 info->head.block,
1551 info->head.sub_block_index);
1552
1553 /* inject on guest isn't allowed, return success directly */
1554 if (amdgpu_sriov_vf(adev))
1555 return 0;
1556
1557 if (!obj)
1558 return -EINVAL;
1559
1560 if (!block_obj || !block_obj->hw_ops) {
1561 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1562 get_ras_block_str(&info->head));
1563 return -EINVAL;
1564 }
1565
1566 /* Calculate XGMI relative offset */
1567 if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1568 info->head.block != AMDGPU_RAS_BLOCK__GFX) {
1569 block_info.address =
1570 amdgpu_xgmi_get_relative_phy_addr(adev,
1571 block_info.address);
1572 }
1573
1574 if (block_obj->hw_ops->ras_error_inject) {
1575 if (info->head.block == AMDGPU_RAS_BLOCK__GFX)
1576 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask);
1577 else /* Special ras_error_inject is defined (e.g: xgmi) */
1578 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info,
1579 info->instance_mask);
1580 } else {
1581 /* default path */
1582 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask);
1583 }
1584
1585 if (ret)
1586 dev_err(adev->dev, "ras inject %s failed %d\n",
1587 get_ras_block_str(&info->head), ret);
1588
1589 return ret;
1590}
1591
1592/**
1593 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP
1594 * @adev: pointer to AMD GPU device
1595 * @ce_count: pointer to an integer to be set to the count of correctible errors.
1596 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors.
1597 * @query_info: pointer to ras_query_if
1598 *
1599 * Return 0 for query success or do nothing, otherwise return an error
1600 * on failures
1601 */
1602static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev,
1603 unsigned long *ce_count,
1604 unsigned long *ue_count,
1605 struct ras_query_if *query_info)
1606{
1607 int ret;
1608
1609 if (!query_info)
1610 /* do nothing if query_info is not specified */
1611 return 0;
1612
1613 ret = amdgpu_ras_query_error_status(adev, query_info);
1614 if (ret)
1615 return ret;
1616
1617 *ce_count += query_info->ce_count;
1618 *ue_count += query_info->ue_count;
1619
1620 /* some hardware/IP supports read to clear
1621 * no need to explictly reset the err status after the query call */
1622 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
1623 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
1624 if (amdgpu_ras_reset_error_status(adev, query_info->head.block))
1625 dev_warn(adev->dev,
1626 "Failed to reset error counter and error status\n");
1627 }
1628
1629 return 0;
1630}
1631
1632/**
1633 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP
1634 * @adev: pointer to AMD GPU device
1635 * @ce_count: pointer to an integer to be set to the count of correctible errors.
1636 * @ue_count: pointer to an integer to be set to the count of uncorrectible
1637 * errors.
1638 * @query_info: pointer to ras_query_if if the query request is only for
1639 * specific ip block; if info is NULL, then the qurey request is for
1640 * all the ip blocks that support query ras error counters/status
1641 *
1642 * If set, @ce_count or @ue_count, count and return the corresponding
1643 * error counts in those integer pointers. Return 0 if the device
1644 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS.
1645 */
1646int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
1647 unsigned long *ce_count,
1648 unsigned long *ue_count,
1649 struct ras_query_if *query_info)
1650{
1651 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1652 struct ras_manager *obj;
1653 unsigned long ce, ue;
1654 int ret;
1655
1656 if (!adev->ras_enabled || !con)
1657 return -EOPNOTSUPP;
1658
1659 /* Don't count since no reporting.
1660 */
1661 if (!ce_count && !ue_count)
1662 return 0;
1663
1664 ce = 0;
1665 ue = 0;
1666 if (!query_info) {
1667 /* query all the ip blocks that support ras query interface */
1668 list_for_each_entry(obj, &con->head, node) {
1669 struct ras_query_if info = {
1670 .head = obj->head,
1671 };
1672
1673 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info);
1674 }
1675 } else {
1676 /* query specific ip block */
1677 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info);
1678 }
1679
1680 if (ret)
1681 return ret;
1682
1683 if (ce_count)
1684 *ce_count = ce;
1685
1686 if (ue_count)
1687 *ue_count = ue;
1688
1689 return 0;
1690}
1691/* query/inject/cure end */
1692
1693
1694/* sysfs begin */
1695
1696static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1697 struct ras_badpage **bps, unsigned int *count);
1698
1699static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
1700{
1701 switch (flags) {
1702 case AMDGPU_RAS_RETIRE_PAGE_RESERVED:
1703 return "R";
1704 case AMDGPU_RAS_RETIRE_PAGE_PENDING:
1705 return "P";
1706 case AMDGPU_RAS_RETIRE_PAGE_FAULT:
1707 default:
1708 return "F";
1709 }
1710}
1711
1712/**
1713 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface
1714 *
1715 * It allows user to read the bad pages of vram on the gpu through
1716 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
1717 *
1718 * It outputs multiple lines, and each line stands for one gpu page.
1719 *
1720 * The format of one line is below,
1721 * gpu pfn : gpu page size : flags
1722 *
1723 * gpu pfn and gpu page size are printed in hex format.
1724 * flags can be one of below character,
1725 *
1726 * R: reserved, this gpu page is reserved and not able to use.
1727 *
1728 * P: pending for reserve, this gpu page is marked as bad, will be reserved
1729 * in next window of page_reserve.
1730 *
1731 * F: unable to reserve. this gpu page can't be reserved due to some reasons.
1732 *
1733 * Examples:
1734 *
1735 * .. code-block:: bash
1736 *
1737 * 0x00000001 : 0x00001000 : R
1738 * 0x00000002 : 0x00001000 : P
1739 *
1740 */
1741
1742static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
1743 struct kobject *kobj, const struct bin_attribute *attr,
1744 char *buf, loff_t ppos, size_t count)
1745{
1746 struct amdgpu_ras *con =
1747 container_of(attr, struct amdgpu_ras, badpages_attr);
1748 struct amdgpu_device *adev = con->adev;
1749 const unsigned int element_size =
1750 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
1751 unsigned int start = div64_ul(ppos + element_size - 1, element_size);
1752 unsigned int end = div64_ul(ppos + count - 1, element_size);
1753 ssize_t s = 0;
1754 struct ras_badpage *bps = NULL;
1755 unsigned int bps_count = 0;
1756
1757 memset(buf, 0, count);
1758
1759 if (amdgpu_ras_badpages_read(adev, &bps, &bps_count))
1760 return 0;
1761
1762 for (; start < end && start < bps_count; start++)
1763 s += scnprintf(&buf[s], element_size + 1,
1764 "0x%08x : 0x%08x : %1s\n",
1765 bps[start].bp,
1766 bps[start].size,
1767 amdgpu_ras_badpage_flags_str(bps[start].flags));
1768
1769 kfree(bps);
1770
1771 return s;
1772}
1773
1774static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
1775 struct device_attribute *attr, char *buf)
1776{
1777 struct amdgpu_ras *con =
1778 container_of(attr, struct amdgpu_ras, features_attr);
1779
1780 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features);
1781}
1782
1783static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev,
1784 struct device_attribute *attr, char *buf)
1785{
1786 struct amdgpu_ras *con =
1787 container_of(attr, struct amdgpu_ras, version_attr);
1788 return sysfs_emit(buf, "table version: 0x%x\n", con->eeprom_control.tbl_hdr.version);
1789}
1790
1791static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev,
1792 struct device_attribute *attr, char *buf)
1793{
1794 struct amdgpu_ras *con =
1795 container_of(attr, struct amdgpu_ras, schema_attr);
1796 return sysfs_emit(buf, "schema: 0x%x\n", con->schema);
1797}
1798
1799static struct {
1800 enum ras_event_type type;
1801 const char *name;
1802} dump_event[] = {
1803 {RAS_EVENT_TYPE_FATAL, "Fatal Error"},
1804 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"},
1805 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"},
1806};
1807
1808static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev,
1809 struct device_attribute *attr, char *buf)
1810{
1811 struct amdgpu_ras *con =
1812 container_of(attr, struct amdgpu_ras, event_state_attr);
1813 struct ras_event_manager *event_mgr = con->event_mgr;
1814 struct ras_event_state *event_state;
1815 int i, size = 0;
1816
1817 if (!event_mgr)
1818 return -EINVAL;
1819
1820 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno));
1821 for (i = 0; i < ARRAY_SIZE(dump_event); i++) {
1822 event_state = &event_mgr->event_state[dump_event[i].type];
1823 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n",
1824 dump_event[i].name,
1825 atomic64_read(&event_state->count),
1826 event_state->last_seqno);
1827 }
1828
1829 return (ssize_t)size;
1830}
1831
1832static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev)
1833{
1834 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1835
1836 if (adev->dev->kobj.sd)
1837 sysfs_remove_file_from_group(&adev->dev->kobj,
1838 &con->badpages_attr.attr,
1839 RAS_FS_NAME);
1840}
1841
1842static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev)
1843{
1844 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1845 struct attribute *attrs[] = {
1846 &con->features_attr.attr,
1847 &con->version_attr.attr,
1848 &con->schema_attr.attr,
1849 &con->event_state_attr.attr,
1850 NULL
1851 };
1852 struct attribute_group group = {
1853 .name = RAS_FS_NAME,
1854 .attrs = attrs,
1855 };
1856
1857 if (adev->dev->kobj.sd)
1858 sysfs_remove_group(&adev->dev->kobj, &group);
1859
1860 return 0;
1861}
1862
1863int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
1864 struct ras_common_if *head)
1865{
1866 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1867
1868 if (amdgpu_aca_is_enabled(adev))
1869 return 0;
1870
1871 if (!obj || obj->attr_inuse)
1872 return -EINVAL;
1873
1874 if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, head->block))
1875 return 0;
1876
1877 get_obj(obj);
1878
1879 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name),
1880 "%s_err_count", head->name);
1881
1882 obj->sysfs_attr = (struct device_attribute){
1883 .attr = {
1884 .name = obj->fs_data.sysfs_name,
1885 .mode = S_IRUGO,
1886 },
1887 .show = amdgpu_ras_sysfs_read,
1888 };
1889 sysfs_attr_init(&obj->sysfs_attr.attr);
1890
1891 if (sysfs_add_file_to_group(&adev->dev->kobj,
1892 &obj->sysfs_attr.attr,
1893 RAS_FS_NAME)) {
1894 put_obj(obj);
1895 return -EINVAL;
1896 }
1897
1898 obj->attr_inuse = 1;
1899
1900 return 0;
1901}
1902
1903int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
1904 struct ras_common_if *head)
1905{
1906 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1907
1908 if (amdgpu_aca_is_enabled(adev))
1909 return 0;
1910
1911 if (!obj || !obj->attr_inuse)
1912 return -EINVAL;
1913
1914 if (adev->dev->kobj.sd)
1915 sysfs_remove_file_from_group(&adev->dev->kobj,
1916 &obj->sysfs_attr.attr,
1917 RAS_FS_NAME);
1918 obj->attr_inuse = 0;
1919 put_obj(obj);
1920
1921 return 0;
1922}
1923
1924static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
1925{
1926 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1927 struct ras_manager *obj, *tmp;
1928
1929 list_for_each_entry_safe(obj, tmp, &con->head, node) {
1930 amdgpu_ras_sysfs_remove(adev, &obj->head);
1931 }
1932
1933 if (amdgpu_bad_page_threshold != 0)
1934 amdgpu_ras_sysfs_remove_bad_page_node(adev);
1935
1936 amdgpu_ras_sysfs_remove_dev_attr_node(adev);
1937
1938 return 0;
1939}
1940/* sysfs end */
1941
1942/**
1943 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors
1944 *
1945 * Normally when there is an uncorrectable error, the driver will reset
1946 * the GPU to recover. However, in the event of an unrecoverable error,
1947 * the driver provides an interface to reboot the system automatically
1948 * in that event.
1949 *
1950 * The following file in debugfs provides that interface:
1951 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot
1952 *
1953 * Usage:
1954 *
1955 * .. code-block:: bash
1956 *
1957 * echo true > .../ras/auto_reboot
1958 *
1959 */
1960/* debugfs begin */
1961static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
1962{
1963 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1964 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control;
1965 struct drm_minor *minor = adev_to_drm(adev)->primary;
1966 struct dentry *dir;
1967
1968 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);
1969 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev,
1970 &amdgpu_ras_debugfs_ctrl_ops);
1971 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev,
1972 &amdgpu_ras_debugfs_eeprom_ops);
1973 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,
1974 &con->bad_page_cnt_threshold);
1975 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs);
1976 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled);
1977 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled);
1978 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev,
1979 &amdgpu_ras_debugfs_eeprom_size_ops);
1980 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table",
1981 S_IRUGO, dir, adev,
1982 &amdgpu_ras_debugfs_eeprom_table_ops);
1983 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control);
1984
1985 /*
1986 * After one uncorrectable error happens, usually GPU recovery will
1987 * be scheduled. But due to the known problem in GPU recovery failing
1988 * to bring GPU back, below interface provides one direct way to
1989 * user to reboot system automatically in such case within
1990 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine
1991 * will never be called.
1992 */
1993 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot);
1994
1995 /*
1996 * User could set this not to clean up hardware's error count register
1997 * of RAS IPs during ras recovery.
1998 */
1999 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir,
2000 &con->disable_ras_err_cnt_harvest);
2001 return dir;
2002}
2003
2004static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
2005 struct ras_fs_if *head,
2006 struct dentry *dir)
2007{
2008 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
2009
2010 if (!obj || !dir)
2011 return;
2012
2013 get_obj(obj);
2014
2015 memcpy(obj->fs_data.debugfs_name,
2016 head->debugfs_name,
2017 sizeof(obj->fs_data.debugfs_name));
2018
2019 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir,
2020 obj, &amdgpu_ras_debugfs_ops);
2021}
2022
2023static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev)
2024{
2025 bool ret;
2026
2027 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
2028 case IP_VERSION(13, 0, 6):
2029 case IP_VERSION(13, 0, 12):
2030 case IP_VERSION(13, 0, 14):
2031 ret = true;
2032 break;
2033 default:
2034 ret = false;
2035 break;
2036 }
2037
2038 return ret;
2039}
2040
2041void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
2042{
2043 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2044 struct dentry *dir;
2045 struct ras_manager *obj;
2046 struct ras_fs_if fs_info;
2047
2048 /*
2049 * it won't be called in resume path, no need to check
2050 * suspend and gpu reset status
2051 */
2052 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con)
2053 return;
2054
2055 dir = amdgpu_ras_debugfs_create_ctrl_node(adev);
2056
2057 list_for_each_entry(obj, &con->head, node) {
2058 if (amdgpu_ras_is_supported(adev, obj->head.block) &&
2059 (obj->attr_inuse == 1)) {
2060 sprintf(fs_info.debugfs_name, "%s_err_inject",
2061 get_ras_block_str(&obj->head));
2062 fs_info.head = obj->head;
2063 amdgpu_ras_debugfs_create(adev, &fs_info, dir);
2064 }
2065 }
2066
2067 if (amdgpu_ras_aca_is_supported(adev)) {
2068 if (amdgpu_aca_is_enabled(adev))
2069 amdgpu_aca_smu_debugfs_init(adev, dir);
2070 else
2071 amdgpu_mca_smu_debugfs_init(adev, dir);
2072 }
2073}
2074
2075/* debugfs end */
2076
2077/* ras fs */
2078static const BIN_ATTR(gpu_vram_bad_pages, S_IRUGO,
2079 amdgpu_ras_sysfs_badpages_read, NULL, 0);
2080static DEVICE_ATTR(features, S_IRUGO,
2081 amdgpu_ras_sysfs_features_read, NULL);
2082static DEVICE_ATTR(version, 0444,
2083 amdgpu_ras_sysfs_version_show, NULL);
2084static DEVICE_ATTR(schema, 0444,
2085 amdgpu_ras_sysfs_schema_show, NULL);
2086static DEVICE_ATTR(event_state, 0444,
2087 amdgpu_ras_sysfs_event_state_show, NULL);
2088static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
2089{
2090 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2091 struct attribute_group group = {
2092 .name = RAS_FS_NAME,
2093 };
2094 struct attribute *attrs[] = {
2095 &con->features_attr.attr,
2096 &con->version_attr.attr,
2097 &con->schema_attr.attr,
2098 &con->event_state_attr.attr,
2099 NULL
2100 };
2101 const struct bin_attribute *bin_attrs[] = {
2102 NULL,
2103 NULL,
2104 };
2105 int r;
2106
2107 group.attrs = attrs;
2108
2109 /* add features entry */
2110 con->features_attr = dev_attr_features;
2111 sysfs_attr_init(attrs[0]);
2112
2113 /* add version entry */
2114 con->version_attr = dev_attr_version;
2115 sysfs_attr_init(attrs[1]);
2116
2117 /* add schema entry */
2118 con->schema_attr = dev_attr_schema;
2119 sysfs_attr_init(attrs[2]);
2120
2121 /* add event_state entry */
2122 con->event_state_attr = dev_attr_event_state;
2123 sysfs_attr_init(attrs[3]);
2124
2125 if (amdgpu_bad_page_threshold != 0) {
2126 /* add bad_page_features entry */
2127 con->badpages_attr = bin_attr_gpu_vram_bad_pages;
2128 sysfs_bin_attr_init(&con->badpages_attr);
2129 bin_attrs[0] = &con->badpages_attr;
2130 group.bin_attrs = bin_attrs;
2131 }
2132
2133 r = sysfs_create_group(&adev->dev->kobj, &group);
2134 if (r)
2135 dev_err(adev->dev, "Failed to create RAS sysfs group!");
2136
2137 return 0;
2138}
2139
2140static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
2141{
2142 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2143 struct ras_manager *con_obj, *ip_obj, *tmp;
2144
2145 if (IS_ENABLED(CONFIG_DEBUG_FS)) {
2146 list_for_each_entry_safe(con_obj, tmp, &con->head, node) {
2147 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head);
2148 if (ip_obj)
2149 put_obj(ip_obj);
2150 }
2151 }
2152
2153 amdgpu_ras_sysfs_remove_all(adev);
2154 return 0;
2155}
2156/* ras fs end */
2157
2158/* ih begin */
2159
2160/* For the hardware that cannot enable bif ring for both ras_controller_irq
2161 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status
2162 * register to check whether the interrupt is triggered or not, and properly
2163 * ack the interrupt if it is there
2164 */
2165void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev)
2166{
2167 /* Fatal error events are handled on host side */
2168 if (amdgpu_sriov_vf(adev))
2169 return;
2170 /*
2171 * If the current interrupt is caused by a non-fatal RAS error, skip
2172 * check for fatal error. For fatal errors, FED status of all devices
2173 * in XGMI hive gets set when the first device gets fatal error
2174 * interrupt. The error gets propagated to other devices as well, so
2175 * make sure to ack the interrupt regardless of FED status.
2176 */
2177 if (!amdgpu_ras_get_fed_status(adev) &&
2178 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY))
2179 return;
2180
2181 if (adev->nbio.ras &&
2182 adev->nbio.ras->handle_ras_controller_intr_no_bifring)
2183 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev);
2184
2185 if (adev->nbio.ras &&
2186 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring)
2187 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev);
2188}
2189
2190static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj,
2191 struct amdgpu_iv_entry *entry)
2192{
2193 bool poison_stat = false;
2194 struct amdgpu_device *adev = obj->adev;
2195 struct amdgpu_ras_block_object *block_obj =
2196 amdgpu_ras_get_ras_block(adev, obj->head.block, 0);
2197 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2198 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION;
2199 u64 event_id;
2200 int ret;
2201
2202 if (!block_obj || !con)
2203 return;
2204
2205 ret = amdgpu_ras_mark_ras_event(adev, type);
2206 if (ret)
2207 return;
2208
2209 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block);
2210 /* both query_poison_status and handle_poison_consumption are optional,
2211 * but at least one of them should be implemented if we need poison
2212 * consumption handler
2213 */
2214 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) {
2215 poison_stat = block_obj->hw_ops->query_poison_status(adev);
2216 if (!poison_stat) {
2217 /* Not poison consumption interrupt, no need to handle it */
2218 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n",
2219 block_obj->ras_comm.name);
2220
2221 return;
2222 }
2223 }
2224
2225 amdgpu_umc_poison_handler(adev, obj->head.block, 0);
2226
2227 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption)
2228 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev);
2229
2230 /* gpu reset is fallback for failed and default cases.
2231 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset.
2232 */
2233 if (poison_stat && !amdgpu_ras_is_rma(adev)) {
2234 event_id = amdgpu_ras_acquire_event_id(adev, type);
2235 RAS_EVENT_LOG(adev, event_id,
2236 "GPU reset for %s RAS poison consumption is issued!\n",
2237 block_obj->ras_comm.name);
2238 amdgpu_ras_reset_gpu(adev);
2239 }
2240
2241 if (!poison_stat)
2242 amdgpu_gfx_poison_consumption_handler(adev, entry);
2243}
2244
2245static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj,
2246 struct amdgpu_iv_entry *entry)
2247{
2248 struct amdgpu_device *adev = obj->adev;
2249 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
2250 u64 event_id;
2251 int ret;
2252
2253 ret = amdgpu_ras_mark_ras_event(adev, type);
2254 if (ret)
2255 return;
2256
2257 event_id = amdgpu_ras_acquire_event_id(adev, type);
2258 RAS_EVENT_LOG(adev, event_id, "Poison is created\n");
2259
2260 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) {
2261 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev);
2262
2263 atomic_inc(&con->page_retirement_req_cnt);
2264 atomic_inc(&con->poison_creation_count);
2265
2266 wake_up(&con->page_retirement_wq);
2267 }
2268}
2269
2270static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
2271 struct amdgpu_iv_entry *entry)
2272{
2273 struct ras_ih_data *data = &obj->ih_data;
2274 struct ras_err_data err_data;
2275 int ret;
2276
2277 if (!data->cb)
2278 return;
2279
2280 ret = amdgpu_ras_error_data_init(&err_data);
2281 if (ret)
2282 return;
2283
2284 /* Let IP handle its data, maybe we need get the output
2285 * from the callback to update the error type/count, etc
2286 */
2287 amdgpu_ras_set_fed(obj->adev, true);
2288 ret = data->cb(obj->adev, &err_data, entry);
2289 /* ue will trigger an interrupt, and in that case
2290 * we need do a reset to recovery the whole system.
2291 * But leave IP do that recovery, here we just dispatch
2292 * the error.
2293 */
2294 if (ret == AMDGPU_RAS_SUCCESS) {
2295 /* these counts could be left as 0 if
2296 * some blocks do not count error number
2297 */
2298 obj->err_data.ue_count += err_data.ue_count;
2299 obj->err_data.ce_count += err_data.ce_count;
2300 obj->err_data.de_count += err_data.de_count;
2301 }
2302
2303 amdgpu_ras_error_data_fini(&err_data);
2304}
2305
2306static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
2307{
2308 struct ras_ih_data *data = &obj->ih_data;
2309 struct amdgpu_iv_entry entry;
2310
2311 while (data->rptr != data->wptr) {
2312 rmb();
2313 memcpy(&entry, &data->ring[data->rptr],
2314 data->element_size);
2315
2316 wmb();
2317 data->rptr = (data->aligned_element_size +
2318 data->rptr) % data->ring_size;
2319
2320 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) {
2321 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
2322 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry);
2323 else
2324 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry);
2325 } else {
2326 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
2327 amdgpu_ras_interrupt_umc_handler(obj, &entry);
2328 else
2329 dev_warn(obj->adev->dev,
2330 "No RAS interrupt handler for non-UMC block with poison disabled.\n");
2331 }
2332 }
2333}
2334
2335static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
2336{
2337 struct ras_ih_data *data =
2338 container_of(work, struct ras_ih_data, ih_work);
2339 struct ras_manager *obj =
2340 container_of(data, struct ras_manager, ih_data);
2341
2342 amdgpu_ras_interrupt_handler(obj);
2343}
2344
2345int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
2346 struct ras_dispatch_if *info)
2347{
2348 struct ras_manager *obj;
2349 struct ras_ih_data *data;
2350
2351 obj = amdgpu_ras_find_obj(adev, &info->head);
2352 if (!obj)
2353 return -EINVAL;
2354
2355 data = &obj->ih_data;
2356
2357 if (data->inuse == 0)
2358 return 0;
2359
2360 /* Might be overflow... */
2361 memcpy(&data->ring[data->wptr], info->entry,
2362 data->element_size);
2363
2364 wmb();
2365 data->wptr = (data->aligned_element_size +
2366 data->wptr) % data->ring_size;
2367
2368 schedule_work(&data->ih_work);
2369
2370 return 0;
2371}
2372
2373int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
2374 struct ras_common_if *head)
2375{
2376 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
2377 struct ras_ih_data *data;
2378
2379 if (!obj)
2380 return -EINVAL;
2381
2382 data = &obj->ih_data;
2383 if (data->inuse == 0)
2384 return 0;
2385
2386 cancel_work_sync(&data->ih_work);
2387
2388 kfree(data->ring);
2389 memset(data, 0, sizeof(*data));
2390 put_obj(obj);
2391
2392 return 0;
2393}
2394
2395int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
2396 struct ras_common_if *head)
2397{
2398 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
2399 struct ras_ih_data *data;
2400 struct amdgpu_ras_block_object *ras_obj;
2401
2402 if (!obj) {
2403 /* in case we registe the IH before enable ras feature */
2404 obj = amdgpu_ras_create_obj(adev, head);
2405 if (!obj)
2406 return -EINVAL;
2407 } else
2408 get_obj(obj);
2409
2410 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm);
2411
2412 data = &obj->ih_data;
2413 /* add the callback.etc */
2414 *data = (struct ras_ih_data) {
2415 .inuse = 0,
2416 .cb = ras_obj->ras_cb,
2417 .element_size = sizeof(struct amdgpu_iv_entry),
2418 .rptr = 0,
2419 .wptr = 0,
2420 };
2421
2422 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
2423
2424 data->aligned_element_size = ALIGN(data->element_size, 8);
2425 /* the ring can store 64 iv entries. */
2426 data->ring_size = 64 * data->aligned_element_size;
2427 data->ring = kmalloc(data->ring_size, GFP_KERNEL);
2428 if (!data->ring) {
2429 put_obj(obj);
2430 return -ENOMEM;
2431 }
2432
2433 /* IH is ready */
2434 data->inuse = 1;
2435
2436 return 0;
2437}
2438
2439static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
2440{
2441 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2442 struct ras_manager *obj, *tmp;
2443
2444 list_for_each_entry_safe(obj, tmp, &con->head, node) {
2445 amdgpu_ras_interrupt_remove_handler(adev, &obj->head);
2446 }
2447
2448 return 0;
2449}
2450/* ih end */
2451
2452/* traversal all IPs except NBIO to query error counter */
2453static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type)
2454{
2455 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2456 struct ras_manager *obj;
2457
2458 if (!adev->ras_enabled || !con)
2459 return;
2460
2461 list_for_each_entry(obj, &con->head, node) {
2462 struct ras_query_if info = {
2463 .head = obj->head,
2464 };
2465
2466 /*
2467 * PCIE_BIF IP has one different isr by ras controller
2468 * interrupt, the specific ras counter query will be
2469 * done in that isr. So skip such block from common
2470 * sync flood interrupt isr calling.
2471 */
2472 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF)
2473 continue;
2474
2475 /*
2476 * this is a workaround for aldebaran, skip send msg to
2477 * smu to get ecc_info table due to smu handle get ecc
2478 * info table failed temporarily.
2479 * should be removed until smu fix handle ecc_info table.
2480 */
2481 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) &&
2482 (amdgpu_ip_version(adev, MP1_HWIP, 0) ==
2483 IP_VERSION(13, 0, 2)))
2484 continue;
2485
2486 amdgpu_ras_query_error_status_with_event(adev, &info, type);
2487
2488 if (amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2489 IP_VERSION(11, 0, 2) &&
2490 amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2491 IP_VERSION(11, 0, 4) &&
2492 amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2493 IP_VERSION(13, 0, 0)) {
2494 if (amdgpu_ras_reset_error_status(adev, info.head.block))
2495 dev_warn(adev->dev, "Failed to reset error counter and error status");
2496 }
2497 }
2498}
2499
2500/* Parse RdRspStatus and WrRspStatus */
2501static void amdgpu_ras_error_status_query(struct amdgpu_device *adev,
2502 struct ras_query_if *info)
2503{
2504 struct amdgpu_ras_block_object *block_obj;
2505 /*
2506 * Only two block need to query read/write
2507 * RspStatus at current state
2508 */
2509 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) &&
2510 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB))
2511 return;
2512
2513 block_obj = amdgpu_ras_get_ras_block(adev,
2514 info->head.block,
2515 info->head.sub_block_index);
2516
2517 if (!block_obj || !block_obj->hw_ops) {
2518 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
2519 get_ras_block_str(&info->head));
2520 return;
2521 }
2522
2523 if (block_obj->hw_ops->query_ras_error_status)
2524 block_obj->hw_ops->query_ras_error_status(adev);
2525
2526}
2527
2528static void amdgpu_ras_query_err_status(struct amdgpu_device *adev)
2529{
2530 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2531 struct ras_manager *obj;
2532
2533 if (!adev->ras_enabled || !con)
2534 return;
2535
2536 list_for_each_entry(obj, &con->head, node) {
2537 struct ras_query_if info = {
2538 .head = obj->head,
2539 };
2540
2541 amdgpu_ras_error_status_query(adev, &info);
2542 }
2543}
2544
2545/* recovery begin */
2546
2547/* return 0 on success.
2548 * caller need free bps.
2549 */
2550static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
2551 struct ras_badpage **bps, unsigned int *count)
2552{
2553 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2554 struct ras_err_handler_data *data;
2555 int i = 0;
2556 int ret = 0, status;
2557
2558 if (!con || !con->eh_data || !bps || !count)
2559 return -EINVAL;
2560
2561 mutex_lock(&con->recovery_lock);
2562 data = con->eh_data;
2563 if (!data || data->count == 0) {
2564 *bps = NULL;
2565 ret = -EINVAL;
2566 goto out;
2567 }
2568
2569 *bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL);
2570 if (!*bps) {
2571 ret = -ENOMEM;
2572 goto out;
2573 }
2574
2575 for (; i < data->count; i++) {
2576 (*bps)[i] = (struct ras_badpage){
2577 .bp = data->bps[i].retired_page,
2578 .size = AMDGPU_GPU_PAGE_SIZE,
2579 .flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED,
2580 };
2581 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr,
2582 data->bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT);
2583 if (status == -EBUSY)
2584 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
2585 else if (status == -ENOENT)
2586 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT;
2587 }
2588
2589 *count = data->count;
2590out:
2591 mutex_unlock(&con->recovery_lock);
2592 return ret;
2593}
2594
2595static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev,
2596 struct amdgpu_hive_info *hive, bool status)
2597{
2598 struct amdgpu_device *tmp_adev;
2599
2600 if (hive) {
2601 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
2602 amdgpu_ras_set_fed(tmp_adev, status);
2603 } else {
2604 amdgpu_ras_set_fed(adev, status);
2605 }
2606}
2607
2608bool amdgpu_ras_in_recovery(struct amdgpu_device *adev)
2609{
2610 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2611 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2612 int hive_ras_recovery = 0;
2613
2614 if (hive) {
2615 hive_ras_recovery = atomic_read(&hive->ras_recovery);
2616 amdgpu_put_xgmi_hive(hive);
2617 }
2618
2619 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery))
2620 return true;
2621
2622 return false;
2623}
2624
2625static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev)
2626{
2627 if (amdgpu_ras_intr_triggered())
2628 return RAS_EVENT_TYPE_FATAL;
2629 else
2630 return RAS_EVENT_TYPE_POISON_CONSUMPTION;
2631}
2632
2633static void amdgpu_ras_do_recovery(struct work_struct *work)
2634{
2635 struct amdgpu_ras *ras =
2636 container_of(work, struct amdgpu_ras, recovery_work);
2637 struct amdgpu_device *remote_adev = NULL;
2638 struct amdgpu_device *adev = ras->adev;
2639 struct list_head device_list, *device_list_handle = NULL;
2640 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2641 enum ras_event_type type;
2642
2643 if (hive) {
2644 atomic_set(&hive->ras_recovery, 1);
2645
2646 /* If any device which is part of the hive received RAS fatal
2647 * error interrupt, set fatal error status on all. This
2648 * condition will need a recovery, and flag will be cleared
2649 * as part of recovery.
2650 */
2651 list_for_each_entry(remote_adev, &hive->device_list,
2652 gmc.xgmi.head)
2653 if (amdgpu_ras_get_fed_status(remote_adev)) {
2654 amdgpu_ras_set_fed_all(adev, hive, true);
2655 break;
2656 }
2657 }
2658 if (!ras->disable_ras_err_cnt_harvest) {
2659
2660 /* Build list of devices to query RAS related errors */
2661 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
2662 device_list_handle = &hive->device_list;
2663 } else {
2664 INIT_LIST_HEAD(&device_list);
2665 list_add_tail(&adev->gmc.xgmi.head, &device_list);
2666 device_list_handle = &device_list;
2667 }
2668
2669 type = amdgpu_ras_get_fatal_error_event(adev);
2670 list_for_each_entry(remote_adev,
2671 device_list_handle, gmc.xgmi.head) {
2672 amdgpu_ras_query_err_status(remote_adev);
2673 amdgpu_ras_log_on_err_counter(remote_adev, type);
2674 }
2675
2676 }
2677
2678 if (amdgpu_device_should_recover_gpu(ras->adev)) {
2679 struct amdgpu_reset_context reset_context;
2680 memset(&reset_context, 0, sizeof(reset_context));
2681
2682 reset_context.method = AMD_RESET_METHOD_NONE;
2683 reset_context.reset_req_dev = adev;
2684 reset_context.src = AMDGPU_RESET_SRC_RAS;
2685 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
2686
2687 /* Perform full reset in fatal error mode */
2688 if (!amdgpu_ras_is_poison_mode_supported(ras->adev))
2689 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2690 else {
2691 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2692
2693 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) {
2694 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET;
2695 reset_context.method = AMD_RESET_METHOD_MODE2;
2696 }
2697
2698 /* Fatal error occurs in poison mode, mode1 reset is used to
2699 * recover gpu.
2700 */
2701 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) {
2702 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET;
2703 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2704
2705 psp_fatal_error_recovery_quirk(&adev->psp);
2706 }
2707 }
2708
2709 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context);
2710 }
2711 atomic_set(&ras->in_recovery, 0);
2712 if (hive) {
2713 atomic_set(&hive->ras_recovery, 0);
2714 amdgpu_put_xgmi_hive(hive);
2715 }
2716}
2717
2718/* alloc/realloc bps array */
2719static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
2720 struct ras_err_handler_data *data, int pages)
2721{
2722 unsigned int old_space = data->count + data->space_left;
2723 unsigned int new_space = old_space + pages;
2724 unsigned int align_space = ALIGN(new_space, 512);
2725 void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL);
2726
2727 if (!bps) {
2728 return -ENOMEM;
2729 }
2730
2731 if (data->bps) {
2732 memcpy(bps, data->bps,
2733 data->count * sizeof(*data->bps));
2734 kfree(data->bps);
2735 }
2736
2737 data->bps = bps;
2738 data->space_left += align_space - old_space;
2739 return 0;
2740}
2741
2742static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev,
2743 struct eeprom_table_record *bps,
2744 struct ras_err_data *err_data)
2745{
2746 struct ta_ras_query_address_input addr_in;
2747 uint32_t socket = 0;
2748 int ret = 0;
2749
2750 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
2751 socket = adev->smuio.funcs->get_socket_id(adev);
2752
2753 /* reinit err_data */
2754 err_data->err_addr_cnt = 0;
2755 err_data->err_addr_len = adev->umc.retire_unit;
2756
2757 memset(&addr_in, 0, sizeof(addr_in));
2758 addr_in.ma.err_addr = bps->address;
2759 addr_in.ma.socket_id = socket;
2760 addr_in.ma.ch_inst = bps->mem_channel;
2761 /* tell RAS TA the node instance is not used */
2762 addr_in.ma.node_inst = TA_RAS_INV_NODE;
2763
2764 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
2765 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data,
2766 &addr_in, NULL, false);
2767
2768 return ret;
2769}
2770
2771static int amdgpu_ras_mca2pa(struct amdgpu_device *adev,
2772 struct eeprom_table_record *bps,
2773 struct ras_err_data *err_data)
2774{
2775 struct ta_ras_query_address_input addr_in;
2776 uint32_t die_id, socket = 0;
2777
2778 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
2779 socket = adev->smuio.funcs->get_socket_id(adev);
2780
2781 /* although die id is gotten from PA in nps1 mode, the id is
2782 * fitable for any nps mode
2783 */
2784 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa)
2785 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address,
2786 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT);
2787 else
2788 return -EINVAL;
2789
2790 /* reinit err_data */
2791 err_data->err_addr_cnt = 0;
2792 err_data->err_addr_len = adev->umc.retire_unit;
2793
2794 memset(&addr_in, 0, sizeof(addr_in));
2795 addr_in.ma.err_addr = bps->address;
2796 addr_in.ma.ch_inst = bps->mem_channel;
2797 addr_in.ma.umc_inst = bps->mcumc_id;
2798 addr_in.ma.node_inst = die_id;
2799 addr_in.ma.socket_id = socket;
2800
2801 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
2802 return adev->umc.ras->convert_ras_err_addr(adev, err_data,
2803 &addr_in, NULL, false);
2804 else
2805 return -EINVAL;
2806}
2807
2808static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev,
2809 struct eeprom_table_record *bps, int count)
2810{
2811 int j;
2812 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2813 struct ras_err_handler_data *data = con->eh_data;
2814
2815 for (j = 0; j < count; j++) {
2816 if (amdgpu_ras_check_bad_page_unlock(con,
2817 bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT))
2818 continue;
2819
2820 if (!data->space_left &&
2821 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
2822 return -ENOMEM;
2823 }
2824
2825 amdgpu_ras_reserve_page(adev, bps[j].retired_page);
2826
2827 memcpy(&data->bps[data->count], &(bps[j]),
2828 sizeof(struct eeprom_table_record));
2829 data->count++;
2830 data->space_left--;
2831 }
2832
2833 return 0;
2834}
2835
2836static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev,
2837 struct eeprom_table_record *bps, struct ras_err_data *err_data,
2838 enum amdgpu_memory_partition nps)
2839{
2840 int i = 0;
2841 enum amdgpu_memory_partition save_nps;
2842
2843 save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK;
2844
2845 /*old asics just have pa in eeprom*/
2846 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) {
2847 memcpy(err_data->err_addr, bps,
2848 sizeof(struct eeprom_table_record) * adev->umc.retire_unit);
2849 goto out;
2850 }
2851
2852 for (i = 0; i < adev->umc.retire_unit; i++)
2853 bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT);
2854
2855 if (save_nps) {
2856 if (save_nps == nps) {
2857 if (amdgpu_umc_pages_in_a_row(adev, err_data,
2858 bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT))
2859 return -EINVAL;
2860 for (i = 0; i < adev->umc.retire_unit; i++) {
2861 err_data->err_addr[i].address = bps[0].address;
2862 err_data->err_addr[i].mem_channel = bps[0].mem_channel;
2863 err_data->err_addr[i].bank = bps[0].bank;
2864 err_data->err_addr[i].err_type = bps[0].err_type;
2865 err_data->err_addr[i].mcumc_id = bps[0].mcumc_id;
2866 }
2867 } else {
2868 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data))
2869 return -EINVAL;
2870 }
2871 } else {
2872 if (bps[0].address == 0) {
2873 /* for specific old eeprom data, mca address is not stored,
2874 * calc it from pa
2875 */
2876 if (amdgpu_umc_pa2mca(adev, bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT,
2877 &(bps[0].address), AMDGPU_NPS1_PARTITION_MODE))
2878 return -EINVAL;
2879 }
2880
2881 if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) {
2882 if (nps == AMDGPU_NPS1_PARTITION_MODE)
2883 memcpy(err_data->err_addr, bps,
2884 sizeof(struct eeprom_table_record) * adev->umc.retire_unit);
2885 else
2886 return -EOPNOTSUPP;
2887 }
2888 }
2889
2890out:
2891 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit);
2892}
2893
2894static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev,
2895 struct eeprom_table_record *bps, struct ras_err_data *err_data,
2896 enum amdgpu_memory_partition nps)
2897{
2898 int i = 0;
2899 enum amdgpu_memory_partition save_nps;
2900
2901 save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK;
2902 bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT);
2903
2904 if (save_nps == nps) {
2905 if (amdgpu_umc_pages_in_a_row(adev, err_data,
2906 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT))
2907 return -EINVAL;
2908 for (i = 0; i < adev->umc.retire_unit; i++) {
2909 err_data->err_addr[i].address = bps->address;
2910 err_data->err_addr[i].mem_channel = bps->mem_channel;
2911 err_data->err_addr[i].bank = bps->bank;
2912 err_data->err_addr[i].err_type = bps->err_type;
2913 err_data->err_addr[i].mcumc_id = bps->mcumc_id;
2914 }
2915 } else {
2916 if (bps->address) {
2917 if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data))
2918 return -EINVAL;
2919 } else {
2920 /* for specific old eeprom data, mca address is not stored,
2921 * calc it from pa
2922 */
2923 if (amdgpu_umc_pa2mca(adev, bps->retired_page << AMDGPU_GPU_PAGE_SHIFT,
2924 &(bps->address), AMDGPU_NPS1_PARTITION_MODE))
2925 return -EINVAL;
2926
2927 if (amdgpu_ras_mca2pa(adev, bps, err_data))
2928 return -EOPNOTSUPP;
2929 }
2930 }
2931
2932 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr,
2933 adev->umc.retire_unit);
2934}
2935
2936/* it deal with vram only. */
2937int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
2938 struct eeprom_table_record *bps, int pages, bool from_rom)
2939{
2940 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2941 struct ras_err_data err_data;
2942 struct amdgpu_ras_eeprom_control *control =
2943 &adev->psp.ras_context.ras->eeprom_control;
2944 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE;
2945 int ret = 0;
2946 uint32_t i = 0;
2947
2948 if (!con || !con->eh_data || !bps || pages <= 0)
2949 return 0;
2950
2951 if (from_rom) {
2952 err_data.err_addr =
2953 kcalloc(adev->umc.retire_unit,
2954 sizeof(struct eeprom_table_record), GFP_KERNEL);
2955 if (!err_data.err_addr) {
2956 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n");
2957 return -ENOMEM;
2958 }
2959
2960 if (adev->gmc.gmc_funcs->query_mem_partition_mode)
2961 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
2962 }
2963
2964 mutex_lock(&con->recovery_lock);
2965
2966 if (from_rom) {
2967 /* there is no pa recs in V3, so skip pa recs processing */
2968 if (control->tbl_hdr.version < RAS_TABLE_VER_V3) {
2969 for (i = 0; i < pages; i++) {
2970 if (control->ras_num_recs - i >= adev->umc.retire_unit) {
2971 if ((bps[i].address == bps[i + 1].address) &&
2972 (bps[i].mem_channel == bps[i + 1].mem_channel)) {
2973 /* deal with retire_unit records a time */
2974 ret = __amdgpu_ras_convert_rec_array_from_rom(adev,
2975 &bps[i], &err_data, nps);
2976 if (ret)
2977 control->ras_num_bad_pages -= adev->umc.retire_unit;
2978 i += (adev->umc.retire_unit - 1);
2979 } else {
2980 break;
2981 }
2982 } else {
2983 break;
2984 }
2985 }
2986 }
2987 for (; i < pages; i++) {
2988 ret = __amdgpu_ras_convert_rec_from_rom(adev,
2989 &bps[i], &err_data, nps);
2990 if (ret)
2991 control->ras_num_bad_pages -= adev->umc.retire_unit;
2992 }
2993 } else {
2994 ret = __amdgpu_ras_restore_bad_pages(adev, bps, pages);
2995 }
2996
2997 if (from_rom)
2998 kfree(err_data.err_addr);
2999 mutex_unlock(&con->recovery_lock);
3000
3001 return ret;
3002}
3003
3004/*
3005 * write error record array to eeprom, the function should be
3006 * protected by recovery_lock
3007 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL
3008 */
3009int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev,
3010 unsigned long *new_cnt)
3011{
3012 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3013 struct ras_err_handler_data *data;
3014 struct amdgpu_ras_eeprom_control *control;
3015 int save_count, unit_num, bad_page_num, i;
3016
3017 if (!con || !con->eh_data) {
3018 if (new_cnt)
3019 *new_cnt = 0;
3020
3021 return 0;
3022 }
3023
3024 if (!con->eeprom_control.is_eeprom_valid) {
3025 dev_warn(adev->dev,
3026 "Failed to save EEPROM table data because of EEPROM data corruption!");
3027 if (new_cnt)
3028 *new_cnt = 0;
3029
3030 return 0;
3031 }
3032
3033 mutex_lock(&con->recovery_lock);
3034 control = &con->eeprom_control;
3035 data = con->eh_data;
3036 bad_page_num = control->ras_num_bad_pages;
3037 save_count = data->count - bad_page_num;
3038 mutex_unlock(&con->recovery_lock);
3039
3040 unit_num = save_count / adev->umc.retire_unit;
3041 if (new_cnt)
3042 *new_cnt = unit_num;
3043
3044 /* only new entries are saved */
3045 if (save_count > 0) {
3046 /*old asics only save pa to eeprom like before*/
3047 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) {
3048 if (amdgpu_ras_eeprom_append(control,
3049 &data->bps[bad_page_num], save_count)) {
3050 dev_err(adev->dev, "Failed to save EEPROM table data!");
3051 return -EIO;
3052 }
3053 } else {
3054 for (i = 0; i < unit_num; i++) {
3055 if (amdgpu_ras_eeprom_append(control,
3056 &data->bps[bad_page_num +
3057 i * adev->umc.retire_unit], 1)) {
3058 dev_err(adev->dev, "Failed to save EEPROM table data!");
3059 return -EIO;
3060 }
3061 }
3062 }
3063
3064 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
3065 }
3066
3067 return 0;
3068}
3069
3070/*
3071 * read error record array in eeprom and reserve enough space for
3072 * storing new bad pages
3073 */
3074static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
3075{
3076 struct amdgpu_ras_eeprom_control *control =
3077 &adev->psp.ras_context.ras->eeprom_control;
3078 struct eeprom_table_record *bps;
3079 int ret, i = 0;
3080
3081 /* no bad page record, skip eeprom access */
3082 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0)
3083 return 0;
3084
3085 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL);
3086 if (!bps)
3087 return -ENOMEM;
3088
3089 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs);
3090 if (ret) {
3091 dev_err(adev->dev, "Failed to load EEPROM table records!");
3092 } else {
3093 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) {
3094 /*In V3, there is no pa recs, and some cases(when address==0) may be parsed
3095 as pa recs, so add verion check to avoid it.
3096 */
3097 if (control->tbl_hdr.version < RAS_TABLE_VER_V3) {
3098 for (i = 0; i < control->ras_num_recs; i++) {
3099 if ((control->ras_num_recs - i) >= adev->umc.retire_unit) {
3100 if ((bps[i].address == bps[i + 1].address) &&
3101 (bps[i].mem_channel == bps[i + 1].mem_channel)) {
3102 control->ras_num_pa_recs += adev->umc.retire_unit;
3103 i += (adev->umc.retire_unit - 1);
3104 } else {
3105 control->ras_num_mca_recs +=
3106 (control->ras_num_recs - i);
3107 break;
3108 }
3109 } else {
3110 control->ras_num_mca_recs += (control->ras_num_recs - i);
3111 break;
3112 }
3113 }
3114 } else {
3115 control->ras_num_mca_recs = control->ras_num_recs;
3116 }
3117 }
3118
3119 ret = amdgpu_ras_eeprom_check(control);
3120 if (ret)
3121 goto out;
3122
3123 /* HW not usable */
3124 if (amdgpu_ras_is_rma(adev)) {
3125 ret = -EHWPOISON;
3126 goto out;
3127 }
3128
3129 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true);
3130 }
3131
3132out:
3133 kfree(bps);
3134 return ret;
3135}
3136
3137static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
3138 uint64_t addr)
3139{
3140 struct ras_err_handler_data *data = con->eh_data;
3141 int i;
3142
3143 addr >>= AMDGPU_GPU_PAGE_SHIFT;
3144 for (i = 0; i < data->count; i++)
3145 if (addr == data->bps[i].retired_page)
3146 return true;
3147
3148 return false;
3149}
3150
3151/*
3152 * check if an address belongs to bad page
3153 *
3154 * Note: this check is only for umc block
3155 */
3156static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
3157 uint64_t addr)
3158{
3159 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3160 bool ret = false;
3161
3162 if (!con || !con->eh_data)
3163 return ret;
3164
3165 mutex_lock(&con->recovery_lock);
3166 ret = amdgpu_ras_check_bad_page_unlock(con, addr);
3167 mutex_unlock(&con->recovery_lock);
3168 return ret;
3169}
3170
3171static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
3172 uint32_t max_count)
3173{
3174 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3175
3176 /*
3177 * amdgpu_bad_page_threshold is used to config
3178 * the threshold for the number of bad pages.
3179 * -1: Threshold is set to default value
3180 * Driver will issue a warning message when threshold is reached
3181 * and continue runtime services.
3182 * 0: Disable bad page retirement
3183 * Driver will not retire bad pages
3184 * which is intended for debugging purpose.
3185 * -2: Threshold is determined by a formula
3186 * that assumes 1 bad page per 100M of local memory.
3187 * Driver will continue runtime services when threhold is reached.
3188 * 0 < threshold < max number of bad page records in EEPROM,
3189 * A user-defined threshold is set
3190 * Driver will halt runtime services when this custom threshold is reached.
3191 */
3192 if (amdgpu_bad_page_threshold == -2) {
3193 u64 val = adev->gmc.mc_vram_size;
3194
3195 do_div(val, RAS_BAD_PAGE_COVER);
3196 con->bad_page_cnt_threshold = min(lower_32_bits(val),
3197 max_count);
3198 } else if (amdgpu_bad_page_threshold == -1) {
3199 con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4;
3200 } else {
3201 con->bad_page_cnt_threshold = min_t(int, max_count,
3202 amdgpu_bad_page_threshold);
3203 }
3204}
3205
3206int amdgpu_ras_put_poison_req(struct amdgpu_device *adev,
3207 enum amdgpu_ras_block block, uint16_t pasid,
3208 pasid_notify pasid_fn, void *data, uint32_t reset)
3209{
3210 int ret = 0;
3211 struct ras_poison_msg poison_msg;
3212 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3213
3214 memset(&poison_msg, 0, sizeof(poison_msg));
3215 poison_msg.block = block;
3216 poison_msg.pasid = pasid;
3217 poison_msg.reset = reset;
3218 poison_msg.pasid_fn = pasid_fn;
3219 poison_msg.data = data;
3220
3221 ret = kfifo_put(&con->poison_fifo, poison_msg);
3222 if (!ret) {
3223 dev_err(adev->dev, "Poison message fifo is full!\n");
3224 return -ENOSPC;
3225 }
3226
3227 return 0;
3228}
3229
3230static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev,
3231 struct ras_poison_msg *poison_msg)
3232{
3233 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3234
3235 return kfifo_get(&con->poison_fifo, poison_msg);
3236}
3237
3238static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log)
3239{
3240 mutex_init(&ecc_log->lock);
3241
3242 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL);
3243 ecc_log->de_queried_count = 0;
3244 ecc_log->prev_de_queried_count = 0;
3245}
3246
3247static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log)
3248{
3249 struct radix_tree_iter iter;
3250 void __rcu **slot;
3251 struct ras_ecc_err *ecc_err;
3252
3253 mutex_lock(&ecc_log->lock);
3254 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) {
3255 ecc_err = radix_tree_deref_slot(slot);
3256 kfree(ecc_err->err_pages.pfn);
3257 kfree(ecc_err);
3258 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot);
3259 }
3260 mutex_unlock(&ecc_log->lock);
3261
3262 mutex_destroy(&ecc_log->lock);
3263 ecc_log->de_queried_count = 0;
3264 ecc_log->prev_de_queried_count = 0;
3265}
3266
3267static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con,
3268 uint32_t delayed_ms)
3269{
3270 int ret;
3271
3272 mutex_lock(&con->umc_ecc_log.lock);
3273 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree,
3274 UMC_ECC_NEW_DETECTED_TAG);
3275 mutex_unlock(&con->umc_ecc_log.lock);
3276
3277 if (ret)
3278 schedule_delayed_work(&con->page_retirement_dwork,
3279 msecs_to_jiffies(delayed_ms));
3280
3281 return ret ? true : false;
3282}
3283
3284static void amdgpu_ras_do_page_retirement(struct work_struct *work)
3285{
3286 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
3287 page_retirement_dwork.work);
3288 struct amdgpu_device *adev = con->adev;
3289 struct ras_err_data err_data;
3290 unsigned long err_cnt;
3291
3292 /* If gpu reset is ongoing, delay retiring the bad pages */
3293 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) {
3294 amdgpu_ras_schedule_retirement_dwork(con,
3295 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3);
3296 return;
3297 }
3298
3299 amdgpu_ras_error_data_init(&err_data);
3300
3301 amdgpu_umc_handle_bad_pages(adev, &err_data);
3302 err_cnt = err_data.err_addr_cnt;
3303
3304 amdgpu_ras_error_data_fini(&err_data);
3305
3306 if (err_cnt && amdgpu_ras_is_rma(adev))
3307 amdgpu_ras_reset_gpu(adev);
3308
3309 amdgpu_ras_schedule_retirement_dwork(con,
3310 AMDGPU_RAS_RETIRE_PAGE_INTERVAL);
3311}
3312
3313static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev,
3314 uint32_t poison_creation_count)
3315{
3316 int ret = 0;
3317 struct ras_ecc_log_info *ecc_log;
3318 struct ras_query_if info;
3319 uint32_t timeout = 0;
3320 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3321 uint64_t de_queried_count;
3322 uint32_t new_detect_count, total_detect_count;
3323 uint32_t need_query_count = poison_creation_count;
3324 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
3325
3326 memset(&info, 0, sizeof(info));
3327 info.head.block = AMDGPU_RAS_BLOCK__UMC;
3328
3329 ecc_log = &ras->umc_ecc_log;
3330 total_detect_count = 0;
3331 do {
3332 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type);
3333 if (ret)
3334 return ret;
3335
3336 de_queried_count = ecc_log->de_queried_count;
3337 if (de_queried_count > ecc_log->prev_de_queried_count) {
3338 new_detect_count = de_queried_count - ecc_log->prev_de_queried_count;
3339 ecc_log->prev_de_queried_count = de_queried_count;
3340 timeout = 0;
3341 } else {
3342 new_detect_count = 0;
3343 }
3344
3345 if (new_detect_count) {
3346 total_detect_count += new_detect_count;
3347 } else {
3348 if (!timeout && need_query_count)
3349 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC;
3350
3351 if (timeout) {
3352 if (!--timeout)
3353 break;
3354 msleep(1);
3355 }
3356 }
3357 } while (total_detect_count < need_query_count);
3358
3359 if (total_detect_count)
3360 schedule_delayed_work(&ras->page_retirement_dwork, 0);
3361
3362 return 0;
3363}
3364
3365static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev)
3366{
3367 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3368 struct ras_poison_msg msg;
3369 int ret;
3370
3371 do {
3372 ret = kfifo_get(&con->poison_fifo, &msg);
3373 } while (ret);
3374}
3375
3376static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev,
3377 uint32_t msg_count, uint32_t *gpu_reset)
3378{
3379 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3380 uint32_t reset_flags = 0, reset = 0;
3381 struct ras_poison_msg msg;
3382 int ret, i;
3383
3384 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
3385
3386 for (i = 0; i < msg_count; i++) {
3387 ret = amdgpu_ras_get_poison_req(adev, &msg);
3388 if (!ret)
3389 continue;
3390
3391 if (msg.pasid_fn)
3392 msg.pasid_fn(adev, msg.pasid, msg.data);
3393
3394 reset_flags |= msg.reset;
3395 }
3396
3397 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */
3398 if (reset_flags && !amdgpu_ras_is_rma(adev)) {
3399 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET)
3400 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET;
3401 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET)
3402 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
3403 else
3404 reset = reset_flags;
3405
3406 flush_delayed_work(&con->page_retirement_dwork);
3407
3408 con->gpu_reset_flags |= reset;
3409 amdgpu_ras_reset_gpu(adev);
3410
3411 *gpu_reset = reset;
3412
3413 /* Wait for gpu recovery to complete */
3414 flush_work(&con->recovery_work);
3415 }
3416
3417 return 0;
3418}
3419
3420static int amdgpu_ras_page_retirement_thread(void *param)
3421{
3422 struct amdgpu_device *adev = (struct amdgpu_device *)param;
3423 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3424 uint32_t poison_creation_count, msg_count;
3425 uint32_t gpu_reset;
3426 int ret;
3427
3428 while (!kthread_should_stop()) {
3429
3430 wait_event_interruptible(con->page_retirement_wq,
3431 kthread_should_stop() ||
3432 atomic_read(&con->page_retirement_req_cnt));
3433
3434 if (kthread_should_stop())
3435 break;
3436
3437 gpu_reset = 0;
3438
3439 do {
3440 poison_creation_count = atomic_read(&con->poison_creation_count);
3441 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count);
3442 if (ret == -EIO)
3443 break;
3444
3445 if (poison_creation_count) {
3446 atomic_sub(poison_creation_count, &con->poison_creation_count);
3447 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt);
3448 }
3449 } while (atomic_read(&con->poison_creation_count));
3450
3451 if (ret != -EIO) {
3452 msg_count = kfifo_len(&con->poison_fifo);
3453 if (msg_count) {
3454 ret = amdgpu_ras_poison_consumption_handler(adev,
3455 msg_count, &gpu_reset);
3456 if ((ret != -EIO) &&
3457 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET))
3458 atomic_sub(msg_count, &con->page_retirement_req_cnt);
3459 }
3460 }
3461
3462 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) {
3463 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */
3464 /* Clear poison creation request */
3465 atomic_set(&con->poison_creation_count, 0);
3466
3467 /* Clear poison fifo */
3468 amdgpu_ras_clear_poison_fifo(adev);
3469
3470 /* Clear all poison requests */
3471 atomic_set(&con->page_retirement_req_cnt, 0);
3472
3473 if (ret == -EIO) {
3474 /* Wait for mode-1 reset to complete */
3475 down_read(&adev->reset_domain->sem);
3476 up_read(&adev->reset_domain->sem);
3477 }
3478
3479 /* Wake up work to save bad pages to eeprom */
3480 schedule_delayed_work(&con->page_retirement_dwork, 0);
3481 } else if (gpu_reset) {
3482 /* gpu just completed mode-2 reset or other reset */
3483 /* Clear poison consumption messages cached in fifo */
3484 msg_count = kfifo_len(&con->poison_fifo);
3485 if (msg_count) {
3486 amdgpu_ras_clear_poison_fifo(adev);
3487 atomic_sub(msg_count, &con->page_retirement_req_cnt);
3488 }
3489
3490 /* Wake up work to save bad pages to eeprom */
3491 schedule_delayed_work(&con->page_retirement_dwork, 0);
3492 }
3493 }
3494
3495 return 0;
3496}
3497
3498int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev)
3499{
3500 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3501 struct amdgpu_ras_eeprom_control *control;
3502 int ret;
3503
3504 if (!con || amdgpu_sriov_vf(adev))
3505 return 0;
3506
3507 control = &con->eeprom_control;
3508 ret = amdgpu_ras_eeprom_init(control);
3509 control->is_eeprom_valid = !ret;
3510
3511 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr)
3512 control->ras_num_pa_recs = control->ras_num_recs;
3513
3514 if (adev->umc.ras &&
3515 adev->umc.ras->get_retire_flip_bits)
3516 adev->umc.ras->get_retire_flip_bits(adev);
3517
3518 if (control->ras_num_recs && control->is_eeprom_valid) {
3519 ret = amdgpu_ras_load_bad_pages(adev);
3520 if (ret) {
3521 control->is_eeprom_valid = false;
3522 return 0;
3523 }
3524
3525 amdgpu_dpm_send_hbm_bad_pages_num(
3526 adev, control->ras_num_bad_pages);
3527
3528 if (con->update_channel_flag == true) {
3529 amdgpu_dpm_send_hbm_bad_channel_flag(
3530 adev, control->bad_channel_bitmap);
3531 con->update_channel_flag = false;
3532 }
3533
3534 /* The format action is only applied to new ASICs */
3535 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 &&
3536 control->tbl_hdr.version < RAS_TABLE_VER_V3)
3537 if (!amdgpu_ras_eeprom_reset_table(control))
3538 if (amdgpu_ras_save_bad_pages(adev, NULL))
3539 dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n");
3540 }
3541
3542 return 0;
3543}
3544
3545int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info)
3546{
3547 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3548 struct ras_err_handler_data **data;
3549 u32 max_eeprom_records_count = 0;
3550 int ret;
3551
3552 if (!con || amdgpu_sriov_vf(adev))
3553 return 0;
3554
3555 /* Allow access to RAS EEPROM via debugfs, when the ASIC
3556 * supports RAS and debugfs is enabled, but when
3557 * adev->ras_enabled is unset, i.e. when "ras_enable"
3558 * module parameter is set to 0.
3559 */
3560 con->adev = adev;
3561
3562 if (!adev->ras_enabled)
3563 return 0;
3564
3565 data = &con->eh_data;
3566 *data = kzalloc(sizeof(**data), GFP_KERNEL);
3567 if (!*data) {
3568 ret = -ENOMEM;
3569 goto out;
3570 }
3571
3572 mutex_init(&con->recovery_lock);
3573 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
3574 atomic_set(&con->in_recovery, 0);
3575 con->eeprom_control.bad_channel_bitmap = 0;
3576
3577 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control);
3578 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count);
3579
3580 if (init_bp_info) {
3581 ret = amdgpu_ras_init_badpage_info(adev);
3582 if (ret)
3583 goto free;
3584 }
3585
3586 mutex_init(&con->page_rsv_lock);
3587 INIT_KFIFO(con->poison_fifo);
3588 mutex_init(&con->page_retirement_lock);
3589 init_waitqueue_head(&con->page_retirement_wq);
3590 atomic_set(&con->page_retirement_req_cnt, 0);
3591 atomic_set(&con->poison_creation_count, 0);
3592 con->page_retirement_thread =
3593 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement");
3594 if (IS_ERR(con->page_retirement_thread)) {
3595 con->page_retirement_thread = NULL;
3596 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n");
3597 }
3598
3599 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement);
3600 amdgpu_ras_ecc_log_init(&con->umc_ecc_log);
3601#ifdef CONFIG_X86_MCE_AMD
3602 if ((adev->asic_type == CHIP_ALDEBARAN) &&
3603 (adev->gmc.xgmi.connected_to_cpu))
3604 amdgpu_register_bad_pages_mca_notifier(adev);
3605#endif
3606 return 0;
3607
3608free:
3609 kfree((*data)->bps);
3610 kfree(*data);
3611 con->eh_data = NULL;
3612out:
3613 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret);
3614
3615 /*
3616 * Except error threshold exceeding case, other failure cases in this
3617 * function would not fail amdgpu driver init.
3618 */
3619 if (!amdgpu_ras_is_rma(adev))
3620 ret = 0;
3621 else
3622 ret = -EINVAL;
3623
3624 return ret;
3625}
3626
3627static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
3628{
3629 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3630 struct ras_err_handler_data *data = con->eh_data;
3631 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES;
3632 bool ret;
3633
3634 /* recovery_init failed to init it, fini is useless */
3635 if (!data)
3636 return 0;
3637
3638 /* Save all cached bad pages to eeprom */
3639 do {
3640 flush_delayed_work(&con->page_retirement_dwork);
3641 ret = amdgpu_ras_schedule_retirement_dwork(con, 0);
3642 } while (ret && max_flush_timeout--);
3643
3644 if (con->page_retirement_thread)
3645 kthread_stop(con->page_retirement_thread);
3646
3647 atomic_set(&con->page_retirement_req_cnt, 0);
3648 atomic_set(&con->poison_creation_count, 0);
3649
3650 mutex_destroy(&con->page_rsv_lock);
3651
3652 cancel_work_sync(&con->recovery_work);
3653
3654 cancel_delayed_work_sync(&con->page_retirement_dwork);
3655
3656 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log);
3657
3658 mutex_lock(&con->recovery_lock);
3659 con->eh_data = NULL;
3660 kfree(data->bps);
3661 kfree(data);
3662 mutex_unlock(&con->recovery_lock);
3663
3664 return 0;
3665}
3666/* recovery end */
3667
3668static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
3669{
3670 if (amdgpu_sriov_vf(adev)) {
3671 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3672 case IP_VERSION(13, 0, 2):
3673 case IP_VERSION(13, 0, 6):
3674 case IP_VERSION(13, 0, 12):
3675 case IP_VERSION(13, 0, 14):
3676 return true;
3677 default:
3678 return false;
3679 }
3680 }
3681
3682 if (adev->asic_type == CHIP_IP_DISCOVERY) {
3683 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3684 case IP_VERSION(13, 0, 0):
3685 case IP_VERSION(13, 0, 6):
3686 case IP_VERSION(13, 0, 10):
3687 case IP_VERSION(13, 0, 12):
3688 case IP_VERSION(13, 0, 14):
3689 case IP_VERSION(14, 0, 3):
3690 return true;
3691 default:
3692 return false;
3693 }
3694 }
3695
3696 return adev->asic_type == CHIP_VEGA10 ||
3697 adev->asic_type == CHIP_VEGA20 ||
3698 adev->asic_type == CHIP_ARCTURUS ||
3699 adev->asic_type == CHIP_ALDEBARAN ||
3700 adev->asic_type == CHIP_SIENNA_CICHLID;
3701}
3702
3703/*
3704 * this is workaround for vega20 workstation sku,
3705 * force enable gfx ras, ignore vbios gfx ras flag
3706 * due to GC EDC can not write
3707 */
3708static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
3709{
3710 struct atom_context *ctx = adev->mode_info.atom_context;
3711
3712 if (!ctx)
3713 return;
3714
3715 if (strnstr(ctx->vbios_pn, "D16406",
3716 sizeof(ctx->vbios_pn)) ||
3717 strnstr(ctx->vbios_pn, "D36002",
3718 sizeof(ctx->vbios_pn)))
3719 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX);
3720}
3721
3722/* Query ras capablity via atomfirmware interface */
3723static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev)
3724{
3725 /* mem_ecc cap */
3726 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
3727 dev_info(adev->dev, "MEM ECC is active.\n");
3728 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC |
3729 1 << AMDGPU_RAS_BLOCK__DF);
3730 } else {
3731 dev_info(adev->dev, "MEM ECC is not presented.\n");
3732 }
3733
3734 /* sram_ecc cap */
3735 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
3736 dev_info(adev->dev, "SRAM ECC is active.\n");
3737 if (!amdgpu_sriov_vf(adev))
3738 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
3739 1 << AMDGPU_RAS_BLOCK__DF);
3740 else
3741 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF |
3742 1 << AMDGPU_RAS_BLOCK__SDMA |
3743 1 << AMDGPU_RAS_BLOCK__GFX);
3744
3745 /*
3746 * VCN/JPEG RAS can be supported on both bare metal and
3747 * SRIOV environment
3748 */
3749 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) ||
3750 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) ||
3751 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3) ||
3752 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(5, 0, 1))
3753 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN |
3754 1 << AMDGPU_RAS_BLOCK__JPEG);
3755 else
3756 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
3757 1 << AMDGPU_RAS_BLOCK__JPEG);
3758
3759 /*
3760 * XGMI RAS is not supported if xgmi num physical nodes
3761 * is zero
3762 */
3763 if (!adev->gmc.xgmi.num_physical_nodes)
3764 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL);
3765 } else {
3766 dev_info(adev->dev, "SRAM ECC is not presented.\n");
3767 }
3768}
3769
3770/* Query poison mode from umc/df IP callbacks */
3771static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev)
3772{
3773 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3774 bool df_poison, umc_poison;
3775
3776 /* poison setting is useless on SRIOV guest */
3777 if (amdgpu_sriov_vf(adev) || !con)
3778 return;
3779
3780 /* Init poison supported flag, the default value is false */
3781 if (adev->gmc.xgmi.connected_to_cpu ||
3782 adev->gmc.is_app_apu) {
3783 /* enabled by default when GPU is connected to CPU */
3784 con->poison_supported = true;
3785 } else if (adev->df.funcs &&
3786 adev->df.funcs->query_ras_poison_mode &&
3787 adev->umc.ras &&
3788 adev->umc.ras->query_ras_poison_mode) {
3789 df_poison =
3790 adev->df.funcs->query_ras_poison_mode(adev);
3791 umc_poison =
3792 adev->umc.ras->query_ras_poison_mode(adev);
3793
3794 /* Only poison is set in both DF and UMC, we can support it */
3795 if (df_poison && umc_poison)
3796 con->poison_supported = true;
3797 else if (df_poison != umc_poison)
3798 dev_warn(adev->dev,
3799 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n",
3800 df_poison, umc_poison);
3801 }
3802}
3803
3804/*
3805 * check hardware's ras ability which will be saved in hw_supported.
3806 * if hardware does not support ras, we can skip some ras initializtion and
3807 * forbid some ras operations from IP.
3808 * if software itself, say boot parameter, limit the ras ability. We still
3809 * need allow IP do some limited operations, like disable. In such case,
3810 * we have to initialize ras as normal. but need check if operation is
3811 * allowed or not in each function.
3812 */
3813static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
3814{
3815 adev->ras_hw_enabled = adev->ras_enabled = 0;
3816
3817 if (!amdgpu_ras_asic_supported(adev))
3818 return;
3819
3820 if (amdgpu_sriov_vf(adev)) {
3821 if (amdgpu_virt_get_ras_capability(adev))
3822 goto init_ras_enabled_flag;
3823 }
3824
3825 /* query ras capability from psp */
3826 if (amdgpu_psp_get_ras_capability(&adev->psp))
3827 goto init_ras_enabled_flag;
3828
3829 /* query ras capablity from bios */
3830 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
3831 amdgpu_ras_query_ras_capablity_from_vbios(adev);
3832 } else {
3833 /* driver only manages a few IP blocks RAS feature
3834 * when GPU is connected cpu through XGMI */
3835 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX |
3836 1 << AMDGPU_RAS_BLOCK__SDMA |
3837 1 << AMDGPU_RAS_BLOCK__MMHUB);
3838 }
3839
3840 /* apply asic specific settings (vega20 only for now) */
3841 amdgpu_ras_get_quirks(adev);
3842
3843 /* query poison mode from umc/df ip callback */
3844 amdgpu_ras_query_poison_mode(adev);
3845
3846init_ras_enabled_flag:
3847 /* hw_supported needs to be aligned with RAS block mask. */
3848 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
3849
3850 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
3851 adev->ras_hw_enabled & amdgpu_ras_mask;
3852
3853 /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */
3854 if (!amdgpu_sriov_vf(adev)) {
3855 adev->aca.is_enabled =
3856 (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) ||
3857 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) ||
3858 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14));
3859 }
3860
3861 /* bad page feature is not applicable to specific app platform */
3862 if (adev->gmc.is_app_apu &&
3863 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0))
3864 amdgpu_bad_page_threshold = 0;
3865}
3866
3867static void amdgpu_ras_counte_dw(struct work_struct *work)
3868{
3869 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
3870 ras_counte_delay_work.work);
3871 struct amdgpu_device *adev = con->adev;
3872 struct drm_device *dev = adev_to_drm(adev);
3873 unsigned long ce_count, ue_count;
3874 int res;
3875
3876 res = pm_runtime_get_sync(dev->dev);
3877 if (res < 0)
3878 goto Out;
3879
3880 /* Cache new values.
3881 */
3882 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) {
3883 atomic_set(&con->ras_ce_count, ce_count);
3884 atomic_set(&con->ras_ue_count, ue_count);
3885 }
3886
3887 pm_runtime_mark_last_busy(dev->dev);
3888Out:
3889 pm_runtime_put_autosuspend(dev->dev);
3890}
3891
3892static int amdgpu_get_ras_schema(struct amdgpu_device *adev)
3893{
3894 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 |
3895 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE |
3896 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE |
3897 AMDGPU_RAS_ERROR__PARITY;
3898}
3899
3900static void ras_event_mgr_init(struct ras_event_manager *mgr)
3901{
3902 struct ras_event_state *event_state;
3903 int i;
3904
3905 memset(mgr, 0, sizeof(*mgr));
3906 atomic64_set(&mgr->seqno, 0);
3907
3908 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) {
3909 event_state = &mgr->event_state[i];
3910 event_state->last_seqno = RAS_EVENT_INVALID_ID;
3911 atomic64_set(&event_state->count, 0);
3912 }
3913}
3914
3915static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev)
3916{
3917 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3918 struct amdgpu_hive_info *hive;
3919
3920 if (!ras)
3921 return;
3922
3923 hive = amdgpu_get_xgmi_hive(adev);
3924 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr;
3925
3926 /* init event manager with node 0 on xgmi system */
3927 if (!amdgpu_reset_in_recovery(adev)) {
3928 if (!hive || adev->gmc.xgmi.node_id == 0)
3929 ras_event_mgr_init(ras->event_mgr);
3930 }
3931
3932 if (hive)
3933 amdgpu_put_xgmi_hive(hive);
3934}
3935
3936static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev)
3937{
3938 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3939
3940 if (!con || (adev->flags & AMD_IS_APU))
3941 return;
3942
3943 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3944 case IP_VERSION(13, 0, 2):
3945 case IP_VERSION(13, 0, 6):
3946 case IP_VERSION(13, 0, 12):
3947 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT;
3948 break;
3949 case IP_VERSION(13, 0, 14):
3950 con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1);
3951 break;
3952 default:
3953 break;
3954 }
3955}
3956
3957int amdgpu_ras_init(struct amdgpu_device *adev)
3958{
3959 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3960 int r;
3961
3962 if (con)
3963 return 0;
3964
3965 con = kzalloc(sizeof(*con) +
3966 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT +
3967 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT,
3968 GFP_KERNEL);
3969 if (!con)
3970 return -ENOMEM;
3971
3972 con->adev = adev;
3973 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw);
3974 atomic_set(&con->ras_ce_count, 0);
3975 atomic_set(&con->ras_ue_count, 0);
3976
3977 con->objs = (struct ras_manager *)(con + 1);
3978
3979 amdgpu_ras_set_context(adev, con);
3980
3981 amdgpu_ras_check_supported(adev);
3982
3983 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) {
3984 /* set gfx block ras context feature for VEGA20 Gaming
3985 * send ras disable cmd to ras ta during ras late init.
3986 */
3987 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
3988 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX);
3989
3990 return 0;
3991 }
3992
3993 r = 0;
3994 goto release_con;
3995 }
3996
3997 con->update_channel_flag = false;
3998 con->features = 0;
3999 con->schema = 0;
4000 INIT_LIST_HEAD(&con->head);
4001 /* Might need get this flag from vbios. */
4002 con->flags = RAS_DEFAULT_FLAGS;
4003
4004 /* initialize nbio ras function ahead of any other
4005 * ras functions so hardware fatal error interrupt
4006 * can be enabled as early as possible */
4007 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) {
4008 case IP_VERSION(7, 4, 0):
4009 case IP_VERSION(7, 4, 1):
4010 case IP_VERSION(7, 4, 4):
4011 if (!adev->gmc.xgmi.connected_to_cpu)
4012 adev->nbio.ras = &nbio_v7_4_ras;
4013 break;
4014 case IP_VERSION(4, 3, 0):
4015 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
4016 /* unlike other generation of nbio ras,
4017 * nbio v4_3 only support fatal error interrupt
4018 * to inform software that DF is freezed due to
4019 * system fatal error event. driver should not
4020 * enable nbio ras in such case. Instead,
4021 * check DF RAS */
4022 adev->nbio.ras = &nbio_v4_3_ras;
4023 break;
4024 case IP_VERSION(6, 3, 1):
4025 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
4026 /* unlike other generation of nbio ras,
4027 * nbif v6_3_1 only support fatal error interrupt
4028 * to inform software that DF is freezed due to
4029 * system fatal error event. driver should not
4030 * enable nbio ras in such case. Instead,
4031 * check DF RAS
4032 */
4033 adev->nbio.ras = &nbif_v6_3_1_ras;
4034 break;
4035 case IP_VERSION(7, 9, 0):
4036 case IP_VERSION(7, 9, 1):
4037 if (!adev->gmc.is_app_apu)
4038 adev->nbio.ras = &nbio_v7_9_ras;
4039 break;
4040 default:
4041 /* nbio ras is not available */
4042 break;
4043 }
4044
4045 /* nbio ras block needs to be enabled ahead of other ras blocks
4046 * to handle fatal error */
4047 r = amdgpu_nbio_ras_sw_init(adev);
4048 if (r)
4049 return r;
4050
4051 if (adev->nbio.ras &&
4052 adev->nbio.ras->init_ras_controller_interrupt) {
4053 r = adev->nbio.ras->init_ras_controller_interrupt(adev);
4054 if (r)
4055 goto release_con;
4056 }
4057
4058 if (adev->nbio.ras &&
4059 adev->nbio.ras->init_ras_err_event_athub_interrupt) {
4060 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev);
4061 if (r)
4062 goto release_con;
4063 }
4064
4065 /* Packed socket_id to ras feature mask bits[31:29] */
4066 if (adev->smuio.funcs &&
4067 adev->smuio.funcs->get_socket_id)
4068 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) <<
4069 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT);
4070
4071 /* Get RAS schema for particular SOC */
4072 con->schema = amdgpu_get_ras_schema(adev);
4073
4074 amdgpu_ras_init_reserved_vram_size(adev);
4075
4076 if (amdgpu_ras_fs_init(adev)) {
4077 r = -EINVAL;
4078 goto release_con;
4079 }
4080
4081 if (amdgpu_ras_aca_is_supported(adev)) {
4082 if (amdgpu_aca_is_enabled(adev))
4083 r = amdgpu_aca_init(adev);
4084 else
4085 r = amdgpu_mca_init(adev);
4086 if (r)
4087 goto release_con;
4088 }
4089
4090 dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
4091 "hardware ability[%x] ras_mask[%x]\n",
4092 adev->ras_hw_enabled, adev->ras_enabled);
4093
4094 return 0;
4095release_con:
4096 amdgpu_ras_set_context(adev, NULL);
4097 kfree(con);
4098
4099 return r;
4100}
4101
4102int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev)
4103{
4104 if (adev->gmc.xgmi.connected_to_cpu ||
4105 adev->gmc.is_app_apu)
4106 return 1;
4107 return 0;
4108}
4109
4110static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev,
4111 struct ras_common_if *ras_block)
4112{
4113 struct ras_query_if info = {
4114 .head = *ras_block,
4115 };
4116
4117 if (!amdgpu_persistent_edc_harvesting_supported(adev))
4118 return 0;
4119
4120 if (amdgpu_ras_query_error_status(adev, &info) != 0)
4121 DRM_WARN("RAS init harvest failure");
4122
4123 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0)
4124 DRM_WARN("RAS init harvest reset failure");
4125
4126 return 0;
4127}
4128
4129bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev)
4130{
4131 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4132
4133 if (!con)
4134 return false;
4135
4136 return con->poison_supported;
4137}
4138
4139/* helper function to handle common stuff in ip late init phase */
4140int amdgpu_ras_block_late_init(struct amdgpu_device *adev,
4141 struct ras_common_if *ras_block)
4142{
4143 struct amdgpu_ras_block_object *ras_obj = NULL;
4144 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4145 struct ras_query_if *query_info;
4146 unsigned long ue_count, ce_count;
4147 int r;
4148
4149 /* disable RAS feature per IP block if it is not supported */
4150 if (!amdgpu_ras_is_supported(adev, ras_block->block)) {
4151 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
4152 return 0;
4153 }
4154
4155 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1);
4156 if (r) {
4157 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) {
4158 /* in resume phase, if fail to enable ras,
4159 * clean up all ras fs nodes, and disable ras */
4160 goto cleanup;
4161 } else
4162 return r;
4163 }
4164
4165 /* check for errors on warm reset edc persisant supported ASIC */
4166 amdgpu_persistent_edc_harvesting(adev, ras_block);
4167
4168 /* in resume phase, no need to create ras fs node */
4169 if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
4170 return 0;
4171
4172 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
4173 if (ras_obj->ras_cb || (ras_obj->hw_ops &&
4174 (ras_obj->hw_ops->query_poison_status ||
4175 ras_obj->hw_ops->handle_poison_consumption))) {
4176 r = amdgpu_ras_interrupt_add_handler(adev, ras_block);
4177 if (r)
4178 goto cleanup;
4179 }
4180
4181 if (ras_obj->hw_ops &&
4182 (ras_obj->hw_ops->query_ras_error_count ||
4183 ras_obj->hw_ops->query_ras_error_status)) {
4184 r = amdgpu_ras_sysfs_create(adev, ras_block);
4185 if (r)
4186 goto interrupt;
4187
4188 /* Those are the cached values at init.
4189 */
4190 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL);
4191 if (!query_info)
4192 return -ENOMEM;
4193 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if));
4194
4195 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) {
4196 atomic_set(&con->ras_ce_count, ce_count);
4197 atomic_set(&con->ras_ue_count, ue_count);
4198 }
4199
4200 kfree(query_info);
4201 }
4202
4203 return 0;
4204
4205interrupt:
4206 if (ras_obj->ras_cb)
4207 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
4208cleanup:
4209 amdgpu_ras_feature_enable(adev, ras_block, 0);
4210 return r;
4211}
4212
4213static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev,
4214 struct ras_common_if *ras_block)
4215{
4216 return amdgpu_ras_block_late_init(adev, ras_block);
4217}
4218
4219/* helper function to remove ras fs node and interrupt handler */
4220void amdgpu_ras_block_late_fini(struct amdgpu_device *adev,
4221 struct ras_common_if *ras_block)
4222{
4223 struct amdgpu_ras_block_object *ras_obj;
4224 if (!ras_block)
4225 return;
4226
4227 amdgpu_ras_sysfs_remove(adev, ras_block);
4228
4229 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
4230 if (ras_obj->ras_cb)
4231 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
4232}
4233
4234static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev,
4235 struct ras_common_if *ras_block)
4236{
4237 return amdgpu_ras_block_late_fini(adev, ras_block);
4238}
4239
4240/* do some init work after IP late init as dependence.
4241 * and it runs in resume/gpu reset/booting up cases.
4242 */
4243void amdgpu_ras_resume(struct amdgpu_device *adev)
4244{
4245 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4246 struct ras_manager *obj, *tmp;
4247
4248 if (!adev->ras_enabled || !con) {
4249 /* clean ras context for VEGA20 Gaming after send ras disable cmd */
4250 amdgpu_release_ras_context(adev);
4251
4252 return;
4253 }
4254
4255 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
4256 /* Set up all other IPs which are not implemented. There is a
4257 * tricky thing that IP's actual ras error type should be
4258 * MULTI_UNCORRECTABLE, but as driver does not handle it, so
4259 * ERROR_NONE make sense anyway.
4260 */
4261 amdgpu_ras_enable_all_features(adev, 1);
4262
4263 /* We enable ras on all hw_supported block, but as boot
4264 * parameter might disable some of them and one or more IP has
4265 * not implemented yet. So we disable them on behalf.
4266 */
4267 list_for_each_entry_safe(obj, tmp, &con->head, node) {
4268 if (!amdgpu_ras_is_supported(adev, obj->head.block)) {
4269 amdgpu_ras_feature_enable(adev, &obj->head, 0);
4270 /* there should be no any reference. */
4271 WARN_ON(alive_obj(obj));
4272 }
4273 }
4274 }
4275}
4276
4277void amdgpu_ras_suspend(struct amdgpu_device *adev)
4278{
4279 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4280
4281 if (!adev->ras_enabled || !con)
4282 return;
4283
4284 amdgpu_ras_disable_all_features(adev, 0);
4285 /* Make sure all ras objects are disabled. */
4286 if (AMDGPU_RAS_GET_FEATURES(con->features))
4287 amdgpu_ras_disable_all_features(adev, 1);
4288}
4289
4290int amdgpu_ras_late_init(struct amdgpu_device *adev)
4291{
4292 struct amdgpu_ras_block_list *node, *tmp;
4293 struct amdgpu_ras_block_object *obj;
4294 int r;
4295
4296 amdgpu_ras_event_mgr_init(adev);
4297
4298 if (amdgpu_ras_aca_is_supported(adev)) {
4299 if (amdgpu_reset_in_recovery(adev)) {
4300 if (amdgpu_aca_is_enabled(adev))
4301 r = amdgpu_aca_reset(adev);
4302 else
4303 r = amdgpu_mca_reset(adev);
4304 if (r)
4305 return r;
4306 }
4307
4308 if (!amdgpu_sriov_vf(adev)) {
4309 if (amdgpu_aca_is_enabled(adev))
4310 amdgpu_ras_set_aca_debug_mode(adev, false);
4311 else
4312 amdgpu_ras_set_mca_debug_mode(adev, false);
4313 }
4314 }
4315
4316 /* Guest side doesn't need init ras feature */
4317 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev))
4318 return 0;
4319
4320 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
4321 obj = node->ras_obj;
4322 if (!obj) {
4323 dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
4324 continue;
4325 }
4326
4327 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block))
4328 continue;
4329
4330 if (obj->ras_late_init) {
4331 r = obj->ras_late_init(adev, &obj->ras_comm);
4332 if (r) {
4333 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n",
4334 obj->ras_comm.name, r);
4335 return r;
4336 }
4337 } else
4338 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm);
4339 }
4340
4341 return 0;
4342}
4343
4344/* do some fini work before IP fini as dependence */
4345int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
4346{
4347 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4348
4349 if (!adev->ras_enabled || !con)
4350 return 0;
4351
4352
4353 /* Need disable ras on all IPs here before ip [hw/sw]fini */
4354 if (AMDGPU_RAS_GET_FEATURES(con->features))
4355 amdgpu_ras_disable_all_features(adev, 0);
4356 amdgpu_ras_recovery_fini(adev);
4357 return 0;
4358}
4359
4360int amdgpu_ras_fini(struct amdgpu_device *adev)
4361{
4362 struct amdgpu_ras_block_list *ras_node, *tmp;
4363 struct amdgpu_ras_block_object *obj = NULL;
4364 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4365
4366 if (!adev->ras_enabled || !con)
4367 return 0;
4368
4369 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) {
4370 if (ras_node->ras_obj) {
4371 obj = ras_node->ras_obj;
4372 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) &&
4373 obj->ras_fini)
4374 obj->ras_fini(adev, &obj->ras_comm);
4375 else
4376 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm);
4377 }
4378
4379 /* Clear ras blocks from ras_list and free ras block list node */
4380 list_del(&ras_node->node);
4381 kfree(ras_node);
4382 }
4383
4384 amdgpu_ras_fs_fini(adev);
4385 amdgpu_ras_interrupt_remove_all(adev);
4386
4387 if (amdgpu_ras_aca_is_supported(adev)) {
4388 if (amdgpu_aca_is_enabled(adev))
4389 amdgpu_aca_fini(adev);
4390 else
4391 amdgpu_mca_fini(adev);
4392 }
4393
4394 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared");
4395
4396 if (AMDGPU_RAS_GET_FEATURES(con->features))
4397 amdgpu_ras_disable_all_features(adev, 0);
4398
4399 cancel_delayed_work_sync(&con->ras_counte_delay_work);
4400
4401 amdgpu_ras_set_context(adev, NULL);
4402 kfree(con);
4403
4404 return 0;
4405}
4406
4407bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev)
4408{
4409 struct amdgpu_ras *ras;
4410
4411 ras = amdgpu_ras_get_context(adev);
4412 if (!ras)
4413 return false;
4414
4415 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4416}
4417
4418void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status)
4419{
4420 struct amdgpu_ras *ras;
4421
4422 ras = amdgpu_ras_get_context(adev);
4423 if (ras) {
4424 if (status)
4425 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4426 else
4427 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4428 }
4429}
4430
4431void amdgpu_ras_clear_err_state(struct amdgpu_device *adev)
4432{
4433 struct amdgpu_ras *ras;
4434
4435 ras = amdgpu_ras_get_context(adev);
4436 if (ras) {
4437 ras->ras_err_state = 0;
4438 ras->gpu_reset_flags = 0;
4439 }
4440}
4441
4442void amdgpu_ras_set_err_poison(struct amdgpu_device *adev,
4443 enum amdgpu_ras_block block)
4444{
4445 struct amdgpu_ras *ras;
4446
4447 ras = amdgpu_ras_get_context(adev);
4448 if (ras)
4449 set_bit(block, &ras->ras_err_state);
4450}
4451
4452bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block)
4453{
4454 struct amdgpu_ras *ras;
4455
4456 ras = amdgpu_ras_get_context(adev);
4457 if (ras) {
4458 if (block == AMDGPU_RAS_BLOCK__ANY)
4459 return (ras->ras_err_state != 0);
4460 else
4461 return test_bit(block, &ras->ras_err_state) ||
4462 test_bit(AMDGPU_RAS_BLOCK__LAST,
4463 &ras->ras_err_state);
4464 }
4465
4466 return false;
4467}
4468
4469static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev)
4470{
4471 struct amdgpu_ras *ras;
4472
4473 ras = amdgpu_ras_get_context(adev);
4474 if (!ras)
4475 return NULL;
4476
4477 return ras->event_mgr;
4478}
4479
4480int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type,
4481 const void *caller)
4482{
4483 struct ras_event_manager *event_mgr;
4484 struct ras_event_state *event_state;
4485 int ret = 0;
4486
4487 if (type >= RAS_EVENT_TYPE_COUNT) {
4488 ret = -EINVAL;
4489 goto out;
4490 }
4491
4492 event_mgr = __get_ras_event_mgr(adev);
4493 if (!event_mgr) {
4494 ret = -EINVAL;
4495 goto out;
4496 }
4497
4498 event_state = &event_mgr->event_state[type];
4499 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno);
4500 atomic64_inc(&event_state->count);
4501
4502out:
4503 if (ret && caller)
4504 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n",
4505 (int)type, caller, ret);
4506
4507 return ret;
4508}
4509
4510u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type)
4511{
4512 struct ras_event_manager *event_mgr;
4513 u64 id;
4514
4515 if (type >= RAS_EVENT_TYPE_COUNT)
4516 return RAS_EVENT_INVALID_ID;
4517
4518 switch (type) {
4519 case RAS_EVENT_TYPE_FATAL:
4520 case RAS_EVENT_TYPE_POISON_CREATION:
4521 case RAS_EVENT_TYPE_POISON_CONSUMPTION:
4522 event_mgr = __get_ras_event_mgr(adev);
4523 if (!event_mgr)
4524 return RAS_EVENT_INVALID_ID;
4525
4526 id = event_mgr->event_state[type].last_seqno;
4527 break;
4528 case RAS_EVENT_TYPE_INVALID:
4529 default:
4530 id = RAS_EVENT_INVALID_ID;
4531 break;
4532 }
4533
4534 return id;
4535}
4536
4537void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
4538{
4539 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
4540 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4541 enum ras_event_type type = RAS_EVENT_TYPE_FATAL;
4542 u64 event_id;
4543
4544 if (amdgpu_ras_mark_ras_event(adev, type)) {
4545 dev_err(adev->dev,
4546 "uncorrectable hardware error (ERREVENT_ATHUB_INTERRUPT) detected!\n");
4547 return;
4548 }
4549
4550 event_id = amdgpu_ras_acquire_event_id(adev, type);
4551
4552 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error"
4553 "(ERREVENT_ATHUB_INTERRUPT) detected!\n");
4554
4555 amdgpu_ras_set_fed(adev, true);
4556 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
4557 amdgpu_ras_reset_gpu(adev);
4558 }
4559}
4560
4561bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev)
4562{
4563 if (adev->asic_type == CHIP_VEGA20 &&
4564 adev->pm.fw_version <= 0x283400) {
4565 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) &&
4566 amdgpu_ras_intr_triggered();
4567 }
4568
4569 return false;
4570}
4571
4572void amdgpu_release_ras_context(struct amdgpu_device *adev)
4573{
4574 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4575
4576 if (!con)
4577 return;
4578
4579 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
4580 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
4581 amdgpu_ras_set_context(adev, NULL);
4582 kfree(con);
4583 }
4584}
4585
4586#ifdef CONFIG_X86_MCE_AMD
4587static struct amdgpu_device *find_adev(uint32_t node_id)
4588{
4589 int i;
4590 struct amdgpu_device *adev = NULL;
4591
4592 for (i = 0; i < mce_adev_list.num_gpu; i++) {
4593 adev = mce_adev_list.devs[i];
4594
4595 if (adev && adev->gmc.xgmi.connected_to_cpu &&
4596 adev->gmc.xgmi.physical_node_id == node_id)
4597 break;
4598 adev = NULL;
4599 }
4600
4601 return adev;
4602}
4603
4604#define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF)
4605#define GET_UMC_INST(m) (((m) >> 21) & 0x7)
4606#define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4))
4607#define GPU_ID_OFFSET 8
4608
4609static int amdgpu_bad_page_notifier(struct notifier_block *nb,
4610 unsigned long val, void *data)
4611{
4612 struct mce *m = (struct mce *)data;
4613 struct amdgpu_device *adev = NULL;
4614 uint32_t gpu_id = 0;
4615 uint32_t umc_inst = 0, ch_inst = 0;
4616
4617 /*
4618 * If the error was generated in UMC_V2, which belongs to GPU UMCs,
4619 * and error occurred in DramECC (Extended error code = 0) then only
4620 * process the error, else bail out.
4621 */
4622 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) &&
4623 (XEC(m->status, 0x3f) == 0x0)))
4624 return NOTIFY_DONE;
4625
4626 /*
4627 * If it is correctable error, return.
4628 */
4629 if (mce_is_correctable(m))
4630 return NOTIFY_OK;
4631
4632 /*
4633 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
4634 */
4635 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET;
4636
4637 adev = find_adev(gpu_id);
4638 if (!adev) {
4639 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__,
4640 gpu_id);
4641 return NOTIFY_DONE;
4642 }
4643
4644 /*
4645 * If it is uncorrectable error, then find out UMC instance and
4646 * channel index.
4647 */
4648 umc_inst = GET_UMC_INST(m->ipid);
4649 ch_inst = GET_CHAN_INDEX(m->ipid);
4650
4651 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d",
4652 umc_inst, ch_inst);
4653
4654 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst))
4655 return NOTIFY_OK;
4656 else
4657 return NOTIFY_DONE;
4658}
4659
4660static struct notifier_block amdgpu_bad_page_nb = {
4661 .notifier_call = amdgpu_bad_page_notifier,
4662 .priority = MCE_PRIO_UC,
4663};
4664
4665static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev)
4666{
4667 /*
4668 * Add the adev to the mce_adev_list.
4669 * During mode2 reset, amdgpu device is temporarily
4670 * removed from the mgpu_info list which can cause
4671 * page retirement to fail.
4672 * Use this list instead of mgpu_info to find the amdgpu
4673 * device on which the UMC error was reported.
4674 */
4675 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev;
4676
4677 /*
4678 * Register the x86 notifier only once
4679 * with MCE subsystem.
4680 */
4681 if (notifier_registered == false) {
4682 mce_register_decode_chain(&amdgpu_bad_page_nb);
4683 notifier_registered = true;
4684 }
4685}
4686#endif
4687
4688struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev)
4689{
4690 if (!adev)
4691 return NULL;
4692
4693 return adev->psp.ras_context.ras;
4694}
4695
4696int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con)
4697{
4698 if (!adev)
4699 return -EINVAL;
4700
4701 adev->psp.ras_context.ras = ras_con;
4702 return 0;
4703}
4704
4705/* check if ras is supported on block, say, sdma, gfx */
4706int amdgpu_ras_is_supported(struct amdgpu_device *adev,
4707 unsigned int block)
4708{
4709 int ret = 0;
4710 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4711
4712 if (block >= AMDGPU_RAS_BLOCK_COUNT)
4713 return 0;
4714
4715 ret = ras && (adev->ras_enabled & (1 << block));
4716
4717 /* For the special asic with mem ecc enabled but sram ecc
4718 * not enabled, even if the ras block is not supported on
4719 * .ras_enabled, if the asic supports poison mode and the
4720 * ras block has ras configuration, it can be considered
4721 * that the ras block supports ras function.
4722 */
4723 if (!ret &&
4724 (block == AMDGPU_RAS_BLOCK__GFX ||
4725 block == AMDGPU_RAS_BLOCK__SDMA ||
4726 block == AMDGPU_RAS_BLOCK__VCN ||
4727 block == AMDGPU_RAS_BLOCK__JPEG) &&
4728 (amdgpu_ras_mask & (1 << block)) &&
4729 amdgpu_ras_is_poison_mode_supported(adev) &&
4730 amdgpu_ras_get_ras_block(adev, block, 0))
4731 ret = 1;
4732
4733 return ret;
4734}
4735
4736int amdgpu_ras_reset_gpu(struct amdgpu_device *adev)
4737{
4738 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4739
4740 /* mode1 is the only selection for RMA status */
4741 if (amdgpu_ras_is_rma(adev)) {
4742 ras->gpu_reset_flags = 0;
4743 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
4744 }
4745
4746 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) {
4747 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
4748 int hive_ras_recovery = 0;
4749
4750 if (hive) {
4751 hive_ras_recovery = atomic_read(&hive->ras_recovery);
4752 amdgpu_put_xgmi_hive(hive);
4753 }
4754 /* In the case of multiple GPUs, after a GPU has started
4755 * resetting all GPUs on hive, other GPUs do not need to
4756 * trigger GPU reset again.
4757 */
4758 if (!hive_ras_recovery)
4759 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
4760 else
4761 atomic_set(&ras->in_recovery, 0);
4762 } else {
4763 flush_work(&ras->recovery_work);
4764 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
4765 }
4766
4767 return 0;
4768}
4769
4770int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable)
4771{
4772 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4773 int ret = 0;
4774
4775 if (con) {
4776 ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
4777 if (!ret)
4778 con->is_aca_debug_mode = enable;
4779 }
4780
4781 return ret;
4782}
4783
4784int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable)
4785{
4786 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4787 int ret = 0;
4788
4789 if (con) {
4790 if (amdgpu_aca_is_enabled(adev))
4791 ret = amdgpu_aca_smu_set_debug_mode(adev, enable);
4792 else
4793 ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
4794 if (!ret)
4795 con->is_aca_debug_mode = enable;
4796 }
4797
4798 return ret;
4799}
4800
4801bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev)
4802{
4803 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4804 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
4805 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
4806
4807 if (!con)
4808 return false;
4809
4810 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) ||
4811 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode))
4812 return con->is_aca_debug_mode;
4813 else
4814 return true;
4815}
4816
4817bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev,
4818 unsigned int *error_query_mode)
4819{
4820 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4821 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
4822 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
4823
4824 if (!con) {
4825 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY;
4826 return false;
4827 }
4828
4829 if (amdgpu_sriov_vf(adev)) {
4830 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY;
4831 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) {
4832 *error_query_mode =
4833 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY;
4834 } else {
4835 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY;
4836 }
4837
4838 return true;
4839}
4840
4841/* Register each ip ras block into amdgpu ras */
4842int amdgpu_ras_register_ras_block(struct amdgpu_device *adev,
4843 struct amdgpu_ras_block_object *ras_block_obj)
4844{
4845 struct amdgpu_ras_block_list *ras_node;
4846 if (!adev || !ras_block_obj)
4847 return -EINVAL;
4848
4849 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL);
4850 if (!ras_node)
4851 return -ENOMEM;
4852
4853 INIT_LIST_HEAD(&ras_node->node);
4854 ras_node->ras_obj = ras_block_obj;
4855 list_add_tail(&ras_node->node, &adev->ras_list);
4856
4857 return 0;
4858}
4859
4860void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name)
4861{
4862 if (!err_type_name)
4863 return;
4864
4865 switch (err_type) {
4866 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE:
4867 sprintf(err_type_name, "correctable");
4868 break;
4869 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE:
4870 sprintf(err_type_name, "uncorrectable");
4871 break;
4872 default:
4873 sprintf(err_type_name, "unknown");
4874 break;
4875 }
4876}
4877
4878bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev,
4879 const struct amdgpu_ras_err_status_reg_entry *reg_entry,
4880 uint32_t instance,
4881 uint32_t *memory_id)
4882{
4883 uint32_t err_status_lo_data, err_status_lo_offset;
4884
4885 if (!reg_entry)
4886 return false;
4887
4888 err_status_lo_offset =
4889 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
4890 reg_entry->seg_lo, reg_entry->reg_lo);
4891 err_status_lo_data = RREG32(err_status_lo_offset);
4892
4893 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) &&
4894 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG))
4895 return false;
4896
4897 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID);
4898
4899 return true;
4900}
4901
4902bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev,
4903 const struct amdgpu_ras_err_status_reg_entry *reg_entry,
4904 uint32_t instance,
4905 unsigned long *err_cnt)
4906{
4907 uint32_t err_status_hi_data, err_status_hi_offset;
4908
4909 if (!reg_entry)
4910 return false;
4911
4912 err_status_hi_offset =
4913 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
4914 reg_entry->seg_hi, reg_entry->reg_hi);
4915 err_status_hi_data = RREG32(err_status_hi_offset);
4916
4917 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) &&
4918 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG))
4919 /* keep the check here in case we need to refer to the result later */
4920 dev_dbg(adev->dev, "Invalid err_info field\n");
4921
4922 /* read err count */
4923 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT);
4924
4925 return true;
4926}
4927
4928void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev,
4929 const struct amdgpu_ras_err_status_reg_entry *reg_list,
4930 uint32_t reg_list_size,
4931 const struct amdgpu_ras_memory_id_entry *mem_list,
4932 uint32_t mem_list_size,
4933 uint32_t instance,
4934 uint32_t err_type,
4935 unsigned long *err_count)
4936{
4937 uint32_t memory_id;
4938 unsigned long err_cnt;
4939 char err_type_name[16];
4940 uint32_t i, j;
4941
4942 for (i = 0; i < reg_list_size; i++) {
4943 /* query memory_id from err_status_lo */
4944 if (!amdgpu_ras_inst_get_memory_id_field(adev, &reg_list[i],
4945 instance, &memory_id))
4946 continue;
4947
4948 /* query err_cnt from err_status_hi */
4949 if (!amdgpu_ras_inst_get_err_cnt_field(adev, &reg_list[i],
4950 instance, &err_cnt) ||
4951 !err_cnt)
4952 continue;
4953
4954 *err_count += err_cnt;
4955
4956 /* log the errors */
4957 amdgpu_ras_get_error_type_name(err_type, err_type_name);
4958 if (!mem_list) {
4959 /* memory_list is not supported */
4960 dev_info(adev->dev,
4961 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n",
4962 err_cnt, err_type_name,
4963 reg_list[i].block_name,
4964 instance, memory_id);
4965 } else {
4966 for (j = 0; j < mem_list_size; j++) {
4967 if (memory_id == mem_list[j].memory_id) {
4968 dev_info(adev->dev,
4969 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n",
4970 err_cnt, err_type_name,
4971 reg_list[i].block_name,
4972 instance, mem_list[j].name);
4973 break;
4974 }
4975 }
4976 }
4977 }
4978}
4979
4980void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev,
4981 const struct amdgpu_ras_err_status_reg_entry *reg_list,
4982 uint32_t reg_list_size,
4983 uint32_t instance)
4984{
4985 uint32_t err_status_lo_offset, err_status_hi_offset;
4986 uint32_t i;
4987
4988 for (i = 0; i < reg_list_size; i++) {
4989 err_status_lo_offset =
4990 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
4991 reg_list[i].seg_lo, reg_list[i].reg_lo);
4992 err_status_hi_offset =
4993 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
4994 reg_list[i].seg_hi, reg_list[i].reg_hi);
4995 WREG32(err_status_lo_offset, 0);
4996 WREG32(err_status_hi_offset, 0);
4997 }
4998}
4999
5000int amdgpu_ras_error_data_init(struct ras_err_data *err_data)
5001{
5002 memset(err_data, 0, sizeof(*err_data));
5003
5004 INIT_LIST_HEAD(&err_data->err_node_list);
5005
5006 return 0;
5007}
5008
5009static void amdgpu_ras_error_node_release(struct ras_err_node *err_node)
5010{
5011 if (!err_node)
5012 return;
5013
5014 list_del(&err_node->node);
5015 kvfree(err_node);
5016}
5017
5018void amdgpu_ras_error_data_fini(struct ras_err_data *err_data)
5019{
5020 struct ras_err_node *err_node, *tmp;
5021
5022 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node)
5023 amdgpu_ras_error_node_release(err_node);
5024}
5025
5026static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data,
5027 struct amdgpu_smuio_mcm_config_info *mcm_info)
5028{
5029 struct ras_err_node *err_node;
5030 struct amdgpu_smuio_mcm_config_info *ref_id;
5031
5032 if (!err_data || !mcm_info)
5033 return NULL;
5034
5035 for_each_ras_error(err_node, err_data) {
5036 ref_id = &err_node->err_info.mcm_info;
5037
5038 if (mcm_info->socket_id == ref_id->socket_id &&
5039 mcm_info->die_id == ref_id->die_id)
5040 return err_node;
5041 }
5042
5043 return NULL;
5044}
5045
5046static struct ras_err_node *amdgpu_ras_error_node_new(void)
5047{
5048 struct ras_err_node *err_node;
5049
5050 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL);
5051 if (!err_node)
5052 return NULL;
5053
5054 INIT_LIST_HEAD(&err_node->node);
5055
5056 return err_node;
5057}
5058
5059static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b)
5060{
5061 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node);
5062 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node);
5063 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info;
5064 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info;
5065
5066 if (unlikely(infoa->socket_id != infob->socket_id))
5067 return infoa->socket_id - infob->socket_id;
5068 else
5069 return infoa->die_id - infob->die_id;
5070
5071 return 0;
5072}
5073
5074static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data,
5075 struct amdgpu_smuio_mcm_config_info *mcm_info)
5076{
5077 struct ras_err_node *err_node;
5078
5079 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info);
5080 if (err_node)
5081 return &err_node->err_info;
5082
5083 err_node = amdgpu_ras_error_node_new();
5084 if (!err_node)
5085 return NULL;
5086
5087 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info));
5088
5089 err_data->err_list_count++;
5090 list_add_tail(&err_node->node, &err_data->err_node_list);
5091 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp);
5092
5093 return &err_node->err_info;
5094}
5095
5096int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data,
5097 struct amdgpu_smuio_mcm_config_info *mcm_info,
5098 u64 count)
5099{
5100 struct ras_err_info *err_info;
5101
5102 if (!err_data || !mcm_info)
5103 return -EINVAL;
5104
5105 if (!count)
5106 return 0;
5107
5108 err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5109 if (!err_info)
5110 return -EINVAL;
5111
5112 err_info->ue_count += count;
5113 err_data->ue_count += count;
5114
5115 return 0;
5116}
5117
5118int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data,
5119 struct amdgpu_smuio_mcm_config_info *mcm_info,
5120 u64 count)
5121{
5122 struct ras_err_info *err_info;
5123
5124 if (!err_data || !mcm_info)
5125 return -EINVAL;
5126
5127 if (!count)
5128 return 0;
5129
5130 err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5131 if (!err_info)
5132 return -EINVAL;
5133
5134 err_info->ce_count += count;
5135 err_data->ce_count += count;
5136
5137 return 0;
5138}
5139
5140int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data,
5141 struct amdgpu_smuio_mcm_config_info *mcm_info,
5142 u64 count)
5143{
5144 struct ras_err_info *err_info;
5145
5146 if (!err_data || !mcm_info)
5147 return -EINVAL;
5148
5149 if (!count)
5150 return 0;
5151
5152 err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5153 if (!err_info)
5154 return -EINVAL;
5155
5156 err_info->de_count += count;
5157 err_data->de_count += count;
5158
5159 return 0;
5160}
5161
5162#define mmMP0_SMN_C2PMSG_92 0x1609C
5163#define mmMP0_SMN_C2PMSG_126 0x160BE
5164static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev,
5165 u32 instance)
5166{
5167 u32 socket_id, aid_id, hbm_id;
5168 u32 fw_status;
5169 u32 boot_error;
5170 u64 reg_addr;
5171
5172 /* The pattern for smn addressing in other SOC could be different from
5173 * the one for aqua_vanjaram. We should revisit the code if the pattern
5174 * is changed. In such case, replace the aqua_vanjaram implementation
5175 * with more common helper */
5176 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
5177 aqua_vanjaram_encode_ext_smn_addressing(instance);
5178 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5179
5180 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) +
5181 aqua_vanjaram_encode_ext_smn_addressing(instance);
5182 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5183
5184 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error);
5185 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error);
5186 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1);
5187
5188 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error))
5189 dev_info(adev->dev,
5190 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n",
5191 socket_id, aid_id, hbm_id, fw_status);
5192
5193 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error))
5194 dev_info(adev->dev,
5195 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n",
5196 socket_id, aid_id, fw_status);
5197
5198 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error))
5199 dev_info(adev->dev,
5200 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n",
5201 socket_id, aid_id, fw_status);
5202
5203 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error))
5204 dev_info(adev->dev,
5205 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n",
5206 socket_id, aid_id, fw_status);
5207
5208 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error))
5209 dev_info(adev->dev,
5210 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n",
5211 socket_id, aid_id, fw_status);
5212
5213 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error))
5214 dev_info(adev->dev,
5215 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n",
5216 socket_id, aid_id, fw_status);
5217
5218 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error))
5219 dev_info(adev->dev,
5220 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n",
5221 socket_id, aid_id, hbm_id, fw_status);
5222
5223 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error))
5224 dev_info(adev->dev,
5225 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n",
5226 socket_id, aid_id, hbm_id, fw_status);
5227
5228 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error))
5229 dev_info(adev->dev,
5230 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n",
5231 socket_id, aid_id, fw_status);
5232
5233 if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error))
5234 dev_info(adev->dev,
5235 "socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n",
5236 socket_id, aid_id, fw_status);
5237}
5238
5239static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev,
5240 u32 instance)
5241{
5242 u64 reg_addr;
5243 u32 reg_data;
5244 int retry_loop;
5245
5246 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
5247 aqua_vanjaram_encode_ext_smn_addressing(instance);
5248
5249 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) {
5250 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5251 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS)
5252 return false;
5253 else
5254 msleep(1);
5255 }
5256
5257 return true;
5258}
5259
5260void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances)
5261{
5262 u32 i;
5263
5264 for (i = 0; i < num_instances; i++) {
5265 if (amdgpu_ras_boot_error_detected(adev, i))
5266 amdgpu_ras_boot_time_error_reporting(adev, i);
5267 }
5268}
5269
5270int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn)
5271{
5272 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5273 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
5274 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT;
5275 int ret = 0;
5276
5277 mutex_lock(&con->page_rsv_lock);
5278 ret = amdgpu_vram_mgr_query_page_status(mgr, start);
5279 if (ret == -ENOENT)
5280 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE);
5281 mutex_unlock(&con->page_rsv_lock);
5282
5283 return ret;
5284}
5285
5286void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id,
5287 const char *fmt, ...)
5288{
5289 struct va_format vaf;
5290 va_list args;
5291
5292 va_start(args, fmt);
5293 vaf.fmt = fmt;
5294 vaf.va = &args;
5295
5296 if (RAS_EVENT_ID_IS_VALID(event_id))
5297 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf);
5298 else
5299 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf);
5300
5301 va_end(args);
5302}
5303
5304bool amdgpu_ras_is_rma(struct amdgpu_device *adev)
5305{
5306 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5307
5308 if (!con)
5309 return false;
5310
5311 return con->is_rma;
5312}
Linux 6.x block layer and io_uring tree(s)