projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/amdgpu: support to convert dedicated umc mca address
[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_ras.c
CommitLineData
c030f2e4 1/*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 *
23 */
24#include <linux/debugfs.h>
25#include <linux/list.h>
26#include <linux/module.h>
f867723b 27#include <linux/uaccess.h>
7c6e68c7
AG		 28#include <linux/reboot.h>
29#include <linux/syscalls.h>
05adfd80 30#include <linux/pm_runtime.h>
f867723b 31
c030f2e4 32#include "amdgpu.h"
33#include "amdgpu_ras.h"
b404ae82 34#include "amdgpu_atomfirmware.h"
19744f5f 35#include "amdgpu_xgmi.h"
4e644fff 36#include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
f50160cf 37#include "atom.h"
25a2b22e
AG		 38#include "amdgpu_reset.h"
39
12b2cab7
MJ		 40#ifdef CONFIG_X86_MCE_AMD
41#include <asm/mce.h>
c030f2e4 42
12b2cab7
MJ		 43static bool notifier_registered;
44#endif
eb0c3cd4
GC		 45static const char *RAS_FS_NAME = "ras";
46
c030f2e4 47const char *ras_error_string[] = {
48 "none",
49 "parity",
50 "single_correctable",
51 "multi_uncorrectable",
52 "poison",
53};
54
55const char *ras_block_string[] = {
56 "umc",
57 "sdma",
58 "gfx",
59 "mmhub",
60 "athub",
61 "pcie_bif",
62 "hdp",
63 "xgmi_wafl",
64 "df",
65 "smn",
66 "sem",
67 "mp0",
68 "mp1",
69 "fuse",
640ae42e 70 "mca",
a3d63c62
MZZ		 71 "vcn",
72 "jpeg",
c030f2e4 73};
74
640ae42e
JC		 75const char *ras_mca_block_string[] = {
76 "mca_mp0",
77 "mca_mp1",
78 "mca_mpio",
79 "mca_iohc",
80};
81
d5e8ff5f 82struct amdgpu_ras_block_list {
83 /* ras block link */
84 struct list_head node;
85
86 struct amdgpu_ras_block_object *ras_obj;
87};
88
640ae42e
JC		 89const char *get_ras_block_str(struct ras_common_if *ras_block)
90{
91 if (!ras_block)
92 return "NULL";
93
94 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT)
95 return "OUT OF RANGE";
96
97 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA)
98 return ras_mca_block_string[ras_block->sub_block_index];
99
100 return ras_block_string[ras_block->block];
101}
102
954ea6aa 103#define ras_block_str(_BLOCK_) \
104 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range")
8b0fb0e9 105
c030f2e4 106#define ras_err_str(i) (ras_error_string[ffs(i)])
c030f2e4 107
108c6a63 108#define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
109
7cdc2ee3
TZ		 110/* inject address is 52 bits */
111#define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52)
112
e4e6a589
LT		 113/* typical ECC bad page rate is 1 bad page per 100MB VRAM */
114#define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL)
c84d4670 115
52dd95f2
GC		 116enum amdgpu_ras_retire_page_reservation {
117 AMDGPU_RAS_RETIRE_PAGE_RESERVED,
118 AMDGPU_RAS_RETIRE_PAGE_PENDING,
119 AMDGPU_RAS_RETIRE_PAGE_FAULT,
120};
7c6e68c7
AG		 121
122atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0);
123
676deb38
DL		 124static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
125 uint64_t addr);
6e4be987
TZ		 126static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
127 uint64_t addr);
12b2cab7 128#ifdef CONFIG_X86_MCE_AMD
91a1a52d
MJ		 129static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev);
130struct mce_notifier_adev_list {
131 struct amdgpu_device *devs[MAX_GPU_INSTANCE];
132 int num_gpu;
133};
134static struct mce_notifier_adev_list mce_adev_list;
12b2cab7 135#endif
6e4be987 136
61380faa
JC		 137void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
138{
a9d82d2f 139 if (adev && amdgpu_ras_get_context(adev))
61380faa
JC		 140 amdgpu_ras_get_context(adev)->error_query_ready = ready;
141}
142
f3167919 143static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev)
61380faa 144{
a9d82d2f 145 if (adev && amdgpu_ras_get_context(adev))
61380faa
JC		 146 return amdgpu_ras_get_context(adev)->error_query_ready;
147
148 return false;
149}
150
cbb8f989
JC		 151static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address)
152{
153 struct ras_err_data err_data = {0, 0, 0, NULL};
154 struct eeprom_table_record err_rec;
155
156 if ((address >= adev->gmc.mc_vram_size) ||
157 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
158 dev_warn(adev->dev,
159 "RAS WARN: input address 0x%llx is invalid.\n",
160 address);
161 return -EINVAL;
162 }
163
164 if (amdgpu_ras_check_bad_page(adev, address)) {
165 dev_warn(adev->dev,
80b0cd0f 166 "RAS WARN: 0x%llx has already been marked as bad page!\n",
cbb8f989
JC		 167 address);
168 return 0;
169 }
170
171 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
cbb8f989 172 err_data.err_addr = &err_rec;
400013b2
TZ		 173 amdgpu_umc_fill_error_record(&err_data, address,
174 (address >> AMDGPU_GPU_PAGE_SHIFT), 0, 0);
cbb8f989
JC		 175
176 if (amdgpu_bad_page_threshold != 0) {
177 amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
178 err_data.err_addr_cnt);
179 amdgpu_ras_save_bad_pages(adev);
180 }
181
182 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n");
183 dev_warn(adev->dev, "Clear EEPROM:\n");
184 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n");
185
186 return 0;
187}
188
c030f2e4 189static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
190 size_t size, loff_t *pos)
191{
192 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
193 struct ras_query_if info = {
194 .head = obj->head,
195 };
196 ssize_t s;
197 char val[128];
198
761d86d3 199 if (amdgpu_ras_query_error_status(obj->adev, &info))
c030f2e4 200 return -EINVAL;
201
2a460963
CL		 202 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */
203 if (obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
204 obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
205 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
206 dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
207 }
208
c030f2e4 209 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n",
210 "ue", info.ue_count,
211 "ce", info.ce_count);
212 if (*pos >= s)
213 return 0;
214
215 s -= *pos;
216 s = min_t(u64, s, size);
217
218
219 if (copy_to_user(buf, &val[*pos], s))
220 return -EINVAL;
221
222 *pos += s;
223
224 return s;
225}
226
c030f2e4 227static const struct file_operations amdgpu_ras_debugfs_ops = {
228 .owner = THIS_MODULE,
229 .read = amdgpu_ras_debugfs_read,
190211ab 230 .write = NULL,
c030f2e4 231 .llseek = default_llseek
232};
233
96ebb307 234static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
235{
236 int i;
237
238 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
239 *block_id = i;
640ae42e 240 if (strcmp(name, ras_block_string[i]) == 0)
96ebb307 241 return 0;
242 }
243 return -EINVAL;
244}
245
246static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
247 const char __user *buf, size_t size,
248 loff_t *pos, struct ras_debug_if *data)
249{
250 ssize_t s = min_t(u64, 64, size);
251 char str[65];
252 char block_name[33];
253 char err[9] = "ue";
254 int op = -1;
255 int block_id;
44494f96 256 uint32_t sub_block;
96ebb307 257 u64 address, value;
258
259 if (*pos)
260 return -EINVAL;
261 *pos = size;
262
263 memset(str, 0, sizeof(str));
264 memset(data, 0, sizeof(*data));
265
266 if (copy_from_user(str, buf, s))
267 return -EINVAL;
268
269 if (sscanf(str, "disable %32s", block_name) == 1)
270 op = 0;
271 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
272 op = 1;
273 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
274 op = 2;
6df23f4c 275 else if (strstr(str, "retire_page") != NULL)
cbb8f989 276 op = 3;
b076296b 277 else if (str[0] && str[1] && str[2] && str[3])
96ebb307 278 /* ascii string, but commands are not matched. */
279 return -EINVAL;
280
281 if (op != -1) {
cbb8f989 282 if (op == 3) {
546aa546
LT		 283 if (sscanf(str, "%*s 0x%llx", &address) != 1 &&
284 sscanf(str, "%*s %llu", &address) != 1)
6cb7a1d4 285 return -EINVAL;
cbb8f989
JC		 286
287 data->op = op;
288 data->inject.address = address;
289
290 return 0;
291 }
292
96ebb307 293 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id))
294 return -EINVAL;
295
296 data->head.block = block_id;
e1063493
TZ		 297 /* only ue and ce errors are supported */
298 if (!memcmp("ue", err, 2))
299 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
300 else if (!memcmp("ce", err, 2))
301 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
302 else
303 return -EINVAL;
304
96ebb307 305 data->op = op;
306
307 if (op == 2) {
546aa546
LT		 308 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx",
309 &sub_block, &address, &value) != 3 &&
310 sscanf(str, "%*s %*s %*s %u %llu %llu",
6cb7a1d4
LT		 311 &sub_block, &address, &value) != 3)
312 return -EINVAL;
44494f96 313 data->head.sub_block_index = sub_block;
96ebb307 314 data->inject.address = address;
315 data->inject.value = value;
316 }
317 } else {
73aa8e1a 318 if (size < sizeof(*data))
96ebb307 319 return -EINVAL;
320
321 if (copy_from_user(data, buf, sizeof(*data)))
322 return -EINVAL;
323 }
324
325 return 0;
326}
7c6e68c7 327
74abc221
TSD		 328/**
329 * DOC: AMDGPU RAS debugfs control interface
36ea1bd2 330 *
737c375b 331 * The control interface accepts struct ras_debug_if which has two members.
36ea1bd2 332 *
333 * First member: ras_debug_if::head or ras_debug_if::inject.
96ebb307 334 *
335 * head is used to indicate which IP block will be under control.
36ea1bd2 336 *
337 * head has four members, they are block, type, sub_block_index, name.
338 * block: which IP will be under control.
339 * type: what kind of error will be enabled/disabled/injected.
340 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
341 * name: the name of IP.
342 *
343 * inject has two more members than head, they are address, value.
344 * As their names indicate, inject operation will write the
345 * value to the address.
346 *
ef177d11 347 * The second member: struct ras_debug_if::op.
c688a06b 348 * It has three kinds of operations.
879e723d
AZ		 349 *
350 * - 0: disable RAS on the block. Take ::head as its data.
351 * - 1: enable RAS on the block. Take ::head as its data.
352 * - 2: inject errors on the block. Take ::inject as its data.
36ea1bd2 353 *
96ebb307 354 * How to use the interface?
ef177d11 355 *
737c375b 356 * In a program
ef177d11 357 *
737c375b
LT		 358 * Copy the struct ras_debug_if in your code and initialize it.
359 * Write the struct to the control interface.
ef177d11 360 *
737c375b 361 * From shell
96ebb307 362 *
879e723d
AZ		 363 * .. code-block:: bash
364 *
737c375b
LT		 365 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
366 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
367 * echo "inject <block> <error> <sub-block> <address> <value> > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
879e723d 368 *
737c375b 369 * Where N, is the card which you want to affect.
ef177d11 370 *
737c375b
LT		 371 * "disable" requires only the block.
372 * "enable" requires the block and error type.
373 * "inject" requires the block, error type, address, and value.
c666bbf0 374 *
737c375b 375 * The block is one of: umc, sdma, gfx, etc.
879e723d 376 * see ras_block_string[] for details
c666bbf0 377 *
737c375b
LT		 378 * The error type is one of: ue, ce, where,
379 * ue is multi-uncorrectable
380 * ce is single-correctable
c666bbf0 381 *
737c375b
LT		 382 * The sub-block is a the sub-block index, pass 0 if there is no sub-block.
383 * The address and value are hexadecimal numbers, leading 0x is optional.
879e723d 384 *
737c375b 385 * For instance,
879e723d
AZ		 386 *
387 * .. code-block:: bash
96ebb307 388 *
44494f96
TZ		 389 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
390 * echo inject umc ce 0 0 0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
96ebb307 391 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
392 *
737c375b 393 * How to check the result of the operation?
36ea1bd2 394 *
737c375b 395 * To check disable/enable, see "ras" features at,
36ea1bd2 396 * /sys/class/drm/card[0/1/2...]/device/ras/features
397 *
737c375b
LT		 398 * To check inject, see the corresponding error count at,
399 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count
36ea1bd2 400 *
879e723d 401 * .. note::
ef177d11 402 * Operations are only allowed on blocks which are supported.
737c375b 403 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask
ef177d11
AD		 404 * to see which blocks support RAS on a particular asic.
405 *
36ea1bd2 406 */
cf696091
LT		 407static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f,
408 const char __user *buf,
409 size_t size, loff_t *pos)
36ea1bd2 410{
411 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
412 struct ras_debug_if data;
413 int ret = 0;
414
61380faa 415 if (!amdgpu_ras_get_error_query_ready(adev)) {
6952e99c
GC		 416 dev_warn(adev->dev, "RAS WARN: error injection "
417 "currently inaccessible\n");
43c4d576
JC		 418 return size;
419 }
420
96ebb307 421 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
422 if (ret)
cf696091 423 return ret;
36ea1bd2 424
80b0cd0f 425 if (data.op == 3) {
cbb8f989 426 ret = amdgpu_reserve_page_direct(adev, data.inject.address);
80b0cd0f 427 if (!ret)
cbb8f989
JC		 428 return size;
429 else
430 return ret;
431 }
432
36ea1bd2 433 if (!amdgpu_ras_is_supported(adev, data.head.block))
434 return -EINVAL;
435
436 switch (data.op) {
437 case 0:
438 ret = amdgpu_ras_feature_enable(adev, &data.head, 0);
439 break;
440 case 1:
441 ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
442 break;
443 case 2:
7cdc2ee3
TZ		 444 if ((data.inject.address >= adev->gmc.mc_vram_size) ||
445 (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
b0d4783a
GC		 446 dev_warn(adev->dev, "RAS WARN: input address "
447 "0x%llx is invalid.",
448 data.inject.address);
7cdc2ee3
TZ		 449 ret = -EINVAL;
450 break;
451 }
452
6e4be987
TZ		 453 /* umc ce/ue error injection for a bad page is not allowed */
454 if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) &&
455 amdgpu_ras_check_bad_page(adev, data.inject.address)) {
c65b0805
LT		 456 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has "
457 "already been marked as bad!\n",
458 data.inject.address);
6e4be987
TZ		 459 break;
460 }
461
7cdc2ee3 462 /* data.inject.address is offset instead of absolute gpu address */
36ea1bd2 463 ret = amdgpu_ras_error_inject(adev, &data.inject);
464 break;
96ebb307 465 default:
466 ret = -EINVAL;
467 break;
374bf7bd 468 }
36ea1bd2 469
470 if (ret)
79c04621 471 return ret;
36ea1bd2 472
473 return size;
474}
475
084fe13b
AG		 476/**
477 * DOC: AMDGPU RAS debugfs EEPROM table reset interface
478 *
f77c7109 479 * Some boards contain an EEPROM which is used to persistently store a list of
ef177d11 480 * bad pages which experiences ECC errors in vram. This interface provides
f77c7109
AD		 481 * a way to reset the EEPROM, e.g., after testing error injection.
482 *
483 * Usage:
484 *
485 * .. code-block:: bash
486 *
487 * echo 1 > ../ras/ras_eeprom_reset
488 *
489 * will reset EEPROM table to 0 entries.
490 *
084fe13b 491 */
cf696091
LT		 492static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f,
493 const char __user *buf,
494 size_t size, loff_t *pos)
084fe13b 495{
bf0b91b7
GC		 496 struct amdgpu_device *adev =
497 (struct amdgpu_device *)file_inode(f)->i_private;
084fe13b
AG		 498 int ret;
499
bf0b91b7 500 ret = amdgpu_ras_eeprom_reset_table(
cf696091 501 &(amdgpu_ras_get_context(adev)->eeprom_control));
084fe13b 502
63d4c081 503 if (!ret) {
cf696091
LT		 504 /* Something was written to EEPROM.
505 */
bf0b91b7
GC		 506 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS;
507 return size;
508 } else {
cf696091 509 return ret;
bf0b91b7 510 }
084fe13b
AG		 511}
512
36ea1bd2 513static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
514 .owner = THIS_MODULE,
515 .read = NULL,
516 .write = amdgpu_ras_debugfs_ctrl_write,
517 .llseek = default_llseek
518};
519
084fe13b
AG		 520static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = {
521 .owner = THIS_MODULE,
522 .read = NULL,
523 .write = amdgpu_ras_debugfs_eeprom_write,
524 .llseek = default_llseek
525};
526
f77c7109
AD		 527/**
528 * DOC: AMDGPU RAS sysfs Error Count Interface
529 *
ef177d11 530 * It allows the user to read the error count for each IP block on the gpu through
f77c7109
AD		 531 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
532 *
533 * It outputs the multiple lines which report the uncorrected (ue) and corrected
534 * (ce) error counts.
535 *
536 * The format of one line is below,
537 *
538 * [ce|ue]: count
539 *
540 * Example:
541 *
542 * .. code-block:: bash
543 *
544 * ue: 0
545 * ce: 1
546 *
547 */
c030f2e4 548static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
549 struct device_attribute *attr, char *buf)
550{
551 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
552 struct ras_query_if info = {
553 .head = obj->head,
554 };
555
61380faa 556 if (!amdgpu_ras_get_error_query_ready(obj->adev))
36000c7a 557 return sysfs_emit(buf, "Query currently inaccessible\n");
43c4d576 558
761d86d3 559 if (amdgpu_ras_query_error_status(obj->adev, &info))
c030f2e4 560 return -EINVAL;
561
2a460963
CL		 562 if (obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
563 obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
1f0d8e37 564 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
2a460963 565 dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
1f0d8e37
MJ		 566 }
567
36000c7a
TT		 568 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count,
569 "ce", info.ce_count);
c030f2e4 570}
571
572/* obj begin */
573
574#define get_obj(obj) do { (obj)->use++; } while (0)
575#define alive_obj(obj) ((obj)->use)
576
577static inline void put_obj(struct ras_manager *obj)
578{
f0872686 579 if (obj && (--obj->use == 0))
c030f2e4 580 list_del(&obj->node);
f0872686 581 if (obj && (obj->use < 0))
640ae42e 582 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head));
c030f2e4 583}
584
585/* make one obj and return it. */
586static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
587 struct ras_common_if *head)
588{
589 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
590 struct ras_manager *obj;
591
8ab0d6f0 592 if (!adev->ras_enabled || !con)
c030f2e4 593 return NULL;
594
595 if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
596 return NULL;
597
640ae42e
JC		 598 if (head->block == AMDGPU_RAS_BLOCK__MCA) {
599 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
600 return NULL;
601
602 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
603 } else
604 obj = &con->objs[head->block];
605
c030f2e4 606 /* already exist. return obj? */
607 if (alive_obj(obj))
608 return NULL;
609
610 obj->head = *head;
611 obj->adev = adev;
612 list_add(&obj->node, &con->head);
613 get_obj(obj);
614
615 return obj;
616}
617
618/* return an obj equal to head, or the first when head is NULL */
f2a79be1 619struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
c030f2e4 620 struct ras_common_if *head)
621{
622 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
623 struct ras_manager *obj;
624 int i;
625
8ab0d6f0 626 if (!adev->ras_enabled || !con)
c030f2e4 627 return NULL;
628
629 if (head) {
630 if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
631 return NULL;
632
640ae42e
JC		 633 if (head->block == AMDGPU_RAS_BLOCK__MCA) {
634 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
635 return NULL;
636
637 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
638 } else
639 obj = &con->objs[head->block];
c030f2e4 640
640ae42e 641 if (alive_obj(obj))
c030f2e4 642 return obj;
c030f2e4 643 } else {
640ae42e 644 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
c030f2e4 645 obj = &con->objs[i];
640ae42e 646 if (alive_obj(obj))
c030f2e4 647 return obj;
c030f2e4 648 }
649 }
650
651 return NULL;
652}
653/* obj end */
654
655/* feature ctl begin */
656static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
e509965e 657 struct ras_common_if *head)
c030f2e4 658{
8ab0d6f0 659 return adev->ras_hw_enabled & BIT(head->block);
c030f2e4 660}
661
662static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
663 struct ras_common_if *head)
664{
665 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
666
667 return con->features & BIT(head->block);
668}
669
670/*
671 * if obj is not created, then create one.
672 * set feature enable flag.
673 */
674static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
675 struct ras_common_if *head, int enable)
676{
677 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
678 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
679
5caf466a 680 /* If hardware does not support ras, then do not create obj.
681 * But if hardware support ras, we can create the obj.
682 * Ras framework checks con->hw_supported to see if it need do
683 * corresponding initialization.
684 * IP checks con->support to see if it need disable ras.
685 */
c030f2e4 686 if (!amdgpu_ras_is_feature_allowed(adev, head))
687 return 0;
c030f2e4 688
689 if (enable) {
690 if (!obj) {
691 obj = amdgpu_ras_create_obj(adev, head);
692 if (!obj)
693 return -EINVAL;
694 } else {
695 /* In case we create obj somewhere else */
696 get_obj(obj);
697 }
698 con->features |= BIT(head->block);
699 } else {
700 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
19d0dfda 701 con->features &= ~BIT(head->block);
c030f2e4 702 put_obj(obj);
703 }
704 }
705
706 return 0;
707}
708
709/* wrapper of psp_ras_enable_features */
710int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
711 struct ras_common_if *head, bool enable)
712{
713 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
7fcffecf 714 union ta_ras_cmd_input *info;
c030f2e4 715 int ret;
716
717 if (!con)
718 return -EINVAL;
719
26093ce1
SY		 720 if (head->block == AMDGPU_RAS_BLOCK__GFX) {
721 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL);
722 if (!info)
723 return -ENOMEM;
724
725 if (!enable) {
726 info->disable_features = (struct ta_ras_disable_features_input) {
727 .block_id = amdgpu_ras_block_to_ta(head->block),
728 .error_type = amdgpu_ras_error_to_ta(head->type),
729 };
730 } else {
731 info->enable_features = (struct ta_ras_enable_features_input) {
732 .block_id = amdgpu_ras_block_to_ta(head->block),
733 .error_type = amdgpu_ras_error_to_ta(head->type),
734 };
735 }
c030f2e4 736 }
737
738 /* Do not enable if it is not allowed. */
739 WARN_ON(enable && !amdgpu_ras_is_feature_allowed(adev, head));
c030f2e4 740
950d6425 741 /* Only enable ras feature operation handle on host side */
26093ce1
SY		 742 if (head->block == AMDGPU_RAS_BLOCK__GFX &&
743 !amdgpu_sriov_vf(adev) &&
950d6425 744 !amdgpu_ras_intr_triggered()) {
7fcffecf 745 ret = psp_ras_enable_features(&adev->psp, info, enable);
bff77e86 746 if (ret) {
e4348849 747 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n",
011907fd 748 enable ? "enable":"disable",
640ae42e 749 get_ras_block_str(head),
e4348849 750 amdgpu_ras_is_poison_mode_supported(adev), ret);
7fcffecf 751 goto out;
bff77e86 752 }
c030f2e4 753 }
754
755 /* setup the obj */
756 __amdgpu_ras_feature_enable(adev, head, enable);
7fcffecf
AB		 757 ret = 0;
758out:
26093ce1
SY		 759 if (head->block == AMDGPU_RAS_BLOCK__GFX)
760 kfree(info);
7fcffecf 761 return ret;
c030f2e4 762}
763
77de502b 764/* Only used in device probe stage and called only once. */
765int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
766 struct ras_common_if *head, bool enable)
767{
768 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
769 int ret;
770
771 if (!con)
772 return -EINVAL;
773
774 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
7af23ebe 775 if (enable) {
776 /* There is no harm to issue a ras TA cmd regardless of
777 * the currecnt ras state.
778 * If current state == target state, it will do nothing
779 * But sometimes it requests driver to reset and repost
780 * with error code -EAGAIN.
781 */
782 ret = amdgpu_ras_feature_enable(adev, head, 1);
783 /* With old ras TA, we might fail to enable ras.
784 * Log it and just setup the object.
785 * TODO need remove this WA in the future.
786 */
787 if (ret == -EINVAL) {
788 ret = __amdgpu_ras_feature_enable(adev, head, 1);
789 if (!ret)
6952e99c
GC		 790 dev_info(adev->dev,
791 "RAS INFO: %s setup object\n",
640ae42e 792 get_ras_block_str(head));
7af23ebe 793 }
794 } else {
795 /* setup the object then issue a ras TA disable cmd.*/
796 ret = __amdgpu_ras_feature_enable(adev, head, 1);
797 if (ret)
798 return ret;
77de502b 799
970fd197
SY		 800 /* gfx block ras dsiable cmd must send to ras-ta */
801 if (head->block == AMDGPU_RAS_BLOCK__GFX)
802 con->features |= BIT(head->block);
803
77de502b 804 ret = amdgpu_ras_feature_enable(adev, head, 0);
19d0dfda
SY		 805
806 /* clean gfx block ras features flag */
8ab0d6f0 807 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
19d0dfda 808 con->features &= ~BIT(head->block);
7af23ebe 809 }
77de502b 810 } else
811 ret = amdgpu_ras_feature_enable(adev, head, enable);
812
813 return ret;
814}
815
c030f2e4 816static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
817 bool bypass)
818{
819 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
820 struct ras_manager *obj, *tmp;
821
822 list_for_each_entry_safe(obj, tmp, &con->head, node) {
823 /* bypass psp.
824 * aka just release the obj and corresponding flags
825 */
826 if (bypass) {
827 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0))
828 break;
829 } else {
830 if (amdgpu_ras_feature_enable(adev, &obj->head, 0))
831 break;
832 }
289d513b 833 }
c030f2e4 834
835 return con->features;
836}
837
838static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
839 bool bypass)
840{
841 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
c030f2e4 842 int i;
640ae42e 843 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE;
c030f2e4 844
640ae42e 845 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
c030f2e4 846 struct ras_common_if head = {
847 .block = i,
191051a1 848 .type = default_ras_type,
c030f2e4 849 .sub_block_index = 0,
850 };
640ae42e
JC		 851
852 if (i == AMDGPU_RAS_BLOCK__MCA)
853 continue;
854
855 if (bypass) {
856 /*
857 * bypass psp. vbios enable ras for us.
858 * so just create the obj
859 */
860 if (__amdgpu_ras_feature_enable(adev, &head, 1))
861 break;
862 } else {
863 if (amdgpu_ras_feature_enable(adev, &head, 1))
864 break;
865 }
866 }
867
868 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
869 struct ras_common_if head = {
870 .block = AMDGPU_RAS_BLOCK__MCA,
871 .type = default_ras_type,
872 .sub_block_index = i,
873 };
874
c030f2e4 875 if (bypass) {
876 /*
877 * bypass psp. vbios enable ras for us.
878 * so just create the obj
879 */
880 if (__amdgpu_ras_feature_enable(adev, &head, 1))
881 break;
882 } else {
883 if (amdgpu_ras_feature_enable(adev, &head, 1))
884 break;
885 }
289d513b 886 }
c030f2e4 887
888 return con->features;
889}
890/* feature ctl end */
891
e3d833f4 892static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj,
893 enum amdgpu_ras_block block)
6492e1b0 894{
b6efdb02 895 if (!block_obj)
6492e1b0 896 return -EINVAL;
897
bdb3489c 898 if (block_obj->ras_comm.block == block)
6492e1b0 899 return 0;
640ae42e 900
6492e1b0 901 return -EINVAL;
902}
903
b6efdb02 904static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev,
6492e1b0 905 enum amdgpu_ras_block block, uint32_t sub_block_index)
640ae42e 906{
d5e8ff5f 907 struct amdgpu_ras_block_list *node, *tmp;
908 struct amdgpu_ras_block_object *obj;
6492e1b0 909
910 if (block >= AMDGPU_RAS_BLOCK__LAST)
911 return NULL;
912
913 if (!amdgpu_ras_is_supported(adev, block))
914 return NULL;
915
d5e8ff5f 916 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
917 if (!node->ras_obj) {
918 dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
919 continue;
920 }
921
922 obj = node->ras_obj;
6492e1b0 923 if (obj->ras_block_match) {
924 if (obj->ras_block_match(obj, block, sub_block_index) == 0)
925 return obj;
926 } else {
927 if (amdgpu_ras_block_match_default(obj, block) == 0)
928 return obj;
929 }
640ae42e 930 }
6492e1b0 931
932 return NULL;
640ae42e
JC		 933}
934
fdcb279d
SY		 935static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data)
936{
937 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
938 int ret = 0;
939
940 /*
941 * choosing right query method according to
942 * whether smu support query error information
943 */
bc143d8b 944 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc));
fdcb279d 945 if (ret == -EOPNOTSUPP) {
efe17d5a 946 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
947 adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
948 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
fdcb279d
SY		 949
950 /* umc query_ras_error_address is also responsible for clearing
951 * error status
952 */
efe17d5a 953 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
954 adev->umc.ras->ras_block.hw_ops->query_ras_error_address)
955 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data);
fdcb279d 956 } else if (!ret) {
efe17d5a 957 if (adev->umc.ras &&
958 adev->umc.ras->ecc_info_query_ras_error_count)
959 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data);
fdcb279d 960
efe17d5a 961 if (adev->umc.ras &&
962 adev->umc.ras->ecc_info_query_ras_error_address)
963 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data);
fdcb279d
SY		 964 }
965}
966
c030f2e4 967/* query/inject/cure begin */
761d86d3 968int amdgpu_ras_query_error_status(struct amdgpu_device *adev,
4d9f771e 969 struct ras_query_if *info)
c030f2e4 970{
b6efdb02 971 struct amdgpu_ras_block_object *block_obj = NULL;
c030f2e4 972 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
6f102dba 973 struct ras_err_data err_data = {0, 0, 0, NULL};
c030f2e4 974
975 if (!obj)
976 return -EINVAL;
c030f2e4 977
7389a5b8 978 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) {
fdcb279d 979 amdgpu_ras_get_ecc_info(adev, &err_data);
7389a5b8 980 } else {
981 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0);
8b0fb0e9 982 if (!block_obj || !block_obj->hw_ops) {
afa37315
LT		 983 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
984 get_ras_block_str(&info->head));
8b0fb0e9 985 return -EINVAL;
3e81ee9a 986 }
761d86d3 987
6c245386 988 if (block_obj->hw_ops->query_ras_error_count)
989 block_obj->hw_ops->query_ras_error_count(adev, &err_data);
7389a5b8 990
991 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) ||
992 (info->head.block == AMDGPU_RAS_BLOCK__GFX) ||
993 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) {
994 if (block_obj->hw_ops->query_ras_error_status)
995 block_obj->hw_ops->query_ras_error_status(adev);
996 }
939e2258 997 }
05a58345
TZ		 998
999 obj->err_data.ue_count += err_data.ue_count;
1000 obj->err_data.ce_count += err_data.ce_count;
1001
c030f2e4 1002 info->ue_count = obj->err_data.ue_count;
1003 info->ce_count = obj->err_data.ce_count;
1004
7c6e68c7 1005 if (err_data.ce_count) {
a30f1286
HZ		 1006 if (adev->smuio.funcs &&
1007 adev->smuio.funcs->get_socket_id &&
1008 adev->smuio.funcs->get_die_id) {
1009 dev_info(adev->dev, "socket: %d, die: %d "
1010 "%ld correctable hardware errors "
6952e99c
GC		 1011 "detected in %s block, no user "
1012 "action is needed.\n",
a30f1286
HZ		 1013 adev->smuio.funcs->get_socket_id(adev),
1014 adev->smuio.funcs->get_die_id(adev),
6952e99c 1015 obj->err_data.ce_count,
640ae42e 1016 get_ras_block_str(&info->head));
a30f1286
HZ		 1017 } else {
1018 dev_info(adev->dev, "%ld correctable hardware errors "
6952e99c
GC		 1019 "detected in %s block, no user "
1020 "action is needed.\n",
1021 obj->err_data.ce_count,
640ae42e 1022 get_ras_block_str(&info->head));
a30f1286 1023 }
7c6e68c7
AG		 1024 }
1025 if (err_data.ue_count) {
a30f1286
HZ		 1026 if (adev->smuio.funcs &&
1027 adev->smuio.funcs->get_socket_id &&
1028 adev->smuio.funcs->get_die_id) {
1029 dev_info(adev->dev, "socket: %d, die: %d "
1030 "%ld uncorrectable hardware errors "
6952e99c 1031 "detected in %s block\n",
a30f1286
HZ		 1032 adev->smuio.funcs->get_socket_id(adev),
1033 adev->smuio.funcs->get_die_id(adev),
6952e99c 1034 obj->err_data.ue_count,
640ae42e 1035 get_ras_block_str(&info->head));
a30f1286
HZ		 1036 } else {
1037 dev_info(adev->dev, "%ld uncorrectable hardware errors "
6952e99c
GC		 1038 "detected in %s block\n",
1039 obj->err_data.ue_count,
640ae42e 1040 get_ras_block_str(&info->head));
a30f1286 1041 }
7c6e68c7 1042 }
05a58345 1043
c030f2e4 1044 return 0;
1045}
1046
761d86d3
DL		 1047int amdgpu_ras_reset_error_status(struct amdgpu_device *adev,
1048 enum amdgpu_ras_block block)
1049{
b6efdb02 1050 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
8b0fb0e9 1051
761d86d3
DL		 1052 if (!amdgpu_ras_is_supported(adev, block))
1053 return -EINVAL;
1054
7389a5b8 1055 if (!block_obj || !block_obj->hw_ops) {
afa37315
LT		 1056 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1057 ras_block_str(block));
7389a5b8 1058 return -EINVAL;
761d86d3
DL		 1059 }
1060
7389a5b8 1061 if (block_obj->hw_ops->reset_ras_error_count)
1062 block_obj->hw_ops->reset_ras_error_count(adev);
5c23e9e0 1063
7389a5b8 1064 if ((block == AMDGPU_RAS_BLOCK__GFX) ||
1065 (block == AMDGPU_RAS_BLOCK__MMHUB)) {
8b0fb0e9 1066 if (block_obj->hw_ops->reset_ras_error_status)
1067 block_obj->hw_ops->reset_ras_error_status(adev);
761d86d3 1068 }
5c23e9e0 1069
761d86d3 1070 return 0;
5c23e9e0
JC		 1071}
1072
c030f2e4 1073/* wrapper of psp_ras_trigger_error */
1074int amdgpu_ras_error_inject(struct amdgpu_device *adev,
1075 struct ras_inject_if *info)
1076{
1077 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1078 struct ta_ras_trigger_error_input block_info = {
828cfa29 1079 .block_id = amdgpu_ras_block_to_ta(info->head.block),
1080 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type),
c030f2e4 1081 .sub_block_index = info->head.sub_block_index,
1082 .address = info->address,
1083 .value = info->value,
1084 };
ab3b9de6
YL		 1085 int ret = -EINVAL;
1086 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev,
1087 info->head.block,
1088 info->head.sub_block_index);
c030f2e4 1089
1090 if (!obj)
1091 return -EINVAL;
1092
22d4ba53 1093 if (!block_obj || !block_obj->hw_ops) {
afa37315
LT		 1094 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1095 get_ras_block_str(&info->head));
22d4ba53 1096 return -EINVAL;
1097 }
1098
a6c44d25
JC		 1099 /* Calculate XGMI relative offset */
1100 if (adev->gmc.xgmi.num_physical_nodes > 1) {
19744f5f
HZ		 1101 block_info.address =
1102 amdgpu_xgmi_get_relative_phy_addr(adev,
1103 block_info.address);
a6c44d25
JC		 1104 }
1105
22d4ba53 1106 if (info->head.block == AMDGPU_RAS_BLOCK__GFX) {
8b0fb0e9 1107 if (block_obj->hw_ops->ras_error_inject)
1108 ret = block_obj->hw_ops->ras_error_inject(adev, info);
22d4ba53 1109 } else {
1110 /* If defined special ras_error_inject(e.g: xgmi), implement special ras_error_inject */
1111 if (block_obj->hw_ops->ras_error_inject)
1112 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info);
1113 else /*If not defined .ras_error_inject, use default ras_error_inject*/
1114 ret = psp_ras_trigger_error(&adev->psp, &block_info);
a5dd40ca
HZ		 1115 }
1116
011907fd
DL		 1117 if (ret)
1118 dev_err(adev->dev, "ras inject %s failed %d\n",
640ae42e 1119 get_ras_block_str(&info->head), ret);
c030f2e4 1120
1121 return ret;
1122}
1123
4d9f771e
LT		 1124/**
1125 * amdgpu_ras_query_error_count -- Get error counts of all IPs
bbe04dec
IB		 1126 * @adev: pointer to AMD GPU device
1127 * @ce_count: pointer to an integer to be set to the count of correctible errors.
1128 * @ue_count: pointer to an integer to be set to the count of uncorrectible
4d9f771e
LT		 1129 * errors.
1130 *
1131 * If set, @ce_count or @ue_count, count and return the corresponding
1132 * error counts in those integer pointers. Return 0 if the device
1133 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS.
1134 */
1135int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
1136 unsigned long *ce_count,
1137 unsigned long *ue_count)
c030f2e4 1138{
1139 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1140 struct ras_manager *obj;
a46751fb 1141 unsigned long ce, ue;
c030f2e4 1142
8ab0d6f0 1143 if (!adev->ras_enabled || !con)
4d9f771e
LT		 1144 return -EOPNOTSUPP;
1145
1146 /* Don't count since no reporting.
1147 */
1148 if (!ce_count && !ue_count)
1149 return 0;
c030f2e4 1150
a46751fb
LT		 1151 ce = 0;
1152 ue = 0;
c030f2e4 1153 list_for_each_entry(obj, &con->head, node) {
1154 struct ras_query_if info = {
1155 .head = obj->head,
1156 };
4d9f771e 1157 int res;
c030f2e4 1158
4d9f771e
LT		 1159 res = amdgpu_ras_query_error_status(adev, &info);
1160 if (res)
1161 return res;
c030f2e4 1162
2a460963
CL		 1163 if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
1164 adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
1165 if (amdgpu_ras_reset_error_status(adev, info.head.block))
1166 dev_warn(adev->dev, "Failed to reset error counter and error status");
1167 }
1168
a46751fb
LT		 1169 ce += info.ce_count;
1170 ue += info.ue_count;
c030f2e4 1171 }
1172
a46751fb
LT		 1173 if (ce_count)
1174 *ce_count = ce;
1175
1176 if (ue_count)
1177 *ue_count = ue;
4d9f771e
LT		 1178
1179 return 0;
c030f2e4 1180}
1181/* query/inject/cure end */
1182
1183
1184/* sysfs begin */
1185
466b1793 1186static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1187 struct ras_badpage **bps, unsigned int *count);
1188
1189static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
1190{
1191 switch (flags) {
52dd95f2 1192 case AMDGPU_RAS_RETIRE_PAGE_RESERVED:
466b1793 1193 return "R";
52dd95f2 1194 case AMDGPU_RAS_RETIRE_PAGE_PENDING:
466b1793 1195 return "P";
52dd95f2 1196 case AMDGPU_RAS_RETIRE_PAGE_FAULT:
466b1793 1197 default:
1198 return "F";
aec576f9 1199 }
466b1793 1200}
1201
f77c7109
AD		 1202/**
1203 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface
466b1793 1204 *
1205 * It allows user to read the bad pages of vram on the gpu through
1206 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
1207 *
1208 * It outputs multiple lines, and each line stands for one gpu page.
1209 *
1210 * The format of one line is below,
1211 * gpu pfn : gpu page size : flags
1212 *
1213 * gpu pfn and gpu page size are printed in hex format.
1214 * flags can be one of below character,
f77c7109 1215 *
466b1793 1216 * R: reserved, this gpu page is reserved and not able to use.
f77c7109 1217 *
466b1793 1218 * P: pending for reserve, this gpu page is marked as bad, will be reserved
f77c7109
AD		 1219 * in next window of page_reserve.
1220 *
466b1793 1221 * F: unable to reserve. this gpu page can't be reserved due to some reasons.
1222 *
f77c7109
AD		 1223 * Examples:
1224 *
1225 * .. code-block:: bash
1226 *
1227 * 0x00000001 : 0x00001000 : R
1228 * 0x00000002 : 0x00001000 : P
1229 *
466b1793 1230 */
1231
1232static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
1233 struct kobject *kobj, struct bin_attribute *attr,
1234 char *buf, loff_t ppos, size_t count)
1235{
1236 struct amdgpu_ras *con =
1237 container_of(attr, struct amdgpu_ras, badpages_attr);
1238 struct amdgpu_device *adev = con->adev;
1239 const unsigned int element_size =
1240 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
d6ee400e
SA		 1241 unsigned int start = div64_ul(ppos + element_size - 1, element_size);
1242 unsigned int end = div64_ul(ppos + count - 1, element_size);
466b1793 1243 ssize_t s = 0;
1244 struct ras_badpage *bps = NULL;
1245 unsigned int bps_count = 0;
1246
1247 memset(buf, 0, count);
1248
1249 if (amdgpu_ras_badpages_read(adev, &bps, &bps_count))
1250 return 0;
1251
1252 for (; start < end && start < bps_count; start++)
1253 s += scnprintf(&buf[s], element_size + 1,
1254 "0x%08x : 0x%08x : %1s\n",
1255 bps[start].bp,
1256 bps[start].size,
1257 amdgpu_ras_badpage_flags_str(bps[start].flags));
1258
1259 kfree(bps);
1260
1261 return s;
1262}
1263
c030f2e4 1264static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
1265 struct device_attribute *attr, char *buf)
1266{
1267 struct amdgpu_ras *con =
1268 container_of(attr, struct amdgpu_ras, features_attr);
c030f2e4 1269
5212a3bd 1270 return scnprintf(buf, PAGE_SIZE, "feature mask: 0x%x\n", con->features);
c030f2e4 1271}
1272
f848159b
GC		 1273static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev)
1274{
1275 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1276
1277 sysfs_remove_file_from_group(&adev->dev->kobj,
1278 &con->badpages_attr.attr,
1279 RAS_FS_NAME);
1280}
1281
c030f2e4 1282static int amdgpu_ras_sysfs_remove_feature_node(struct amdgpu_device *adev)
1283{
1284 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1285 struct attribute *attrs[] = {
1286 &con->features_attr.attr,
1287 NULL
1288 };
1289 struct attribute_group group = {
eb0c3cd4 1290 .name = RAS_FS_NAME,
c030f2e4 1291 .attrs = attrs,
1292 };
1293
1294 sysfs_remove_group(&adev->dev->kobj, &group);
1295
1296 return 0;
1297}
1298
1299int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
9252d33d 1300 struct ras_common_if *head)
c030f2e4 1301{
9252d33d 1302 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
c030f2e4 1303
1304 if (!obj || obj->attr_inuse)
1305 return -EINVAL;
1306
1307 get_obj(obj);
1308
9252d33d 1309 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name),
1310 "%s_err_count", head->name);
c030f2e4 1311
1312 obj->sysfs_attr = (struct device_attribute){
1313 .attr = {
1314 .name = obj->fs_data.sysfs_name,
1315 .mode = S_IRUGO,
1316 },
1317 .show = amdgpu_ras_sysfs_read,
1318 };
163def43 1319 sysfs_attr_init(&obj->sysfs_attr.attr);
c030f2e4 1320
1321 if (sysfs_add_file_to_group(&adev->dev->kobj,
1322 &obj->sysfs_attr.attr,
eb0c3cd4 1323 RAS_FS_NAME)) {
c030f2e4 1324 put_obj(obj);
1325 return -EINVAL;
1326 }
1327
1328 obj->attr_inuse = 1;
1329
1330 return 0;
1331}
1332
1333int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
1334 struct ras_common_if *head)
1335{
1336 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1337
1338 if (!obj || !obj->attr_inuse)
1339 return -EINVAL;
1340
1341 sysfs_remove_file_from_group(&adev->dev->kobj,
1342 &obj->sysfs_attr.attr,
eb0c3cd4 1343 RAS_FS_NAME);
c030f2e4 1344 obj->attr_inuse = 0;
1345 put_obj(obj);
1346
1347 return 0;
1348}
1349
1350static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
1351{
1352 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1353 struct ras_manager *obj, *tmp;
1354
1355 list_for_each_entry_safe(obj, tmp, &con->head, node) {
1356 amdgpu_ras_sysfs_remove(adev, &obj->head);
1357 }
1358
f848159b
GC		 1359 if (amdgpu_bad_page_threshold != 0)
1360 amdgpu_ras_sysfs_remove_bad_page_node(adev);
1361
c030f2e4 1362 amdgpu_ras_sysfs_remove_feature_node(adev);
1363
1364 return 0;
1365}
1366/* sysfs end */
1367
ef177d11
AD		 1368/**
1369 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors
1370 *
1371 * Normally when there is an uncorrectable error, the driver will reset
1372 * the GPU to recover. However, in the event of an unrecoverable error,
1373 * the driver provides an interface to reboot the system automatically
1374 * in that event.
1375 *
1376 * The following file in debugfs provides that interface:
1377 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot
1378 *
1379 * Usage:
1380 *
1381 * .. code-block:: bash
1382 *
1383 * echo true > .../ras/auto_reboot
1384 *
1385 */
c030f2e4 1386/* debugfs begin */
ea1b8c9b 1387static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
36ea1bd2 1388{
1389 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
ef0d7d20
LT		 1390 struct drm_minor *minor = adev_to_drm(adev)->primary;
1391 struct dentry *dir;
36ea1bd2 1392
88293c03
ND		 1393 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);
1394 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev,
1395 &amdgpu_ras_debugfs_ctrl_ops);
1396 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev,
1397 &amdgpu_ras_debugfs_eeprom_ops);
7fb64071
LT		 1398 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,
1399 &con->bad_page_cnt_threshold);
ef0d7d20
LT		 1400 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled);
1401 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled);
c65b0805
LT		 1402 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev,
1403 &amdgpu_ras_debugfs_eeprom_size_ops);
1404 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table",
1405 S_IRUGO, dir, adev,
1406 &amdgpu_ras_debugfs_eeprom_table_ops);
1407 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control);
c688a06b
GC		 1408
1409 /*
1410 * After one uncorrectable error happens, usually GPU recovery will
1411 * be scheduled. But due to the known problem in GPU recovery failing
1412 * to bring GPU back, below interface provides one direct way to
1413 * user to reboot system automatically in such case within
1414 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine
1415 * will never be called.
1416 */
88293c03 1417 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot);
66459e1d
GC		 1418
1419 /*
1420 * User could set this not to clean up hardware's error count register
1421 * of RAS IPs during ras recovery.
1422 */
88293c03
ND		 1423 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir,
1424 &con->disable_ras_err_cnt_harvest);
1425 return dir;
36ea1bd2 1426}
1427
cedf7884 1428static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
88293c03
ND		 1429 struct ras_fs_if *head,
1430 struct dentry *dir)
c030f2e4 1431{
c030f2e4 1432 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
c030f2e4 1433
88293c03 1434 if (!obj || !dir)
450f30ea 1435 return;
c030f2e4 1436
1437 get_obj(obj);
1438
1439 memcpy(obj->fs_data.debugfs_name,
1440 head->debugfs_name,
1441 sizeof(obj->fs_data.debugfs_name));
1442
88293c03
ND		 1443 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir,
1444 obj, &amdgpu_ras_debugfs_ops);
c030f2e4 1445}
1446
f9317014
TZ		 1447void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
1448{
1449 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
88293c03 1450 struct dentry *dir;
c1509f3f 1451 struct ras_manager *obj;
f9317014
TZ		 1452 struct ras_fs_if fs_info;
1453
1454 /*
1455 * it won't be called in resume path, no need to check
1456 * suspend and gpu reset status
1457 */
cedf7884 1458 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con)
f9317014
TZ		 1459 return;
1460
88293c03 1461 dir = amdgpu_ras_debugfs_create_ctrl_node(adev);
f9317014 1462
c1509f3f 1463 list_for_each_entry(obj, &con->head, node) {
f9317014
TZ		 1464 if (amdgpu_ras_is_supported(adev, obj->head.block) &&
1465 (obj->attr_inuse == 1)) {
1466 sprintf(fs_info.debugfs_name, "%s_err_inject",
640ae42e 1467 get_ras_block_str(&obj->head));
f9317014 1468 fs_info.head = obj->head;
88293c03 1469 amdgpu_ras_debugfs_create(adev, &fs_info, dir);
f9317014
TZ		 1470 }
1471 }
1472}
1473
c030f2e4 1474/* debugfs end */
1475
1476/* ras fs */
c3d4d45d
GC		 1477static BIN_ATTR(gpu_vram_bad_pages, S_IRUGO,
1478 amdgpu_ras_sysfs_badpages_read, NULL, 0);
1479static DEVICE_ATTR(features, S_IRUGO,
1480 amdgpu_ras_sysfs_features_read, NULL);
c030f2e4 1481static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
1482{
c3d4d45d
GC		 1483 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1484 struct attribute_group group = {
1485 .name = RAS_FS_NAME,
1486 };
1487 struct attribute *attrs[] = {
1488 &con->features_attr.attr,
1489 NULL
1490 };
1491 struct bin_attribute *bin_attrs[] = {
1492 NULL,
1493 NULL,
1494 };
a069a9eb 1495 int r;
c030f2e4 1496
c3d4d45d
GC		 1497 /* add features entry */
1498 con->features_attr = dev_attr_features;
1499 group.attrs = attrs;
1500 sysfs_attr_init(attrs[0]);
1501
1502 if (amdgpu_bad_page_threshold != 0) {
1503 /* add bad_page_features entry */
1504 bin_attr_gpu_vram_bad_pages.private = NULL;
1505 con->badpages_attr = bin_attr_gpu_vram_bad_pages;
1506 bin_attrs[0] = &con->badpages_attr;
1507 group.bin_attrs = bin_attrs;
1508 sysfs_bin_attr_init(bin_attrs[0]);
1509 }
1510
a069a9eb
AD		 1511 r = sysfs_create_group(&adev->dev->kobj, &group);
1512 if (r)
1513 dev_err(adev->dev, "Failed to create RAS sysfs group!");
f848159b 1514
c030f2e4 1515 return 0;
1516}
1517
1518static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
1519{
88293c03
ND		 1520 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1521 struct ras_manager *con_obj, *ip_obj, *tmp;
1522
1523 if (IS_ENABLED(CONFIG_DEBUG_FS)) {
1524 list_for_each_entry_safe(con_obj, tmp, &con->head, node) {
1525 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head);
1526 if (ip_obj)
1527 put_obj(ip_obj);
1528 }
1529 }
1530
c030f2e4 1531 amdgpu_ras_sysfs_remove_all(adev);
1532 return 0;
1533}
1534/* ras fs end */
1535
1536/* ih begin */
b3c76814
TZ		 1537
1538/* For the hardware that cannot enable bif ring for both ras_controller_irq
1539 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status
1540 * register to check whether the interrupt is triggered or not, and properly
1541 * ack the interrupt if it is there
1542 */
1543void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev)
1544{
950d6425
SY		 1545 /* Fatal error events are handled on host side */
1546 if (amdgpu_sriov_vf(adev) ||
1547 !amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__PCIE_BIF))
b3c76814
TZ		 1548 return;
1549
1550 if (adev->nbio.ras &&
1551 adev->nbio.ras->handle_ras_controller_intr_no_bifring)
1552 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev);
1553
1554 if (adev->nbio.ras &&
1555 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring)
1556 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev);
1557}
1558
66f87949
TZ		 1559static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj,
1560 struct amdgpu_iv_entry *entry)
1561{
b63ac5d3 1562 bool poison_stat = false;
66f87949
TZ		 1563 struct amdgpu_device *adev = obj->adev;
1564 struct ras_err_data err_data = {0, 0, 0, NULL};
1565 struct amdgpu_ras_block_object *block_obj =
1566 amdgpu_ras_get_ras_block(adev, obj->head.block, 0);
1567
b63ac5d3
TZ		 1568 if (!block_obj || !block_obj->hw_ops)
1569 return;
66f87949 1570
b63ac5d3
TZ		 1571 /* both query_poison_status and handle_poison_consumption are optional,
1572 * but at least one of them should be implemented if we need poison
1573 * consumption handler
1574 */
1575 if (block_obj->hw_ops->query_poison_status) {
1576 poison_stat = block_obj->hw_ops->query_poison_status(adev);
1577 if (!poison_stat) {
1578 /* Not poison consumption interrupt, no need to handle it */
1579 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n",
1580 block_obj->ras_comm.name);
1581
1582 return;
66f87949
TZ		 1583 }
1584 }
1585
b63ac5d3
TZ		 1586 if (!adev->gmc.xgmi.connected_to_cpu)
1587 amdgpu_umc_poison_handler(adev, &err_data, false);
1588
1589 if (block_obj->hw_ops->handle_poison_consumption)
1590 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev);
1591
1592 /* gpu reset is fallback for failed and default cases */
1593 if (poison_stat) {
1594 dev_info(adev->dev, "GPU reset for %s RAS poison consumption is issued!\n",
1595 block_obj->ras_comm.name);
66f87949 1596 amdgpu_ras_reset_gpu(adev);
b63ac5d3 1597 }
66f87949
TZ		 1598}
1599
50a7d025
TZ		 1600static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj,
1601 struct amdgpu_iv_entry *entry)
1602{
1603 dev_info(obj->adev->dev,
1604 "Poison is created, no user action is needed.\n");
1605}
1606
1607static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
1608 struct amdgpu_iv_entry *entry)
1609{
1610 struct ras_ih_data *data = &obj->ih_data;
1611 struct ras_err_data err_data = {0, 0, 0, NULL};
1612 int ret;
1613
1614 if (!data->cb)
1615 return;
1616
1617 /* Let IP handle its data, maybe we need get the output
1618 * from the callback to update the error type/count, etc
1619 */
1620 ret = data->cb(obj->adev, &err_data, entry);
1621 /* ue will trigger an interrupt, and in that case
1622 * we need do a reset to recovery the whole system.
1623 * But leave IP do that recovery, here we just dispatch
1624 * the error.
1625 */
1626 if (ret == AMDGPU_RAS_SUCCESS) {
1627 /* these counts could be left as 0 if
1628 * some blocks do not count error number
1629 */
1630 obj->err_data.ue_count += err_data.ue_count;
1631 obj->err_data.ce_count += err_data.ce_count;
1632 }
1633}
1634
c030f2e4 1635static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
1636{
1637 struct ras_ih_data *data = &obj->ih_data;
1638 struct amdgpu_iv_entry entry;
c030f2e4 1639
1640 while (data->rptr != data->wptr) {
1641 rmb();
1642 memcpy(&entry, &data->ring[data->rptr],
1643 data->element_size);
1644
1645 wmb();
1646 data->rptr = (data->aligned_element_size +
1647 data->rptr) % data->ring_size;
1648
50a7d025
TZ		 1649 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) {
1650 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
1651 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry);
66f87949
TZ		 1652 else
1653 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry);
50a7d025
TZ		 1654 } else {
1655 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
1656 amdgpu_ras_interrupt_umc_handler(obj, &entry);
1657 else
1658 dev_warn(obj->adev->dev,
1659 "No RAS interrupt handler for non-UMC block with poison disabled.\n");
c030f2e4 1660 }
1661 }
1662}
1663
1664static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
1665{
1666 struct ras_ih_data *data =
1667 container_of(work, struct ras_ih_data, ih_work);
1668 struct ras_manager *obj =
1669 container_of(data, struct ras_manager, ih_data);
1670
1671 amdgpu_ras_interrupt_handler(obj);
1672}
1673
1674int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
1675 struct ras_dispatch_if *info)
1676{
1677 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1678 struct ras_ih_data *data = &obj->ih_data;
1679
1680 if (!obj)
1681 return -EINVAL;
1682
1683 if (data->inuse == 0)
1684 return 0;
1685
1686 /* Might be overflow... */
1687 memcpy(&data->ring[data->wptr], info->entry,
1688 data->element_size);
1689
1690 wmb();
1691 data->wptr = (data->aligned_element_size +
1692 data->wptr) % data->ring_size;
1693
1694 schedule_work(&data->ih_work);
1695
1696 return 0;
1697}
1698
1699int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
9252d33d 1700 struct ras_common_if *head)
c030f2e4 1701{
9252d33d 1702 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
c030f2e4 1703 struct ras_ih_data *data;
1704
1705 if (!obj)
1706 return -EINVAL;
1707
1708 data = &obj->ih_data;
1709 if (data->inuse == 0)
1710 return 0;
1711
1712 cancel_work_sync(&data->ih_work);
1713
1714 kfree(data->ring);
1715 memset(data, 0, sizeof(*data));
1716 put_obj(obj);
1717
1718 return 0;
1719}
1720
1721int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
9252d33d 1722 struct ras_common_if *head)
c030f2e4 1723{
9252d33d 1724 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
c030f2e4 1725 struct ras_ih_data *data;
9252d33d 1726 struct amdgpu_ras_block_object *ras_obj;
c030f2e4 1727
1728 if (!obj) {
1729 /* in case we registe the IH before enable ras feature */
9252d33d 1730 obj = amdgpu_ras_create_obj(adev, head);
c030f2e4 1731 if (!obj)
1732 return -EINVAL;
1733 } else
1734 get_obj(obj);
1735
9252d33d 1736 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm);
1737
c030f2e4 1738 data = &obj->ih_data;
1739 /* add the callback.etc */
1740 *data = (struct ras_ih_data) {
1741 .inuse = 0,
9252d33d 1742 .cb = ras_obj->ras_cb,
c030f2e4 1743 .element_size = sizeof(struct amdgpu_iv_entry),
1744 .rptr = 0,
1745 .wptr = 0,
1746 };
1747
1748 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
1749
1750 data->aligned_element_size = ALIGN(data->element_size, 8);
1751 /* the ring can store 64 iv entries. */
1752 data->ring_size = 64 * data->aligned_element_size;
1753 data->ring = kmalloc(data->ring_size, GFP_KERNEL);
1754 if (!data->ring) {
1755 put_obj(obj);
1756 return -ENOMEM;
1757 }
1758
1759 /* IH is ready */
1760 data->inuse = 1;
1761
1762 return 0;
1763}
1764
1765static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
1766{
1767 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1768 struct ras_manager *obj, *tmp;
1769
1770 list_for_each_entry_safe(obj, tmp, &con->head, node) {
9252d33d 1771 amdgpu_ras_interrupt_remove_handler(adev, &obj->head);
c030f2e4 1772 }
1773
1774 return 0;
1775}
1776/* ih end */
1777
313c8fd3
GC		 1778/* traversal all IPs except NBIO to query error counter */
1779static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev)
1780{
1781 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1782 struct ras_manager *obj;
1783
8ab0d6f0 1784 if (!adev->ras_enabled || !con)
313c8fd3
GC		 1785 return;
1786
1787 list_for_each_entry(obj, &con->head, node) {
1788 struct ras_query_if info = {
1789 .head = obj->head,
1790 };
1791
1792 /*
1793 * PCIE_BIF IP has one different isr by ras controller
1794 * interrupt, the specific ras counter query will be
1795 * done in that isr. So skip such block from common
1796 * sync flood interrupt isr calling.
1797 */
1798 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF)
1799 continue;
1800
cf63b702
SY		 1801 /*
1802 * this is a workaround for aldebaran, skip send msg to
1803 * smu to get ecc_info table due to smu handle get ecc
1804 * info table failed temporarily.
1805 * should be removed until smu fix handle ecc_info table.
1806 */
1807 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) &&
1808 (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 2)))
1809 continue;
1810
761d86d3 1811 amdgpu_ras_query_error_status(adev, &info);
2a460963
CL		 1812
1813 if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
6da15a23
CL		 1814 adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4) &&
1815 adev->ip_versions[MP0_HWIP][0] != IP_VERSION(13, 0, 0)) {
2a460963
CL		 1816 if (amdgpu_ras_reset_error_status(adev, info.head.block))
1817 dev_warn(adev->dev, "Failed to reset error counter and error status");
1818 }
313c8fd3
GC		 1819 }
1820}
1821
3f975d0f 1822/* Parse RdRspStatus and WrRspStatus */
cd92df93
LJ		 1823static void amdgpu_ras_error_status_query(struct amdgpu_device *adev,
1824 struct ras_query_if *info)
3f975d0f 1825{
8eb53bb2 1826 struct amdgpu_ras_block_object *block_obj;
3f975d0f
SY		 1827 /*
1828 * Only two block need to query read/write
1829 * RspStatus at current state
1830 */
5e67bba3 1831 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) &&
1832 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB))
b6efdb02 1833 return;
1834
1835 block_obj = amdgpu_ras_get_ras_block(adev,
1836 info->head.block,
1837 info->head.sub_block_index);
5e67bba3 1838
5e67bba3 1839 if (!block_obj || !block_obj->hw_ops) {
afa37315
LT		 1840 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1841 get_ras_block_str(&info->head));
b6efdb02 1842 return;
3f975d0f 1843 }
5e67bba3 1844
1845 if (block_obj->hw_ops->query_ras_error_status)
ab3b9de6 1846 block_obj->hw_ops->query_ras_error_status(adev);
5e67bba3 1847
3f975d0f
SY		 1848}
1849
1850static void amdgpu_ras_query_err_status(struct amdgpu_device *adev)
1851{
1852 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1853 struct ras_manager *obj;
1854
8ab0d6f0 1855 if (!adev->ras_enabled || !con)
3f975d0f
SY		 1856 return;
1857
1858 list_for_each_entry(obj, &con->head, node) {
1859 struct ras_query_if info = {
1860 .head = obj->head,
1861 };
1862
1863 amdgpu_ras_error_status_query(adev, &info);
1864 }
1865}
1866
c030f2e4 1867/* recovery begin */
466b1793 1868
1869/* return 0 on success.
1870 * caller need free bps.
1871 */
1872static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1873 struct ras_badpage **bps, unsigned int *count)
1874{
1875 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1876 struct ras_err_handler_data *data;
1877 int i = 0;
732f2a30 1878 int ret = 0, status;
466b1793 1879
1880 if (!con || !con->eh_data || !bps || !count)
1881 return -EINVAL;
1882
1883 mutex_lock(&con->recovery_lock);
1884 data = con->eh_data;
1885 if (!data || data->count == 0) {
1886 *bps = NULL;
46cf2fec 1887 ret = -EINVAL;
466b1793 1888 goto out;
1889 }
1890
1891 *bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL);
1892 if (!*bps) {
1893 ret = -ENOMEM;
1894 goto out;
1895 }
1896
1897 for (; i < data->count; i++) {
1898 (*bps)[i] = (struct ras_badpage){
9dc23a63 1899 .bp = data->bps[i].retired_page,
466b1793 1900 .size = AMDGPU_GPU_PAGE_SIZE,
52dd95f2 1901 .flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED,
466b1793 1902 };
ec6aae97 1903 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr,
676deb38 1904 data->bps[i].retired_page);
732f2a30 1905 if (status == -EBUSY)
52dd95f2 1906 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
732f2a30 1907 else if (status == -ENOENT)
52dd95f2 1908 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT;
466b1793 1909 }
1910
1911 *count = data->count;
1912out:
1913 mutex_unlock(&con->recovery_lock);
1914 return ret;
1915}
1916
c030f2e4 1917static void amdgpu_ras_do_recovery(struct work_struct *work)
1918{
1919 struct amdgpu_ras *ras =
1920 container_of(work, struct amdgpu_ras, recovery_work);
b3dbd6d3
JC		 1921 struct amdgpu_device *remote_adev = NULL;
1922 struct amdgpu_device *adev = ras->adev;
1923 struct list_head device_list, *device_list_handle = NULL;
b3dbd6d3 1924
f75e94d8 1925 if (!ras->disable_ras_err_cnt_harvest) {
d95e8e97
DL		 1926 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
1927
f75e94d8
GC		 1928 /* Build list of devices to query RAS related errors */
1929 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
1930 device_list_handle = &hive->device_list;
1931 } else {
1932 INIT_LIST_HEAD(&device_list);
1933 list_add_tail(&adev->gmc.xgmi.head, &device_list);
1934 device_list_handle = &device_list;
1935 }
c030f2e4 1936
f75e94d8 1937 list_for_each_entry(remote_adev,
3f975d0f
SY		 1938 device_list_handle, gmc.xgmi.head) {
1939 amdgpu_ras_query_err_status(remote_adev);
f75e94d8 1940 amdgpu_ras_log_on_err_counter(remote_adev);
3f975d0f 1941 }
d95e8e97
DL		 1942
1943 amdgpu_put_xgmi_hive(hive);
b3dbd6d3 1944 }
313c8fd3 1945
f1549c09
LG		 1946 if (amdgpu_device_should_recover_gpu(ras->adev)) {
1947 struct amdgpu_reset_context reset_context;
1948 memset(&reset_context, 0, sizeof(reset_context));
1949
1950 reset_context.method = AMD_RESET_METHOD_NONE;
1951 reset_context.reset_req_dev = adev;
1952 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
dac6b808 1953 clear_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags);
f1549c09
LG		 1954
1955 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context);
1956 }
c030f2e4 1957 atomic_set(&ras->in_recovery, 0);
1958}
1959
c030f2e4 1960/* alloc/realloc bps array */
1961static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
1962 struct ras_err_handler_data *data, int pages)
1963{
1964 unsigned int old_space = data->count + data->space_left;
1965 unsigned int new_space = old_space + pages;
9dc23a63
TZ		 1966 unsigned int align_space = ALIGN(new_space, 512);
1967 void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL);
9dc23a63 1968
676deb38 1969 if (!bps) {
c030f2e4 1970 return -ENOMEM;
9dc23a63 1971 }
c030f2e4 1972
1973 if (data->bps) {
9dc23a63 1974 memcpy(bps, data->bps,
c030f2e4 1975 data->count * sizeof(*data->bps));
1976 kfree(data->bps);
1977 }
1978
9dc23a63 1979 data->bps = bps;
c030f2e4 1980 data->space_left += align_space - old_space;
1981 return 0;
1982}
1983
1984/* it deal with vram only. */
1985int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
9dc23a63 1986 struct eeprom_table_record *bps, int pages)
c030f2e4 1987{
1988 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
73aa8e1a 1989 struct ras_err_handler_data *data;
c030f2e4 1990 int ret = 0;
676deb38 1991 uint32_t i;
c030f2e4 1992
73aa8e1a 1993 if (!con || !con->eh_data || !bps || pages <= 0)
c030f2e4 1994 return 0;
1995
1996 mutex_lock(&con->recovery_lock);
73aa8e1a 1997 data = con->eh_data;
c030f2e4 1998 if (!data)
1999 goto out;
2000
676deb38
DL		 2001 for (i = 0; i < pages; i++) {
2002 if (amdgpu_ras_check_bad_page_unlock(con,
2003 bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT))
2004 continue;
2005
2006 if (!data->space_left &&
2007 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
c030f2e4 2008 ret = -ENOMEM;
2009 goto out;
2010 }
2011
ec6aae97 2012 amdgpu_vram_mgr_reserve_range(&adev->mman.vram_mgr,
676deb38
DL		 2013 bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT,
2014 AMDGPU_GPU_PAGE_SIZE);
9dc23a63 2015
676deb38
DL		 2016 memcpy(&data->bps[data->count], &bps[i], sizeof(*data->bps));
2017 data->count++;
2018 data->space_left--;
2019 }
c030f2e4 2020out:
2021 mutex_unlock(&con->recovery_lock);
2022
2023 return ret;
2024}
2025
78ad00c9
TZ		 2026/*
2027 * write error record array to eeprom, the function should be
2028 * protected by recovery_lock
2029 */
22503d80 2030int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev)
78ad00c9
TZ		 2031{
2032 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2033 struct ras_err_handler_data *data;
8a3e801f 2034 struct amdgpu_ras_eeprom_control *control;
78ad00c9
TZ		 2035 int save_count;
2036
2037 if (!con || !con->eh_data)
2038 return 0;
2039
d9a69fe5 2040 mutex_lock(&con->recovery_lock);
8a3e801f 2041 control = &con->eeprom_control;
78ad00c9 2042 data = con->eh_data;
0686627b 2043 save_count = data->count - control->ras_num_recs;
d9a69fe5 2044 mutex_unlock(&con->recovery_lock);
78ad00c9 2045 /* only new entries are saved */
b1628425 2046 if (save_count > 0) {
63d4c081
LT		 2047 if (amdgpu_ras_eeprom_append(control,
2048 &data->bps[control->ras_num_recs],
2049 save_count)) {
6952e99c 2050 dev_err(adev->dev, "Failed to save EEPROM table data!");
78ad00c9
TZ		 2051 return -EIO;
2052 }
2053
b1628425
GC		 2054 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
2055 }
2056
78ad00c9
TZ		 2057 return 0;
2058}
2059
2060/*
2061 * read error record array in eeprom and reserve enough space for
2062 * storing new bad pages
2063 */
2064static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
2065{
2066 struct amdgpu_ras_eeprom_control *control =
6457205c 2067 &adev->psp.ras_context.ras->eeprom_control;
e4e6a589
LT		 2068 struct eeprom_table_record *bps;
2069 int ret;
78ad00c9
TZ		 2070
2071 /* no bad page record, skip eeprom access */
0686627b 2072 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0)
e4e6a589 2073 return 0;
78ad00c9 2074
0686627b 2075 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL);
78ad00c9
TZ		 2076 if (!bps)
2077 return -ENOMEM;
2078
0686627b 2079 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs);
e4e6a589 2080 if (ret)
6952e99c 2081 dev_err(adev->dev, "Failed to load EEPROM table records!");
e4e6a589 2082 else
0686627b 2083 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs);
78ad00c9 2084
78ad00c9
TZ		 2085 kfree(bps);
2086 return ret;
2087}
2088
676deb38
DL		 2089static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
2090 uint64_t addr)
2091{
2092 struct ras_err_handler_data *data = con->eh_data;
2093 int i;
2094
2095 addr >>= AMDGPU_GPU_PAGE_SHIFT;
2096 for (i = 0; i < data->count; i++)
2097 if (addr == data->bps[i].retired_page)
2098 return true;
2099
2100 return false;
2101}
2102
6e4be987
TZ		 2103/*
2104 * check if an address belongs to bad page
2105 *
2106 * Note: this check is only for umc block
2107 */
2108static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
2109 uint64_t addr)
2110{
2111 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
6e4be987
TZ		 2112 bool ret = false;
2113
2114 if (!con || !con->eh_data)
2115 return ret;
2116
2117 mutex_lock(&con->recovery_lock);
676deb38 2118 ret = amdgpu_ras_check_bad_page_unlock(con, addr);
6e4be987
TZ		 2119 mutex_unlock(&con->recovery_lock);
2120 return ret;
2121}
2122
e5c04edf 2123static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
e4e6a589 2124 uint32_t max_count)
c84d4670 2125{
e5c04edf 2126 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
c84d4670
GC		 2127
2128 /*
2129 * Justification of value bad_page_cnt_threshold in ras structure
2130 *
2131 * Generally, -1 <= amdgpu_bad_page_threshold <= max record length
2132 * in eeprom, and introduce two scenarios accordingly.
2133 *
2134 * Bad page retirement enablement:
2135 * - If amdgpu_bad_page_threshold = -1,
2136 * bad_page_cnt_threshold = typical value by formula.
2137 *
2138 * - When the value from user is 0 < amdgpu_bad_page_threshold <
2139 * max record length in eeprom, use it directly.
2140 *
2141 * Bad page retirement disablement:
2142 * - If amdgpu_bad_page_threshold = 0, bad page retirement
2143 * functionality is disabled, and bad_page_cnt_threshold will
2144 * take no effect.
2145 */
2146
e4e6a589
LT		 2147 if (amdgpu_bad_page_threshold < 0) {
2148 u64 val = adev->gmc.mc_vram_size;
c84d4670 2149
e4e6a589 2150 do_div(val, RAS_BAD_PAGE_COVER);
e5c04edf 2151 con->bad_page_cnt_threshold = min(lower_32_bits(val),
e4e6a589 2152 max_count);
e5c04edf 2153 } else {
e4e6a589
LT		 2154 con->bad_page_cnt_threshold = min_t(int, max_count,
2155 amdgpu_bad_page_threshold);
c84d4670
GC		 2156 }
2157}
2158
1a6fc071 2159int amdgpu_ras_recovery_init(struct amdgpu_device *adev)
c030f2e4 2160{
2161 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4d1337d2 2162 struct ras_err_handler_data **data;
e4e6a589 2163 u32 max_eeprom_records_count = 0;
b82e65a9 2164 bool exc_err_limit = false;
78ad00c9 2165 int ret;
c030f2e4 2166
e0e146d5 2167 if (!con || amdgpu_sriov_vf(adev))
1d9d2ca8
LT		 2168 return 0;
2169
2170 /* Allow access to RAS EEPROM via debugfs, when the ASIC
2171 * supports RAS and debugfs is enabled, but when
2172 * adev->ras_enabled is unset, i.e. when "ras_enable"
2173 * module parameter is set to 0.
2174 */
2175 con->adev = adev;
2176
2177 if (!adev->ras_enabled)
4d1337d2
AG		 2178 return 0;
2179
1d9d2ca8 2180 data = &con->eh_data;
1a6fc071
TZ		 2181 *data = kmalloc(sizeof(**data), GFP_KERNEL | __GFP_ZERO);
2182 if (!*data) {
2183 ret = -ENOMEM;
2184 goto out;
2185 }
c030f2e4 2186
2187 mutex_init(&con->recovery_lock);
2188 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
2189 atomic_set(&con->in_recovery, 0);
69691c82 2190 con->eeprom_control.bad_channel_bitmap = 0;
c030f2e4 2191
e4e6a589
LT		 2192 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count();
2193 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count);
c84d4670 2194
e5086659 2195 /* Todo: During test the SMU might fail to read the eeprom through I2C
2196 * when the GPU is pending on XGMI reset during probe time
2197 * (Mostly after second bus reset), skip it now
2198 */
2199 if (adev->gmc.xgmi.pending_reset)
2200 return 0;
b82e65a9
GC		 2201 ret = amdgpu_ras_eeprom_init(&con->eeprom_control, &exc_err_limit);
2202 /*
2203 * This calling fails when exc_err_limit is true or
2204 * ret != 0.
2205 */
2206 if (exc_err_limit || ret)
1a6fc071 2207 goto free;
78ad00c9 2208
0686627b 2209 if (con->eeprom_control.ras_num_recs) {
78ad00c9
TZ		 2210 ret = amdgpu_ras_load_bad_pages(adev);
2211 if (ret)
1a6fc071 2212 goto free;
513befa6 2213
bc143d8b 2214 amdgpu_dpm_send_hbm_bad_pages_num(adev, con->eeprom_control.ras_num_recs);
69691c82
SY		 2215
2216 if (con->update_channel_flag == true) {
2217 amdgpu_dpm_send_hbm_bad_channel_flag(adev, con->eeprom_control.bad_channel_bitmap);
2218 con->update_channel_flag = false;
2219 }
78ad00c9 2220 }
c030f2e4 2221
12b2cab7
MJ		 2222#ifdef CONFIG_X86_MCE_AMD
2223 if ((adev->asic_type == CHIP_ALDEBARAN) &&
2224 (adev->gmc.xgmi.connected_to_cpu))
91a1a52d 2225 amdgpu_register_bad_pages_mca_notifier(adev);
12b2cab7 2226#endif
c030f2e4 2227 return 0;
1a6fc071 2228
1a6fc071 2229free:
1a6fc071 2230 kfree((*data)->bps);
1a6fc071 2231 kfree(*data);
1995b3a3 2232 con->eh_data = NULL;
1a6fc071 2233out:
cf696091 2234 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret);
1a6fc071 2235
b82e65a9
GC		 2236 /*
2237 * Except error threshold exceeding case, other failure cases in this
2238 * function would not fail amdgpu driver init.
2239 */
2240 if (!exc_err_limit)
2241 ret = 0;
2242 else
2243 ret = -EINVAL;
2244
1a6fc071 2245 return ret;
c030f2e4 2246}
2247
2248static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
2249{
2250 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2251 struct ras_err_handler_data *data = con->eh_data;
2252
1a6fc071
TZ		 2253 /* recovery_init failed to init it, fini is useless */
2254 if (!data)
2255 return 0;
2256
c030f2e4 2257 cancel_work_sync(&con->recovery_work);
c030f2e4 2258
2259 mutex_lock(&con->recovery_lock);
2260 con->eh_data = NULL;
2261 kfree(data->bps);
2262 kfree(data);
2263 mutex_unlock(&con->recovery_lock);
2264
2265 return 0;
2266}
2267/* recovery end */
2268
084e2640 2269static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
5436ab94 2270{
084e2640
LT		 2271 return adev->asic_type == CHIP_VEGA10 ||
2272 adev->asic_type == CHIP_VEGA20 ||
2273 adev->asic_type == CHIP_ARCTURUS ||
75f06251 2274 adev->asic_type == CHIP_ALDEBARAN ||
084e2640 2275 adev->asic_type == CHIP_SIENNA_CICHLID;
5436ab94
SY		 2276}
2277
f50160cf
SY		 2278/*
2279 * this is workaround for vega20 workstation sku,
2280 * force enable gfx ras, ignore vbios gfx ras flag
2281 * due to GC EDC can not write
2282 */
e509965e 2283static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
f50160cf
SY		 2284{
2285 struct atom_context *ctx = adev->mode_info.atom_context;
2286
2287 if (!ctx)
2288 return;
2289
2290 if (strnstr(ctx->vbios_version, "D16406",
e11d5e0d
SY		 2291 sizeof(ctx->vbios_version)) ||
2292 strnstr(ctx->vbios_version, "D36002",
2293 sizeof(ctx->vbios_version)))
8ab0d6f0 2294 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX);
f50160cf
SY		 2295}
2296
5caf466a 2297/*
2298 * check hardware's ras ability which will be saved in hw_supported.
2299 * if hardware does not support ras, we can skip some ras initializtion and
2300 * forbid some ras operations from IP.
2301 * if software itself, say boot parameter, limit the ras ability. We still
2302 * need allow IP do some limited operations, like disable. In such case,
2303 * we have to initialize ras as normal. but need check if operation is
2304 * allowed or not in each function.
2305 */
e509965e 2306static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
c030f2e4 2307{
8ab0d6f0 2308 adev->ras_hw_enabled = adev->ras_enabled = 0;
c030f2e4 2309
950d6425 2310 if (!adev->is_atom_fw ||
084e2640 2311 !amdgpu_ras_asic_supported(adev))
5caf466a 2312 return;
b404ae82 2313
28caf8c4
SY		 2314 /* If driver run on sriov guest side, only enable ras for aldebaran */
2315 if (amdgpu_sriov_vf(adev) &&
2316 adev->ip_versions[MP1_HWIP][0] != IP_VERSION(13, 0, 2))
950d6425
SY		 2317 return;
2318
75f06251
HZ		 2319 if (!adev->gmc.xgmi.connected_to_cpu) {
2320 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
2321 dev_info(adev->dev, "MEM ECC is active.\n");
8ab0d6f0 2322 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC |
e509965e 2323 1 << AMDGPU_RAS_BLOCK__DF);
75f06251
HZ		 2324 } else {
2325 dev_info(adev->dev, "MEM ECC is not presented.\n");
2326 }
88474cca 2327
75f06251
HZ		 2328 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
2329 dev_info(adev->dev, "SRAM ECC is active.\n");
950d6425
SY		 2330 if (!amdgpu_sriov_vf(adev)) {
2331 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
2332 1 << AMDGPU_RAS_BLOCK__DF);
2333
2334 if (adev->ip_versions[VCN_HWIP][0] == IP_VERSION(2, 6, 0))
2335 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN |
2336 1 << AMDGPU_RAS_BLOCK__JPEG);
2337 else
2338 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
2339 1 << AMDGPU_RAS_BLOCK__JPEG);
2340 } else {
2341 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF |
2342 1 << AMDGPU_RAS_BLOCK__SDMA |
2343 1 << AMDGPU_RAS_BLOCK__GFX);
2344 }
75f06251
HZ		 2345 } else {
2346 dev_info(adev->dev, "SRAM ECC is not presented.\n");
2347 }
2348 } else {
2349 /* driver only manages a few IP blocks RAS feature
2350 * when GPU is connected cpu through XGMI */
8ab0d6f0 2351 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX |
e509965e
LT		 2352 1 << AMDGPU_RAS_BLOCK__SDMA |
2353 1 << AMDGPU_RAS_BLOCK__MMHUB);
75f06251 2354 }
88474cca 2355
e509965e 2356 amdgpu_ras_get_quirks(adev);
f50160cf 2357
88474cca 2358 /* hw_supported needs to be aligned with RAS block mask. */
8ab0d6f0 2359 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
b404ae82 2360
8ab0d6f0
LT		 2361 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
2362 adev->ras_hw_enabled & amdgpu_ras_mask;
c030f2e4 2363}
2364
05adfd80
LT		 2365static void amdgpu_ras_counte_dw(struct work_struct *work)
2366{
2367 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
2368 ras_counte_delay_work.work);
2369 struct amdgpu_device *adev = con->adev;
a3fbb0d8 2370 struct drm_device *dev = adev_to_drm(adev);
05adfd80
LT		 2371 unsigned long ce_count, ue_count;
2372 int res;
2373
2374 res = pm_runtime_get_sync(dev->dev);
2375 if (res < 0)
2376 goto Out;
2377
2378 /* Cache new values.
2379 */
4d9f771e
LT		 2380 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count) == 0) {
2381 atomic_set(&con->ras_ce_count, ce_count);
2382 atomic_set(&con->ras_ue_count, ue_count);
2383 }
05adfd80
LT		 2384
2385 pm_runtime_mark_last_busy(dev->dev);
2386Out:
2387 pm_runtime_put_autosuspend(dev->dev);
2388}
2389
c030f2e4 2390int amdgpu_ras_init(struct amdgpu_device *adev)
2391{
2392 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4e644fff 2393 int r;
e4348849 2394 bool df_poison, umc_poison;
c030f2e4 2395
b404ae82 2396 if (con)
c030f2e4 2397 return 0;
2398
2399 con = kmalloc(sizeof(struct amdgpu_ras) +
640ae42e
JC		 2400 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT +
2401 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT,
c030f2e4 2402 GFP_KERNEL|__GFP_ZERO);
2403 if (!con)
2404 return -ENOMEM;
2405
05adfd80
LT		 2406 con->adev = adev;
2407 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw);
2408 atomic_set(&con->ras_ce_count, 0);
2409 atomic_set(&con->ras_ue_count, 0);
2410
c030f2e4 2411 con->objs = (struct ras_manager *)(con + 1);
2412
2413 amdgpu_ras_set_context(adev, con);
2414
e509965e
LT		 2415 amdgpu_ras_check_supported(adev);
2416
7ddd9770 2417 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) {
970fd197
SY		 2418 /* set gfx block ras context feature for VEGA20 Gaming
2419 * send ras disable cmd to ras ta during ras late init.
2420 */
8ab0d6f0 2421 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
970fd197
SY		 2422 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX);
2423
2424 return 0;
2425 }
2426
5e91160a 2427 r = 0;
5436ab94 2428 goto release_con;
fb2a3607
HZ		 2429 }
2430
69691c82 2431 con->update_channel_flag = false;
c030f2e4 2432 con->features = 0;
2433 INIT_LIST_HEAD(&con->head);
108c6a63 2434 /* Might need get this flag from vbios. */
2435 con->flags = RAS_DEFAULT_FLAGS;
c030f2e4 2436
6e36f231
HZ		 2437 /* initialize nbio ras function ahead of any other
2438 * ras functions so hardware fatal error interrupt
2439 * can be enabled as early as possible */
2440 switch (adev->asic_type) {
2441 case CHIP_VEGA20:
2442 case CHIP_ARCTURUS:
2443 case CHIP_ALDEBARAN:
2e54fe5d 2444 if (!adev->gmc.xgmi.connected_to_cpu) {
2445 adev->nbio.ras = &nbio_v7_4_ras;
2446 amdgpu_ras_register_ras_block(adev, &adev->nbio.ras->ras_block);
80ed77f9 2447 adev->nbio.ras_if = &adev->nbio.ras->ras_block.ras_comm;
2e54fe5d 2448 }
6e36f231
HZ		 2449 break;
2450 default:
2451 /* nbio ras is not available */
2452 break;
2453 }
2454
2e54fe5d 2455 if (adev->nbio.ras &&
2456 adev->nbio.ras->init_ras_controller_interrupt) {
2457 r = adev->nbio.ras->init_ras_controller_interrupt(adev);
4e644fff 2458 if (r)
5436ab94 2459 goto release_con;
4e644fff
HZ		 2460 }
2461
2e54fe5d 2462 if (adev->nbio.ras &&
2463 adev->nbio.ras->init_ras_err_event_athub_interrupt) {
2464 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev);
4e644fff 2465 if (r)
5436ab94 2466 goto release_con;
4e644fff
HZ		 2467 }
2468
e4348849 2469 /* Init poison supported flag, the default value is false */
655ff353
TZ		 2470 if (adev->gmc.xgmi.connected_to_cpu) {
2471 /* enabled by default when GPU is connected to CPU */
2472 con->poison_supported = true;
2473 }
2474 else if (adev->df.funcs &&
e4348849 2475 adev->df.funcs->query_ras_poison_mode &&
efe17d5a 2476 adev->umc.ras &&
2477 adev->umc.ras->query_ras_poison_mode) {
e4348849
TZ		 2478 df_poison =
2479 adev->df.funcs->query_ras_poison_mode(adev);
2480 umc_poison =
efe17d5a 2481 adev->umc.ras->query_ras_poison_mode(adev);
e4348849
TZ		 2482 /* Only poison is set in both DF and UMC, we can support it */
2483 if (df_poison && umc_poison)
2484 con->poison_supported = true;
2485 else if (df_poison != umc_poison)
2486 dev_warn(adev->dev, "Poison setting is inconsistent in DF/UMC(%d:%d)!\n",
2487 df_poison, umc_poison);
2488 }
2489
5e91160a
GC		 2490 if (amdgpu_ras_fs_init(adev)) {
2491 r = -EINVAL;
5436ab94 2492 goto release_con;
5e91160a 2493 }
c030f2e4 2494
6952e99c 2495 dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
e509965e 2496 "hardware ability[%x] ras_mask[%x]\n",
8ab0d6f0 2497 adev->ras_hw_enabled, adev->ras_enabled);
e509965e 2498
c030f2e4 2499 return 0;
5436ab94 2500release_con:
c030f2e4 2501 amdgpu_ras_set_context(adev, NULL);
2502 kfree(con);
2503
5e91160a 2504 return r;
c030f2e4 2505}
2506
8f6368a9 2507int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev)
134d16d5
JC		 2508{
2509 if (adev->gmc.xgmi.connected_to_cpu)
2510 return 1;
2511 return 0;
2512}
2513
2514static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev,
2515 struct ras_common_if *ras_block)
2516{
2517 struct ras_query_if info = {
2518 .head = *ras_block,
2519 };
2520
2521 if (!amdgpu_persistent_edc_harvesting_supported(adev))
2522 return 0;
2523
2524 if (amdgpu_ras_query_error_status(adev, &info) != 0)
2525 DRM_WARN("RAS init harvest failure");
2526
2527 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0)
2528 DRM_WARN("RAS init harvest reset failure");
2529
2530 return 0;
2531}
2532
e4348849
TZ		 2533bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev)
2534{
2535 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2536
2537 if (!con)
2538 return false;
2539
2540 return con->poison_supported;
2541}
2542
b293e891 2543/* helper function to handle common stuff in ip late init phase */
563285c8 2544int amdgpu_ras_block_late_init(struct amdgpu_device *adev,
2545 struct ras_common_if *ras_block)
b293e891 2546{
29c9b6cd 2547 struct amdgpu_ras_block_object *ras_obj = NULL;
05adfd80
LT		 2548 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2549 unsigned long ue_count, ce_count;
b293e891
HZ		 2550 int r;
2551
2552 /* disable RAS feature per IP block if it is not supported */
2553 if (!amdgpu_ras_is_supported(adev, ras_block->block)) {
2554 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
2555 return 0;
2556 }
2557
2558 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1);
2559 if (r) {
9080a18f 2560 if (adev->in_suspend || amdgpu_in_reset(adev)) {
b293e891
HZ		 2561 /* in resume phase, if fail to enable ras,
2562 * clean up all ras fs nodes, and disable ras */
2563 goto cleanup;
2564 } else
2565 return r;
2566 }
2567
134d16d5
JC		 2568 /* check for errors on warm reset edc persisant supported ASIC */
2569 amdgpu_persistent_edc_harvesting(adev, ras_block);
2570
b293e891 2571 /* in resume phase, no need to create ras fs node */
53b3f8f4 2572 if (adev->in_suspend || amdgpu_in_reset(adev))
b293e891
HZ		 2573 return 0;
2574
563285c8 2575 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
36780606
TZ		 2576 if (ras_obj->ras_cb || (ras_obj->hw_ops &&
2577 (ras_obj->hw_ops->query_poison_status ||
2578 ras_obj->hw_ops->handle_poison_consumption))) {
9252d33d 2579 r = amdgpu_ras_interrupt_add_handler(adev, ras_block);
b293e891 2580 if (r)
779596ce 2581 goto cleanup;
b293e891
HZ		 2582 }
2583
9252d33d 2584 r = amdgpu_ras_sysfs_create(adev, ras_block);
b293e891 2585 if (r)
779596ce 2586 goto interrupt;
b293e891 2587
05adfd80
LT		 2588 /* Those are the cached values at init.
2589 */
4d9f771e
LT		 2590 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count) == 0) {
2591 atomic_set(&con->ras_ce_count, ce_count);
2592 atomic_set(&con->ras_ue_count, ue_count);
2593 }
05adfd80 2594
b293e891 2595 return 0;
779596ce
TR		 2596
2597interrupt:
563285c8 2598 if (ras_obj->ras_cb)
9252d33d 2599 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
779596ce 2600cleanup:
b293e891
HZ		 2601 amdgpu_ras_feature_enable(adev, ras_block, 0);
2602 return r;
2603}
2604
d41ff22a 2605static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev,
418abce2 2606 struct ras_common_if *ras_block)
2607{
2608 return amdgpu_ras_block_late_init(adev, ras_block);
2609}
2610
b293e891 2611/* helper function to remove ras fs node and interrupt handler */
bdb3489c 2612void amdgpu_ras_block_late_fini(struct amdgpu_device *adev,
2613 struct ras_common_if *ras_block)
2614{
563285c8 2615 struct amdgpu_ras_block_object *ras_obj;
bdb3489c 2616 if (!ras_block)
2617 return;
2618
563285c8 2619 amdgpu_ras_sysfs_remove(adev, ras_block);
bdb3489c 2620
563285c8 2621 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
2622 if (ras_obj->ras_cb)
2623 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
bdb3489c 2624}
2625
80e0c2cb 2626static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev,
2627 struct ras_common_if *ras_block)
2628{
2629 return amdgpu_ras_block_late_fini(adev, ras_block);
2630}
2631
a564808e 2632/* do some init work after IP late init as dependence.
511fdbc3 2633 * and it runs in resume/gpu reset/booting up cases.
a564808e 2634 */
511fdbc3 2635void amdgpu_ras_resume(struct amdgpu_device *adev)
108c6a63 2636{
2637 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2638 struct ras_manager *obj, *tmp;
2639
8ab0d6f0 2640 if (!adev->ras_enabled || !con) {
970fd197
SY		 2641 /* clean ras context for VEGA20 Gaming after send ras disable cmd */
2642 amdgpu_release_ras_context(adev);
2643
108c6a63 2644 return;
970fd197 2645 }
108c6a63 2646
108c6a63 2647 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
191051a1 2648 /* Set up all other IPs which are not implemented. There is a
2649 * tricky thing that IP's actual ras error type should be
2650 * MULTI_UNCORRECTABLE, but as driver does not handle it, so
2651 * ERROR_NONE make sense anyway.
2652 */
2653 amdgpu_ras_enable_all_features(adev, 1);
2654
2655 /* We enable ras on all hw_supported block, but as boot
2656 * parameter might disable some of them and one or more IP has
2657 * not implemented yet. So we disable them on behalf.
2658 */
108c6a63 2659 list_for_each_entry_safe(obj, tmp, &con->head, node) {
2660 if (!amdgpu_ras_is_supported(adev, obj->head.block)) {
2661 amdgpu_ras_feature_enable(adev, &obj->head, 0);
2662 /* there should be no any reference. */
2663 WARN_ON(alive_obj(obj));
2664 }
191051a1 2665 }
108c6a63 2666 }
2667}
2668
511fdbc3 2669void amdgpu_ras_suspend(struct amdgpu_device *adev)
2670{
2671 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2672
8ab0d6f0 2673 if (!adev->ras_enabled || !con)
511fdbc3 2674 return;
2675
2676 amdgpu_ras_disable_all_features(adev, 0);
2677 /* Make sure all ras objects are disabled. */
2678 if (con->features)
2679 amdgpu_ras_disable_all_features(adev, 1);
2680}
2681
867e24ca 2682int amdgpu_ras_late_init(struct amdgpu_device *adev)
2683{
2684 struct amdgpu_ras_block_list *node, *tmp;
2685 struct amdgpu_ras_block_object *obj;
2686 int r;
2687
950d6425
SY		 2688 /* Guest side doesn't need init ras feature */
2689 if (amdgpu_sriov_vf(adev))
2690 return 0;
2691
867e24ca 2692 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
2693 if (!node->ras_obj) {
2694 dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
2695 continue;
2696 }
418abce2 2697
867e24ca 2698 obj = node->ras_obj;
2699 if (obj->ras_late_init) {
2700 r = obj->ras_late_init(adev, &obj->ras_comm);
2701 if (r) {
2702 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n",
2703 obj->ras_comm.name, r);
2704 return r;
2705 }
418abce2 2706 } else
2707 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm);
867e24ca 2708 }
2709
2710 return 0;
2711}
2712
c030f2e4 2713/* do some fini work before IP fini as dependence */
2714int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
2715{
2716 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2717
8ab0d6f0 2718 if (!adev->ras_enabled || !con)
c030f2e4 2719 return 0;
2720
72c8c97b 2721
c030f2e4 2722 /* Need disable ras on all IPs here before ip [hw/sw]fini */
642c0401
YC		 2723 if (con->features)
2724 amdgpu_ras_disable_all_features(adev, 0);
c030f2e4 2725 amdgpu_ras_recovery_fini(adev);
2726 return 0;
2727}
2728
2729int amdgpu_ras_fini(struct amdgpu_device *adev)
2730{
d5e8ff5f 2731 struct amdgpu_ras_block_list *ras_node, *tmp;
1f211a82 2732 struct amdgpu_ras_block_object *obj = NULL;
c030f2e4 2733 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2734
8ab0d6f0 2735 if (!adev->ras_enabled || !con)
c030f2e4 2736 return 0;
2737
1f211a82 2738 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) {
2739 if (ras_node->ras_obj) {
2740 obj = ras_node->ras_obj;
2741 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) &&
2742 obj->ras_fini)
2743 obj->ras_fini(adev, &obj->ras_comm);
80e0c2cb 2744 else
2745 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm);
1f211a82 2746 }
2747
2748 /* Clear ras blocks from ras_list and free ras block list node */
2749 list_del(&ras_node->node);
2750 kfree(ras_node);
2751 }
2752
c030f2e4 2753 amdgpu_ras_fs_fini(adev);
2754 amdgpu_ras_interrupt_remove_all(adev);
2755
2756 WARN(con->features, "Feature mask is not cleared");
2757
2758 if (con->features)
2759 amdgpu_ras_disable_all_features(adev, 1);
2760
05adfd80
LT		 2761 cancel_delayed_work_sync(&con->ras_counte_delay_work);
2762
c030f2e4 2763 amdgpu_ras_set_context(adev, NULL);
2764 kfree(con);
2765
2766 return 0;
2767}
7c6e68c7
AG		 2768
2769void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
2770{
e509965e 2771 amdgpu_ras_check_supported(adev);
8ab0d6f0 2772 if (!adev->ras_hw_enabled)
ed606f8a
AG		 2773 return;
2774
7c6e68c7 2775 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
6952e99c
GC		 2776 dev_info(adev->dev, "uncorrectable hardware error"
2777 "(ERREVENT_ATHUB_INTERRUPT) detected!\n");
d5ea093e 2778
61934624 2779 amdgpu_ras_reset_gpu(adev);
7c6e68c7
AG		 2780 }
2781}
bb5c7235
WS		 2782
2783bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev)
2784{
2785 if (adev->asic_type == CHIP_VEGA20 &&
2786 adev->pm.fw_version <= 0x283400) {
2787 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) &&
2788 amdgpu_ras_intr_triggered();
2789 }
2790
2791 return false;
2792}
970fd197
SY		 2793
2794void amdgpu_release_ras_context(struct amdgpu_device *adev)
2795{
2796 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2797
2798 if (!con)
2799 return;
2800
8ab0d6f0 2801 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
970fd197
SY		 2802 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
2803 amdgpu_ras_set_context(adev, NULL);
2804 kfree(con);
2805 }
2806}
12b2cab7
MJ		 2807
2808#ifdef CONFIG_X86_MCE_AMD
2809static struct amdgpu_device *find_adev(uint32_t node_id)
2810{
12b2cab7
MJ		 2811 int i;
2812 struct amdgpu_device *adev = NULL;
2813
91a1a52d
MJ		 2814 for (i = 0; i < mce_adev_list.num_gpu; i++) {
2815 adev = mce_adev_list.devs[i];
12b2cab7 2816
91a1a52d 2817 if (adev && adev->gmc.xgmi.connected_to_cpu &&
12b2cab7
MJ		 2818 adev->gmc.xgmi.physical_node_id == node_id)
2819 break;
2820 adev = NULL;
2821 }
2822
12b2cab7
MJ		 2823 return adev;
2824}
2825
2826#define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF)
2827#define GET_UMC_INST(m) (((m) >> 21) & 0x7)
2828#define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4))
2829#define GPU_ID_OFFSET 8
2830
2831static int amdgpu_bad_page_notifier(struct notifier_block *nb,
2832 unsigned long val, void *data)
2833{
2834 struct mce *m = (struct mce *)data;
2835 struct amdgpu_device *adev = NULL;
2836 uint32_t gpu_id = 0;
2837 uint32_t umc_inst = 0;
2838 uint32_t ch_inst, channel_index = 0;
2839 struct ras_err_data err_data = {0, 0, 0, NULL};
2840 struct eeprom_table_record err_rec;
2841 uint64_t retired_page;
2842
2843 /*
2844 * If the error was generated in UMC_V2, which belongs to GPU UMCs,
2845 * and error occurred in DramECC (Extended error code = 0) then only
2846 * process the error, else bail out.
2847 */
91f75eb4 2848 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) &&
12b2cab7
MJ		 2849 (XEC(m->status, 0x3f) == 0x0)))
2850 return NOTIFY_DONE;
2851
2852 /*
2853 * If it is correctable error, return.
2854 */
2855 if (mce_is_correctable(m))
2856 return NOTIFY_OK;
2857
2858 /*
2859 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
2860 */
2861 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET;
2862
2863 adev = find_adev(gpu_id);
2864 if (!adev) {
2865 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__,
2866 gpu_id);
2867 return NOTIFY_DONE;
2868 }
2869
2870 /*
2871 * If it is uncorrectable error, then find out UMC instance and
2872 * channel index.
2873 */
2874 umc_inst = GET_UMC_INST(m->ipid);
2875 ch_inst = GET_CHAN_INDEX(m->ipid);
2876
2877 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d",
2878 umc_inst, ch_inst);
2879
12b2cab7
MJ		 2880 /*
2881 * Translate UMC channel address to Physical address
2882 */
2883 channel_index =
2884 adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num
2885 + ch_inst];
2886
2887 retired_page = ADDR_OF_8KB_BLOCK(m->addr) |
2888 ADDR_OF_256B_BLOCK(channel_index) |
2889 OFFSET_IN_256B_BLOCK(m->addr);
2890
400013b2 2891 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
12b2cab7 2892 err_data.err_addr = &err_rec;
400013b2
TZ		 2893 amdgpu_umc_fill_error_record(&err_data, m->addr,
2894 retired_page, channel_index, umc_inst);
12b2cab7
MJ		 2895
2896 if (amdgpu_bad_page_threshold != 0) {
2897 amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
2898 err_data.err_addr_cnt);
2899 amdgpu_ras_save_bad_pages(adev);
2900 }
2901
2902 return NOTIFY_OK;
2903}
2904
2905static struct notifier_block amdgpu_bad_page_nb = {
2906 .notifier_call = amdgpu_bad_page_notifier,
2907 .priority = MCE_PRIO_UC,
2908};
2909
91a1a52d 2910static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev)
12b2cab7 2911{
91a1a52d
MJ		 2912 /*
2913 * Add the adev to the mce_adev_list.
2914 * During mode2 reset, amdgpu device is temporarily
2915 * removed from the mgpu_info list which can cause
2916 * page retirement to fail.
2917 * Use this list instead of mgpu_info to find the amdgpu
2918 * device on which the UMC error was reported.
2919 */
2920 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev;
2921
12b2cab7
MJ		 2922 /*
2923 * Register the x86 notifier only once
2924 * with MCE subsystem.
2925 */
2926 if (notifier_registered == false) {
2927 mce_register_decode_chain(&amdgpu_bad_page_nb);
2928 notifier_registered = true;
2929 }
2930}
2931#endif
7cab2124 2932
b6efdb02 2933struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev)
7cab2124 2934{
2935 if (!adev)
2936 return NULL;
2937
2938 return adev->psp.ras_context.ras;
2939}
2940
b6efdb02 2941int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con)
7cab2124 2942{
2943 if (!adev)
69f91d32 2944 return -EINVAL;
7cab2124 2945
2946 adev->psp.ras_context.ras = ras_con;
2947 return 0;
2948}
2949
2950/* check if ras is supported on block, say, sdma, gfx */
2951int amdgpu_ras_is_supported(struct amdgpu_device *adev,
2952 unsigned int block)
2953{
2954 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2955
2956 if (block >= AMDGPU_RAS_BLOCK_COUNT)
2957 return 0;
2958 return ras && (adev->ras_enabled & (1 << block));
2959}
2960
2961int amdgpu_ras_reset_gpu(struct amdgpu_device *adev)
2962{
2963 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2964
2965 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0)
25a2b22e 2966 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
7cab2124 2967 return 0;
2968}
2969
2970
6492e1b0 2971/* Register each ip ras block into amdgpu ras */
2972int amdgpu_ras_register_ras_block(struct amdgpu_device *adev,
b6efdb02 2973 struct amdgpu_ras_block_object *ras_block_obj)
6492e1b0 2974{
d5e8ff5f 2975 struct amdgpu_ras_block_list *ras_node;
6492e1b0 2976 if (!adev || !ras_block_obj)
2977 return -EINVAL;
2978
df01fe73 2979 if (!amdgpu_ras_asic_supported(adev))
2980 return 0;
2981
d5e8ff5f 2982 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL);
2983 if (!ras_node)
2984 return -ENOMEM;
2985
2986 INIT_LIST_HEAD(&ras_node->node);
2987 ras_node->ras_obj = ras_block_obj;
2988 list_add_tail(&ras_node->node, &adev->ras_list);
6492e1b0 2989
2990 return 0;
2991}
Linux 6.x block layer and io_uring tree(s)