projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'io_uring-6.16-20250630' of git://git.kernel.dk/linux
[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_ras.c
CommitLineData
c030f2e4 1/*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 *
23 */
24#include <linux/debugfs.h>
25#include <linux/list.h>
26#include <linux/module.h>
f867723b 27#include <linux/uaccess.h>
7c6e68c7
AG		 28#include <linux/reboot.h>
29#include <linux/syscalls.h>
05adfd80 30#include <linux/pm_runtime.h>
dbf3850d 31#include <linux/list_sort.h>
f867723b 32
c030f2e4 33#include "amdgpu.h"
34#include "amdgpu_ras.h"
b404ae82 35#include "amdgpu_atomfirmware.h"
19744f5f 36#include "amdgpu_xgmi.h"
4e644fff 37#include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
9af357bc 38#include "nbio_v4_3.h"
ecd1191e 39#include "nbif_v6_3_1.h"
7692e1ee 40#include "nbio_v7_9.h"
f50160cf 41#include "atom.h"
25a2b22e 42#include "amdgpu_reset.h"
4e2965bd 43#include "amdgpu_psp.h"
25a2b22e 44
12b2cab7
MJ		 45#ifdef CONFIG_X86_MCE_AMD
46#include <asm/mce.h>
c030f2e4 47
12b2cab7
MJ		 48static bool notifier_registered;
49#endif
eb0c3cd4
GC		 50static const char *RAS_FS_NAME = "ras";
51
c030f2e4 52const char *ras_error_string[] = {
53 "none",
54 "parity",
55 "single_correctable",
56 "multi_uncorrectable",
57 "poison",
58};
59
60const char *ras_block_string[] = {
61 "umc",
62 "sdma",
63 "gfx",
64 "mmhub",
65 "athub",
66 "pcie_bif",
67 "hdp",
68 "xgmi_wafl",
69 "df",
70 "smn",
71 "sem",
72 "mp0",
73 "mp1",
74 "fuse",
640ae42e 75 "mca",
a3d63c62
MZZ		 76 "vcn",
77 "jpeg",
7ed97155
YW		 78 "ih",
79 "mpio",
cc11dffc 80 "mmsch",
c030f2e4 81};
82
640ae42e
JC		 83const char *ras_mca_block_string[] = {
84 "mca_mp0",
85 "mca_mp1",
86 "mca_mpio",
87 "mca_iohc",
88};
89
d5e8ff5f 90struct amdgpu_ras_block_list {
91 /* ras block link */
92 struct list_head node;
93
94 struct amdgpu_ras_block_object *ras_obj;
95};
96
640ae42e
JC		 97const char *get_ras_block_str(struct ras_common_if *ras_block)
98{
99 if (!ras_block)
100 return "NULL";
101
7ed97155
YW		 102 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT ||
103 ras_block->block >= ARRAY_SIZE(ras_block_string))
640ae42e
JC		 104 return "OUT OF RANGE";
105
106 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA)
107 return ras_mca_block_string[ras_block->sub_block_index];
108
109 return ras_block_string[ras_block->block];
110}
111
954ea6aa 112#define ras_block_str(_BLOCK_) \
113 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range")
8b0fb0e9 114
c030f2e4 115#define ras_err_str(i) (ras_error_string[ffs(i)])
c030f2e4 116
108c6a63 117#define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
118
7cdc2ee3
TZ		 119/* inject address is 52 bits */
120#define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52)
121
e4e6a589
LT		 122/* typical ECC bad page rate is 1 bad page per 100MB VRAM */
123#define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL)
c84d4670 124
78146c1d 125#define MAX_UMC_POISON_POLLING_TIME_ASYNC 300 //ms
6c23f3d1 126
2cf8e50e
YC		 127#define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms
128
c0470691
YC		 129#define MAX_FLUSH_RETIRE_DWORK_TIMES 100
130
52dd95f2
GC		 131enum amdgpu_ras_retire_page_reservation {
132 AMDGPU_RAS_RETIRE_PAGE_RESERVED,
133 AMDGPU_RAS_RETIRE_PAGE_PENDING,
134 AMDGPU_RAS_RETIRE_PAGE_FAULT,
135};
7c6e68c7
AG		 136
137atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0);
138
676deb38
DL		 139static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
140 uint64_t addr);
6e4be987
TZ		 141static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
142 uint64_t addr);
12b2cab7 143#ifdef CONFIG_X86_MCE_AMD
91a1a52d
MJ		 144static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev);
145struct mce_notifier_adev_list {
146 struct amdgpu_device *devs[MAX_GPU_INSTANCE];
147 int num_gpu;
148};
149static struct mce_notifier_adev_list mce_adev_list;
12b2cab7 150#endif
6e4be987 151
61380faa
JC		 152void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
153{
a9d82d2f 154 if (adev && amdgpu_ras_get_context(adev))
61380faa
JC		 155 amdgpu_ras_get_context(adev)->error_query_ready = ready;
156}
157
f3167919 158static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev)
61380faa 159{
a9d82d2f 160 if (adev && amdgpu_ras_get_context(adev))
61380faa
JC		 161 return amdgpu_ras_get_context(adev)->error_query_ready;
162
163 return false;
164}
165
cbb8f989
JC		 166static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address)
167{
5b1270be 168 struct ras_err_data err_data;
cbb8f989 169 struct eeprom_table_record err_rec;
5b1270be 170 int ret;
cbb8f989
JC		 171
172 if ((address >= adev->gmc.mc_vram_size) ||
173 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
174 dev_warn(adev->dev,
175 "RAS WARN: input address 0x%llx is invalid.\n",
176 address);
177 return -EINVAL;
178 }
179
180 if (amdgpu_ras_check_bad_page(adev, address)) {
181 dev_warn(adev->dev,
80b0cd0f 182 "RAS WARN: 0x%llx has already been marked as bad page!\n",
cbb8f989
JC		 183 address);
184 return 0;
185 }
186
5b1270be
YW		 187 ret = amdgpu_ras_error_data_init(&err_data);
188 if (ret)
189 return ret;
190
cbb8f989 191 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
cbb8f989 192 err_data.err_addr = &err_rec;
71344a71 193 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0);
cbb8f989
JC		 194
195 if (amdgpu_bad_page_threshold != 0) {
196 amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
a8d133e6 197 err_data.err_addr_cnt, false);
4d33e0f1 198 amdgpu_ras_save_bad_pages(adev, NULL);
cbb8f989
JC		 199 }
200
5b1270be
YW		 201 amdgpu_ras_error_data_fini(&err_data);
202
cbb8f989
JC		 203 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n");
204 dev_warn(adev->dev, "Clear EEPROM:\n");
205 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n");
206
207 return 0;
208}
209
c030f2e4 210static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
211 size_t size, loff_t *pos)
212{
213 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
214 struct ras_query_if info = {
215 .head = obj->head,
216 };
217 ssize_t s;
218 char val[128];
219
761d86d3 220 if (amdgpu_ras_query_error_status(obj->adev, &info))
c030f2e4 221 return -EINVAL;
222
2a460963 223 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */
4e8303cf
LL		 224 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
225 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
2a460963
CL		 226 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
227 dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
228 }
229
c030f2e4 230 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n",
231 "ue", info.ue_count,
232 "ce", info.ce_count);
233 if (*pos >= s)
234 return 0;
235
236 s -= *pos;
237 s = min_t(u64, s, size);
238
239
240 if (copy_to_user(buf, &val[*pos], s))
241 return -EINVAL;
242
243 *pos += s;
244
245 return s;
246}
247
c030f2e4 248static const struct file_operations amdgpu_ras_debugfs_ops = {
249 .owner = THIS_MODULE,
250 .read = amdgpu_ras_debugfs_read,
190211ab 251 .write = NULL,
c030f2e4 252 .llseek = default_llseek
253};
254
96ebb307 255static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
256{
257 int i;
258
259 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
260 *block_id = i;
640ae42e 261 if (strcmp(name, ras_block_string[i]) == 0)
96ebb307 262 return 0;
263 }
264 return -EINVAL;
265}
266
267static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
268 const char __user *buf, size_t size,
269 loff_t *pos, struct ras_debug_if *data)
270{
271 ssize_t s = min_t(u64, 64, size);
272 char str[65];
273 char block_name[33];
274 char err[9] = "ue";
275 int op = -1;
276 int block_id;
44494f96 277 uint32_t sub_block;
96ebb307 278 u64 address, value;
2c22ed0b
TZ		 279 /* default value is 0 if the mask is not set by user */
280 u32 instance_mask = 0;
96ebb307 281
282 if (*pos)
283 return -EINVAL;
284 *pos = size;
285
286 memset(str, 0, sizeof(str));
287 memset(data, 0, sizeof(*data));
288
289 if (copy_from_user(str, buf, s))
290 return -EINVAL;
291
292 if (sscanf(str, "disable %32s", block_name) == 1)
293 op = 0;
294 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
295 op = 1;
296 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
297 op = 2;
6df23f4c 298 else if (strstr(str, "retire_page") != NULL)
cbb8f989 299 op = 3;
b076296b 300 else if (str[0] && str[1] && str[2] && str[3])
96ebb307 301 /* ascii string, but commands are not matched. */
302 return -EINVAL;
303
304 if (op != -1) {
cbb8f989 305 if (op == 3) {
546aa546
LT		 306 if (sscanf(str, "%*s 0x%llx", &address) != 1 &&
307 sscanf(str, "%*s %llu", &address) != 1)
6cb7a1d4 308 return -EINVAL;
cbb8f989
JC		 309
310 data->op = op;
311 data->inject.address = address;
312
313 return 0;
314 }
315
96ebb307 316 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id))
317 return -EINVAL;
318
319 data->head.block = block_id;
fb1e9171 320 /* only ue, ce and poison errors are supported */
e1063493
TZ		 321 if (!memcmp("ue", err, 2))
322 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
323 else if (!memcmp("ce", err, 2))
324 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
fb1e9171
CL		 325 else if (!memcmp("poison", err, 6))
326 data->head.type = AMDGPU_RAS_ERROR__POISON;
e1063493
TZ		 327 else
328 return -EINVAL;
329
96ebb307 330 data->op = op;
331
332 if (op == 2) {
2c22ed0b
TZ		 333 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x",
334 &sub_block, &address, &value, &instance_mask) != 4 &&
335 sscanf(str, "%*s %*s %*s %u %llu %llu %u",
336 &sub_block, &address, &value, &instance_mask) != 4 &&
337 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx",
546aa546
LT		 338 &sub_block, &address, &value) != 3 &&
339 sscanf(str, "%*s %*s %*s %u %llu %llu",
6cb7a1d4
LT		 340 &sub_block, &address, &value) != 3)
341 return -EINVAL;
44494f96 342 data->head.sub_block_index = sub_block;
96ebb307 343 data->inject.address = address;
344 data->inject.value = value;
2c22ed0b 345 data->inject.instance_mask = instance_mask;
96ebb307 346 }
347 } else {
73aa8e1a 348 if (size < sizeof(*data))
96ebb307 349 return -EINVAL;
350
351 if (copy_from_user(data, buf, sizeof(*data)))
352 return -EINVAL;
353 }
354
355 return 0;
356}
7c6e68c7 357
f464c5dd
TZ		 358static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev,
359 struct ras_debug_if *data)
360{
361 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
362 uint32_t mask, inst_mask = data->inject.instance_mask;
363
364 /* no need to set instance mask if there is only one instance */
365 if (num_xcc <= 1 && inst_mask) {
366 data->inject.instance_mask = 0;
367 dev_dbg(adev->dev,
368 "RAS inject mask(0x%x) isn't supported and force it to 0.\n",
369 inst_mask);
370
371 return;
372 }
373
374 switch (data->head.block) {
375 case AMDGPU_RAS_BLOCK__GFX:
376 mask = GENMASK(num_xcc - 1, 0);
377 break;
378 case AMDGPU_RAS_BLOCK__SDMA:
379 mask = GENMASK(adev->sdma.num_instances - 1, 0);
380 break;
e3959cb5
SY		 381 case AMDGPU_RAS_BLOCK__VCN:
382 case AMDGPU_RAS_BLOCK__JPEG:
383 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0);
384 break;
f464c5dd 385 default:
e3959cb5 386 mask = inst_mask;
f464c5dd
TZ		 387 break;
388 }
389
390 /* remove invalid bits in instance mask */
391 data->inject.instance_mask &= mask;
392 if (inst_mask != data->inject.instance_mask)
393 dev_dbg(adev->dev,
394 "Adjust RAS inject mask 0x%x to 0x%x\n",
395 inst_mask, data->inject.instance_mask);
396}
397
74abc221
TSD		 398/**
399 * DOC: AMDGPU RAS debugfs control interface
36ea1bd2 400 *
737c375b 401 * The control interface accepts struct ras_debug_if which has two members.
36ea1bd2 402 *
403 * First member: ras_debug_if::head or ras_debug_if::inject.
96ebb307 404 *
405 * head is used to indicate which IP block will be under control.
36ea1bd2 406 *
407 * head has four members, they are block, type, sub_block_index, name.
408 * block: which IP will be under control.
409 * type: what kind of error will be enabled/disabled/injected.
410 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
411 * name: the name of IP.
412 *
2c22ed0b 413 * inject has three more members than head, they are address, value and mask.
36ea1bd2 414 * As their names indicate, inject operation will write the
415 * value to the address.
416 *
ef177d11 417 * The second member: struct ras_debug_if::op.
c688a06b 418 * It has three kinds of operations.
879e723d
AZ		 419 *
420 * - 0: disable RAS on the block. Take ::head as its data.
421 * - 1: enable RAS on the block. Take ::head as its data.
422 * - 2: inject errors on the block. Take ::inject as its data.
36ea1bd2 423 *
96ebb307 424 * How to use the interface?
ef177d11 425 *
737c375b 426 * In a program
ef177d11 427 *
737c375b
LT		 428 * Copy the struct ras_debug_if in your code and initialize it.
429 * Write the struct to the control interface.
ef177d11 430 *
737c375b 431 * From shell
96ebb307 432 *
879e723d
AZ		 433 * .. code-block:: bash
434 *
737c375b
LT		 435 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
436 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
2c22ed0b 437 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
879e723d 438 *
737c375b 439 * Where N, is the card which you want to affect.
ef177d11 440 *
737c375b
LT		 441 * "disable" requires only the block.
442 * "enable" requires the block and error type.
443 * "inject" requires the block, error type, address, and value.
c666bbf0 444 *
737c375b 445 * The block is one of: umc, sdma, gfx, etc.
879e723d 446 * see ras_block_string[] for details
c666bbf0 447 *
fb1e9171 448 * The error type is one of: ue, ce and poison where,
737c375b
LT		 449 * ue is multi-uncorrectable
450 * ce is single-correctable
fb1e9171 451 * poison is poison
c666bbf0 452 *
737c375b
LT		 453 * The sub-block is a the sub-block index, pass 0 if there is no sub-block.
454 * The address and value are hexadecimal numbers, leading 0x is optional.
2c22ed0b 455 * The mask means instance mask, is optional, default value is 0x1.
879e723d 456 *
737c375b 457 * For instance,
879e723d
AZ		 458 *
459 * .. code-block:: bash
96ebb307 460 *
44494f96 461 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
2c22ed0b 462 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl
96ebb307 463 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
464 *
737c375b 465 * How to check the result of the operation?
36ea1bd2 466 *
737c375b 467 * To check disable/enable, see "ras" features at,
36ea1bd2 468 * /sys/class/drm/card[0/1/2...]/device/ras/features
469 *
737c375b
LT		 470 * To check inject, see the corresponding error count at,
471 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count
36ea1bd2 472 *
879e723d 473 * .. note::
ef177d11 474 * Operations are only allowed on blocks which are supported.
737c375b 475 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask
ef177d11
AD		 476 * to see which blocks support RAS on a particular asic.
477 *
36ea1bd2 478 */
cf696091
LT		 479static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f,
480 const char __user *buf,
481 size_t size, loff_t *pos)
36ea1bd2 482{
483 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
484 struct ras_debug_if data;
485 int ret = 0;
486
61380faa 487 if (!amdgpu_ras_get_error_query_ready(adev)) {
6952e99c
GC		 488 dev_warn(adev->dev, "RAS WARN: error injection "
489 "currently inaccessible\n");
43c4d576
JC		 490 return size;
491 }
492
96ebb307 493 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
494 if (ret)
cf696091 495 return ret;
36ea1bd2 496
80b0cd0f 497 if (data.op == 3) {
cbb8f989 498 ret = amdgpu_reserve_page_direct(adev, data.inject.address);
80b0cd0f 499 if (!ret)
cbb8f989
JC		 500 return size;
501 else
502 return ret;
503 }
504
36ea1bd2 505 if (!amdgpu_ras_is_supported(adev, data.head.block))
506 return -EINVAL;
507
508 switch (data.op) {
509 case 0:
510 ret = amdgpu_ras_feature_enable(adev, &data.head, 0);
511 break;
512 case 1:
513 ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
514 break;
515 case 2:
43aedbf4
SY		 516 if ((data.inject.address >= adev->gmc.mc_vram_size &&
517 adev->gmc.mc_vram_size) ||
7cdc2ee3 518 (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
b0d4783a
GC		 519 dev_warn(adev->dev, "RAS WARN: input address "
520 "0x%llx is invalid.",
521 data.inject.address);
7cdc2ee3
TZ		 522 ret = -EINVAL;
523 break;
524 }
525
6e4be987
TZ		 526 /* umc ce/ue error injection for a bad page is not allowed */
527 if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) &&
528 amdgpu_ras_check_bad_page(adev, data.inject.address)) {
c65b0805
LT		 529 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has "
530 "already been marked as bad!\n",
531 data.inject.address);
6e4be987
TZ		 532 break;
533 }
534
f464c5dd
TZ		 535 amdgpu_ras_instance_mask_check(adev, &data);
536
7cdc2ee3 537 /* data.inject.address is offset instead of absolute gpu address */
36ea1bd2 538 ret = amdgpu_ras_error_inject(adev, &data.inject);
539 break;
96ebb307 540 default:
541 ret = -EINVAL;
542 break;
374bf7bd 543 }
36ea1bd2 544
545 if (ret)
79c04621 546 return ret;
36ea1bd2 547
548 return size;
549}
550
084fe13b
AG		 551/**
552 * DOC: AMDGPU RAS debugfs EEPROM table reset interface
553 *
f77c7109 554 * Some boards contain an EEPROM which is used to persistently store a list of
ef177d11 555 * bad pages which experiences ECC errors in vram. This interface provides
f77c7109
AD		 556 * a way to reset the EEPROM, e.g., after testing error injection.
557 *
558 * Usage:
559 *
560 * .. code-block:: bash
561 *
562 * echo 1 > ../ras/ras_eeprom_reset
563 *
564 * will reset EEPROM table to 0 entries.
565 *
084fe13b 566 */
cf696091
LT		 567static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f,
568 const char __user *buf,
569 size_t size, loff_t *pos)
084fe13b 570{
bf0b91b7
GC		 571 struct amdgpu_device *adev =
572 (struct amdgpu_device *)file_inode(f)->i_private;
084fe13b
AG		 573 int ret;
574
bf0b91b7 575 ret = amdgpu_ras_eeprom_reset_table(
cf696091 576 &(amdgpu_ras_get_context(adev)->eeprom_control));
084fe13b 577
63d4c081 578 if (!ret) {
cf696091
LT		 579 /* Something was written to EEPROM.
580 */
bf0b91b7
GC		 581 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS;
582 return size;
583 } else {
cf696091 584 return ret;
bf0b91b7 585 }
084fe13b
AG		 586}
587
36ea1bd2 588static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
589 .owner = THIS_MODULE,
590 .read = NULL,
591 .write = amdgpu_ras_debugfs_ctrl_write,
592 .llseek = default_llseek
593};
594
084fe13b
AG		 595static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = {
596 .owner = THIS_MODULE,
597 .read = NULL,
598 .write = amdgpu_ras_debugfs_eeprom_write,
599 .llseek = default_llseek
600};
601
f77c7109
AD		 602/**
603 * DOC: AMDGPU RAS sysfs Error Count Interface
604 *
ef177d11 605 * It allows the user to read the error count for each IP block on the gpu through
f77c7109
AD		 606 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
607 *
608 * It outputs the multiple lines which report the uncorrected (ue) and corrected
609 * (ce) error counts.
610 *
611 * The format of one line is below,
612 *
613 * [ce|ue]: count
614 *
615 * Example:
616 *
617 * .. code-block:: bash
618 *
619 * ue: 0
620 * ce: 1
621 *
622 */
c030f2e4 623static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
624 struct device_attribute *attr, char *buf)
625{
626 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
627 struct ras_query_if info = {
628 .head = obj->head,
629 };
630
61380faa 631 if (!amdgpu_ras_get_error_query_ready(obj->adev))
36000c7a 632 return sysfs_emit(buf, "Query currently inaccessible\n");
43c4d576 633
761d86d3 634 if (amdgpu_ras_query_error_status(obj->adev, &info))
c030f2e4 635 return -EINVAL;
636
4e8303cf
LL		 637 if (amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
638 amdgpu_ip_version(obj->adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
1f0d8e37 639 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
2a460963 640 dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
1f0d8e37
MJ		 641 }
642
2c7a1560
SY		 643 if (info.head.block == AMDGPU_RAS_BLOCK__UMC)
644 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
645 "ce", info.ce_count, "de", info.de_count);
646 else
647 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count,
648 "ce", info.ce_count);
c030f2e4 649}
650
651/* obj begin */
652
653#define get_obj(obj) do { (obj)->use++; } while (0)
654#define alive_obj(obj) ((obj)->use)
655
656static inline void put_obj(struct ras_manager *obj)
657{
ec3e0a91 658 if (obj && (--obj->use == 0)) {
c030f2e4 659 list_del(&obj->node);
ec3e0a91
YW		 660 amdgpu_ras_error_data_fini(&obj->err_data);
661 }
662
f0872686 663 if (obj && (obj->use < 0))
640ae42e 664 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head));
c030f2e4 665}
666
667/* make one obj and return it. */
668static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
669 struct ras_common_if *head)
670{
671 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
672 struct ras_manager *obj;
673
8ab0d6f0 674 if (!adev->ras_enabled || !con)
c030f2e4 675 return NULL;
676
677 if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
678 return NULL;
679
640ae42e
JC		 680 if (head->block == AMDGPU_RAS_BLOCK__MCA) {
681 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
682 return NULL;
683
684 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
685 } else
686 obj = &con->objs[head->block];
687
c030f2e4 688 /* already exist. return obj? */
689 if (alive_obj(obj))
690 return NULL;
691
ec3e0a91
YW		 692 if (amdgpu_ras_error_data_init(&obj->err_data))
693 return NULL;
694
c030f2e4 695 obj->head = *head;
696 obj->adev = adev;
697 list_add(&obj->node, &con->head);
698 get_obj(obj);
699
700 return obj;
701}
702
703/* return an obj equal to head, or the first when head is NULL */
f2a79be1 704struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
c030f2e4 705 struct ras_common_if *head)
706{
707 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
708 struct ras_manager *obj;
709 int i;
710
8ab0d6f0 711 if (!adev->ras_enabled || !con)
c030f2e4 712 return NULL;
713
714 if (head) {
715 if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
716 return NULL;
717
640ae42e
JC		 718 if (head->block == AMDGPU_RAS_BLOCK__MCA) {
719 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
720 return NULL;
721
722 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
723 } else
724 obj = &con->objs[head->block];
c030f2e4 725
640ae42e 726 if (alive_obj(obj))
c030f2e4 727 return obj;
c030f2e4 728 } else {
640ae42e 729 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
c030f2e4 730 obj = &con->objs[i];
640ae42e 731 if (alive_obj(obj))
c030f2e4 732 return obj;
c030f2e4 733 }
734 }
735
736 return NULL;
737}
738/* obj end */
739
740/* feature ctl begin */
741static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
e509965e 742 struct ras_common_if *head)
c030f2e4 743{
8ab0d6f0 744 return adev->ras_hw_enabled & BIT(head->block);
c030f2e4 745}
746
747static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
748 struct ras_common_if *head)
749{
750 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
751
752 return con->features & BIT(head->block);
753}
754
755/*
756 * if obj is not created, then create one.
757 * set feature enable flag.
758 */
759static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
760 struct ras_common_if *head, int enable)
761{
762 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
763 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
764
5caf466a 765 /* If hardware does not support ras, then do not create obj.
766 * But if hardware support ras, we can create the obj.
767 * Ras framework checks con->hw_supported to see if it need do
768 * corresponding initialization.
769 * IP checks con->support to see if it need disable ras.
770 */
c030f2e4 771 if (!amdgpu_ras_is_feature_allowed(adev, head))
772 return 0;
c030f2e4 773
774 if (enable) {
775 if (!obj) {
776 obj = amdgpu_ras_create_obj(adev, head);
777 if (!obj)
778 return -EINVAL;
779 } else {
780 /* In case we create obj somewhere else */
781 get_obj(obj);
782 }
783 con->features |= BIT(head->block);
784 } else {
785 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
19d0dfda 786 con->features &= ~BIT(head->block);
c030f2e4 787 put_obj(obj);
788 }
789 }
790
791 return 0;
792}
793
794/* wrapper of psp_ras_enable_features */
795int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
796 struct ras_common_if *head, bool enable)
797{
798 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
7fcffecf 799 union ta_ras_cmd_input *info;
bf7aa8be 800 int ret;
c030f2e4 801
802 if (!con)
803 return -EINVAL;
804
ec70578c
HZ		 805 /* For non-gfx ip, do not enable ras feature if it is not allowed */
806 /* For gfx ip, regardless of feature support status, */
807 /* Force issue enable or disable ras feature commands */
808 if (head->block != AMDGPU_RAS_BLOCK__GFX &&
6fc9d92c 809 !amdgpu_ras_is_feature_allowed(adev, head))
bf7aa8be 810 return 0;
6fc9d92c
HZ		 811
812 /* Only enable gfx ras feature from host side */
813 if (head->block == AMDGPU_RAS_BLOCK__GFX &&
814 !amdgpu_sriov_vf(adev) &&
815 !amdgpu_ras_intr_triggered()) {
26093ce1
SY		 816 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL);
817 if (!info)
818 return -ENOMEM;
819
820 if (!enable) {
821 info->disable_features = (struct ta_ras_disable_features_input) {
822 .block_id = amdgpu_ras_block_to_ta(head->block),
823 .error_type = amdgpu_ras_error_to_ta(head->type),
824 };
825 } else {
826 info->enable_features = (struct ta_ras_enable_features_input) {
827 .block_id = amdgpu_ras_block_to_ta(head->block),
828 .error_type = amdgpu_ras_error_to_ta(head->type),
829 };
830 }
c030f2e4 831
7fcffecf 832 ret = psp_ras_enable_features(&adev->psp, info, enable);
bff77e86 833 if (ret) {
e4348849 834 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n",
011907fd 835 enable ? "enable":"disable",
640ae42e 836 get_ras_block_str(head),
e4348849 837 amdgpu_ras_is_poison_mode_supported(adev), ret);
5838f74c 838 kfree(info);
bf7aa8be 839 return ret;
bff77e86 840 }
bf7aa8be
HZ		 841
842 kfree(info);
c030f2e4 843 }
844
845 /* setup the obj */
846 __amdgpu_ras_feature_enable(adev, head, enable);
bf7aa8be
HZ		 847
848 return 0;
c030f2e4 849}
850
77de502b 851/* Only used in device probe stage and called only once. */
852int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
853 struct ras_common_if *head, bool enable)
854{
855 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
856 int ret;
857
858 if (!con)
859 return -EINVAL;
860
861 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
7af23ebe 862 if (enable) {
863 /* There is no harm to issue a ras TA cmd regardless of
864 * the currecnt ras state.
865 * If current state == target state, it will do nothing
866 * But sometimes it requests driver to reset and repost
867 * with error code -EAGAIN.
868 */
869 ret = amdgpu_ras_feature_enable(adev, head, 1);
870 /* With old ras TA, we might fail to enable ras.
871 * Log it and just setup the object.
872 * TODO need remove this WA in the future.
873 */
874 if (ret == -EINVAL) {
875 ret = __amdgpu_ras_feature_enable(adev, head, 1);
876 if (!ret)
6952e99c
GC		 877 dev_info(adev->dev,
878 "RAS INFO: %s setup object\n",
640ae42e 879 get_ras_block_str(head));
7af23ebe 880 }
881 } else {
882 /* setup the object then issue a ras TA disable cmd.*/
883 ret = __amdgpu_ras_feature_enable(adev, head, 1);
884 if (ret)
885 return ret;
77de502b 886
0110ac11 887 /* gfx block ras disable cmd must send to ras-ta */
970fd197
SY		 888 if (head->block == AMDGPU_RAS_BLOCK__GFX)
889 con->features |= BIT(head->block);
890
77de502b 891 ret = amdgpu_ras_feature_enable(adev, head, 0);
19d0dfda
SY		 892
893 /* clean gfx block ras features flag */
8ab0d6f0 894 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
19d0dfda 895 con->features &= ~BIT(head->block);
7af23ebe 896 }
77de502b 897 } else
898 ret = amdgpu_ras_feature_enable(adev, head, enable);
899
900 return ret;
901}
902
c030f2e4 903static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
904 bool bypass)
905{
906 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
907 struct ras_manager *obj, *tmp;
908
909 list_for_each_entry_safe(obj, tmp, &con->head, node) {
910 /* bypass psp.
911 * aka just release the obj and corresponding flags
912 */
913 if (bypass) {
914 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0))
915 break;
916 } else {
917 if (amdgpu_ras_feature_enable(adev, &obj->head, 0))
918 break;
919 }
289d513b 920 }
c030f2e4 921
922 return con->features;
923}
924
925static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
926 bool bypass)
927{
928 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
c030f2e4 929 int i;
640ae42e 930 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE;
c030f2e4 931
640ae42e 932 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
c030f2e4 933 struct ras_common_if head = {
934 .block = i,
191051a1 935 .type = default_ras_type,
c030f2e4 936 .sub_block_index = 0,
937 };
640ae42e
JC		 938
939 if (i == AMDGPU_RAS_BLOCK__MCA)
940 continue;
941
942 if (bypass) {
943 /*
944 * bypass psp. vbios enable ras for us.
945 * so just create the obj
946 */
947 if (__amdgpu_ras_feature_enable(adev, &head, 1))
948 break;
949 } else {
950 if (amdgpu_ras_feature_enable(adev, &head, 1))
951 break;
952 }
953 }
954
955 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
956 struct ras_common_if head = {
957 .block = AMDGPU_RAS_BLOCK__MCA,
958 .type = default_ras_type,
959 .sub_block_index = i,
960 };
961
c030f2e4 962 if (bypass) {
963 /*
964 * bypass psp. vbios enable ras for us.
965 * so just create the obj
966 */
967 if (__amdgpu_ras_feature_enable(adev, &head, 1))
968 break;
969 } else {
970 if (amdgpu_ras_feature_enable(adev, &head, 1))
971 break;
972 }
289d513b 973 }
c030f2e4 974
975 return con->features;
976}
977/* feature ctl end */
978
e3d833f4 979static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj,
980 enum amdgpu_ras_block block)
6492e1b0 981{
b6efdb02 982 if (!block_obj)
6492e1b0 983 return -EINVAL;
984
bdb3489c 985 if (block_obj->ras_comm.block == block)
6492e1b0 986 return 0;
640ae42e 987
6492e1b0 988 return -EINVAL;
989}
990
b6efdb02 991static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev,
6492e1b0 992 enum amdgpu_ras_block block, uint32_t sub_block_index)
640ae42e 993{
d5e8ff5f 994 struct amdgpu_ras_block_list *node, *tmp;
995 struct amdgpu_ras_block_object *obj;
6492e1b0 996
997 if (block >= AMDGPU_RAS_BLOCK__LAST)
998 return NULL;
999
d5e8ff5f 1000 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
1001 if (!node->ras_obj) {
1002 dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
1003 continue;
1004 }
1005
1006 obj = node->ras_obj;
6492e1b0 1007 if (obj->ras_block_match) {
1008 if (obj->ras_block_match(obj, block, sub_block_index) == 0)
1009 return obj;
1010 } else {
1011 if (amdgpu_ras_block_match_default(obj, block) == 0)
1012 return obj;
1013 }
640ae42e 1014 }
6492e1b0 1015
1016 return NULL;
640ae42e
JC		 1017}
1018
fdcb279d
SY		 1019static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data)
1020{
1021 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
1022 int ret = 0;
1023
1024 /*
1025 * choosing right query method according to
1026 * whether smu support query error information
1027 */
bc143d8b 1028 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc));
fdcb279d 1029 if (ret == -EOPNOTSUPP) {
efe17d5a 1030 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1031 adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
1032 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
fdcb279d
SY		 1033
1034 /* umc query_ras_error_address is also responsible for clearing
1035 * error status
1036 */
efe17d5a 1037 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1038 adev->umc.ras->ras_block.hw_ops->query_ras_error_address)
1039 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data);
fdcb279d 1040 } else if (!ret) {
efe17d5a 1041 if (adev->umc.ras &&
1042 adev->umc.ras->ecc_info_query_ras_error_count)
1043 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data);
fdcb279d 1044
efe17d5a 1045 if (adev->umc.ras &&
1046 adev->umc.ras->ecc_info_query_ras_error_address)
1047 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data);
fdcb279d
SY		 1048 }
1049}
1050
5b1270be 1051static void amdgpu_ras_error_print_error_data(struct amdgpu_device *adev,
ec3e0a91 1052 struct ras_manager *ras_mgr,
5b1270be 1053 struct ras_err_data *err_data,
9dc57c2a 1054 struct ras_query_context *qctx,
ec3e0a91 1055 const char *blk_name,
46e2231c
CL		 1056 bool is_ue,
1057 bool is_de)
5b1270be 1058{
5b1270be
YW		 1059 struct amdgpu_smuio_mcm_config_info *mcm_info;
1060 struct ras_err_node *err_node;
1061 struct ras_err_info *err_info;
75ac6a25 1062 u64 event_id = qctx->evid.event_id;
5b1270be 1063
ec3e0a91
YW		 1064 if (is_ue) {
1065 for_each_ras_error(err_node, err_data) {
1066 err_info = &err_node->err_info;
1067 mcm_info = &err_info->mcm_info;
1068 if (err_info->ue_count) {
9dc57c2a
YW		 1069 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1070 "%lld new uncorrectable hardware errors detected in %s block\n",
1071 mcm_info->socket_id,
1072 mcm_info->die_id,
1073 err_info->ue_count,
1074 blk_name);
ec3e0a91
YW		 1075 }
1076 }
1077
1078 for_each_ras_error(err_node, &ras_mgr->err_data) {
1079 err_info = &err_node->err_info;
1080 mcm_info = &err_info->mcm_info;
9dc57c2a
YW		 1081 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1082 "%lld uncorrectable hardware errors detected in total in %s block\n",
1083 mcm_info->socket_id, mcm_info->die_id, err_info->ue_count, blk_name);
ec3e0a91
YW		 1084 }
1085
1086 } else {
46e2231c
CL		 1087 if (is_de) {
1088 for_each_ras_error(err_node, err_data) {
1089 err_info = &err_node->err_info;
1090 mcm_info = &err_info->mcm_info;
1091 if (err_info->de_count) {
9dc57c2a
YW		 1092 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1093 "%lld new deferred hardware errors detected in %s block\n",
1094 mcm_info->socket_id,
1095 mcm_info->die_id,
1096 err_info->de_count,
1097 blk_name);
46e2231c
CL		 1098 }
1099 }
1100
1101 for_each_ras_error(err_node, &ras_mgr->err_data) {
1102 err_info = &err_node->err_info;
1103 mcm_info = &err_info->mcm_info;
9dc57c2a
YW		 1104 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1105 "%lld deferred hardware errors detected in total in %s block\n",
1106 mcm_info->socket_id, mcm_info->die_id,
1107 err_info->de_count, blk_name);
46e2231c
CL		 1108 }
1109 } else {
1110 for_each_ras_error(err_node, err_data) {
1111 err_info = &err_node->err_info;
1112 mcm_info = &err_info->mcm_info;
1113 if (err_info->ce_count) {
9dc57c2a
YW		 1114 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1115 "%lld new correctable hardware errors detected in %s block\n",
1116 mcm_info->socket_id,
1117 mcm_info->die_id,
1118 err_info->ce_count,
1119 blk_name);
46e2231c 1120 }
ec3e0a91 1121 }
ec3e0a91 1122
46e2231c
CL		 1123 for_each_ras_error(err_node, &ras_mgr->err_data) {
1124 err_info = &err_node->err_info;
1125 mcm_info = &err_info->mcm_info;
9dc57c2a
YW		 1126 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d, "
1127 "%lld correctable hardware errors detected in total in %s block\n",
1128 mcm_info->socket_id, mcm_info->die_id,
1129 err_info->ce_count, blk_name);
46e2231c 1130 }
5b1270be
YW		 1131 }
1132 }
1133}
1134
ec3e0a91
YW		 1135static inline bool err_data_has_source_info(struct ras_err_data *data)
1136{
1137 return !list_empty(&data->err_node_list);
1138}
1139
5b1270be
YW		 1140static void amdgpu_ras_error_generate_report(struct amdgpu_device *adev,
1141 struct ras_query_if *query_if,
9dc57c2a
YW		 1142 struct ras_err_data *err_data,
1143 struct ras_query_context *qctx)
5b1270be
YW		 1144{
1145 struct ras_manager *ras_mgr = amdgpu_ras_find_obj(adev, &query_if->head);
1146 const char *blk_name = get_ras_block_str(&query_if->head);
75ac6a25 1147 u64 event_id = qctx->evid.event_id;
5b1270be
YW		 1148
1149 if (err_data->ce_count) {
ec3e0a91 1150 if (err_data_has_source_info(err_data)) {
9dc57c2a 1151 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
46e2231c 1152 blk_name, false, false);
5b1270be
YW		 1153 } else if (!adev->aid_mask &&
1154 adev->smuio.funcs &&
1155 adev->smuio.funcs->get_socket_id &&
1156 adev->smuio.funcs->get_die_id) {
9dc57c2a
YW		 1157 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1158 "%ld correctable hardware errors "
1159 "detected in %s block\n",
1160 adev->smuio.funcs->get_socket_id(adev),
1161 adev->smuio.funcs->get_die_id(adev),
1162 ras_mgr->err_data.ce_count,
1163 blk_name);
5b1270be 1164 } else {
9dc57c2a
YW		 1165 RAS_EVENT_LOG(adev, event_id, "%ld correctable hardware errors "
1166 "detected in %s block\n",
1167 ras_mgr->err_data.ce_count,
1168 blk_name);
5b1270be
YW		 1169 }
1170 }
1171
1172 if (err_data->ue_count) {
ec3e0a91 1173 if (err_data_has_source_info(err_data)) {
9dc57c2a 1174 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
46e2231c 1175 blk_name, true, false);
5b1270be
YW		 1176 } else if (!adev->aid_mask &&
1177 adev->smuio.funcs &&
1178 adev->smuio.funcs->get_socket_id &&
1179 adev->smuio.funcs->get_die_id) {
9dc57c2a
YW		 1180 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1181 "%ld uncorrectable hardware errors "
1182 "detected in %s block\n",
1183 adev->smuio.funcs->get_socket_id(adev),
1184 adev->smuio.funcs->get_die_id(adev),
1185 ras_mgr->err_data.ue_count,
1186 blk_name);
5b1270be 1187 } else {
9dc57c2a
YW		 1188 RAS_EVENT_LOG(adev, event_id, "%ld uncorrectable hardware errors "
1189 "detected in %s block\n",
1190 ras_mgr->err_data.ue_count,
1191 blk_name);
5b1270be
YW		 1192 }
1193 }
1194
46e2231c
CL		 1195 if (err_data->de_count) {
1196 if (err_data_has_source_info(err_data)) {
9dc57c2a 1197 amdgpu_ras_error_print_error_data(adev, ras_mgr, err_data, qctx,
46e2231c
CL		 1198 blk_name, false, true);
1199 } else if (!adev->aid_mask &&
1200 adev->smuio.funcs &&
1201 adev->smuio.funcs->get_socket_id &&
1202 adev->smuio.funcs->get_die_id) {
9dc57c2a
YW		 1203 RAS_EVENT_LOG(adev, event_id, "socket: %d, die: %d "
1204 "%ld deferred hardware errors "
1205 "detected in %s block\n",
1206 adev->smuio.funcs->get_socket_id(adev),
1207 adev->smuio.funcs->get_die_id(adev),
1208 ras_mgr->err_data.de_count,
1209 blk_name);
46e2231c 1210 } else {
9dc57c2a
YW		 1211 RAS_EVENT_LOG(adev, event_id, "%ld deferred hardware errors "
1212 "detected in %s block\n",
1213 ras_mgr->err_data.de_count,
1214 blk_name);
46e2231c
CL		 1215 }
1216 }
5b1270be
YW		 1217}
1218
84a2947e
VS		 1219static void amdgpu_ras_virt_error_generate_report(struct amdgpu_device *adev,
1220 struct ras_query_if *query_if,
1221 struct ras_err_data *err_data,
1222 struct ras_query_context *qctx)
1223{
1224 unsigned long new_ue, new_ce, new_de;
1225 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &query_if->head);
1226 const char *blk_name = get_ras_block_str(&query_if->head);
1227 u64 event_id = qctx->evid.event_id;
1228
1229 new_ce = err_data->ce_count - obj->err_data.ce_count;
1230 new_ue = err_data->ue_count - obj->err_data.ue_count;
1231 new_de = err_data->de_count - obj->err_data.de_count;
1232
1233 if (new_ce) {
1234 RAS_EVENT_LOG(adev, event_id, "%lu correctable hardware errors "
1235 "detected in %s block\n",
1236 new_ce,
1237 blk_name);
1238 }
1239
1240 if (new_ue) {
1241 RAS_EVENT_LOG(adev, event_id, "%lu uncorrectable hardware errors "
1242 "detected in %s block\n",
1243 new_ue,
1244 blk_name);
1245 }
1246
1247 if (new_de) {
1248 RAS_EVENT_LOG(adev, event_id, "%lu deferred hardware errors "
1249 "detected in %s block\n",
1250 new_de,
1251 blk_name);
1252 }
1253}
1254
ec3e0a91
YW		 1255static void amdgpu_rasmgr_error_data_statistic_update(struct ras_manager *obj, struct ras_err_data *err_data)
1256{
1257 struct ras_err_node *err_node;
1258 struct ras_err_info *err_info;
1259
1260 if (err_data_has_source_info(err_data)) {
1261 for_each_ras_error(err_node, err_data) {
1262 err_info = &err_node->err_info;
46e2231c 1263 amdgpu_ras_error_statistic_de_count(&obj->err_data,
671af066 1264 &err_info->mcm_info, err_info->de_count);
9f91e983 1265 amdgpu_ras_error_statistic_ce_count(&obj->err_data,
671af066 1266 &err_info->mcm_info, err_info->ce_count);
9f91e983 1267 amdgpu_ras_error_statistic_ue_count(&obj->err_data,
671af066 1268 &err_info->mcm_info, err_info->ue_count);
ec3e0a91
YW		 1269 }
1270 } else {
1271 /* for legacy asic path which doesn't has error source info */
1272 obj->err_data.ue_count += err_data->ue_count;
1273 obj->err_data.ce_count += err_data->ce_count;
46e2231c 1274 obj->err_data.de_count += err_data->de_count;
ec3e0a91
YW		 1275 }
1276}
1277
84a2947e
VS		 1278static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager *obj,
1279 struct ras_err_data *err_data)
1280{
1281 /* Host reports absolute counts */
1282 obj->err_data.ue_count = err_data->ue_count;
1283 obj->err_data.ce_count = err_data->ce_count;
1284 obj->err_data.de_count = err_data->de_count;
1285}
1286
04c4fcd2
YW		 1287static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
1288{
1289 struct ras_common_if head;
1290
1291 memset(&head, 0, sizeof(head));
1292 head.block = blk;
1293
1294 return amdgpu_ras_find_obj(adev, &head);
1295}
1296
1297int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
1298 const struct aca_info *aca_info, void *data)
1299{
1300 struct ras_manager *obj;
1301
6f3b6913 1302 /* in resume phase, no need to create aca fs node */
e283f4fb 1303 if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
6f3b6913
YC		 1304 return 0;
1305
04c4fcd2
YW		 1306 obj = get_ras_manager(adev, blk);
1307 if (!obj)
1308 return -EINVAL;
1309
1310 return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data);
1311}
1312
1313int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk)
1314{
1315 struct ras_manager *obj;
1316
1317 obj = get_ras_manager(adev, blk);
1318 if (!obj)
1319 return -EINVAL;
1320
1321 amdgpu_aca_remove_handle(&obj->aca_handle);
1322
1323 return 0;
1324}
1325
1326static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk,
31fd330b
YW		 1327 enum aca_error_type type, struct ras_err_data *err_data,
1328 struct ras_query_context *qctx)
04c4fcd2
YW		 1329{
1330 struct ras_manager *obj;
1331
1332 obj = get_ras_manager(adev, blk);
1333 if (!obj)
1334 return -EINVAL;
1335
31fd330b 1336 return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx);
04c4fcd2
YW		 1337}
1338
37973b69
YW		 1339ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr,
1340 struct aca_handle *handle, char *buf, void *data)
1341{
1342 struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle);
1343 struct ras_query_if info = {
1344 .head = obj->head,
1345 };
1346
78347b65
YC		 1347 if (!amdgpu_ras_get_error_query_ready(obj->adev))
1348 return sysfs_emit(buf, "Query currently inaccessible\n");
1349
37973b69
YW		 1350 if (amdgpu_ras_query_error_status(obj->adev, &info))
1351 return -EINVAL;
1352
865d3397 1353 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count,
3c603b1f 1354 "ce", info.ce_count, "de", info.de_count);
37973b69
YW		 1355}
1356
8cc0f566
HZ		 1357static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev,
1358 struct ras_query_if *info,
1359 struct ras_err_data *err_data,
9dc57c2a 1360 struct ras_query_context *qctx,
8cc0f566 1361 unsigned int error_query_mode)
c030f2e4 1362{
8cc0f566 1363 enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT;
b6efdb02 1364 struct amdgpu_ras_block_object *block_obj = NULL;
04c4fcd2 1365 int ret;
8cc0f566 1366
b8d55a90
SS		 1367 if (blk == AMDGPU_RAS_BLOCK_COUNT)
1368 return -EINVAL;
1369
8cc0f566
HZ		 1370 if (error_query_mode == AMDGPU_RAS_INVALID_ERROR_QUERY)
1371 return -EINVAL;
1372
84a2947e
VS		 1373 if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
1374 return amdgpu_virt_req_ras_err_count(adev, blk, err_data);
1375 } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) {
8cc0f566
HZ		 1376 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) {
1377 amdgpu_ras_get_ecc_info(adev, err_data);
1378 } else {
1379 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0);
1380 if (!block_obj || !block_obj->hw_ops) {
1381 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1382 get_ras_block_str(&info->head));
1383 return -EINVAL;
1384 }
1385
1386 if (block_obj->hw_ops->query_ras_error_count)
07ee43fa 1387 block_obj->hw_ops->query_ras_error_count(adev, err_data);
8cc0f566
HZ		 1388
1389 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) ||
1390 (info->head.block == AMDGPU_RAS_BLOCK__GFX) ||
1391 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) {
1392 if (block_obj->hw_ops->query_ras_error_status)
1393 block_obj->hw_ops->query_ras_error_status(adev);
1394 }
1395 }
1396 } else {
04c4fcd2 1397 if (amdgpu_aca_is_enabled(adev)) {
31fd330b 1398 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx);
04c4fcd2
YW		 1399 if (ret)
1400 return ret;
1401
31fd330b 1402 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx);
04c4fcd2
YW		 1403 if (ret)
1404 return ret;
865d3397 1405
31fd330b 1406 ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx);
865d3397
YW		 1407 if (ret)
1408 return ret;
04c4fcd2
YW		 1409 } else {
1410 /* FIXME: add code to check return value later */
9dc57c2a
YW		 1411 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx);
1412 amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx);
04c4fcd2 1413 }
8cc0f566
HZ		 1414 }
1415
1416 return 0;
1417}
1418
1419/* query/inject/cure begin */
75ac6a25
YW		 1420static int amdgpu_ras_query_error_status_with_event(struct amdgpu_device *adev,
1421 struct ras_query_if *info,
1422 enum ras_event_type type)
8cc0f566 1423{
c030f2e4 1424 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
5b1270be 1425 struct ras_err_data err_data;
9dc57c2a 1426 struct ras_query_context qctx;
8cc0f566 1427 unsigned int error_query_mode;
5b1270be 1428 int ret;
c030f2e4 1429
1430 if (!obj)
1431 return -EINVAL;
c030f2e4 1432
5b1270be
YW		 1433 ret = amdgpu_ras_error_data_init(&err_data);
1434 if (ret)
1435 return ret;
1436
8cc0f566
HZ		 1437 if (!amdgpu_ras_get_error_query_mode(adev, &error_query_mode))
1438 return -EINVAL;
7389a5b8 1439
9dc57c2a 1440 memset(&qctx, 0, sizeof(qctx));
75ac6a25
YW		 1441 qctx.evid.type = type;
1442 qctx.evid.event_id = amdgpu_ras_acquire_event_id(adev, type);
f852c979
YC		 1443
1444 if (!down_read_trylock(&adev->reset_domain->sem)) {
1445 ret = -EIO;
1446 goto out_fini_err_data;
1447 }
1448
8cc0f566
HZ		 1449 ret = amdgpu_ras_query_error_status_helper(adev, info,
1450 &err_data,
9dc57c2a 1451 &qctx,
8cc0f566 1452 error_query_mode);
f852c979 1453 up_read(&adev->reset_domain->sem);
8cc0f566
HZ		 1454 if (ret)
1455 goto out_fini_err_data;
05a58345 1456
84a2947e
VS		 1457 if (error_query_mode != AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) {
1458 amdgpu_rasmgr_error_data_statistic_update(obj, &err_data);
1459 amdgpu_ras_error_generate_report(adev, info, &err_data, &qctx);
1460 } else {
1461 /* Host provides absolute error counts. First generate the report
1462 * using the previous VF internal count against new host count.
1463 * Then Update VF internal count.
1464 */
1465 amdgpu_ras_virt_error_generate_report(adev, info, &err_data, &qctx);
1466 amdgpu_ras_mgr_virt_error_data_statistics_update(obj, &err_data);
1467 }
05a58345 1468
c030f2e4 1469 info->ue_count = obj->err_data.ue_count;
1470 info->ce_count = obj->err_data.ce_count;
46e2231c 1471 info->de_count = obj->err_data.de_count;
c030f2e4 1472
5b1270be
YW		 1473out_fini_err_data:
1474 amdgpu_ras_error_data_fini(&err_data);
1475
1476 return ret;
c030f2e4 1477}
1478
75ac6a25
YW		 1479int amdgpu_ras_query_error_status(struct amdgpu_device *adev, struct ras_query_if *info)
1480{
1481 return amdgpu_ras_query_error_status_with_event(adev, info, RAS_EVENT_TYPE_INVALID);
1482}
1483
472c5fb2 1484int amdgpu_ras_reset_error_count(struct amdgpu_device *adev,
761d86d3
DL		 1485 enum amdgpu_ras_block block)
1486{
b6efdb02 1487 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
73582be1 1488 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
04c4fcd2 1489 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
8b0fb0e9 1490
a83f2bf1 1491 if (!block_obj || !block_obj->hw_ops) {
afa37315 1492 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
472c5fb2
TZ		 1493 ras_block_str(block));
1494 return -EOPNOTSUPP;
761d86d3
DL		 1495 }
1496
d1d4c0b7 1497 if (!amdgpu_ras_is_supported(adev, block) ||
04c4fcd2 1498 !amdgpu_ras_get_aca_debug_mode(adev))
d1d4c0b7
TZ		 1499 return -EOPNOTSUPP;
1500
5045c6c6
EP		 1501 if (amdgpu_sriov_vf(adev))
1502 return -EOPNOTSUPP;
1503
73582be1 1504 /* skip ras error reset in gpu reset */
7e437167 1505 if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) &&
04c4fcd2
YW		 1506 ((smu_funcs && smu_funcs->set_debug_mode) ||
1507 (mca_funcs && mca_funcs->mca_set_debug_mode)))
73582be1
TZ		 1508 return -EOPNOTSUPP;
1509
7389a5b8 1510 if (block_obj->hw_ops->reset_ras_error_count)
1511 block_obj->hw_ops->reset_ras_error_count(adev);
5c23e9e0 1512
472c5fb2
TZ		 1513 return 0;
1514}
1515
1516int amdgpu_ras_reset_error_status(struct amdgpu_device *adev,
1517 enum amdgpu_ras_block block)
1518{
1519 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
1520
1521 if (amdgpu_ras_reset_error_count(adev, block) == -EOPNOTSUPP)
1522 return 0;
1523
7389a5b8 1524 if ((block == AMDGPU_RAS_BLOCK__GFX) ||
1525 (block == AMDGPU_RAS_BLOCK__MMHUB)) {
8b0fb0e9 1526 if (block_obj->hw_ops->reset_ras_error_status)
1527 block_obj->hw_ops->reset_ras_error_status(adev);
761d86d3 1528 }
5c23e9e0 1529
761d86d3 1530 return 0;
5c23e9e0
JC		 1531}
1532
c030f2e4 1533/* wrapper of psp_ras_trigger_error */
1534int amdgpu_ras_error_inject(struct amdgpu_device *adev,
1535 struct ras_inject_if *info)
1536{
1537 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1538 struct ta_ras_trigger_error_input block_info = {
828cfa29 1539 .block_id = amdgpu_ras_block_to_ta(info->head.block),
1540 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type),
c030f2e4 1541 .sub_block_index = info->head.sub_block_index,
1542 .address = info->address,
1543 .value = info->value,
1544 };
ab3b9de6
YL		 1545 int ret = -EINVAL;
1546 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev,
1547 info->head.block,
1548 info->head.sub_block_index);
c030f2e4 1549
248c9635
TZ		 1550 /* inject on guest isn't allowed, return success directly */
1551 if (amdgpu_sriov_vf(adev))
1552 return 0;
1553
c030f2e4 1554 if (!obj)
1555 return -EINVAL;
1556
22d4ba53 1557 if (!block_obj || !block_obj->hw_ops) {
afa37315
LT		 1558 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1559 get_ras_block_str(&info->head));
22d4ba53 1560 return -EINVAL;
1561 }
1562
a6c44d25 1563 /* Calculate XGMI relative offset */
a80fe1a6
TZ		 1564 if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1565 info->head.block != AMDGPU_RAS_BLOCK__GFX) {
19744f5f
HZ		 1566 block_info.address =
1567 amdgpu_xgmi_get_relative_phy_addr(adev,
1568 block_info.address);
a6c44d25
JC		 1569 }
1570
27c5f295
TZ		 1571 if (block_obj->hw_ops->ras_error_inject) {
1572 if (info->head.block == AMDGPU_RAS_BLOCK__GFX)
2c22ed0b 1573 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask);
27c5f295 1574 else /* Special ras_error_inject is defined (e.g: xgmi) */
2c22ed0b
TZ		 1575 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info,
1576 info->instance_mask);
27c5f295
TZ		 1577 } else {
1578 /* default path */
1579 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask);
a5dd40ca
HZ		 1580 }
1581
011907fd
DL		 1582 if (ret)
1583 dev_err(adev->dev, "ras inject %s failed %d\n",
640ae42e 1584 get_ras_block_str(&info->head), ret);
c030f2e4 1585
1586 return ret;
1587}
1588
4d9f771e 1589/**
4a1c9a44
HZ		 1590 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP
1591 * @adev: pointer to AMD GPU device
1592 * @ce_count: pointer to an integer to be set to the count of correctible errors.
1593 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors.
1594 * @query_info: pointer to ras_query_if
1595 *
1596 * Return 0 for query success or do nothing, otherwise return an error
1597 * on failures
1598 */
1599static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev,
1600 unsigned long *ce_count,
1601 unsigned long *ue_count,
1602 struct ras_query_if *query_info)
1603{
1604 int ret;
1605
1606 if (!query_info)
1607 /* do nothing if query_info is not specified */
1608 return 0;
1609
1610 ret = amdgpu_ras_query_error_status(adev, query_info);
1611 if (ret)
1612 return ret;
1613
1614 *ce_count += query_info->ce_count;
1615 *ue_count += query_info->ue_count;
1616
1617 /* some hardware/IP supports read to clear
1618 * no need to explictly reset the err status after the query call */
4e8303cf
LL		 1619 if (amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 2) &&
1620 amdgpu_ip_version(adev, MP0_HWIP, 0) != IP_VERSION(11, 0, 4)) {
4a1c9a44
HZ		 1621 if (amdgpu_ras_reset_error_status(adev, query_info->head.block))
1622 dev_warn(adev->dev,
1623 "Failed to reset error counter and error status\n");
1624 }
1625
1626 return 0;
1627}
1628
1629/**
1630 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP
bbe04dec
IB		 1631 * @adev: pointer to AMD GPU device
1632 * @ce_count: pointer to an integer to be set to the count of correctible errors.
1633 * @ue_count: pointer to an integer to be set to the count of uncorrectible
4d9f771e 1634 * errors.
4a1c9a44
HZ		 1635 * @query_info: pointer to ras_query_if if the query request is only for
1636 * specific ip block; if info is NULL, then the qurey request is for
1637 * all the ip blocks that support query ras error counters/status
4d9f771e
LT		 1638 *
1639 * If set, @ce_count or @ue_count, count and return the corresponding
1640 * error counts in those integer pointers. Return 0 if the device
1641 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS.
1642 */
1643int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
1644 unsigned long *ce_count,
4a1c9a44
HZ		 1645 unsigned long *ue_count,
1646 struct ras_query_if *query_info)
c030f2e4 1647{
1648 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1649 struct ras_manager *obj;
a46751fb 1650 unsigned long ce, ue;
4a1c9a44 1651 int ret;
c030f2e4 1652
8ab0d6f0 1653 if (!adev->ras_enabled || !con)
4d9f771e
LT		 1654 return -EOPNOTSUPP;
1655
1656 /* Don't count since no reporting.
1657 */
1658 if (!ce_count && !ue_count)
1659 return 0;
c030f2e4 1660
a46751fb
LT		 1661 ce = 0;
1662 ue = 0;
4a1c9a44
HZ		 1663 if (!query_info) {
1664 /* query all the ip blocks that support ras query interface */
1665 list_for_each_entry(obj, &con->head, node) {
1666 struct ras_query_if info = {
1667 .head = obj->head,
1668 };
c030f2e4 1669
4a1c9a44 1670 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info);
2a460963 1671 }
4a1c9a44
HZ		 1672 } else {
1673 /* query specific ip block */
1674 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info);
c030f2e4 1675 }
1676
4a1c9a44
HZ		 1677 if (ret)
1678 return ret;
1679
a46751fb
LT		 1680 if (ce_count)
1681 *ce_count = ce;
1682
1683 if (ue_count)
1684 *ue_count = ue;
4d9f771e
LT		 1685
1686 return 0;
c030f2e4 1687}
1688/* query/inject/cure end */
1689
1690
1691/* sysfs begin */
1692
466b1793 1693static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1694 struct ras_badpage **bps, unsigned int *count);
1695
1696static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
1697{
1698 switch (flags) {
52dd95f2 1699 case AMDGPU_RAS_RETIRE_PAGE_RESERVED:
466b1793 1700 return "R";
52dd95f2 1701 case AMDGPU_RAS_RETIRE_PAGE_PENDING:
466b1793 1702 return "P";
52dd95f2 1703 case AMDGPU_RAS_RETIRE_PAGE_FAULT:
466b1793 1704 default:
1705 return "F";
aec576f9 1706 }
466b1793 1707}
1708
f77c7109
AD		 1709/**
1710 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface
466b1793 1711 *
1712 * It allows user to read the bad pages of vram on the gpu through
1713 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
1714 *
1715 * It outputs multiple lines, and each line stands for one gpu page.
1716 *
1717 * The format of one line is below,
1718 * gpu pfn : gpu page size : flags
1719 *
1720 * gpu pfn and gpu page size are printed in hex format.
1721 * flags can be one of below character,
f77c7109 1722 *
466b1793 1723 * R: reserved, this gpu page is reserved and not able to use.
f77c7109 1724 *
466b1793 1725 * P: pending for reserve, this gpu page is marked as bad, will be reserved
f77c7109
AD		 1726 * in next window of page_reserve.
1727 *
466b1793 1728 * F: unable to reserve. this gpu page can't be reserved due to some reasons.
1729 *
f77c7109
AD		 1730 * Examples:
1731 *
1732 * .. code-block:: bash
1733 *
1734 * 0x00000001 : 0x00001000 : R
1735 * 0x00000002 : 0x00001000 : P
1736 *
466b1793 1737 */
1738
1739static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
2d0f5001 1740 struct kobject *kobj, const struct bin_attribute *attr,
466b1793 1741 char *buf, loff_t ppos, size_t count)
1742{
1743 struct amdgpu_ras *con =
1744 container_of(attr, struct amdgpu_ras, badpages_attr);
1745 struct amdgpu_device *adev = con->adev;
1746 const unsigned int element_size =
1747 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
d6ee400e
SA		 1748 unsigned int start = div64_ul(ppos + element_size - 1, element_size);
1749 unsigned int end = div64_ul(ppos + count - 1, element_size);
466b1793 1750 ssize_t s = 0;
1751 struct ras_badpage *bps = NULL;
1752 unsigned int bps_count = 0;
1753
1754 memset(buf, 0, count);
1755
1756 if (amdgpu_ras_badpages_read(adev, &bps, &bps_count))
1757 return 0;
1758
1759 for (; start < end && start < bps_count; start++)
1760 s += scnprintf(&buf[s], element_size + 1,
1761 "0x%08x : 0x%08x : %1s\n",
1762 bps[start].bp,
1763 bps[start].size,
1764 amdgpu_ras_badpage_flags_str(bps[start].flags));
1765
1766 kfree(bps);
1767
1768 return s;
1769}
1770
c030f2e4 1771static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
1772 struct device_attribute *attr, char *buf)
1773{
1774 struct amdgpu_ras *con =
1775 container_of(attr, struct amdgpu_ras, features_attr);
c030f2e4 1776
2cffcb66 1777 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features);
c030f2e4 1778}
1779
625e5f38
AK		 1780static ssize_t amdgpu_ras_sysfs_version_show(struct device *dev,
1781 struct device_attribute *attr, char *buf)
1782{
1783 struct amdgpu_ras *con =
1784 container_of(attr, struct amdgpu_ras, version_attr);
1785 return sysfs_emit(buf, "table version: 0x%x\n", con->eeprom_control.tbl_hdr.version);
1786}
1787
1788static ssize_t amdgpu_ras_sysfs_schema_show(struct device *dev,
1789 struct device_attribute *attr, char *buf)
1790{
1791 struct amdgpu_ras *con =
1792 container_of(attr, struct amdgpu_ras, schema_attr);
1793 return sysfs_emit(buf, "schema: 0x%x\n", con->schema);
1794}
1795
59f488be
YW		 1796static struct {
1797 enum ras_event_type type;
1798 const char *name;
1799} dump_event[] = {
1800 {RAS_EVENT_TYPE_FATAL, "Fatal Error"},
1801 {RAS_EVENT_TYPE_POISON_CREATION, "Poison Creation"},
1802 {RAS_EVENT_TYPE_POISON_CONSUMPTION, "Poison Consumption"},
1803};
1804
1805static ssize_t amdgpu_ras_sysfs_event_state_show(struct device *dev,
1806 struct device_attribute *attr, char *buf)
1807{
1808 struct amdgpu_ras *con =
1809 container_of(attr, struct amdgpu_ras, event_state_attr);
1810 struct ras_event_manager *event_mgr = con->event_mgr;
1811 struct ras_event_state *event_state;
1812 int i, size = 0;
1813
1814 if (!event_mgr)
1815 return -EINVAL;
1816
1817 size += sysfs_emit_at(buf, size, "current seqno: %llu\n", atomic64_read(&event_mgr->seqno));
1818 for (i = 0; i < ARRAY_SIZE(dump_event); i++) {
1819 event_state = &event_mgr->event_state[dump_event[i].type];
1820 size += sysfs_emit_at(buf, size, "%s: count:%llu, last_seqno:%llu\n",
1821 dump_event[i].name,
1822 atomic64_read(&event_state->count),
1823 event_state->last_seqno);
1824 }
1825
1826 return (ssize_t)size;
1827}
1828
f848159b
GC		 1829static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev)
1830{
1831 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1832
4638e0c2
VP		 1833 if (adev->dev->kobj.sd)
1834 sysfs_remove_file_from_group(&adev->dev->kobj,
f848159b
GC		 1835 &con->badpages_attr.attr,
1836 RAS_FS_NAME);
1837}
1838
625e5f38 1839static int amdgpu_ras_sysfs_remove_dev_attr_node(struct amdgpu_device *adev)
c030f2e4 1840{
1841 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1842 struct attribute *attrs[] = {
1843 &con->features_attr.attr,
625e5f38
AK		 1844 &con->version_attr.attr,
1845 &con->schema_attr.attr,
59f488be 1846 &con->event_state_attr.attr,
c030f2e4 1847 NULL
1848 };
1849 struct attribute_group group = {
eb0c3cd4 1850 .name = RAS_FS_NAME,
c030f2e4 1851 .attrs = attrs,
1852 };
1853
4638e0c2
VP		 1854 if (adev->dev->kobj.sd)
1855 sysfs_remove_group(&adev->dev->kobj, &group);
c030f2e4 1856
1857 return 0;
1858}
1859
1860int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
9252d33d 1861 struct ras_common_if *head)
c030f2e4 1862{
9252d33d 1863 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
c030f2e4 1864
9262f411
YW		 1865 if (amdgpu_aca_is_enabled(adev))
1866 return 0;
1867
c030f2e4 1868 if (!obj || obj->attr_inuse)
1869 return -EINVAL;
1870
04893397
VS		 1871 if (amdgpu_sriov_vf(adev) && !amdgpu_virt_ras_telemetry_block_en(adev, head->block))
1872 return 0;
1873
c030f2e4 1874 get_obj(obj);
1875
9252d33d 1876 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name),
1877 "%s_err_count", head->name);
c030f2e4 1878
1879 obj->sysfs_attr = (struct device_attribute){
1880 .attr = {
1881 .name = obj->fs_data.sysfs_name,
1882 .mode = S_IRUGO,
1883 },
1884 .show = amdgpu_ras_sysfs_read,
1885 };
163def43 1886 sysfs_attr_init(&obj->sysfs_attr.attr);
c030f2e4 1887
1888 if (sysfs_add_file_to_group(&adev->dev->kobj,
1889 &obj->sysfs_attr.attr,
eb0c3cd4 1890 RAS_FS_NAME)) {
c030f2e4 1891 put_obj(obj);
1892 return -EINVAL;
1893 }
1894
1895 obj->attr_inuse = 1;
1896
1897 return 0;
1898}
1899
1900int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
1901 struct ras_common_if *head)
1902{
1903 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1904
9262f411
YW		 1905 if (amdgpu_aca_is_enabled(adev))
1906 return 0;
1907
c030f2e4 1908 if (!obj || !obj->attr_inuse)
1909 return -EINVAL;
1910
4638e0c2
VP		 1911 if (adev->dev->kobj.sd)
1912 sysfs_remove_file_from_group(&adev->dev->kobj,
c030f2e4 1913 &obj->sysfs_attr.attr,
eb0c3cd4 1914 RAS_FS_NAME);
c030f2e4 1915 obj->attr_inuse = 0;
1916 put_obj(obj);
1917
1918 return 0;
1919}
1920
1921static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
1922{
1923 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1924 struct ras_manager *obj, *tmp;
1925
1926 list_for_each_entry_safe(obj, tmp, &con->head, node) {
1927 amdgpu_ras_sysfs_remove(adev, &obj->head);
1928 }
1929
f848159b
GC		 1930 if (amdgpu_bad_page_threshold != 0)
1931 amdgpu_ras_sysfs_remove_bad_page_node(adev);
1932
625e5f38 1933 amdgpu_ras_sysfs_remove_dev_attr_node(adev);
c030f2e4 1934
1935 return 0;
1936}
1937/* sysfs end */
1938
ef177d11
AD		 1939/**
1940 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors
1941 *
1942 * Normally when there is an uncorrectable error, the driver will reset
1943 * the GPU to recover. However, in the event of an unrecoverable error,
1944 * the driver provides an interface to reboot the system automatically
1945 * in that event.
1946 *
1947 * The following file in debugfs provides that interface:
1948 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot
1949 *
1950 * Usage:
1951 *
1952 * .. code-block:: bash
1953 *
1954 * echo true > .../ras/auto_reboot
1955 *
1956 */
c030f2e4 1957/* debugfs begin */
ea1b8c9b 1958static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
36ea1bd2 1959{
1960 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
740f42a2 1961 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control;
ef0d7d20
LT		 1962 struct drm_minor *minor = adev_to_drm(adev)->primary;
1963 struct dentry *dir;
36ea1bd2 1964
88293c03
ND		 1965 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);
1966 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev,
1967 &amdgpu_ras_debugfs_ctrl_ops);
1968 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev,
1969 &amdgpu_ras_debugfs_eeprom_ops);
7fb64071
LT		 1970 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,
1971 &con->bad_page_cnt_threshold);
740f42a2 1972 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs);
ef0d7d20
LT		 1973 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled);
1974 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled);
c65b0805
LT		 1975 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev,
1976 &amdgpu_ras_debugfs_eeprom_size_ops);
1977 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table",
1978 S_IRUGO, dir, adev,
1979 &amdgpu_ras_debugfs_eeprom_table_ops);
1980 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control);
c688a06b
GC		 1981
1982 /*
1983 * After one uncorrectable error happens, usually GPU recovery will
1984 * be scheduled. But due to the known problem in GPU recovery failing
1985 * to bring GPU back, below interface provides one direct way to
1986 * user to reboot system automatically in such case within
1987 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine
1988 * will never be called.
1989 */
88293c03 1990 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot);
66459e1d
GC		 1991
1992 /*
1993 * User could set this not to clean up hardware's error count register
1994 * of RAS IPs during ras recovery.
1995 */
88293c03
ND		 1996 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir,
1997 &con->disable_ras_err_cnt_harvest);
1998 return dir;
36ea1bd2 1999}
2000
cedf7884 2001static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
88293c03
ND		 2002 struct ras_fs_if *head,
2003 struct dentry *dir)
c030f2e4 2004{
c030f2e4 2005 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
c030f2e4 2006
88293c03 2007 if (!obj || !dir)
450f30ea 2008 return;
c030f2e4 2009
2010 get_obj(obj);
2011
2012 memcpy(obj->fs_data.debugfs_name,
2013 head->debugfs_name,
2014 sizeof(obj->fs_data.debugfs_name));
2015
88293c03
ND		 2016 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir,
2017 obj, &amdgpu_ras_debugfs_ops);
c030f2e4 2018}
2019
9817f061
YW		 2020static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev)
2021{
2022 bool ret;
2023
2024 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
2025 case IP_VERSION(13, 0, 6):
9a826c4a 2026 case IP_VERSION(13, 0, 12):
9817f061
YW		 2027 case IP_VERSION(13, 0, 14):
2028 ret = true;
2029 break;
2030 default:
2031 ret = false;
2032 break;
2033 }
2034
2035 return ret;
2036}
2037
f9317014
TZ		 2038void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
2039{
2040 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
88293c03 2041 struct dentry *dir;
c1509f3f 2042 struct ras_manager *obj;
f9317014
TZ		 2043 struct ras_fs_if fs_info;
2044
2045 /*
2046 * it won't be called in resume path, no need to check
2047 * suspend and gpu reset status
2048 */
cedf7884 2049 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con)
f9317014
TZ		 2050 return;
2051
88293c03 2052 dir = amdgpu_ras_debugfs_create_ctrl_node(adev);
f9317014 2053
c1509f3f 2054 list_for_each_entry(obj, &con->head, node) {
f9317014
TZ		 2055 if (amdgpu_ras_is_supported(adev, obj->head.block) &&
2056 (obj->attr_inuse == 1)) {
2057 sprintf(fs_info.debugfs_name, "%s_err_inject",
640ae42e 2058 get_ras_block_str(&obj->head));
f9317014 2059 fs_info.head = obj->head;
88293c03 2060 amdgpu_ras_debugfs_create(adev, &fs_info, dir);
f9317014
TZ		 2061 }
2062 }
4051844c 2063
9817f061
YW		 2064 if (amdgpu_ras_aca_is_supported(adev)) {
2065 if (amdgpu_aca_is_enabled(adev))
2066 amdgpu_aca_smu_debugfs_init(adev, dir);
2067 else
2068 amdgpu_mca_smu_debugfs_init(adev, dir);
2069 }
f9317014
TZ		 2070}
2071
c030f2e4 2072/* debugfs end */
2073
2074/* ras fs */
2d0f5001
TW		 2075static const BIN_ATTR(gpu_vram_bad_pages, S_IRUGO,
2076 amdgpu_ras_sysfs_badpages_read, NULL, 0);
c3d4d45d
GC		 2077static DEVICE_ATTR(features, S_IRUGO,
2078 amdgpu_ras_sysfs_features_read, NULL);
625e5f38
AK		 2079static DEVICE_ATTR(version, 0444,
2080 amdgpu_ras_sysfs_version_show, NULL);
2081static DEVICE_ATTR(schema, 0444,
2082 amdgpu_ras_sysfs_schema_show, NULL);
59f488be
YW		 2083static DEVICE_ATTR(event_state, 0444,
2084 amdgpu_ras_sysfs_event_state_show, NULL);
c030f2e4 2085static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
2086{
c3d4d45d
GC		 2087 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2088 struct attribute_group group = {
2089 .name = RAS_FS_NAME,
2090 };
2091 struct attribute *attrs[] = {
2092 &con->features_attr.attr,
625e5f38
AK		 2093 &con->version_attr.attr,
2094 &con->schema_attr.attr,
59f488be 2095 &con->event_state_attr.attr,
c3d4d45d
GC		 2096 NULL
2097 };
2d0f5001 2098 const struct bin_attribute *bin_attrs[] = {
c3d4d45d
GC		 2099 NULL,
2100 NULL,
2101 };
a069a9eb 2102 int r;
c030f2e4 2103
625e5f38
AK		 2104 group.attrs = attrs;
2105
c3d4d45d
GC		 2106 /* add features entry */
2107 con->features_attr = dev_attr_features;
c3d4d45d
GC		 2108 sysfs_attr_init(attrs[0]);
2109
625e5f38
AK		 2110 /* add version entry */
2111 con->version_attr = dev_attr_version;
2112 sysfs_attr_init(attrs[1]);
2113
2114 /* add schema entry */
2115 con->schema_attr = dev_attr_schema;
2116 sysfs_attr_init(attrs[2]);
2117
59f488be
YW		 2118 /* add event_state entry */
2119 con->event_state_attr = dev_attr_event_state;
2120 sysfs_attr_init(attrs[3]);
2121
c3d4d45d
GC		 2122 if (amdgpu_bad_page_threshold != 0) {
2123 /* add bad_page_features entry */
c3d4d45d 2124 con->badpages_attr = bin_attr_gpu_vram_bad_pages;
2d0f5001 2125 sysfs_bin_attr_init(&con->badpages_attr);
c3d4d45d 2126 bin_attrs[0] = &con->badpages_attr;
2d0f5001 2127 group.bin_attrs_new = bin_attrs;
c3d4d45d
GC		 2128 }
2129
a069a9eb
AD		 2130 r = sysfs_create_group(&adev->dev->kobj, &group);
2131 if (r)
2132 dev_err(adev->dev, "Failed to create RAS sysfs group!");
f848159b 2133
c030f2e4 2134 return 0;
2135}
2136
2137static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
2138{
88293c03
ND		 2139 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2140 struct ras_manager *con_obj, *ip_obj, *tmp;
2141
2142 if (IS_ENABLED(CONFIG_DEBUG_FS)) {
2143 list_for_each_entry_safe(con_obj, tmp, &con->head, node) {
2144 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head);
2145 if (ip_obj)
2146 put_obj(ip_obj);
2147 }
2148 }
2149
c030f2e4 2150 amdgpu_ras_sysfs_remove_all(adev);
2151 return 0;
2152}
2153/* ras fs end */
2154
2155/* ih begin */
b3c76814
TZ		 2156
2157/* For the hardware that cannot enable bif ring for both ras_controller_irq
2158 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status
2159 * register to check whether the interrupt is triggered or not, and properly
2160 * ack the interrupt if it is there
2161 */
2162void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev)
2163{
950d6425 2164 /* Fatal error events are handled on host side */
8eba7205 2165 if (amdgpu_sriov_vf(adev))
b3c76814 2166 return;
0105725e 2167 /*
e1ee2111
LL		 2168 * If the current interrupt is caused by a non-fatal RAS error, skip
2169 * check for fatal error. For fatal errors, FED status of all devices
2170 * in XGMI hive gets set when the first device gets fatal error
2171 * interrupt. The error gets propagated to other devices as well, so
2172 * make sure to ack the interrupt regardless of FED status.
2173 */
2174 if (!amdgpu_ras_get_fed_status(adev) &&
2175 amdgpu_ras_is_err_state(adev, AMDGPU_RAS_BLOCK__ANY))
2176 return;
b3c76814
TZ		 2177
2178 if (adev->nbio.ras &&
2179 adev->nbio.ras->handle_ras_controller_intr_no_bifring)
2180 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev);
2181
2182 if (adev->nbio.ras &&
2183 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring)
2184 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev);
2185}
2186
66f87949
TZ		 2187static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj,
2188 struct amdgpu_iv_entry *entry)
2189{
b63ac5d3 2190 bool poison_stat = false;
66f87949 2191 struct amdgpu_device *adev = obj->adev;
66f87949
TZ		 2192 struct amdgpu_ras_block_object *block_obj =
2193 amdgpu_ras_get_ras_block(adev, obj->head.block, 0);
5f7697bb 2194 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
12b435a4
YW		 2195 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CONSUMPTION;
2196 u64 event_id;
2197 int ret;
66f87949 2198
5f7697bb 2199 if (!block_obj || !con)
b63ac5d3 2200 return;
66f87949 2201
12b435a4
YW		 2202 ret = amdgpu_ras_mark_ras_event(adev, type);
2203 if (ret)
2204 return;
2205
e1ee2111 2206 amdgpu_ras_set_err_poison(adev, block_obj->ras_comm.block);
b63ac5d3
TZ		 2207 /* both query_poison_status and handle_poison_consumption are optional,
2208 * but at least one of them should be implemented if we need poison
2209 * consumption handler
2210 */
ac7b25d9 2211 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) {
b63ac5d3
TZ		 2212 poison_stat = block_obj->hw_ops->query_poison_status(adev);
2213 if (!poison_stat) {
2214 /* Not poison consumption interrupt, no need to handle it */
2215 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n",
2216 block_obj->ras_comm.name);
2217
2218 return;
66f87949
TZ		 2219 }
2220 }
2221
2fc46e0b 2222 amdgpu_umc_poison_handler(adev, obj->head.block, 0);
b63ac5d3 2223
ac7b25d9 2224 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption)
b63ac5d3
TZ		 2225 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev);
2226
5f7697bb
TZ		 2227 /* gpu reset is fallback for failed and default cases.
2228 * For RMA case, amdgpu_umc_poison_handler will handle gpu reset.
2229 */
792be2e2 2230 if (poison_stat && !amdgpu_ras_is_rma(adev)) {
12b435a4
YW		 2231 event_id = amdgpu_ras_acquire_event_id(adev, type);
2232 RAS_EVENT_LOG(adev, event_id,
2233 "GPU reset for %s RAS poison consumption is issued!\n",
2234 block_obj->ras_comm.name);
66f87949 2235 amdgpu_ras_reset_gpu(adev);
b63ac5d3 2236 }
5f7697bb
TZ		 2237
2238 if (!poison_stat)
2239 amdgpu_gfx_poison_consumption_handler(adev, entry);
66f87949
TZ		 2240}
2241
50a7d025
TZ		 2242static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj,
2243 struct amdgpu_iv_entry *entry)
2244{
5b9de259
YW		 2245 struct amdgpu_device *adev = obj->adev;
2246 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
2247 u64 event_id;
2248 int ret;
2249
2250 ret = amdgpu_ras_mark_ras_event(adev, type);
2251 if (ret)
2252 return;
2253
2254 event_id = amdgpu_ras_acquire_event_id(adev, type);
2255 RAS_EVENT_LOG(adev, event_id, "Poison is created\n");
a734adfb
YC		 2256
2257 if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) {
2258 struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev);
2259
a734adfb 2260 atomic_inc(&con->page_retirement_req_cnt);
5f08275c 2261 atomic_inc(&con->poison_creation_count);
a734adfb
YC		 2262
2263 wake_up(&con->page_retirement_wq);
2264 }
50a7d025
TZ		 2265}
2266
2267static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
2268 struct amdgpu_iv_entry *entry)
2269{
2270 struct ras_ih_data *data = &obj->ih_data;
5b1270be 2271 struct ras_err_data err_data;
50a7d025
TZ		 2272 int ret;
2273
2274 if (!data->cb)
2275 return;
2276
5b1270be
YW		 2277 ret = amdgpu_ras_error_data_init(&err_data);
2278 if (ret)
2279 return;
2280
50a7d025
TZ		 2281 /* Let IP handle its data, maybe we need get the output
2282 * from the callback to update the error type/count, etc
2283 */
09a3d820 2284 amdgpu_ras_set_fed(obj->adev, true);
50a7d025
TZ		 2285 ret = data->cb(obj->adev, &err_data, entry);
2286 /* ue will trigger an interrupt, and in that case
2287 * we need do a reset to recovery the whole system.
2288 * But leave IP do that recovery, here we just dispatch
2289 * the error.
2290 */
2291 if (ret == AMDGPU_RAS_SUCCESS) {
2292 /* these counts could be left as 0 if
2293 * some blocks do not count error number
2294 */
2295 obj->err_data.ue_count += err_data.ue_count;
2296 obj->err_data.ce_count += err_data.ce_count;
46e2231c 2297 obj->err_data.de_count += err_data.de_count;
50a7d025 2298 }
5b1270be
YW		 2299
2300 amdgpu_ras_error_data_fini(&err_data);
50a7d025
TZ		 2301}
2302
c030f2e4 2303static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
2304{
2305 struct ras_ih_data *data = &obj->ih_data;
2306 struct amdgpu_iv_entry entry;
c030f2e4 2307
2308 while (data->rptr != data->wptr) {
2309 rmb();
2310 memcpy(&entry, &data->ring[data->rptr],
2311 data->element_size);
2312
2313 wmb();
2314 data->rptr = (data->aligned_element_size +
2315 data->rptr) % data->ring_size;
2316
50a7d025
TZ		 2317 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) {
2318 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
2319 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry);
66f87949
TZ		 2320 else
2321 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry);
50a7d025
TZ		 2322 } else {
2323 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
2324 amdgpu_ras_interrupt_umc_handler(obj, &entry);
2325 else
2326 dev_warn(obj->adev->dev,
2327 "No RAS interrupt handler for non-UMC block with poison disabled.\n");
c030f2e4 2328 }
2329 }
2330}
2331
2332static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
2333{
2334 struct ras_ih_data *data =
2335 container_of(work, struct ras_ih_data, ih_work);
2336 struct ras_manager *obj =
2337 container_of(data, struct ras_manager, ih_data);
2338
2339 amdgpu_ras_interrupt_handler(obj);
2340}
2341
2342int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
2343 struct ras_dispatch_if *info)
2344{
4c11d30c
MJ		 2345 struct ras_manager *obj;
2346 struct ras_ih_data *data;
c030f2e4 2347
4c11d30c 2348 obj = amdgpu_ras_find_obj(adev, &info->head);
c030f2e4 2349 if (!obj)
2350 return -EINVAL;
2351
4c11d30c
MJ		 2352 data = &obj->ih_data;
2353
c030f2e4 2354 if (data->inuse == 0)
2355 return 0;
2356
2357 /* Might be overflow... */
2358 memcpy(&data->ring[data->wptr], info->entry,
2359 data->element_size);
2360
2361 wmb();
2362 data->wptr = (data->aligned_element_size +
2363 data->wptr) % data->ring_size;
2364
2365 schedule_work(&data->ih_work);
2366
2367 return 0;
2368}
2369
2370int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
9252d33d 2371 struct ras_common_if *head)
c030f2e4 2372{
9252d33d 2373 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
c030f2e4 2374 struct ras_ih_data *data;
2375
2376 if (!obj)
2377 return -EINVAL;
2378
2379 data = &obj->ih_data;
2380 if (data->inuse == 0)
2381 return 0;
2382
2383 cancel_work_sync(&data->ih_work);
2384
2385 kfree(data->ring);
2386 memset(data, 0, sizeof(*data));
2387 put_obj(obj);
2388
2389 return 0;
2390}
2391
2392int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
9252d33d 2393 struct ras_common_if *head)
c030f2e4 2394{
9252d33d 2395 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
c030f2e4 2396 struct ras_ih_data *data;
9252d33d 2397 struct amdgpu_ras_block_object *ras_obj;
c030f2e4 2398
2399 if (!obj) {
2400 /* in case we registe the IH before enable ras feature */
9252d33d 2401 obj = amdgpu_ras_create_obj(adev, head);
c030f2e4 2402 if (!obj)
2403 return -EINVAL;
2404 } else
2405 get_obj(obj);
2406
9252d33d 2407 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm);
2408
c030f2e4 2409 data = &obj->ih_data;
2410 /* add the callback.etc */
2411 *data = (struct ras_ih_data) {
2412 .inuse = 0,
9252d33d 2413 .cb = ras_obj->ras_cb,
c030f2e4 2414 .element_size = sizeof(struct amdgpu_iv_entry),
2415 .rptr = 0,
2416 .wptr = 0,
2417 };
2418
2419 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
2420
2421 data->aligned_element_size = ALIGN(data->element_size, 8);
2422 /* the ring can store 64 iv entries. */
2423 data->ring_size = 64 * data->aligned_element_size;
2424 data->ring = kmalloc(data->ring_size, GFP_KERNEL);
2425 if (!data->ring) {
2426 put_obj(obj);
2427 return -ENOMEM;
2428 }
2429
2430 /* IH is ready */
2431 data->inuse = 1;
2432
2433 return 0;
2434}
2435
2436static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
2437{
2438 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2439 struct ras_manager *obj, *tmp;
2440
2441 list_for_each_entry_safe(obj, tmp, &con->head, node) {
9252d33d 2442 amdgpu_ras_interrupt_remove_handler(adev, &obj->head);
c030f2e4 2443 }
2444
2445 return 0;
2446}
2447/* ih end */
2448
313c8fd3 2449/* traversal all IPs except NBIO to query error counter */
75ac6a25 2450static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev, enum ras_event_type type)
313c8fd3
GC		 2451{
2452 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2453 struct ras_manager *obj;
2454
8ab0d6f0 2455 if (!adev->ras_enabled || !con)
313c8fd3
GC		 2456 return;
2457
2458 list_for_each_entry(obj, &con->head, node) {
2459 struct ras_query_if info = {
2460 .head = obj->head,
2461 };
2462
2463 /*
2464 * PCIE_BIF IP has one different isr by ras controller
2465 * interrupt, the specific ras counter query will be
2466 * done in that isr. So skip such block from common
2467 * sync flood interrupt isr calling.
2468 */
2469 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF)
2470 continue;
2471
cf63b702
SY		 2472 /*
2473 * this is a workaround for aldebaran, skip send msg to
2474 * smu to get ecc_info table due to smu handle get ecc
2475 * info table failed temporarily.
2476 * should be removed until smu fix handle ecc_info table.
2477 */
2478 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) &&
4e8303cf
LL		 2479 (amdgpu_ip_version(adev, MP1_HWIP, 0) ==
2480 IP_VERSION(13, 0, 2)))
cf63b702
SY		 2481 continue;
2482
75ac6a25 2483 amdgpu_ras_query_error_status_with_event(adev, &info, type);
2a460963 2484
4e8303cf
LL		 2485 if (amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2486 IP_VERSION(11, 0, 2) &&
2487 amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2488 IP_VERSION(11, 0, 4) &&
2489 amdgpu_ip_version(adev, MP0_HWIP, 0) !=
2490 IP_VERSION(13, 0, 0)) {
2a460963
CL		 2491 if (amdgpu_ras_reset_error_status(adev, info.head.block))
2492 dev_warn(adev->dev, "Failed to reset error counter and error status");
2493 }
313c8fd3
GC		 2494 }
2495}
2496
3f975d0f 2497/* Parse RdRspStatus and WrRspStatus */
cd92df93
LJ		 2498static void amdgpu_ras_error_status_query(struct amdgpu_device *adev,
2499 struct ras_query_if *info)
3f975d0f 2500{
8eb53bb2 2501 struct amdgpu_ras_block_object *block_obj;
3f975d0f
SY		 2502 /*
2503 * Only two block need to query read/write
2504 * RspStatus at current state
2505 */
5e67bba3 2506 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) &&
2507 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB))
b6efdb02 2508 return;
2509
2510 block_obj = amdgpu_ras_get_ras_block(adev,
2511 info->head.block,
2512 info->head.sub_block_index);
5e67bba3 2513
5e67bba3 2514 if (!block_obj || !block_obj->hw_ops) {
afa37315
LT		 2515 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
2516 get_ras_block_str(&info->head));
b6efdb02 2517 return;
3f975d0f 2518 }
5e67bba3 2519
2520 if (block_obj->hw_ops->query_ras_error_status)
ab3b9de6 2521 block_obj->hw_ops->query_ras_error_status(adev);
5e67bba3 2522
3f975d0f
SY		 2523}
2524
2525static void amdgpu_ras_query_err_status(struct amdgpu_device *adev)
2526{
2527 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2528 struct ras_manager *obj;
2529
8ab0d6f0 2530 if (!adev->ras_enabled || !con)
3f975d0f
SY		 2531 return;
2532
2533 list_for_each_entry(obj, &con->head, node) {
2534 struct ras_query_if info = {
2535 .head = obj->head,
2536 };
2537
2538 amdgpu_ras_error_status_query(adev, &info);
2539 }
2540}
2541
c030f2e4 2542/* recovery begin */
466b1793 2543
2544/* return 0 on success.
2545 * caller need free bps.
2546 */
2547static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
2548 struct ras_badpage **bps, unsigned int *count)
2549{
2550 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2551 struct ras_err_handler_data *data;
2552 int i = 0;
732f2a30 2553 int ret = 0, status;
466b1793 2554
2555 if (!con || !con->eh_data || !bps || !count)
2556 return -EINVAL;
2557
2558 mutex_lock(&con->recovery_lock);
2559 data = con->eh_data;
2560 if (!data || data->count == 0) {
2561 *bps = NULL;
46cf2fec 2562 ret = -EINVAL;
466b1793 2563 goto out;
2564 }
2565
2566 *bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL);
2567 if (!*bps) {
2568 ret = -ENOMEM;
2569 goto out;
2570 }
2571
2572 for (; i < data->count; i++) {
2573 (*bps)[i] = (struct ras_badpage){
9dc23a63 2574 .bp = data->bps[i].retired_page,
466b1793 2575 .size = AMDGPU_GPU_PAGE_SIZE,
52dd95f2 2576 .flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED,
466b1793 2577 };
ec6aae97 2578 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr,
bcc09348 2579 data->bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT);
732f2a30 2580 if (status == -EBUSY)
52dd95f2 2581 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
732f2a30 2582 else if (status == -ENOENT)
52dd95f2 2583 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT;
466b1793 2584 }
2585
2586 *count = data->count;
2587out:
2588 mutex_unlock(&con->recovery_lock);
2589 return ret;
2590}
2591
b41f742d
LL		 2592static void amdgpu_ras_set_fed_all(struct amdgpu_device *adev,
2593 struct amdgpu_hive_info *hive, bool status)
2594{
2595 struct amdgpu_device *tmp_adev;
2596
2597 if (hive) {
2598 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
2599 amdgpu_ras_set_fed(tmp_adev, status);
2600 } else {
2601 amdgpu_ras_set_fed(adev, status);
2602 }
2603}
2604
7e437167
TZ		 2605bool amdgpu_ras_in_recovery(struct amdgpu_device *adev)
2606{
2607 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2608 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2609 int hive_ras_recovery = 0;
2610
2611 if (hive) {
2612 hive_ras_recovery = atomic_read(&hive->ras_recovery);
2613 amdgpu_put_xgmi_hive(hive);
2614 }
2615
2616 if (ras && (atomic_read(&ras->in_recovery) || hive_ras_recovery))
2617 return true;
2618
2619 return false;
2620}
2621
75ac6a25
YW		 2622static enum ras_event_type amdgpu_ras_get_fatal_error_event(struct amdgpu_device *adev)
2623{
2624 if (amdgpu_ras_intr_triggered())
2625 return RAS_EVENT_TYPE_FATAL;
2626 else
12b435a4 2627 return RAS_EVENT_TYPE_POISON_CONSUMPTION;
75ac6a25
YW		 2628}
2629
c030f2e4 2630static void amdgpu_ras_do_recovery(struct work_struct *work)
2631{
2632 struct amdgpu_ras *ras =
2633 container_of(work, struct amdgpu_ras, recovery_work);
b3dbd6d3
JC		 2634 struct amdgpu_device *remote_adev = NULL;
2635 struct amdgpu_device *adev = ras->adev;
2636 struct list_head device_list, *device_list_handle = NULL;
53dd920c 2637 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
75ac6a25 2638 enum ras_event_type type;
b3dbd6d3 2639
b41f742d 2640 if (hive) {
53dd920c 2641 atomic_set(&hive->ras_recovery, 1);
b41f742d
LL		 2642
2643 /* If any device which is part of the hive received RAS fatal
2644 * error interrupt, set fatal error status on all. This
2645 * condition will need a recovery, and flag will be cleared
2646 * as part of recovery.
2647 */
2648 list_for_each_entry(remote_adev, &hive->device_list,
2649 gmc.xgmi.head)
2650 if (amdgpu_ras_get_fed_status(remote_adev)) {
2651 amdgpu_ras_set_fed_all(adev, hive, true);
2652 break;
2653 }
2654 }
f75e94d8 2655 if (!ras->disable_ras_err_cnt_harvest) {
d95e8e97 2656
f75e94d8
GC		 2657 /* Build list of devices to query RAS related errors */
2658 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
2659 device_list_handle = &hive->device_list;
2660 } else {
2661 INIT_LIST_HEAD(&device_list);
2662 list_add_tail(&adev->gmc.xgmi.head, &device_list);
2663 device_list_handle = &device_list;
2664 }
c030f2e4 2665
75ac6a25 2666 type = amdgpu_ras_get_fatal_error_event(adev);
f75e94d8 2667 list_for_each_entry(remote_adev,
3f975d0f
SY		 2668 device_list_handle, gmc.xgmi.head) {
2669 amdgpu_ras_query_err_status(remote_adev);
75ac6a25 2670 amdgpu_ras_log_on_err_counter(remote_adev, type);
3f975d0f 2671 }
d95e8e97 2672
b3dbd6d3 2673 }
313c8fd3 2674
f1549c09
LG		 2675 if (amdgpu_device_should_recover_gpu(ras->adev)) {
2676 struct amdgpu_reset_context reset_context;
2677 memset(&reset_context, 0, sizeof(reset_context));
2678
2679 reset_context.method = AMD_RESET_METHOD_NONE;
2680 reset_context.reset_req_dev = adev;
bac640dd 2681 reset_context.src = AMDGPU_RESET_SRC_RAS;
81db4eab 2682 set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
1a11a65d
YC		 2683
2684 /* Perform full reset in fatal error mode */
2685 if (!amdgpu_ras_is_poison_mode_supported(ras->adev))
2686 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
6c47a79b 2687 else {
1a11a65d 2688 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
f1549c09 2689
6c47a79b
YC		 2690 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) {
2691 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET;
2692 reset_context.method = AMD_RESET_METHOD_MODE2;
2693 }
2c7cd280
YC		 2694
2695 /* Fatal error occurs in poison mode, mode1 reset is used to
2696 * recover gpu.
2697 */
2698 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) {
2699 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET;
2700 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
1b98a5f8
YC		 2701
2702 psp_fatal_error_recovery_quirk(&adev->psp);
2c7cd280 2703 }
6c47a79b
YC		 2704 }
2705
f1549c09
LG		 2706 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context);
2707 }
c030f2e4 2708 atomic_set(&ras->in_recovery, 0);
53dd920c
AK		 2709 if (hive) {
2710 atomic_set(&hive->ras_recovery, 0);
2711 amdgpu_put_xgmi_hive(hive);
2712 }
c030f2e4 2713}
2714
c030f2e4 2715/* alloc/realloc bps array */
2716static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
2717 struct ras_err_handler_data *data, int pages)
2718{
2719 unsigned int old_space = data->count + data->space_left;
2720 unsigned int new_space = old_space + pages;
9dc23a63
TZ		 2721 unsigned int align_space = ALIGN(new_space, 512);
2722 void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL);
9dc23a63 2723
676deb38 2724 if (!bps) {
c030f2e4 2725 return -ENOMEM;
9dc23a63 2726 }
c030f2e4 2727
2728 if (data->bps) {
9dc23a63 2729 memcpy(bps, data->bps,
c030f2e4 2730 data->count * sizeof(*data->bps));
2731 kfree(data->bps);
2732 }
2733
9dc23a63 2734 data->bps = bps;
c030f2e4 2735 data->space_left += align_space - old_space;
2736 return 0;
2737}
2738
a8d133e6 2739static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev,
0eecff79
TZ		 2740 struct eeprom_table_record *bps,
2741 struct ras_err_data *err_data)
2742{
2743 struct ta_ras_query_address_input addr_in;
2744 uint32_t socket = 0;
2745 int ret = 0;
2746
2747 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
2748 socket = adev->smuio.funcs->get_socket_id(adev);
2749
2750 /* reinit err_data */
2751 err_data->err_addr_cnt = 0;
2752 err_data->err_addr_len = adev->umc.retire_unit;
2753
2754 memset(&addr_in, 0, sizeof(addr_in));
2755 addr_in.ma.err_addr = bps->address;
2756 addr_in.ma.socket_id = socket;
2757 addr_in.ma.ch_inst = bps->mem_channel;
2758 /* tell RAS TA the node instance is not used */
2759 addr_in.ma.node_inst = TA_RAS_INV_NODE;
2760
2761 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
2762 ret = adev->umc.ras->convert_ras_err_addr(adev, err_data,
2763 &addr_in, NULL, false);
2764
2765 return ret;
2766}
2767
a8d133e6
TZ		 2768static int amdgpu_ras_mca2pa(struct amdgpu_device *adev,
2769 struct eeprom_table_record *bps,
2770 struct ras_err_data *err_data)
2771{
2772 struct ta_ras_query_address_input addr_in;
2773 uint32_t die_id, socket = 0;
2774
2775 if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id)
2776 socket = adev->smuio.funcs->get_socket_id(adev);
2777
2778 /* although die id is gotten from PA in nps1 mode, the id is
2779 * fitable for any nps mode
2780 */
2781 if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa)
2782 die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address,
2783 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT);
2784 else
2785 return -EINVAL;
2786
2787 /* reinit err_data */
2788 err_data->err_addr_cnt = 0;
2789 err_data->err_addr_len = adev->umc.retire_unit;
2790
2791 memset(&addr_in, 0, sizeof(addr_in));
2792 addr_in.ma.err_addr = bps->address;
2793 addr_in.ma.ch_inst = bps->mem_channel;
2794 addr_in.ma.umc_inst = bps->mcumc_id;
2795 addr_in.ma.node_inst = die_id;
2796 addr_in.ma.socket_id = socket;
2797
2798 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr)
2799 return adev->umc.ras->convert_ras_err_addr(adev, err_data,
2800 &addr_in, NULL, false);
2801 else
2802 return -EINVAL;
2803}
2804
0153d276 2805static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev,
2806 struct eeprom_table_record *bps, int count)
2807{
2808 int j;
2809 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2810 struct ras_err_handler_data *data = con->eh_data;
2811
2812 for (j = 0; j < count; j++) {
2813 if (amdgpu_ras_check_bad_page_unlock(con,
2814 bps[j].retired_page << AMDGPU_GPU_PAGE_SHIFT))
2815 continue;
2816
2817 if (!data->space_left &&
2818 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
2819 return -ENOMEM;
2820 }
2821
2822 amdgpu_ras_reserve_page(adev, bps[j].retired_page);
2823
2824 memcpy(&data->bps[data->count], &(bps[j]),
2825 sizeof(struct eeprom_table_record));
2826 data->count++;
2827 data->space_left--;
2828 }
2829
2830 return 0;
2831}
2832
2833static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev,
2834 struct eeprom_table_record *bps, struct ras_err_data *err_data,
2835 enum amdgpu_memory_partition nps)
2836{
2837 int i = 0;
2838 enum amdgpu_memory_partition save_nps;
2839
2840 save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK;
2841
a4b6e990 2842 /*old asics just have pa in eeprom*/
2843 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) {
2844 memcpy(err_data->err_addr, bps,
2845 sizeof(struct eeprom_table_record) * adev->umc.retire_unit);
2846 goto out;
2847 }
2848
0153d276 2849 for (i = 0; i < adev->umc.retire_unit; i++)
2850 bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT);
2851
2852 if (save_nps) {
2853 if (save_nps == nps) {
2854 if (amdgpu_umc_pages_in_a_row(adev, err_data,
2855 bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT))
2856 return -EINVAL;
2857 } else {
2858 if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data))
2859 return -EINVAL;
2860 }
2861 } else {
fce0afca 2862 if (bps[0].address == 0) {
2863 /* for specific old eeprom data, mca address is not stored,
2864 * calc it from pa
2865 */
2866 if (amdgpu_umc_pa2mca(adev, bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT,
2867 &(bps[0].address), AMDGPU_NPS1_PARTITION_MODE))
2868 return -EINVAL;
2869 }
2870
0153d276 2871 if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) {
2872 if (nps == AMDGPU_NPS1_PARTITION_MODE)
2873 memcpy(err_data->err_addr, bps,
2874 sizeof(struct eeprom_table_record) * adev->umc.retire_unit);
2875 else
2876 return -EOPNOTSUPP;
2877 }
2878 }
2879
a4b6e990 2880out:
0153d276 2881 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit);
2882}
2883
2884static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev,
2885 struct eeprom_table_record *bps, struct ras_err_data *err_data,
2886 enum amdgpu_memory_partition nps)
2887{
2888 enum amdgpu_memory_partition save_nps;
2889
2890 save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK;
2891 bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT);
2892
2893 if (save_nps == nps) {
2894 if (amdgpu_umc_pages_in_a_row(adev, err_data,
2895 bps->retired_page << AMDGPU_GPU_PAGE_SHIFT))
2896 return -EINVAL;
2897 } else {
31e837d2 2898 if (bps->address) {
2899 if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data))
2900 return -EINVAL;
2901 } else {
2902 /* for specific old eeprom data, mca address is not stored,
2903 * calc it from pa
2904 */
2905 if (amdgpu_umc_pa2mca(adev, bps->retired_page << AMDGPU_GPU_PAGE_SHIFT,
2906 &(bps->address), AMDGPU_NPS1_PARTITION_MODE))
2907 return -EINVAL;
2908
2909 if (amdgpu_ras_mca2pa(adev, bps, err_data))
2910 return -EOPNOTSUPP;
2911 }
0153d276 2912 }
f5db5906 2913
0153d276 2914 return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr,
2915 adev->umc.retire_unit);
2916}
2917
c030f2e4 2918/* it deal with vram only. */
2919int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
a8d133e6 2920 struct eeprom_table_record *bps, int pages, bool from_rom)
c030f2e4 2921{
2922 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
0eecff79 2923 struct ras_err_data err_data;
d08fb663
TZ		 2924 struct amdgpu_ras_eeprom_control *control =
2925 &adev->psp.ras_context.ras->eeprom_control;
07dd49e1 2926 enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE;
c030f2e4 2927 int ret = 0;
f5db5906 2928 uint32_t i = 0;
c030f2e4 2929
73aa8e1a 2930 if (!con || !con->eh_data || !bps || pages <= 0)
c030f2e4 2931 return 0;
2932
a8d133e6 2933 if (from_rom) {
0eecff79
TZ		 2934 err_data.err_addr =
2935 kcalloc(adev->umc.retire_unit,
2936 sizeof(struct eeprom_table_record), GFP_KERNEL);
2937 if (!err_data.err_addr) {
2938 dev_warn(adev->dev, "Failed to alloc UMC error address record in mca2pa conversion!\n");
0153d276 2939 return -ENOMEM;
c030f2e4 2940 }
2941
07dd49e1
TZ		 2942 if (adev->gmc.gmc_funcs->query_mem_partition_mode)
2943 nps = adev->gmc.gmc_funcs->query_mem_partition_mode(adev);
0eecff79
TZ		 2944 }
2945
a8d133e6 2946 mutex_lock(&con->recovery_lock);
0153d276 2947
2948 if (from_rom) {
f5db5906 2949 /* there is no pa recs in V3, so skip pa recs processing */
2950 if (control->tbl_hdr.version < RAS_TABLE_VER_V3) {
2951 for (i = 0; i < pages; i++) {
2952 if (control->ras_num_recs - i >= adev->umc.retire_unit) {
2953 if ((bps[i].address == bps[i + 1].address) &&
2954 (bps[i].mem_channel == bps[i + 1].mem_channel)) {
2955 /* deal with retire_unit records a time */
2956 ret = __amdgpu_ras_convert_rec_array_from_rom(adev,
2957 &bps[i], &err_data, nps);
2958 if (ret)
2959 control->ras_num_bad_pages -= adev->umc.retire_unit;
2960 i += (adev->umc.retire_unit - 1);
2961 } else {
2962 break;
2963 }
a8d133e6 2964 } else {
0153d276 2965 break;
a8d133e6 2966 }
a8d133e6 2967 }
0eecff79 2968 }
0153d276 2969 for (; i < pages; i++) {
2970 ret = __amdgpu_ras_convert_rec_from_rom(adev,
2971 &bps[i], &err_data, nps);
2972 if (ret)
f5db5906 2973 control->ras_num_bad_pages -= adev->umc.retire_unit;
0eecff79 2974 }
0153d276 2975 } else {
2976 ret = __amdgpu_ras_restore_bad_pages(adev, bps, pages);
676deb38 2977 }
0eecff79 2978
a8d133e6 2979 if (from_rom)
0eecff79 2980 kfree(err_data.err_addr);
c030f2e4 2981 mutex_unlock(&con->recovery_lock);
2982
2983 return ret;
2984}
2985
78ad00c9
TZ		 2986/*
2987 * write error record array to eeprom, the function should be
2988 * protected by recovery_lock
4d33e0f1 2989 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL
78ad00c9 2990 */
4d33e0f1
TZ		 2991int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev,
2992 unsigned long *new_cnt)
78ad00c9
TZ		 2993{
2994 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2995 struct ras_err_handler_data *data;
8a3e801f 2996 struct amdgpu_ras_eeprom_control *control;
c3d4acf0 2997 int save_count, unit_num, bad_page_num, i;
78ad00c9 2998
4d33e0f1
TZ		 2999 if (!con || !con->eh_data) {
3000 if (new_cnt)
3001 *new_cnt = 0;
3002
78ad00c9 3003 return 0;
4d33e0f1 3004 }
78ad00c9 3005
d9a69fe5 3006 mutex_lock(&con->recovery_lock);
8a3e801f 3007 control = &con->eeprom_control;
78ad00c9 3008 data = con->eh_data;
ae756cd8 3009 bad_page_num = control->ras_num_bad_pages;
c3d4acf0 3010 save_count = data->count - bad_page_num;
d9a69fe5 3011 mutex_unlock(&con->recovery_lock);
4d33e0f1 3012
c3d4acf0 3013 unit_num = save_count / adev->umc.retire_unit;
4d33e0f1 3014 if (new_cnt)
c3d4acf0 3015 *new_cnt = unit_num;
4d33e0f1 3016
78ad00c9 3017 /* only new entries are saved */
b1628425 3018 if (save_count > 0) {
a4b6e990 3019 /*old asics only save pa to eeprom like before*/
3020 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) {
c3d4acf0 3021 if (amdgpu_ras_eeprom_append(control,
a4b6e990 3022 &data->bps[bad_page_num], save_count)) {
c3d4acf0
TZ		 3023 dev_err(adev->dev, "Failed to save EEPROM table data!");
3024 return -EIO;
3025 }
a4b6e990 3026 } else {
3027 for (i = 0; i < unit_num; i++) {
3028 if (amdgpu_ras_eeprom_append(control,
3029 &data->bps[bad_page_num +
3030 i * adev->umc.retire_unit], 1)) {
3031 dev_err(adev->dev, "Failed to save EEPROM table data!");
3032 return -EIO;
3033 }
3034 }
78ad00c9 3035 }
a4b6e990 3036
b1628425
GC		 3037 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
3038 }
3039
78ad00c9
TZ		 3040 return 0;
3041}
3042
3043/*
3044 * read error record array in eeprom and reserve enough space for
3045 * storing new bad pages
3046 */
3047static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
3048{
3049 struct amdgpu_ras_eeprom_control *control =
6457205c 3050 &adev->psp.ras_context.ras->eeprom_control;
e4e6a589 3051 struct eeprom_table_record *bps;
a8f921a1 3052 int ret, i = 0;
78ad00c9
TZ		 3053
3054 /* no bad page record, skip eeprom access */
0686627b 3055 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0)
e4e6a589 3056 return 0;
78ad00c9 3057
0686627b 3058 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL);
78ad00c9
TZ		 3059 if (!bps)
3060 return -ENOMEM;
3061
0686627b 3062 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs);
772df3df 3063 if (ret) {
6952e99c 3064 dev_err(adev->dev, "Failed to load EEPROM table records!");
772df3df 3065 } else {
a8f921a1 3066 if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) {
f5db5906 3067 /*In V3, there is no pa recs, and some cases(when address==0) may be parsed
3068 as pa recs, so add verion check to avoid it.
3069 */
3070 if (control->tbl_hdr.version < RAS_TABLE_VER_V3) {
3071 for (i = 0; i < control->ras_num_recs; i++) {
3072 if ((control->ras_num_recs - i) >= adev->umc.retire_unit) {
3073 if ((bps[i].address == bps[i + 1].address) &&
3074 (bps[i].mem_channel == bps[i + 1].mem_channel)) {
3075 control->ras_num_pa_recs += adev->umc.retire_unit;
3076 i += (adev->umc.retire_unit - 1);
3077 } else {
3078 control->ras_num_mca_recs +=
3079 (control->ras_num_recs - i);
3080 break;
3081 }
a8f921a1 3082 } else {
f5db5906 3083 control->ras_num_mca_recs += (control->ras_num_recs - i);
a8f921a1 3084 break;
3085 }
a8f921a1 3086 }
f5db5906 3087 } else {
3088 control->ras_num_mca_recs = control->ras_num_recs;
a8f921a1 3089 }
772df3df
TZ		 3090 }
3091
1f06e7f3
TZ		 3092 ret = amdgpu_ras_eeprom_check(control);
3093 if (ret)
3094 goto out;
3095
3096 /* HW not usable */
3097 if (amdgpu_ras_is_rma(adev)) {
3098 ret = -EHWPOISON;
3099 goto out;
3100 }
3101
a8d133e6 3102 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true);
772df3df 3103 }
78ad00c9 3104
1f06e7f3 3105out:
78ad00c9
TZ		 3106 kfree(bps);
3107 return ret;
3108}
3109
676deb38
DL		 3110static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
3111 uint64_t addr)
3112{
3113 struct ras_err_handler_data *data = con->eh_data;
3114 int i;
3115
3116 addr >>= AMDGPU_GPU_PAGE_SHIFT;
3117 for (i = 0; i < data->count; i++)
3118 if (addr == data->bps[i].retired_page)
3119 return true;
3120
3121 return false;
3122}
3123
6e4be987
TZ		 3124/*
3125 * check if an address belongs to bad page
3126 *
3127 * Note: this check is only for umc block
3128 */
3129static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
3130 uint64_t addr)
3131{
3132 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
6e4be987
TZ		 3133 bool ret = false;
3134
3135 if (!con || !con->eh_data)
3136 return ret;
3137
3138 mutex_lock(&con->recovery_lock);
676deb38 3139 ret = amdgpu_ras_check_bad_page_unlock(con, addr);
6e4be987
TZ		 3140 mutex_unlock(&con->recovery_lock);
3141 return ret;
3142}
3143
e5c04edf 3144static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
e4e6a589 3145 uint32_t max_count)
c84d4670 3146{
e5c04edf 3147 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
c84d4670
GC		 3148
3149 /*
16b85a09
HZ		 3150 * amdgpu_bad_page_threshold is used to config
3151 * the threshold for the number of bad pages.
3152 * -1: Threshold is set to default value
3153 * Driver will issue a warning message when threshold is reached
3154 * and continue runtime services.
3155 * 0: Disable bad page retirement
3156 * Driver will not retire bad pages
3157 * which is intended for debugging purpose.
3158 * -2: Threshold is determined by a formula
3159 * that assumes 1 bad page per 100M of local memory.
3160 * Driver will continue runtime services when threhold is reached.
3161 * 0 < threshold < max number of bad page records in EEPROM,
3162 * A user-defined threshold is set
3163 * Driver will halt runtime services when this custom threshold is reached.
c84d4670 3164 */
16b85a09 3165 if (amdgpu_bad_page_threshold == -2) {
e4e6a589 3166 u64 val = adev->gmc.mc_vram_size;
c84d4670 3167
e4e6a589 3168 do_div(val, RAS_BAD_PAGE_COVER);
e5c04edf 3169 con->bad_page_cnt_threshold = min(lower_32_bits(val),
e4e6a589 3170 max_count);
16b85a09
HZ		 3171 } else if (amdgpu_bad_page_threshold == -1) {
3172 con->bad_page_cnt_threshold = ((con->reserved_pages_in_bytes) >> 21) << 4;
e5c04edf 3173 } else {
e4e6a589
LT		 3174 con->bad_page_cnt_threshold = min_t(int, max_count,
3175 amdgpu_bad_page_threshold);
c84d4670
GC		 3176 }
3177}
3178
98b5bc87
YC		 3179int amdgpu_ras_put_poison_req(struct amdgpu_device *adev,
3180 enum amdgpu_ras_block block, uint16_t pasid,
3181 pasid_notify pasid_fn, void *data, uint32_t reset)
3182{
3183 int ret = 0;
3184 struct ras_poison_msg poison_msg;
3185 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3186
3187 memset(&poison_msg, 0, sizeof(poison_msg));
3188 poison_msg.block = block;
3189 poison_msg.pasid = pasid;
3190 poison_msg.reset = reset;
3191 poison_msg.pasid_fn = pasid_fn;
3192 poison_msg.data = data;
3193
3194 ret = kfifo_put(&con->poison_fifo, poison_msg);
3195 if (!ret) {
3196 dev_err(adev->dev, "Poison message fifo is full!\n");
3197 return -ENOSPC;
3198 }
3199
3200 return 0;
3201}
3202
98b5bc87
YC		 3203static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev,
3204 struct ras_poison_msg *poison_msg)
3205{
3206 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3207
3208 return kfifo_get(&con->poison_fifo, poison_msg);
3209}
98b5bc87 3210
f493dd64
YC		 3211static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log)
3212{
3213 mutex_init(&ecc_log->lock);
3214
f493dd64 3215 INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL);
78146c1d
YC		 3216 ecc_log->de_queried_count = 0;
3217 ecc_log->prev_de_queried_count = 0;
f493dd64
YC		 3218}
3219
3220static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log)
3221{
3222 struct radix_tree_iter iter;
3223 void __rcu **slot;
3224 struct ras_ecc_err *ecc_err;
3225
3226 mutex_lock(&ecc_log->lock);
3227 radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) {
3228 ecc_err = radix_tree_deref_slot(slot);
3229 kfree(ecc_err->err_pages.pfn);
3230 kfree(ecc_err);
3231 radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot);
3232 }
3233 mutex_unlock(&ecc_log->lock);
3234
3235 mutex_destroy(&ecc_log->lock);
78146c1d
YC		 3236 ecc_log->de_queried_count = 0;
3237 ecc_log->prev_de_queried_count = 0;
f493dd64 3238}
a734adfb 3239
c0470691
YC		 3240static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con,
3241 uint32_t delayed_ms)
3242{
3243 int ret;
3244
3245 mutex_lock(&con->umc_ecc_log.lock);
3246 ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree,
3247 UMC_ECC_NEW_DETECTED_TAG);
3248 mutex_unlock(&con->umc_ecc_log.lock);
3249
3250 if (ret)
3251 schedule_delayed_work(&con->page_retirement_dwork,
3252 msecs_to_jiffies(delayed_ms));
3253
3254 return ret ? true : false;
3255}
3256
2cf8e50e
YC		 3257static void amdgpu_ras_do_page_retirement(struct work_struct *work)
3258{
3259 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
3260 page_retirement_dwork.work);
3261 struct amdgpu_device *adev = con->adev;
3262 struct ras_err_data err_data;
5f7697bb 3263 unsigned long err_cnt;
2cf8e50e 3264
e23300df
YC		 3265 /* If gpu reset is ongoing, delay retiring the bad pages */
3266 if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) {
3267 amdgpu_ras_schedule_retirement_dwork(con,
3268 AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3);
2cf8e50e 3269 return;
e23300df 3270 }
2cf8e50e
YC		 3271
3272 amdgpu_ras_error_data_init(&err_data);
3273
3274 amdgpu_umc_handle_bad_pages(adev, &err_data);
5f7697bb 3275 err_cnt = err_data.err_addr_cnt;
2cf8e50e
YC		 3276
3277 amdgpu_ras_error_data_fini(&err_data);
3278
792be2e2 3279 if (err_cnt && amdgpu_ras_is_rma(adev))
5f7697bb
TZ		 3280 amdgpu_ras_reset_gpu(adev);
3281
c0470691
YC		 3282 amdgpu_ras_schedule_retirement_dwork(con,
3283 AMDGPU_RAS_RETIRE_PAGE_INTERVAL);
2cf8e50e
YC		 3284}
3285
78146c1d
YC		 3286static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev,
3287 uint32_t poison_creation_count)
a734adfb
YC		 3288{
3289 int ret = 0;
3290 struct ras_ecc_log_info *ecc_log;
3291 struct ras_query_if info;
78146c1d 3292 uint32_t timeout = 0;
a734adfb 3293 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
78146c1d
YC		 3294 uint64_t de_queried_count;
3295 uint32_t new_detect_count, total_detect_count;
3296 uint32_t need_query_count = poison_creation_count;
3297 bool query_data_timeout = false;
5b9de259 3298 enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION;
a734adfb
YC		 3299
3300 memset(&info, 0, sizeof(info));
3ca73073 3301 info.head.block = AMDGPU_RAS_BLOCK__UMC;
a734adfb
YC		 3302
3303 ecc_log = &ras->umc_ecc_log;
78146c1d 3304 total_detect_count = 0;
a734adfb 3305 do {
5b9de259 3306 ret = amdgpu_ras_query_error_status_with_event(adev, &info, type);
78146c1d
YC		 3307 if (ret)
3308 return ret;
3309
3310 de_queried_count = ecc_log->de_queried_count;
3311 if (de_queried_count > ecc_log->prev_de_queried_count) {
3312 new_detect_count = de_queried_count - ecc_log->prev_de_queried_count;
3313 ecc_log->prev_de_queried_count = de_queried_count;
3314 timeout = 0;
3315 } else {
3316 new_detect_count = 0;
a734adfb
YC		 3317 }
3318
78146c1d
YC		 3319 if (new_detect_count) {
3320 total_detect_count += new_detect_count;
3321 } else {
3322 if (!timeout && need_query_count)
3323 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC;
3324
3325 if (timeout) {
3326 if (!--timeout) {
3327 query_data_timeout = true;
3328 break;
3329 }
3330 msleep(1);
3331 }
a734adfb 3332 }
78146c1d 3333 } while (total_detect_count < need_query_count);
a734adfb 3334
78146c1d
YC		 3335 if (query_data_timeout) {
3336 dev_warn(adev->dev, "Can't find deferred error! count: %u\n",
3337 (need_query_count - total_detect_count));
3338 return -ENOENT;
a734adfb
YC		 3339 }
3340
78146c1d 3341 if (total_detect_count)
3ca73073 3342 schedule_delayed_work(&ras->page_retirement_dwork, 0);
78146c1d
YC		 3343
3344 return 0;
a734adfb
YC		 3345}
3346
f852c979
YC		 3347static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev)
3348{
3349 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3350 struct ras_poison_msg msg;
3351 int ret;
3352
3353 do {
3354 ret = kfifo_get(&con->poison_fifo, &msg);
3355 } while (ret);
3356}
3357
370fbff4 3358static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev,
e278849c 3359 uint32_t msg_count, uint32_t *gpu_reset)
370fbff4
YC		 3360{
3361 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
e278849c
YC		 3362 uint32_t reset_flags = 0, reset = 0;
3363 struct ras_poison_msg msg;
3364 int ret, i;
370fbff4
YC		 3365
3366 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);
3367
e278849c
YC		 3368 for (i = 0; i < msg_count; i++) {
3369 ret = amdgpu_ras_get_poison_req(adev, &msg);
3370 if (!ret)
3371 continue;
3372
3373 if (msg.pasid_fn)
3374 msg.pasid_fn(adev, msg.pasid, msg.data);
3375
3376 reset_flags |= msg.reset;
3377 }
370fbff4 3378
5f7697bb 3379 /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */
792be2e2 3380 if (reset_flags && !amdgpu_ras_is_rma(adev)) {
e278849c
YC		 3381 if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET)
3382 reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET;
3383 else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET)
3384 reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET;
3385 else
3386 reset = reset_flags;
3387
370fbff4
YC		 3388 flush_delayed_work(&con->page_retirement_dwork);
3389
3390 con->gpu_reset_flags |= reset;
3391 amdgpu_ras_reset_gpu(adev);
e278849c
YC		 3392
3393 *gpu_reset = reset;
f852c979
YC		 3394
3395 /* Wait for gpu recovery to complete */
3396 flush_work(&con->recovery_work);
370fbff4
YC		 3397 }
3398
3399 return 0;
3400}
3401
3fdcd0a3
YC		 3402static int amdgpu_ras_page_retirement_thread(void *param)
3403{
3404 struct amdgpu_device *adev = (struct amdgpu_device *)param;
3405 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
e278849c
YC		 3406 uint32_t poison_creation_count, msg_count;
3407 uint32_t gpu_reset;
5f08275c 3408 int ret;
3fdcd0a3
YC		 3409
3410 while (!kthread_should_stop()) {
3411
3412 wait_event_interruptible(con->page_retirement_wq,
c84a7e21 3413 kthread_should_stop() ||
3fdcd0a3
YC		 3414 atomic_read(&con->page_retirement_req_cnt));
3415
c84a7e21
MJ		 3416 if (kthread_should_stop())
3417 break;
3418
e278849c 3419 gpu_reset = 0;
5f08275c
YC		 3420
3421 do {
3422 poison_creation_count = atomic_read(&con->poison_creation_count);
3423 ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count);
3424 if (ret == -EIO)
3425 break;
3426
3427 if (poison_creation_count) {
3428 atomic_sub(poison_creation_count, &con->poison_creation_count);
3429 atomic_sub(poison_creation_count, &con->page_retirement_req_cnt);
3430 }
3431 } while (atomic_read(&con->poison_creation_count));
6c23f3d1 3432
e278849c
YC		 3433 if (ret != -EIO) {
3434 msg_count = kfifo_len(&con->poison_fifo);
3435 if (msg_count) {
3436 ret = amdgpu_ras_poison_consumption_handler(adev,
3437 msg_count, &gpu_reset);
3438 if ((ret != -EIO) &&
3439 (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET))
3440 atomic_sub(msg_count, &con->page_retirement_req_cnt);
3441 }
3442 }
f852c979
YC		 3443
3444 if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) {
3445 /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */
3446 /* Clear poison creation request */
3447 atomic_set(&con->poison_creation_count, 0);
3448
3449 /* Clear poison fifo */
3450 amdgpu_ras_clear_poison_fifo(adev);
3451
3452 /* Clear all poison requests */
3453 atomic_set(&con->page_retirement_req_cnt, 0);
3454
3455 if (ret == -EIO) {
3456 /* Wait for mode-1 reset to complete */
3457 down_read(&adev->reset_domain->sem);
3458 up_read(&adev->reset_domain->sem);
3459 }
3460
3461 /* Wake up work to save bad pages to eeprom */
3462 schedule_delayed_work(&con->page_retirement_dwork, 0);
3463 } else if (gpu_reset) {
3464 /* gpu just completed mode-2 reset or other reset */
3465 /* Clear poison consumption messages cached in fifo */
3466 msg_count = kfifo_len(&con->poison_fifo);
3467 if (msg_count) {
3468 amdgpu_ras_clear_poison_fifo(adev);
3469 atomic_sub(msg_count, &con->page_retirement_req_cnt);
3470 }
3471
3472 /* Wake up work to save bad pages to eeprom */
3473 schedule_delayed_work(&con->page_retirement_dwork, 0);
3474 }
3fdcd0a3
YC		 3475 }
3476
3477 return 0;
3478}
3479
b17f8732
LL		 3480int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev)
3481{
3482 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
772df3df 3483 struct amdgpu_ras_eeprom_control *control;
b17f8732
LL		 3484 int ret;
3485
3486 if (!con || amdgpu_sriov_vf(adev))
3487 return 0;
3488
772df3df
TZ		 3489 control = &con->eeprom_control;
3490 ret = amdgpu_ras_eeprom_init(control);
b17f8732
LL		 3491 if (ret)
3492 return ret;
3493
772df3df 3494 if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr)
a8f921a1 3495 control->ras_num_pa_recs = control->ras_num_recs;
772df3df 3496
699bff37
TZ		 3497 if (adev->umc.ras &&
3498 adev->umc.ras->get_retire_flip_bits)
3499 adev->umc.ras->get_retire_flip_bits(adev);
3500
772df3df 3501 if (control->ras_num_recs) {
b17f8732
LL		 3502 ret = amdgpu_ras_load_bad_pages(adev);
3503 if (ret)
3504 return ret;
3505
3506 amdgpu_dpm_send_hbm_bad_pages_num(
ae756cd8 3507 adev, control->ras_num_bad_pages);
b17f8732
LL		 3508
3509 if (con->update_channel_flag == true) {
3510 amdgpu_dpm_send_hbm_bad_channel_flag(
772df3df 3511 adev, control->bad_channel_bitmap);
b17f8732
LL		 3512 con->update_channel_flag = false;
3513 }
05d50ea3
TZ		 3514
3515 /* The format action is only applied to new ASICs */
3516 if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 &&
3517 control->tbl_hdr.version < RAS_TABLE_VER_V3)
3518 if (!amdgpu_ras_eeprom_reset_table(control))
3519 if (amdgpu_ras_save_bad_pages(adev, NULL))
3520 dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n");
b17f8732
LL		 3521 }
3522
3523 return ret;
3524}
3525
3526int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info)
c030f2e4 3527{
3528 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4d1337d2 3529 struct ras_err_handler_data **data;
e4e6a589 3530 u32 max_eeprom_records_count = 0;
78ad00c9 3531 int ret;
c030f2e4 3532
e0e146d5 3533 if (!con || amdgpu_sriov_vf(adev))
1d9d2ca8
LT		 3534 return 0;
3535
3536 /* Allow access to RAS EEPROM via debugfs, when the ASIC
3537 * supports RAS and debugfs is enabled, but when
3538 * adev->ras_enabled is unset, i.e. when "ras_enable"
3539 * module parameter is set to 0.
3540 */
3541 con->adev = adev;
3542
3543 if (!adev->ras_enabled)
4d1337d2
AG		 3544 return 0;
3545
1d9d2ca8 3546 data = &con->eh_data;
091411be 3547 *data = kzalloc(sizeof(**data), GFP_KERNEL);
1a6fc071
TZ		 3548 if (!*data) {
3549 ret = -ENOMEM;
3550 goto out;
3551 }
c030f2e4 3552
3553 mutex_init(&con->recovery_lock);
3554 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
3555 atomic_set(&con->in_recovery, 0);
69691c82 3556 con->eeprom_control.bad_channel_bitmap = 0;
c030f2e4 3557
7f599fed 3558 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control);
e4e6a589 3559 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count);
c84d4670 3560
b17f8732
LL		 3561 if (init_bp_info) {
3562 ret = amdgpu_ras_init_badpage_info(adev);
78ad00c9 3563 if (ret)
1a6fc071 3564 goto free;
78ad00c9 3565 }
c030f2e4 3566
af730e08 3567 mutex_init(&con->page_rsv_lock);
98b5bc87 3568 INIT_KFIFO(con->poison_fifo);
3fdcd0a3
YC		 3569 mutex_init(&con->page_retirement_lock);
3570 init_waitqueue_head(&con->page_retirement_wq);
3571 atomic_set(&con->page_retirement_req_cnt, 0);
5f08275c 3572 atomic_set(&con->poison_creation_count, 0);
3fdcd0a3
YC		 3573 con->page_retirement_thread =
3574 kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement");
3575 if (IS_ERR(con->page_retirement_thread)) {
3576 con->page_retirement_thread = NULL;
3577 dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n");
3578 }
3579
2cf8e50e 3580 INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement);
f493dd64 3581 amdgpu_ras_ecc_log_init(&con->umc_ecc_log);
12b2cab7
MJ		 3582#ifdef CONFIG_X86_MCE_AMD
3583 if ((adev->asic_type == CHIP_ALDEBARAN) &&
3584 (adev->gmc.xgmi.connected_to_cpu))
91a1a52d 3585 amdgpu_register_bad_pages_mca_notifier(adev);
12b2cab7 3586#endif
c030f2e4 3587 return 0;
1a6fc071 3588
1a6fc071 3589free:
1a6fc071 3590 kfree((*data)->bps);
1a6fc071 3591 kfree(*data);
1995b3a3 3592 con->eh_data = NULL;
1a6fc071 3593out:
cf696091 3594 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret);
1a6fc071 3595
b82e65a9
GC		 3596 /*
3597 * Except error threshold exceeding case, other failure cases in this
3598 * function would not fail amdgpu driver init.
3599 */
792be2e2 3600 if (!amdgpu_ras_is_rma(adev))
b82e65a9
GC		 3601 ret = 0;
3602 else
3603 ret = -EINVAL;
3604
1a6fc071 3605 return ret;
c030f2e4 3606}
3607
3608static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
3609{
3610 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3611 struct ras_err_handler_data *data = con->eh_data;
c0470691
YC		 3612 int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES;
3613 bool ret;
c030f2e4 3614
1a6fc071
TZ		 3615 /* recovery_init failed to init it, fini is useless */
3616 if (!data)
3617 return 0;
3618
c0470691
YC		 3619 /* Save all cached bad pages to eeprom */
3620 do {
3621 flush_delayed_work(&con->page_retirement_dwork);
3622 ret = amdgpu_ras_schedule_retirement_dwork(con, 0);
3623 } while (ret && max_flush_timeout--);
3624
3fdcd0a3
YC		 3625 if (con->page_retirement_thread)
3626 kthread_stop(con->page_retirement_thread);
3627
3628 atomic_set(&con->page_retirement_req_cnt, 0);
5f08275c 3629 atomic_set(&con->poison_creation_count, 0);
3fdcd0a3 3630
af730e08
YC		 3631 mutex_destroy(&con->page_rsv_lock);
3632
c030f2e4 3633 cancel_work_sync(&con->recovery_work);
c030f2e4 3634
2cf8e50e
YC		 3635 cancel_delayed_work_sync(&con->page_retirement_dwork);
3636
f493dd64
YC		 3637 amdgpu_ras_ecc_log_fini(&con->umc_ecc_log);
3638
c030f2e4 3639 mutex_lock(&con->recovery_lock);
3640 con->eh_data = NULL;
3641 kfree(data->bps);
3642 kfree(data);
3643 mutex_unlock(&con->recovery_lock);
3644
3645 return 0;
3646}
3647/* recovery end */
3648
084e2640 3649static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
5436ab94 3650{
82835055 3651 if (amdgpu_sriov_vf(adev)) {
4e8303cf 3652 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
82835055 3653 case IP_VERSION(13, 0, 2):
80578f16 3654 case IP_VERSION(13, 0, 6):
9a826c4a 3655 case IP_VERSION(13, 0, 12):
1dbd59f3 3656 case IP_VERSION(13, 0, 14):
82835055
YC		 3657 return true;
3658 default:
3659 return false;
3660 }
3661 }
3662
073285ef 3663 if (adev->asic_type == CHIP_IP_DISCOVERY) {
4e8303cf 3664 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
073285ef 3665 case IP_VERSION(13, 0, 0):
cb906ce3 3666 case IP_VERSION(13, 0, 6):
073285ef 3667 case IP_VERSION(13, 0, 10):
9a826c4a 3668 case IP_VERSION(13, 0, 12):
1dbd59f3 3669 case IP_VERSION(13, 0, 14):
d1ebe307 3670 case IP_VERSION(14, 0, 3):
073285ef
YC		 3671 return true;
3672 default:
3673 return false;
3674 }
3675 }
3676
084e2640
LT		 3677 return adev->asic_type == CHIP_VEGA10 ||
3678 adev->asic_type == CHIP_VEGA20 ||
3679 adev->asic_type == CHIP_ARCTURUS ||
75f06251 3680 adev->asic_type == CHIP_ALDEBARAN ||
084e2640 3681 adev->asic_type == CHIP_SIENNA_CICHLID;
5436ab94
SY		 3682}
3683
f50160cf
SY		 3684/*
3685 * this is workaround for vega20 workstation sku,
3686 * force enable gfx ras, ignore vbios gfx ras flag
3687 * due to GC EDC can not write
3688 */
e509965e 3689static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
f50160cf
SY		 3690{
3691 struct atom_context *ctx = adev->mode_info.atom_context;
3692
3693 if (!ctx)
3694 return;
3695
adf64e21
ML		 3696 if (strnstr(ctx->vbios_pn, "D16406",
3697 sizeof(ctx->vbios_pn)) ||
3698 strnstr(ctx->vbios_pn, "D36002",
3699 sizeof(ctx->vbios_pn)))
8ab0d6f0 3700 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX);
f50160cf
SY		 3701}
3702
4e2965bd
HZ		 3703/* Query ras capablity via atomfirmware interface */
3704static void amdgpu_ras_query_ras_capablity_from_vbios(struct amdgpu_device *adev)
3705{
3706 /* mem_ecc cap */
3707 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
3708 dev_info(adev->dev, "MEM ECC is active.\n");
3709 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC |
3710 1 << AMDGPU_RAS_BLOCK__DF);
3711 } else {
3712 dev_info(adev->dev, "MEM ECC is not presented.\n");
3713 }
3714
3715 /* sram_ecc cap */
3716 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
3717 dev_info(adev->dev, "SRAM ECC is active.\n");
3718 if (!amdgpu_sriov_vf(adev))
3719 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
3720 1 << AMDGPU_RAS_BLOCK__DF);
3721 else
3722 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF |
3723 1 << AMDGPU_RAS_BLOCK__SDMA |
3724 1 << AMDGPU_RAS_BLOCK__GFX);
3725
3726 /*
3727 * VCN/JPEG RAS can be supported on both bare metal and
3728 * SRIOV environment
3729 */
3730 if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(2, 6, 0) ||
3731 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 0) ||
b758667f
MG		 3732 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 3) ||
3733 amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(5, 0, 1))
4e2965bd
HZ		 3734 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN |
3735 1 << AMDGPU_RAS_BLOCK__JPEG);
3736 else
3737 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
3738 1 << AMDGPU_RAS_BLOCK__JPEG);
3739
3740 /*
3741 * XGMI RAS is not supported if xgmi num physical nodes
3742 * is zero
3743 */
3744 if (!adev->gmc.xgmi.num_physical_nodes)
3745 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL);
3746 } else {
3747 dev_info(adev->dev, "SRAM ECC is not presented.\n");
3748 }
3749}
3750
3751/* Query poison mode from umc/df IP callbacks */
3752static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev)
3753{
3754 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3755 bool df_poison, umc_poison;
3756
3757 /* poison setting is useless on SRIOV guest */
3758 if (amdgpu_sriov_vf(adev) || !con)
3759 return;
3760
3761 /* Init poison supported flag, the default value is false */
3762 if (adev->gmc.xgmi.connected_to_cpu ||
3763 adev->gmc.is_app_apu) {
3764 /* enabled by default when GPU is connected to CPU */
3765 con->poison_supported = true;
3766 } else if (adev->df.funcs &&
3767 adev->df.funcs->query_ras_poison_mode &&
3768 adev->umc.ras &&
3769 adev->umc.ras->query_ras_poison_mode) {
3770 df_poison =
3771 adev->df.funcs->query_ras_poison_mode(adev);
3772 umc_poison =
3773 adev->umc.ras->query_ras_poison_mode(adev);
3774
3775 /* Only poison is set in both DF and UMC, we can support it */
3776 if (df_poison && umc_poison)
3777 con->poison_supported = true;
3778 else if (df_poison != umc_poison)
3779 dev_warn(adev->dev,
3780 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n",
3781 df_poison, umc_poison);
3782 }
3783}
3784
5caf466a 3785/*
3786 * check hardware's ras ability which will be saved in hw_supported.
3787 * if hardware does not support ras, we can skip some ras initializtion and
3788 * forbid some ras operations from IP.
3789 * if software itself, say boot parameter, limit the ras ability. We still
3790 * need allow IP do some limited operations, like disable. In such case,
3791 * we have to initialize ras as normal. but need check if operation is
3792 * allowed or not in each function.
3793 */
e509965e 3794static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
c030f2e4 3795{
8ab0d6f0 3796 adev->ras_hw_enabled = adev->ras_enabled = 0;
c030f2e4 3797
38298ce6 3798 if (!amdgpu_ras_asic_supported(adev))
5caf466a 3799 return;
b404ae82 3800
907fec2d
VS		 3801 if (amdgpu_sriov_vf(adev)) {
3802 if (amdgpu_virt_get_ras_capability(adev))
3803 goto init_ras_enabled_flag;
3804 }
3805
4e2965bd
HZ		 3806 /* query ras capability from psp */
3807 if (amdgpu_psp_get_ras_capability(&adev->psp))
3808 goto init_ras_enabled_flag;
58bc2a9c 3809
4e2965bd
HZ		 3810 /* query ras capablity from bios */
3811 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
3812 amdgpu_ras_query_ras_capablity_from_vbios(adev);
75f06251
HZ		 3813 } else {
3814 /* driver only manages a few IP blocks RAS feature
3815 * when GPU is connected cpu through XGMI */
8ab0d6f0 3816 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX |
e509965e
LT		 3817 1 << AMDGPU_RAS_BLOCK__SDMA |
3818 1 << AMDGPU_RAS_BLOCK__MMHUB);
75f06251 3819 }
88474cca 3820
4e2965bd 3821 /* apply asic specific settings (vega20 only for now) */
e509965e 3822 amdgpu_ras_get_quirks(adev);
f50160cf 3823
4e2965bd
HZ		 3824 /* query poison mode from umc/df ip callback */
3825 amdgpu_ras_query_poison_mode(adev);
3826
3827init_ras_enabled_flag:
88474cca 3828 /* hw_supported needs to be aligned with RAS block mask. */
8ab0d6f0 3829 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
b404ae82 3830
66d64e4e
SY		 3831 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
3832 adev->ras_hw_enabled & amdgpu_ras_mask;
04c4fcd2 3833
13c13bdd 3834 /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */
3394069e
VS		 3835 if (!amdgpu_sriov_vf(adev)) {
3836 adev->aca.is_enabled =
3837 (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) ||
3838 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) ||
3839 amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14));
3840 }
c389a060
TZ		 3841
3842 /* bad page feature is not applicable to specific app platform */
3843 if (adev->gmc.is_app_apu &&
3844 amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0))
3845 amdgpu_bad_page_threshold = 0;
c030f2e4 3846}
3847
05adfd80
LT		 3848static void amdgpu_ras_counte_dw(struct work_struct *work)
3849{
3850 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
3851 ras_counte_delay_work.work);
3852 struct amdgpu_device *adev = con->adev;
a3fbb0d8 3853 struct drm_device *dev = adev_to_drm(adev);
05adfd80
LT		 3854 unsigned long ce_count, ue_count;
3855 int res;
3856
3857 res = pm_runtime_get_sync(dev->dev);
3858 if (res < 0)
3859 goto Out;
3860
3861 /* Cache new values.
3862 */
4a1c9a44 3863 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) {
4d9f771e
LT		 3864 atomic_set(&con->ras_ce_count, ce_count);
3865 atomic_set(&con->ras_ue_count, ue_count);
3866 }
05adfd80
LT		 3867
3868 pm_runtime_mark_last_busy(dev->dev);
3869Out:
3870 pm_runtime_put_autosuspend(dev->dev);
3871}
3872
625e5f38
AK		 3873static int amdgpu_get_ras_schema(struct amdgpu_device *adev)
3874{
3875 return amdgpu_ras_is_poison_mode_supported(adev) ? AMDGPU_RAS_ERROR__POISON : 0 |
3876 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE |
3877 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE |
3878 AMDGPU_RAS_ERROR__PARITY;
3879}
3880
9dc57c2a
YW		 3881static void ras_event_mgr_init(struct ras_event_manager *mgr)
3882{
59f488be 3883 struct ras_event_state *event_state;
9dc57c2a
YW		 3884 int i;
3885
75ac6a25
YW		 3886 memset(mgr, 0, sizeof(*mgr));
3887 atomic64_set(&mgr->seqno, 0);
3888
59f488be
YW		 3889 for (i = 0; i < ARRAY_SIZE(mgr->event_state); i++) {
3890 event_state = &mgr->event_state[i];
3891 event_state->last_seqno = RAS_EVENT_INVALID_ID;
3892 atomic64_set(&event_state->count, 0);
3893 }
9dc57c2a
YW		 3894}
3895
3896static void amdgpu_ras_event_mgr_init(struct amdgpu_device *adev)
3897{
3898 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3899 struct amdgpu_hive_info *hive;
3900
3901 if (!ras)
3902 return;
3903
3904 hive = amdgpu_get_xgmi_hive(adev);
3905 ras->event_mgr = hive ? &hive->event_mgr : &ras->__event_mgr;
3906
3907 /* init event manager with node 0 on xgmi system */
e283f4fb 3908 if (!amdgpu_reset_in_recovery(adev)) {
9dc57c2a
YW		 3909 if (!hive || adev->gmc.xgmi.node_id == 0)
3910 ras_event_mgr_init(ras->event_mgr);
3911 }
3912
3913 if (hive)
3914 amdgpu_put_xgmi_hive(hive);
3915}
3916
473af28d
HZ		 3917static void amdgpu_ras_init_reserved_vram_size(struct amdgpu_device *adev)
3918{
3919 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
3920
3921 if (!con || (adev->flags & AMD_IS_APU))
3922 return;
3923
3924 switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) {
3925 case IP_VERSION(13, 0, 2):
3926 case IP_VERSION(13, 0, 6):
9a826c4a 3927 case IP_VERSION(13, 0, 12):
16b85a09
HZ		 3928 con->reserved_pages_in_bytes = AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT;
3929 break;
473af28d 3930 case IP_VERSION(13, 0, 14):
16b85a09 3931 con->reserved_pages_in_bytes = (AMDGPU_RAS_RESERVED_VRAM_SIZE_DEFAULT << 1);
473af28d
HZ		 3932 break;
3933 default:
3934 break;
3935 }
3936}
3937
c030f2e4 3938int amdgpu_ras_init(struct amdgpu_device *adev)
3939{
3940 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4e644fff 3941 int r;
c030f2e4 3942
b404ae82 3943 if (con)
c030f2e4 3944 return 0;
3945
091411be 3946 con = kzalloc(sizeof(*con) +
640ae42e
JC		 3947 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT +
3948 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT,
091411be 3949 GFP_KERNEL);
c030f2e4 3950 if (!con)
3951 return -ENOMEM;
3952
05adfd80
LT		 3953 con->adev = adev;
3954 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw);
3955 atomic_set(&con->ras_ce_count, 0);
3956 atomic_set(&con->ras_ue_count, 0);
3957
c030f2e4 3958 con->objs = (struct ras_manager *)(con + 1);
3959
3960 amdgpu_ras_set_context(adev, con);
3961
e509965e
LT		 3962 amdgpu_ras_check_supported(adev);
3963
7ddd9770 3964 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) {
970fd197
SY		 3965 /* set gfx block ras context feature for VEGA20 Gaming
3966 * send ras disable cmd to ras ta during ras late init.
3967 */
8ab0d6f0 3968 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
970fd197
SY		 3969 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX);
3970
3971 return 0;
3972 }
3973
5e91160a 3974 r = 0;
5436ab94 3975 goto release_con;
fb2a3607
HZ		 3976 }
3977
69691c82 3978 con->update_channel_flag = false;
c030f2e4 3979 con->features = 0;
625e5f38 3980 con->schema = 0;
c030f2e4 3981 INIT_LIST_HEAD(&con->head);
108c6a63 3982 /* Might need get this flag from vbios. */
3983 con->flags = RAS_DEFAULT_FLAGS;
c030f2e4 3984
6e36f231
HZ		 3985 /* initialize nbio ras function ahead of any other
3986 * ras functions so hardware fatal error interrupt
3987 * can be enabled as early as possible */
4e8303cf 3988 switch (amdgpu_ip_version(adev, NBIO_HWIP, 0)) {
fdc94d3a
HZ		 3989 case IP_VERSION(7, 4, 0):
3990 case IP_VERSION(7, 4, 1):
3991 case IP_VERSION(7, 4, 4):
3992 if (!adev->gmc.xgmi.connected_to_cpu)
2e54fe5d 3993 adev->nbio.ras = &nbio_v7_4_ras;
6e36f231 3994 break;
9af357bc
HZ		 3995 case IP_VERSION(4, 3, 0):
3996 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
3997 /* unlike other generation of nbio ras,
3998 * nbio v4_3 only support fatal error interrupt
3999 * to inform software that DF is freezed due to
4000 * system fatal error event. driver should not
4001 * enable nbio ras in such case. Instead,
4002 * check DF RAS */
4003 adev->nbio.ras = &nbio_v4_3_ras;
4004 break;
ecd1191e
CL		 4005 case IP_VERSION(6, 3, 1):
4006 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
4007 /* unlike other generation of nbio ras,
4008 * nbif v6_3_1 only support fatal error interrupt
4009 * to inform software that DF is freezed due to
4010 * system fatal error event. driver should not
4011 * enable nbio ras in such case. Instead,
4012 * check DF RAS
4013 */
4014 adev->nbio.ras = &nbif_v6_3_1_ras;
4015 break;
7692e1ee 4016 case IP_VERSION(7, 9, 0):
9a826c4a 4017 case IP_VERSION(7, 9, 1):
7692e1ee
TZ		 4018 if (!adev->gmc.is_app_apu)
4019 adev->nbio.ras = &nbio_v7_9_ras;
4020 break;
6e36f231
HZ		 4021 default:
4022 /* nbio ras is not available */
4023 break;
4024 }
4025
fdc94d3a
HZ		 4026 /* nbio ras block needs to be enabled ahead of other ras blocks
4027 * to handle fatal error */
4028 r = amdgpu_nbio_ras_sw_init(adev);
4029 if (r)
4030 return r;
4031
2e54fe5d 4032 if (adev->nbio.ras &&
4033 adev->nbio.ras->init_ras_controller_interrupt) {
4034 r = adev->nbio.ras->init_ras_controller_interrupt(adev);
4e644fff 4035 if (r)
5436ab94 4036 goto release_con;
4e644fff
HZ		 4037 }
4038
2e54fe5d 4039 if (adev->nbio.ras &&
4040 adev->nbio.ras->init_ras_err_event_athub_interrupt) {
4041 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev);
4e644fff 4042 if (r)
5436ab94 4043 goto release_con;
4e644fff
HZ		 4044 }
4045
73cb81dc
HZ		 4046 /* Packed socket_id to ras feature mask bits[31:29] */
4047 if (adev->smuio.funcs &&
4048 adev->smuio.funcs->get_socket_id)
ee9c3031
SY		 4049 con->features |= ((adev->smuio.funcs->get_socket_id(adev)) <<
4050 AMDGPU_RAS_FEATURES_SOCKETID_SHIFT);
73cb81dc 4051
625e5f38
AK		 4052 /* Get RAS schema for particular SOC */
4053 con->schema = amdgpu_get_ras_schema(adev);
4054
473af28d
HZ		 4055 amdgpu_ras_init_reserved_vram_size(adev);
4056
5e91160a
GC		 4057 if (amdgpu_ras_fs_init(adev)) {
4058 r = -EINVAL;
5436ab94 4059 goto release_con;
5e91160a 4060 }
c030f2e4 4061
9817f061
YW		 4062 if (amdgpu_ras_aca_is_supported(adev)) {
4063 if (amdgpu_aca_is_enabled(adev))
4064 r = amdgpu_aca_init(adev);
4065 else
4066 r = amdgpu_mca_init(adev);
4067 if (r)
4068 goto release_con;
4069 }
4070
6952e99c 4071 dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
e509965e 4072 "hardware ability[%x] ras_mask[%x]\n",
8ab0d6f0 4073 adev->ras_hw_enabled, adev->ras_enabled);
e509965e 4074
c030f2e4 4075 return 0;
5436ab94 4076release_con:
c030f2e4 4077 amdgpu_ras_set_context(adev, NULL);
4078 kfree(con);
4079
5e91160a 4080 return r;
c030f2e4 4081}
4082
8f6368a9 4083int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev)
134d16d5 4084{
8107e499
HZ		 4085 if (adev->gmc.xgmi.connected_to_cpu ||
4086 adev->gmc.is_app_apu)
134d16d5
JC		 4087 return 1;
4088 return 0;
4089}
4090
4091static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev,
4092 struct ras_common_if *ras_block)
4093{
4094 struct ras_query_if info = {
4095 .head = *ras_block,
4096 };
4097
4098 if (!amdgpu_persistent_edc_harvesting_supported(adev))
4099 return 0;
4100
4101 if (amdgpu_ras_query_error_status(adev, &info) != 0)
4102 DRM_WARN("RAS init harvest failure");
4103
4104 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0)
4105 DRM_WARN("RAS init harvest reset failure");
4106
4107 return 0;
4108}
4109
e4348849
TZ		 4110bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev)
4111{
4112 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4113
4114 if (!con)
4115 return false;
4116
4117 return con->poison_supported;
4118}
4119
b293e891 4120/* helper function to handle common stuff in ip late init phase */
563285c8 4121int amdgpu_ras_block_late_init(struct amdgpu_device *adev,
4122 struct ras_common_if *ras_block)
b293e891 4123{
29c9b6cd 4124 struct amdgpu_ras_block_object *ras_obj = NULL;
05adfd80 4125 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4a1c9a44 4126 struct ras_query_if *query_info;
05adfd80 4127 unsigned long ue_count, ce_count;
b293e891
HZ		 4128 int r;
4129
4130 /* disable RAS feature per IP block if it is not supported */
4131 if (!amdgpu_ras_is_supported(adev, ras_block->block)) {
4132 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
4133 return 0;
4134 }
4135
4136 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1);
4137 if (r) {
e283f4fb 4138 if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) {
b293e891
HZ		 4139 /* in resume phase, if fail to enable ras,
4140 * clean up all ras fs nodes, and disable ras */
4141 goto cleanup;
4142 } else
4143 return r;
4144 }
4145
134d16d5
JC		 4146 /* check for errors on warm reset edc persisant supported ASIC */
4147 amdgpu_persistent_edc_harvesting(adev, ras_block);
4148
b293e891 4149 /* in resume phase, no need to create ras fs node */
e283f4fb 4150 if (adev->in_suspend || amdgpu_reset_in_recovery(adev))
b293e891
HZ		 4151 return 0;
4152
563285c8 4153 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
36780606
TZ		 4154 if (ras_obj->ras_cb || (ras_obj->hw_ops &&
4155 (ras_obj->hw_ops->query_poison_status ||
4156 ras_obj->hw_ops->handle_poison_consumption))) {
9252d33d 4157 r = amdgpu_ras_interrupt_add_handler(adev, ras_block);
b293e891 4158 if (r)
779596ce 4159 goto cleanup;
b293e891
HZ		 4160 }
4161
f957138c
HZ		 4162 if (ras_obj->hw_ops &&
4163 (ras_obj->hw_ops->query_ras_error_count ||
4164 ras_obj->hw_ops->query_ras_error_status)) {
4165 r = amdgpu_ras_sysfs_create(adev, ras_block);
4166 if (r)
4167 goto interrupt;
b293e891 4168
f957138c
HZ		 4169 /* Those are the cached values at init.
4170 */
4171 query_info = kzalloc(sizeof(*query_info), GFP_KERNEL);
4172 if (!query_info)
4173 return -ENOMEM;
4174 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if));
4a1c9a44 4175
f957138c
HZ		 4176 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) {
4177 atomic_set(&con->ras_ce_count, ce_count);
4178 atomic_set(&con->ras_ue_count, ue_count);
4179 }
4180
4181 kfree(query_info);
4d9f771e 4182 }
05adfd80 4183
b293e891 4184 return 0;
779596ce
TR		 4185
4186interrupt:
563285c8 4187 if (ras_obj->ras_cb)
9252d33d 4188 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
779596ce 4189cleanup:
b293e891
HZ		 4190 amdgpu_ras_feature_enable(adev, ras_block, 0);
4191 return r;
4192}
4193
d41ff22a 4194static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev,
418abce2 4195 struct ras_common_if *ras_block)
4196{
4197 return amdgpu_ras_block_late_init(adev, ras_block);
4198}
4199
b293e891 4200/* helper function to remove ras fs node and interrupt handler */
bdb3489c 4201void amdgpu_ras_block_late_fini(struct amdgpu_device *adev,
4202 struct ras_common_if *ras_block)
4203{
563285c8 4204 struct amdgpu_ras_block_object *ras_obj;
bdb3489c 4205 if (!ras_block)
4206 return;
4207
563285c8 4208 amdgpu_ras_sysfs_remove(adev, ras_block);
bdb3489c 4209
563285c8 4210 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
4211 if (ras_obj->ras_cb)
4212 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
bdb3489c 4213}
4214
80e0c2cb 4215static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev,
4216 struct ras_common_if *ras_block)
4217{
4218 return amdgpu_ras_block_late_fini(adev, ras_block);
4219}
4220
a564808e 4221/* do some init work after IP late init as dependence.
511fdbc3 4222 * and it runs in resume/gpu reset/booting up cases.
a564808e 4223 */
511fdbc3 4224void amdgpu_ras_resume(struct amdgpu_device *adev)
108c6a63 4225{
4226 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4227 struct ras_manager *obj, *tmp;
4228
8ab0d6f0 4229 if (!adev->ras_enabled || !con) {
970fd197
SY		 4230 /* clean ras context for VEGA20 Gaming after send ras disable cmd */
4231 amdgpu_release_ras_context(adev);
4232
108c6a63 4233 return;
970fd197 4234 }
108c6a63 4235
108c6a63 4236 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
191051a1 4237 /* Set up all other IPs which are not implemented. There is a
4238 * tricky thing that IP's actual ras error type should be
4239 * MULTI_UNCORRECTABLE, but as driver does not handle it, so
4240 * ERROR_NONE make sense anyway.
4241 */
4242 amdgpu_ras_enable_all_features(adev, 1);
4243
4244 /* We enable ras on all hw_supported block, but as boot
4245 * parameter might disable some of them and one or more IP has
4246 * not implemented yet. So we disable them on behalf.
4247 */
108c6a63 4248 list_for_each_entry_safe(obj, tmp, &con->head, node) {
4249 if (!amdgpu_ras_is_supported(adev, obj->head.block)) {
4250 amdgpu_ras_feature_enable(adev, &obj->head, 0);
4251 /* there should be no any reference. */
4252 WARN_ON(alive_obj(obj));
4253 }
191051a1 4254 }
108c6a63 4255 }
4256}
4257
511fdbc3 4258void amdgpu_ras_suspend(struct amdgpu_device *adev)
4259{
4260 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4261
8ab0d6f0 4262 if (!adev->ras_enabled || !con)
511fdbc3 4263 return;
4264
4265 amdgpu_ras_disable_all_features(adev, 0);
4266 /* Make sure all ras objects are disabled. */
ee9c3031 4267 if (AMDGPU_RAS_GET_FEATURES(con->features))
511fdbc3 4268 amdgpu_ras_disable_all_features(adev, 1);
4269}
4270
867e24ca 4271int amdgpu_ras_late_init(struct amdgpu_device *adev)
4272{
4273 struct amdgpu_ras_block_list *node, *tmp;
4274 struct amdgpu_ras_block_object *obj;
4275 int r;
4276
9dc57c2a
YW		 4277 amdgpu_ras_event_mgr_init(adev);
4278
9817f061 4279 if (amdgpu_ras_aca_is_supported(adev)) {
e283f4fb 4280 if (amdgpu_reset_in_recovery(adev)) {
9817f061
YW		 4281 if (amdgpu_aca_is_enabled(adev))
4282 r = amdgpu_aca_reset(adev);
4283 else
4284 r = amdgpu_mca_reset(adev);
062a7ce6
YW		 4285 if (r)
4286 return r;
4287 }
c0c48f0d 4288
9817f061
YW		 4289 if (!amdgpu_sriov_vf(adev)) {
4290 if (amdgpu_aca_is_enabled(adev))
4291 amdgpu_ras_set_aca_debug_mode(adev, false);
4292 else
4293 amdgpu_ras_set_mca_debug_mode(adev, false);
4294 }
c0c48f0d 4295 }
201761b5 4296
329cec8f 4297 /* Guest side doesn't need init ras feature */
84a2947e 4298 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev))
329cec8f
YW		 4299 return 0;
4300
867e24ca 4301 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
2866a454
YW		 4302 obj = node->ras_obj;
4303 if (!obj) {
867e24ca 4304 dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
4305 continue;
4306 }
418abce2 4307
2866a454
YW		 4308 if (!amdgpu_ras_is_supported(adev, obj->ras_comm.block))
4309 continue;
4310
867e24ca 4311 if (obj->ras_late_init) {
4312 r = obj->ras_late_init(adev, &obj->ras_comm);
4313 if (r) {
4314 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n",
4315 obj->ras_comm.name, r);
4316 return r;
4317 }
418abce2 4318 } else
4319 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm);
867e24ca 4320 }
4321
4322 return 0;
4323}
4324
c030f2e4 4325/* do some fini work before IP fini as dependence */
4326int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
4327{
4328 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4329
8ab0d6f0 4330 if (!adev->ras_enabled || !con)
c030f2e4 4331 return 0;
4332
72c8c97b 4333
c030f2e4 4334 /* Need disable ras on all IPs here before ip [hw/sw]fini */
ee9c3031 4335 if (AMDGPU_RAS_GET_FEATURES(con->features))
642c0401 4336 amdgpu_ras_disable_all_features(adev, 0);
c030f2e4 4337 amdgpu_ras_recovery_fini(adev);
4338 return 0;
4339}
4340
4341int amdgpu_ras_fini(struct amdgpu_device *adev)
4342{
d5e8ff5f 4343 struct amdgpu_ras_block_list *ras_node, *tmp;
1f211a82 4344 struct amdgpu_ras_block_object *obj = NULL;
c030f2e4 4345 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4346
8ab0d6f0 4347 if (!adev->ras_enabled || !con)
c030f2e4 4348 return 0;
4349
1f211a82 4350 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) {
4351 if (ras_node->ras_obj) {
4352 obj = ras_node->ras_obj;
4353 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) &&
4354 obj->ras_fini)
4355 obj->ras_fini(adev, &obj->ras_comm);
80e0c2cb 4356 else
4357 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm);
1f211a82 4358 }
4359
4360 /* Clear ras blocks from ras_list and free ras block list node */
4361 list_del(&ras_node->node);
4362 kfree(ras_node);
4363 }
4364
c030f2e4 4365 amdgpu_ras_fs_fini(adev);
4366 amdgpu_ras_interrupt_remove_all(adev);
4367
9817f061
YW		 4368 if (amdgpu_ras_aca_is_supported(adev)) {
4369 if (amdgpu_aca_is_enabled(adev))
4370 amdgpu_aca_fini(adev);
4371 else
4372 amdgpu_mca_fini(adev);
4373 }
c0c48f0d 4374
ee9c3031 4375 WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared");
c030f2e4 4376
ee9c3031 4377 if (AMDGPU_RAS_GET_FEATURES(con->features))
edfdde90 4378 amdgpu_ras_disable_all_features(adev, 0);
c030f2e4 4379
05adfd80
LT		 4380 cancel_delayed_work_sync(&con->ras_counte_delay_work);
4381
c030f2e4 4382 amdgpu_ras_set_context(adev, NULL);
4383 kfree(con);
4384
4385 return 0;
4386}
7c6e68c7 4387
1b6ef74b
LL		 4388bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev)
4389{
4390 struct amdgpu_ras *ras;
4391
4392 ras = amdgpu_ras_get_context(adev);
4393 if (!ras)
4394 return false;
4395
e1ee2111 4396 return test_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
1b6ef74b
LL		 4397}
4398
4399void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status)
4400{
4401 struct amdgpu_ras *ras;
4402
e1ee2111
LL		 4403 ras = amdgpu_ras_get_context(adev);
4404 if (ras) {
4405 if (status)
4406 set_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4407 else
4408 clear_bit(AMDGPU_RAS_BLOCK__LAST, &ras->ras_err_state);
4409 }
4410}
4411
4412void amdgpu_ras_clear_err_state(struct amdgpu_device *adev)
4413{
4414 struct amdgpu_ras *ras;
4415
1b6ef74b
LL		 4416 ras = amdgpu_ras_get_context(adev);
4417 if (ras)
e1ee2111
LL		 4418 ras->ras_err_state = 0;
4419}
4420
4421void amdgpu_ras_set_err_poison(struct amdgpu_device *adev,
4422 enum amdgpu_ras_block block)
4423{
4424 struct amdgpu_ras *ras;
4425
4426 ras = amdgpu_ras_get_context(adev);
4427 if (ras)
4428 set_bit(block, &ras->ras_err_state);
4429}
4430
4431bool amdgpu_ras_is_err_state(struct amdgpu_device *adev, int block)
4432{
4433 struct amdgpu_ras *ras;
4434
4435 ras = amdgpu_ras_get_context(adev);
4436 if (ras) {
4437 if (block == AMDGPU_RAS_BLOCK__ANY)
4438 return (ras->ras_err_state != 0);
4439 else
4440 return test_bit(block, &ras->ras_err_state) ||
4441 test_bit(AMDGPU_RAS_BLOCK__LAST,
4442 &ras->ras_err_state);
4443 }
4444
4445 return false;
1b6ef74b
LL		 4446}
4447
75ac6a25 4448static struct ras_event_manager *__get_ras_event_mgr(struct amdgpu_device *adev)
9dc57c2a 4449{
75ac6a25
YW		 4450 struct amdgpu_ras *ras;
4451
4452 ras = amdgpu_ras_get_context(adev);
4453 if (!ras)
4454 return NULL;
4455
4456 return ras->event_mgr;
4457}
4458
4459int amdgpu_ras_mark_ras_event_caller(struct amdgpu_device *adev, enum ras_event_type type,
4460 const void *caller)
4461{
4462 struct ras_event_manager *event_mgr;
59f488be 4463 struct ras_event_state *event_state;
75ac6a25
YW		 4464 int ret = 0;
4465
4466 if (type >= RAS_EVENT_TYPE_COUNT) {
4467 ret = -EINVAL;
4468 goto out;
4469 }
4470
4471 event_mgr = __get_ras_event_mgr(adev);
4472 if (!event_mgr) {
4473 ret = -EINVAL;
4474 goto out;
4475 }
4476
59f488be
YW		 4477 event_state = &event_mgr->event_state[type];
4478 event_state->last_seqno = atomic64_inc_return(&event_mgr->seqno);
4479 atomic64_inc(&event_state->count);
75ac6a25
YW		 4480
4481out:
4482 if (ret && caller)
4483 dev_warn(adev->dev, "failed mark ras event (%d) in %ps, ret:%d\n",
4484 (int)type, caller, ret);
4485
4486 return ret;
9dc57c2a
YW		 4487}
4488
4489u64 amdgpu_ras_acquire_event_id(struct amdgpu_device *adev, enum ras_event_type type)
4490{
75ac6a25 4491 struct ras_event_manager *event_mgr;
9dc57c2a
YW		 4492 u64 id;
4493
75ac6a25
YW		 4494 if (type >= RAS_EVENT_TYPE_COUNT)
4495 return RAS_EVENT_INVALID_ID;
4496
9dc57c2a 4497 switch (type) {
75ac6a25 4498 case RAS_EVENT_TYPE_FATAL:
5b9de259 4499 case RAS_EVENT_TYPE_POISON_CREATION:
12b435a4 4500 case RAS_EVENT_TYPE_POISON_CONSUMPTION:
75ac6a25
YW		 4501 event_mgr = __get_ras_event_mgr(adev);
4502 if (!event_mgr)
4503 return RAS_EVENT_INVALID_ID;
4504
59f488be 4505 id = event_mgr->event_state[type].last_seqno;
9dc57c2a
YW		 4506 break;
4507 case RAS_EVENT_TYPE_INVALID:
4508 default:
75ac6a25 4509 id = RAS_EVENT_INVALID_ID;
9dc57c2a
YW		 4510 break;
4511 }
4512
4513 return id;
4514}
4515
7c6e68c7
AG		 4516void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
4517{
4518 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
2c7cd280 4519 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
75ac6a25
YW		 4520 enum ras_event_type type = RAS_EVENT_TYPE_FATAL;
4521 u64 event_id;
4522
937467b7
LL		 4523 if (amdgpu_ras_mark_ras_event(adev, type)) {
4524 dev_err(adev->dev,
4525 "uncorrectable hardware error (ERREVENT_ATHUB_INTERRUPT) detected!\n");
75ac6a25 4526 return;
937467b7 4527 }
75ac6a25
YW		 4528
4529 event_id = amdgpu_ras_acquire_event_id(adev, type);
2c7cd280 4530
9dc57c2a
YW		 4531 RAS_EVENT_LOG(adev, event_id, "uncorrectable hardware error"
4532 "(ERREVENT_ATHUB_INTERRUPT) detected!\n");
d5ea093e 4533
b41f742d 4534 amdgpu_ras_set_fed(adev, true);
2c7cd280 4535 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
61934624 4536 amdgpu_ras_reset_gpu(adev);
7c6e68c7
AG		 4537 }
4538}
bb5c7235
WS		 4539
4540bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev)
4541{
4542 if (adev->asic_type == CHIP_VEGA20 &&
4543 adev->pm.fw_version <= 0x283400) {
4544 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) &&
4545 amdgpu_ras_intr_triggered();
4546 }
4547
4548 return false;
4549}
970fd197
SY		 4550
4551void amdgpu_release_ras_context(struct amdgpu_device *adev)
4552{
4553 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4554
4555 if (!con)
4556 return;
4557
8ab0d6f0 4558 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
970fd197
SY		 4559 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
4560 amdgpu_ras_set_context(adev, NULL);
4561 kfree(con);
4562 }
4563}
12b2cab7
MJ		 4564
4565#ifdef CONFIG_X86_MCE_AMD
4566static struct amdgpu_device *find_adev(uint32_t node_id)
4567{
12b2cab7
MJ		 4568 int i;
4569 struct amdgpu_device *adev = NULL;
4570
91a1a52d
MJ		 4571 for (i = 0; i < mce_adev_list.num_gpu; i++) {
4572 adev = mce_adev_list.devs[i];
12b2cab7 4573
91a1a52d 4574 if (adev && adev->gmc.xgmi.connected_to_cpu &&
12b2cab7
MJ		 4575 adev->gmc.xgmi.physical_node_id == node_id)
4576 break;
4577 adev = NULL;
4578 }
4579
12b2cab7
MJ		 4580 return adev;
4581}
4582
4583#define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF)
4584#define GET_UMC_INST(m) (((m) >> 21) & 0x7)
4585#define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4))
4586#define GPU_ID_OFFSET 8
4587
4588static int amdgpu_bad_page_notifier(struct notifier_block *nb,
4589 unsigned long val, void *data)
4590{
4591 struct mce *m = (struct mce *)data;
4592 struct amdgpu_device *adev = NULL;
4593 uint32_t gpu_id = 0;
cd4c99f1 4594 uint32_t umc_inst = 0, ch_inst = 0;
12b2cab7
MJ		 4595
4596 /*
4597 * If the error was generated in UMC_V2, which belongs to GPU UMCs,
4598 * and error occurred in DramECC (Extended error code = 0) then only
4599 * process the error, else bail out.
4600 */
91f75eb4 4601 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) &&
12b2cab7
MJ		 4602 (XEC(m->status, 0x3f) == 0x0)))
4603 return NOTIFY_DONE;
4604
4605 /*
4606 * If it is correctable error, return.
4607 */
4608 if (mce_is_correctable(m))
4609 return NOTIFY_OK;
4610
4611 /*
4612 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
4613 */
4614 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET;
4615
4616 adev = find_adev(gpu_id);
4617 if (!adev) {
4618 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__,
4619 gpu_id);
4620 return NOTIFY_DONE;
4621 }
4622
4623 /*
4624 * If it is uncorrectable error, then find out UMC instance and
4625 * channel index.
4626 */
4627 umc_inst = GET_UMC_INST(m->ipid);
4628 ch_inst = GET_CHAN_INDEX(m->ipid);
4629
4630 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d",
4631 umc_inst, ch_inst);
4632
24b82292
TZ		 4633 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst))
4634 return NOTIFY_OK;
4635 else
6c0ca748 4636 return NOTIFY_DONE;
12b2cab7
MJ		 4637}
4638
4639static struct notifier_block amdgpu_bad_page_nb = {
4640 .notifier_call = amdgpu_bad_page_notifier,
4641 .priority = MCE_PRIO_UC,
4642};
4643
91a1a52d 4644static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev)
12b2cab7 4645{
91a1a52d
MJ		 4646 /*
4647 * Add the adev to the mce_adev_list.
4648 * During mode2 reset, amdgpu device is temporarily
4649 * removed from the mgpu_info list which can cause
4650 * page retirement to fail.
4651 * Use this list instead of mgpu_info to find the amdgpu
4652 * device on which the UMC error was reported.
4653 */
4654 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev;
4655
12b2cab7
MJ		 4656 /*
4657 * Register the x86 notifier only once
4658 * with MCE subsystem.
4659 */
4660 if (notifier_registered == false) {
4661 mce_register_decode_chain(&amdgpu_bad_page_nb);
4662 notifier_registered = true;
4663 }
4664}
4665#endif
7cab2124 4666
b6efdb02 4667struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev)
7cab2124 4668{
4669 if (!adev)
4670 return NULL;
4671
4672 return adev->psp.ras_context.ras;
4673}
4674
b6efdb02 4675int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con)
7cab2124 4676{
4677 if (!adev)
69f91d32 4678 return -EINVAL;
7cab2124 4679
4680 adev->psp.ras_context.ras = ras_con;
4681 return 0;
4682}
4683
4684/* check if ras is supported on block, say, sdma, gfx */
4685int amdgpu_ras_is_supported(struct amdgpu_device *adev,
4686 unsigned int block)
4687{
8f453c51 4688 int ret = 0;
7cab2124 4689 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4690
4691 if (block >= AMDGPU_RAS_BLOCK_COUNT)
4692 return 0;
8f453c51
YC		 4693
4694 ret = ras && (adev->ras_enabled & (1 << block));
4695
4696 /* For the special asic with mem ecc enabled but sram ecc
4697 * not enabled, even if the ras block is not supported on
4698 * .ras_enabled, if the asic supports poison mode and the
4699 * ras block has ras configuration, it can be considered
4700 * that the ras block supports ras function.
4701 */
4702 if (!ret &&
bc0f8080
CL		 4703 (block == AMDGPU_RAS_BLOCK__GFX ||
4704 block == AMDGPU_RAS_BLOCK__SDMA ||
4705 block == AMDGPU_RAS_BLOCK__VCN ||
4706 block == AMDGPU_RAS_BLOCK__JPEG) &&
7ec11c2f 4707 (amdgpu_ras_mask & (1 << block)) &&
8f453c51
YC		 4708 amdgpu_ras_is_poison_mode_supported(adev) &&
4709 amdgpu_ras_get_ras_block(adev, block, 0))
4710 ret = 1;
4711
4712 return ret;
7cab2124 4713}
4714
4715int amdgpu_ras_reset_gpu(struct amdgpu_device *adev)
4716{
4717 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
4718
5f7697bb 4719 /* mode1 is the only selection for RMA status */
792be2e2 4720 if (amdgpu_ras_is_rma(adev)) {
5f7697bb
TZ		 4721 ras->gpu_reset_flags = 0;
4722 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
4723 }
4724
9e0feb79
YC		 4725 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0) {
4726 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
4727 int hive_ras_recovery = 0;
4728
4729 if (hive) {
4730 hive_ras_recovery = atomic_read(&hive->ras_recovery);
4731 amdgpu_put_xgmi_hive(hive);
4732 }
4733 /* In the case of multiple GPUs, after a GPU has started
4734 * resetting all GPUs on hive, other GPUs do not need to
4735 * trigger GPU reset again.
4736 */
4737 if (!hive_ras_recovery)
4738 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
4739 else
4740 atomic_set(&ras->in_recovery, 0);
4741 } else {
4742 flush_work(&ras->recovery_work);
25a2b22e 4743 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
9e0feb79
YC		 4744 }
4745
7cab2124 4746 return 0;
4747}
4748
201761b5 4749int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable)
8096df76
TZ		 4750{
4751 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
201761b5 4752 int ret = 0;
8096df76 4753
201761b5
LL		 4754 if (con) {
4755 ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
4756 if (!ret)
04c4fcd2 4757 con->is_aca_debug_mode = enable;
201761b5
LL		 4758 }
4759
4760 return ret;
8096df76
TZ		 4761}
4762
33dcda51
YW		 4763int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable)
4764{
4765 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4766 int ret = 0;
4767
4768 if (con) {
04c4fcd2
YW		 4769 if (amdgpu_aca_is_enabled(adev))
4770 ret = amdgpu_aca_smu_set_debug_mode(adev, enable);
4771 else
4772 ret = amdgpu_mca_smu_set_debug_mode(adev, enable);
33dcda51 4773 if (!ret)
04c4fcd2 4774 con->is_aca_debug_mode = enable;
33dcda51
YW		 4775 }
4776
4777 return ret;
4778}
4779
04c4fcd2 4780bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev)
8096df76
TZ		 4781{
4782 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
04c4fcd2 4783 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
8096df76
TZ		 4784 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
4785
4786 if (!con)
4787 return false;
4788
04c4fcd2
YW		 4789 if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) ||
4790 (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode))
4791 return con->is_aca_debug_mode;
8096df76
TZ		 4792 else
4793 return true;
4794}
7cab2124 4795
8cc0f566
HZ		 4796bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev,
4797 unsigned int *error_query_mode)
4798{
4799 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4800 const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs;
04c4fcd2 4801 const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs;
8cc0f566
HZ		 4802
4803 if (!con) {
4804 *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY;
4805 return false;
4806 }
4807
84a2947e
VS		 4808 if (amdgpu_sriov_vf(adev)) {
4809 *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY;
4810 } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) {
8cc0f566 4811 *error_query_mode =
04c4fcd2 4812 (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY;
84a2947e 4813 } else {
8cc0f566 4814 *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY;
84a2947e 4815 }
8cc0f566
HZ		 4816
4817 return true;
4818}
4819
6492e1b0 4820/* Register each ip ras block into amdgpu ras */
4821int amdgpu_ras_register_ras_block(struct amdgpu_device *adev,
b6efdb02 4822 struct amdgpu_ras_block_object *ras_block_obj)
6492e1b0 4823{
d5e8ff5f 4824 struct amdgpu_ras_block_list *ras_node;
6492e1b0 4825 if (!adev || !ras_block_obj)
4826 return -EINVAL;
4827
d5e8ff5f 4828 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL);
4829 if (!ras_node)
4830 return -ENOMEM;
4831
4832 INIT_LIST_HEAD(&ras_node->node);
4833 ras_node->ras_obj = ras_block_obj;
4834 list_add_tail(&ras_node->node, &adev->ras_list);
6492e1b0 4835
4836 return 0;
4837}
322a7e00
HZ		 4838
4839void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name)
4840{
4841 if (!err_type_name)
4842 return;
4843
4844 switch (err_type) {
4845 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE:
4846 sprintf(err_type_name, "correctable");
4847 break;
4848 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE:
4849 sprintf(err_type_name, "uncorrectable");
4850 break;
4851 default:
4852 sprintf(err_type_name, "unknown");
4853 break;
4854 }
4855}
4856
4857bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev,
4858 const struct amdgpu_ras_err_status_reg_entry *reg_entry,
4859 uint32_t instance,
4860 uint32_t *memory_id)
4861{
4862 uint32_t err_status_lo_data, err_status_lo_offset;
4863
4864 if (!reg_entry)
4865 return false;
4866
4867 err_status_lo_offset =
4868 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
4869 reg_entry->seg_lo, reg_entry->reg_lo);
4870 err_status_lo_data = RREG32(err_status_lo_offset);
4871
4872 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) &&
4873 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG))
4874 return false;
4875
4876 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID);
4877
4878 return true;
4879}
4880
4881bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev,
4882 const struct amdgpu_ras_err_status_reg_entry *reg_entry,
4883 uint32_t instance,
4884 unsigned long *err_cnt)
4885{
4886 uint32_t err_status_hi_data, err_status_hi_offset;
4887
4888 if (!reg_entry)
4889 return false;
4890
4891 err_status_hi_offset =
4892 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
4893 reg_entry->seg_hi, reg_entry->reg_hi);
4894 err_status_hi_data = RREG32(err_status_hi_offset);
4895
4896 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) &&
4897 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG))
9b337b7d
HZ		 4898 /* keep the check here in case we need to refer to the result later */
4899 dev_dbg(adev->dev, "Invalid err_info field\n");
322a7e00
HZ		 4900
4901 /* read err count */
4902 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT);
4903
4904 return true;
4905}
4906
4907void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev,
4908 const struct amdgpu_ras_err_status_reg_entry *reg_list,
4909 uint32_t reg_list_size,
4910 const struct amdgpu_ras_memory_id_entry *mem_list,
4911 uint32_t mem_list_size,
4912 uint32_t instance,
4913 uint32_t err_type,
4914 unsigned long *err_count)
4915{
4916 uint32_t memory_id;
4917 unsigned long err_cnt;
4918 char err_type_name[16];
4919 uint32_t i, j;
4920
4921 for (i = 0; i < reg_list_size; i++) {
9b337b7d
HZ		 4922 /* query memory_id from err_status_lo */
4923 if (!amdgpu_ras_inst_get_memory_id_field(adev, &reg_list[i],
4924 instance, &memory_id))
4925 continue;
4926
322a7e00
HZ		 4927 /* query err_cnt from err_status_hi */
4928 if (!amdgpu_ras_inst_get_err_cnt_field(adev, &reg_list[i],
4929 instance, &err_cnt) ||
4930 !err_cnt)
4931 continue;
4932
322a7e00
HZ		 4933 *err_count += err_cnt;
4934
4935 /* log the errors */
4936 amdgpu_ras_get_error_type_name(err_type, err_type_name);
4937 if (!mem_list) {
4938 /* memory_list is not supported */
4939 dev_info(adev->dev,
4940 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n",
4941 err_cnt, err_type_name,
4942 reg_list[i].block_name,
4943 instance, memory_id);
4944 } else {
4945 for (j = 0; j < mem_list_size; j++) {
4946 if (memory_id == mem_list[j].memory_id) {
4947 dev_info(adev->dev,
4948 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n",
4949 err_cnt, err_type_name,
4950 reg_list[i].block_name,
4951 instance, mem_list[j].name);
4952 break;
4953 }
4954 }
4955 }
4956 }
4957}
e53a3250
HZ		 4958
4959void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev,
4960 const struct amdgpu_ras_err_status_reg_entry *reg_list,
4961 uint32_t reg_list_size,
4962 uint32_t instance)
4963{
4964 uint32_t err_status_lo_offset, err_status_hi_offset;
4965 uint32_t i;
4966
4967 for (i = 0; i < reg_list_size; i++) {
4968 err_status_lo_offset =
4969 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
4970 reg_list[i].seg_lo, reg_list[i].reg_lo);
4971 err_status_hi_offset =
4972 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
4973 reg_list[i].seg_hi, reg_list[i].reg_hi);
4974 WREG32(err_status_lo_offset, 0);
4975 WREG32(err_status_hi_offset, 0);
4976 }
4977}
5b1270be
YW		 4978
4979int amdgpu_ras_error_data_init(struct ras_err_data *err_data)
4980{
4981 memset(err_data, 0, sizeof(*err_data));
4982
4983 INIT_LIST_HEAD(&err_data->err_node_list);
4984
4985 return 0;
4986}
4987
4988static void amdgpu_ras_error_node_release(struct ras_err_node *err_node)
4989{
4990 if (!err_node)
4991 return;
4992
4993 list_del(&err_node->node);
4994 kvfree(err_node);
4995}
4996
4997void amdgpu_ras_error_data_fini(struct ras_err_data *err_data)
4998{
4999 struct ras_err_node *err_node, *tmp;
5000
8a656611 5001 list_for_each_entry_safe(err_node, tmp, &err_data->err_node_list, node)
5b1270be 5002 amdgpu_ras_error_node_release(err_node);
5b1270be
YW		 5003}
5004
5005static struct ras_err_node *amdgpu_ras_error_find_node_by_id(struct ras_err_data *err_data,
5006 struct amdgpu_smuio_mcm_config_info *mcm_info)
5007{
5008 struct ras_err_node *err_node;
5009 struct amdgpu_smuio_mcm_config_info *ref_id;
5010
5011 if (!err_data || !mcm_info)
5012 return NULL;
5013
5014 for_each_ras_error(err_node, err_data) {
5015 ref_id = &err_node->err_info.mcm_info;
5b1270be 5016
53d4d779
YW		 5017 if (mcm_info->socket_id == ref_id->socket_id &&
5018 mcm_info->die_id == ref_id->die_id)
5019 return err_node;
5b1270be
YW		 5020 }
5021
5022 return NULL;
5023}
5024
5025static struct ras_err_node *amdgpu_ras_error_node_new(void)
5026{
5027 struct ras_err_node *err_node;
5028
5029 err_node = kvzalloc(sizeof(*err_node), GFP_KERNEL);
5030 if (!err_node)
5031 return NULL;
5032
5033 INIT_LIST_HEAD(&err_node->node);
5034
5035 return err_node;
5036}
5037
dbf3850d
YW		 5038static int ras_err_info_cmp(void *priv, const struct list_head *a, const struct list_head *b)
5039{
5040 struct ras_err_node *nodea = container_of(a, struct ras_err_node, node);
5041 struct ras_err_node *nodeb = container_of(b, struct ras_err_node, node);
5042 struct amdgpu_smuio_mcm_config_info *infoa = &nodea->err_info.mcm_info;
5043 struct amdgpu_smuio_mcm_config_info *infob = &nodeb->err_info.mcm_info;
5044
5045 if (unlikely(infoa->socket_id != infob->socket_id))
5046 return infoa->socket_id - infob->socket_id;
5047 else
5048 return infoa->die_id - infob->die_id;
5049
5050 return 0;
5051}
5052
5b1270be 5053static struct ras_err_info *amdgpu_ras_error_get_info(struct ras_err_data *err_data,
0795b5d2 5054 struct amdgpu_smuio_mcm_config_info *mcm_info)
5b1270be
YW		 5055{
5056 struct ras_err_node *err_node;
5057
5058 err_node = amdgpu_ras_error_find_node_by_id(err_data, mcm_info);
5059 if (err_node)
5060 return &err_node->err_info;
5061
5062 err_node = amdgpu_ras_error_node_new();
5063 if (!err_node)
5064 return NULL;
5065
0795b5d2 5066 memcpy(&err_node->err_info.mcm_info, mcm_info, sizeof(*mcm_info));
9f91e983 5067
5b1270be
YW		 5068 err_data->err_list_count++;
5069 list_add_tail(&err_node->node, &err_data->err_node_list);
dbf3850d 5070 list_sort(NULL, &err_data->err_node_list, ras_err_info_cmp);
5b1270be
YW		 5071
5072 return &err_node->err_info;
5073}
5074
5075int amdgpu_ras_error_statistic_ue_count(struct ras_err_data *err_data,
671af066
YW		 5076 struct amdgpu_smuio_mcm_config_info *mcm_info,
5077 u64 count)
5b1270be
YW		 5078{
5079 struct ras_err_info *err_info;
5080
5081 if (!err_data || !mcm_info)
5082 return -EINVAL;
5083
5084 if (!count)
5085 return 0;
5086
0795b5d2 5087 err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5b1270be
YW		 5088 if (!err_info)
5089 return -EINVAL;
5090
5091 err_info->ue_count += count;
5092 err_data->ue_count += count;
5093
5094 return 0;
5095}
5096
5097int amdgpu_ras_error_statistic_ce_count(struct ras_err_data *err_data,
671af066
YW		 5098 struct amdgpu_smuio_mcm_config_info *mcm_info,
5099 u64 count)
5b1270be
YW		 5100{
5101 struct ras_err_info *err_info;
5102
5103 if (!err_data || !mcm_info)
5104 return -EINVAL;
5105
5106 if (!count)
5107 return 0;
5108
0795b5d2 5109 err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
5b1270be
YW		 5110 if (!err_info)
5111 return -EINVAL;
5112
5113 err_info->ce_count += count;
5114 err_data->ce_count += count;
5115
5116 return 0;
5117}
cce4febb 5118
46e2231c 5119int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data,
671af066
YW		 5120 struct amdgpu_smuio_mcm_config_info *mcm_info,
5121 u64 count)
46e2231c
CL		 5122{
5123 struct ras_err_info *err_info;
5124
5125 if (!err_data || !mcm_info)
5126 return -EINVAL;
5127
5128 if (!count)
5129 return 0;
5130
0795b5d2 5131 err_info = amdgpu_ras_error_get_info(err_data, mcm_info);
46e2231c
CL		 5132 if (!err_info)
5133 return -EINVAL;
5134
5135 err_info->de_count += count;
5136 err_data->de_count += count;
5137
5138 return 0;
5139}
5140
cce4febb
HZ		 5141#define mmMP0_SMN_C2PMSG_92 0x1609C
5142#define mmMP0_SMN_C2PMSG_126 0x160BE
5143static void amdgpu_ras_boot_time_error_reporting(struct amdgpu_device *adev,
a474161e 5144 u32 instance)
cce4febb
HZ		 5145{
5146 u32 socket_id, aid_id, hbm_id;
a474161e
HZ		 5147 u32 fw_status;
5148 u32 boot_error;
cce4febb
HZ		 5149 u64 reg_addr;
5150
cce4febb
HZ		 5151 /* The pattern for smn addressing in other SOC could be different from
5152 * the one for aqua_vanjaram. We should revisit the code if the pattern
5153 * is changed. In such case, replace the aqua_vanjaram implementation
5154 * with more common helper */
5155 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
5156 aqua_vanjaram_encode_ext_smn_addressing(instance);
a474161e
HZ		 5157 fw_status = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
5158
5159 reg_addr = (mmMP0_SMN_C2PMSG_126 << 2) +
5160 aqua_vanjaram_encode_ext_smn_addressing(instance);
5161 boot_error = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
cce4febb 5162
a474161e
HZ		 5163 socket_id = AMDGPU_RAS_GPU_ERR_SOCKET_ID(boot_error);
5164 aid_id = AMDGPU_RAS_GPU_ERR_AID_ID(boot_error);
d3dbccac 5165 hbm_id = ((1 == AMDGPU_RAS_GPU_ERR_HBM_ID(boot_error)) ? 0 : 1);
cce4febb
HZ		 5166
5167 if (AMDGPU_RAS_GPU_ERR_MEM_TRAINING(boot_error))
a474161e
HZ		 5168 dev_info(adev->dev,
5169 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, memory training failed\n",
5170 socket_id, aid_id, hbm_id, fw_status);
cce4febb
HZ		 5171
5172 if (AMDGPU_RAS_GPU_ERR_FW_LOAD(boot_error))
a474161e
HZ		 5173 dev_info(adev->dev,
5174 "socket: %d, aid: %d, fw_status: 0x%x, firmware load failed at boot time\n",
5175 socket_id, aid_id, fw_status);
cce4febb
HZ		 5176
5177 if (AMDGPU_RAS_GPU_ERR_WAFL_LINK_TRAINING(boot_error))
a474161e
HZ		 5178 dev_info(adev->dev,
5179 "socket: %d, aid: %d, fw_status: 0x%x, wafl link training failed\n",
5180 socket_id, aid_id, fw_status);
cce4febb
HZ		 5181
5182 if (AMDGPU_RAS_GPU_ERR_XGMI_LINK_TRAINING(boot_error))
a474161e
HZ		 5183 dev_info(adev->dev,
5184 "socket: %d, aid: %d, fw_status: 0x%x, xgmi link training failed\n",
5185 socket_id, aid_id, fw_status);
cce4febb
HZ		 5186
5187 if (AMDGPU_RAS_GPU_ERR_USR_CP_LINK_TRAINING(boot_error))
a474161e
HZ		 5188 dev_info(adev->dev,
5189 "socket: %d, aid: %d, fw_status: 0x%x, usr cp link training failed\n",
5190 socket_id, aid_id, fw_status);
cce4febb
HZ		 5191
5192 if (AMDGPU_RAS_GPU_ERR_USR_DP_LINK_TRAINING(boot_error))
a474161e
HZ		 5193 dev_info(adev->dev,
5194 "socket: %d, aid: %d, fw_status: 0x%x, usr dp link training failed\n",
5195 socket_id, aid_id, fw_status);
cce4febb
HZ		 5196
5197 if (AMDGPU_RAS_GPU_ERR_HBM_MEM_TEST(boot_error))
a474161e
HZ		 5198 dev_info(adev->dev,
5199 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm memory test failed\n",
5200 socket_id, aid_id, hbm_id, fw_status);
cce4febb
HZ		 5201
5202 if (AMDGPU_RAS_GPU_ERR_HBM_BIST_TEST(boot_error))
a474161e
HZ		 5203 dev_info(adev->dev,
5204 "socket: %d, aid: %d, hbm: %d, fw_status: 0x%x, hbm bist test failed\n",
5205 socket_id, aid_id, hbm_id, fw_status);
dfe9d047
HZ		 5206
5207 if (AMDGPU_RAS_GPU_ERR_DATA_ABORT(boot_error))
5208 dev_info(adev->dev,
5209 "socket: %d, aid: %d, fw_status: 0x%x, data abort exception\n",
5210 socket_id, aid_id, fw_status);
5211
d4bd7a50 5212 if (AMDGPU_RAS_GPU_ERR_GENERIC(boot_error))
dfe9d047 5213 dev_info(adev->dev,
d4bd7a50 5214 "socket: %d, aid: %d, fw_status: 0x%x, Boot Controller Generic Error\n",
dfe9d047 5215 socket_id, aid_id, fw_status);
cce4febb
HZ		 5216}
5217
a474161e
HZ		 5218static bool amdgpu_ras_boot_error_detected(struct amdgpu_device *adev,
5219 u32 instance)
cce4febb 5220{
a474161e 5221 u64 reg_addr;
cce4febb
HZ		 5222 u32 reg_data;
5223 int retry_loop;
5224
1731ba9b
HZ		 5225 reg_addr = (mmMP0_SMN_C2PMSG_92 << 2) +
5226 aqua_vanjaram_encode_ext_smn_addressing(instance);
5227
5228 for (retry_loop = 0; retry_loop < AMDGPU_RAS_BOOT_STATUS_POLLING_LIMIT; retry_loop++) {
5229 reg_data = amdgpu_device_indirect_rreg_ext(adev, reg_addr);
a474161e
HZ		 5230 if ((reg_data & AMDGPU_RAS_BOOT_STATUS_MASK) == AMDGPU_RAS_BOOT_STEADY_STATUS)
5231 return false;
5232 else
5233 msleep(1);
cce4febb
HZ		 5234 }
5235
a474161e 5236 return true;
cce4febb
HZ		 5237}
5238
5239void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances)
5240{
cce4febb
HZ		 5241 u32 i;
5242
5243 for (i = 0; i < num_instances; i++) {
a474161e
HZ		 5244 if (amdgpu_ras_boot_error_detected(adev, i))
5245 amdgpu_ras_boot_time_error_reporting(adev, i);
cce4febb
HZ		 5246 }
5247}
af730e08
YC		 5248
5249int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn)
5250{
5251 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5252 struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
5253 uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT;
5254 int ret = 0;
5255
5256 mutex_lock(&con->page_rsv_lock);
5257 ret = amdgpu_vram_mgr_query_page_status(mgr, start);
5258 if (ret == -ENOENT)
5259 ret = amdgpu_vram_mgr_reserve_range(mgr, start, AMDGPU_GPU_PAGE_SIZE);
5260 mutex_unlock(&con->page_rsv_lock);
5261
5262 return ret;
5263}
b712d7c2
YW		 5264
5265void amdgpu_ras_event_log_print(struct amdgpu_device *adev, u64 event_id,
5266 const char *fmt, ...)
5267{
5268 struct va_format vaf;
5269 va_list args;
5270
5271 va_start(args, fmt);
5272 vaf.fmt = fmt;
5273 vaf.va = &args;
5274
75ac6a25 5275 if (RAS_EVENT_ID_IS_VALID(event_id))
b712d7c2
YW		 5276 dev_printk(KERN_INFO, adev->dev, "{%llu}%pV", event_id, &vaf);
5277 else
5278 dev_printk(KERN_INFO, adev->dev, "%pV", &vaf);
5279
5280 va_end(args);
5281}
792be2e2
TZ		 5282
5283bool amdgpu_ras_is_rma(struct amdgpu_device *adev)
5284{
5285 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5286
5287 if (!con)
5288 return false;
5289
5290 return con->is_rma;
5291}
Linux 6.x block layer and io_uring tree(s)