projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/amdgpu: Update invalid PTE flag setting
[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_ras.c
CommitLineData
c030f2e4 1/*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 *
23 */
24#include <linux/debugfs.h>
25#include <linux/list.h>
26#include <linux/module.h>
f867723b 27#include <linux/uaccess.h>
7c6e68c7
AG		 28#include <linux/reboot.h>
29#include <linux/syscalls.h>
05adfd80 30#include <linux/pm_runtime.h>
f867723b 31
c030f2e4 32#include "amdgpu.h"
33#include "amdgpu_ras.h"
b404ae82 34#include "amdgpu_atomfirmware.h"
19744f5f 35#include "amdgpu_xgmi.h"
4e644fff 36#include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
9af357bc 37#include "nbio_v4_3.h"
f50160cf 38#include "atom.h"
25a2b22e
AG		 39#include "amdgpu_reset.h"
40
12b2cab7
MJ		 41#ifdef CONFIG_X86_MCE_AMD
42#include <asm/mce.h>
c030f2e4 43
12b2cab7
MJ		 44static bool notifier_registered;
45#endif
eb0c3cd4
GC		 46static const char *RAS_FS_NAME = "ras";
47
c030f2e4 48const char *ras_error_string[] = {
49 "none",
50 "parity",
51 "single_correctable",
52 "multi_uncorrectable",
53 "poison",
54};
55
56const char *ras_block_string[] = {
57 "umc",
58 "sdma",
59 "gfx",
60 "mmhub",
61 "athub",
62 "pcie_bif",
63 "hdp",
64 "xgmi_wafl",
65 "df",
66 "smn",
67 "sem",
68 "mp0",
69 "mp1",
70 "fuse",
640ae42e 71 "mca",
a3d63c62
MZZ		 72 "vcn",
73 "jpeg",
c030f2e4 74};
75
640ae42e
JC		 76const char *ras_mca_block_string[] = {
77 "mca_mp0",
78 "mca_mp1",
79 "mca_mpio",
80 "mca_iohc",
81};
82
d5e8ff5f 83struct amdgpu_ras_block_list {
84 /* ras block link */
85 struct list_head node;
86
87 struct amdgpu_ras_block_object *ras_obj;
88};
89
640ae42e
JC		 90const char *get_ras_block_str(struct ras_common_if *ras_block)
91{
92 if (!ras_block)
93 return "NULL";
94
95 if (ras_block->block >= AMDGPU_RAS_BLOCK_COUNT)
96 return "OUT OF RANGE";
97
98 if (ras_block->block == AMDGPU_RAS_BLOCK__MCA)
99 return ras_mca_block_string[ras_block->sub_block_index];
100
101 return ras_block_string[ras_block->block];
102}
103
954ea6aa 104#define ras_block_str(_BLOCK_) \
105 (((_BLOCK_) < ARRAY_SIZE(ras_block_string)) ? ras_block_string[_BLOCK_] : "Out Of Range")
8b0fb0e9 106
c030f2e4 107#define ras_err_str(i) (ras_error_string[ffs(i)])
c030f2e4 108
108c6a63 109#define RAS_DEFAULT_FLAGS (AMDGPU_RAS_FLAG_INIT_BY_VBIOS)
110
7cdc2ee3
TZ		 111/* inject address is 52 bits */
112#define RAS_UMC_INJECT_ADDR_LIMIT (0x1ULL << 52)
113
e4e6a589
LT		 114/* typical ECC bad page rate is 1 bad page per 100MB VRAM */
115#define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL)
c84d4670 116
52dd95f2
GC		 117enum amdgpu_ras_retire_page_reservation {
118 AMDGPU_RAS_RETIRE_PAGE_RESERVED,
119 AMDGPU_RAS_RETIRE_PAGE_PENDING,
120 AMDGPU_RAS_RETIRE_PAGE_FAULT,
121};
7c6e68c7
AG		 122
123atomic_t amdgpu_ras_in_intr = ATOMIC_INIT(0);
124
676deb38
DL		 125static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
126 uint64_t addr);
6e4be987
TZ		 127static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
128 uint64_t addr);
12b2cab7 129#ifdef CONFIG_X86_MCE_AMD
91a1a52d
MJ		 130static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev);
131struct mce_notifier_adev_list {
132 struct amdgpu_device *devs[MAX_GPU_INSTANCE];
133 int num_gpu;
134};
135static struct mce_notifier_adev_list mce_adev_list;
12b2cab7 136#endif
6e4be987 137
61380faa
JC		 138void amdgpu_ras_set_error_query_ready(struct amdgpu_device *adev, bool ready)
139{
a9d82d2f 140 if (adev && amdgpu_ras_get_context(adev))
61380faa
JC		 141 amdgpu_ras_get_context(adev)->error_query_ready = ready;
142}
143
f3167919 144static bool amdgpu_ras_get_error_query_ready(struct amdgpu_device *adev)
61380faa 145{
a9d82d2f 146 if (adev && amdgpu_ras_get_context(adev))
61380faa
JC		 147 return amdgpu_ras_get_context(adev)->error_query_ready;
148
149 return false;
150}
151
cbb8f989
JC		 152static int amdgpu_reserve_page_direct(struct amdgpu_device *adev, uint64_t address)
153{
154 struct ras_err_data err_data = {0, 0, 0, NULL};
155 struct eeprom_table_record err_rec;
156
157 if ((address >= adev->gmc.mc_vram_size) ||
158 (address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
159 dev_warn(adev->dev,
160 "RAS WARN: input address 0x%llx is invalid.\n",
161 address);
162 return -EINVAL;
163 }
164
165 if (amdgpu_ras_check_bad_page(adev, address)) {
166 dev_warn(adev->dev,
80b0cd0f 167 "RAS WARN: 0x%llx has already been marked as bad page!\n",
cbb8f989
JC		 168 address);
169 return 0;
170 }
171
172 memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
cbb8f989 173 err_data.err_addr = &err_rec;
71344a71 174 amdgpu_umc_fill_error_record(&err_data, address, address, 0, 0);
cbb8f989
JC		 175
176 if (amdgpu_bad_page_threshold != 0) {
177 amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
178 err_data.err_addr_cnt);
4d33e0f1 179 amdgpu_ras_save_bad_pages(adev, NULL);
cbb8f989
JC		 180 }
181
182 dev_warn(adev->dev, "WARNING: THIS IS ONLY FOR TEST PURPOSES AND WILL CORRUPT RAS EEPROM\n");
183 dev_warn(adev->dev, "Clear EEPROM:\n");
184 dev_warn(adev->dev, " echo 1 > /sys/kernel/debug/dri/0/ras/ras_eeprom_reset\n");
185
186 return 0;
187}
188
c030f2e4 189static ssize_t amdgpu_ras_debugfs_read(struct file *f, char __user *buf,
190 size_t size, loff_t *pos)
191{
192 struct ras_manager *obj = (struct ras_manager *)file_inode(f)->i_private;
193 struct ras_query_if info = {
194 .head = obj->head,
195 };
196 ssize_t s;
197 char val[128];
198
761d86d3 199 if (amdgpu_ras_query_error_status(obj->adev, &info))
c030f2e4 200 return -EINVAL;
201
2a460963
CL		 202 /* Hardware counter will be reset automatically after the query on Vega20 and Arcturus */
203 if (obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
204 obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
205 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
206 dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
207 }
208
c030f2e4 209 s = snprintf(val, sizeof(val), "%s: %lu\n%s: %lu\n",
210 "ue", info.ue_count,
211 "ce", info.ce_count);
212 if (*pos >= s)
213 return 0;
214
215 s -= *pos;
216 s = min_t(u64, s, size);
217
218
219 if (copy_to_user(buf, &val[*pos], s))
220 return -EINVAL;
221
222 *pos += s;
223
224 return s;
225}
226
c030f2e4 227static const struct file_operations amdgpu_ras_debugfs_ops = {
228 .owner = THIS_MODULE,
229 .read = amdgpu_ras_debugfs_read,
190211ab 230 .write = NULL,
c030f2e4 231 .llseek = default_llseek
232};
233
96ebb307 234static int amdgpu_ras_find_block_id_by_name(const char *name, int *block_id)
235{
236 int i;
237
238 for (i = 0; i < ARRAY_SIZE(ras_block_string); i++) {
239 *block_id = i;
640ae42e 240 if (strcmp(name, ras_block_string[i]) == 0)
96ebb307 241 return 0;
242 }
243 return -EINVAL;
244}
245
246static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f,
247 const char __user *buf, size_t size,
248 loff_t *pos, struct ras_debug_if *data)
249{
250 ssize_t s = min_t(u64, 64, size);
251 char str[65];
252 char block_name[33];
253 char err[9] = "ue";
254 int op = -1;
255 int block_id;
44494f96 256 uint32_t sub_block;
96ebb307 257 u64 address, value;
2c22ed0b
TZ		 258 /* default value is 0 if the mask is not set by user */
259 u32 instance_mask = 0;
96ebb307 260
261 if (*pos)
262 return -EINVAL;
263 *pos = size;
264
265 memset(str, 0, sizeof(str));
266 memset(data, 0, sizeof(*data));
267
268 if (copy_from_user(str, buf, s))
269 return -EINVAL;
270
271 if (sscanf(str, "disable %32s", block_name) == 1)
272 op = 0;
273 else if (sscanf(str, "enable %32s %8s", block_name, err) == 2)
274 op = 1;
275 else if (sscanf(str, "inject %32s %8s", block_name, err) == 2)
276 op = 2;
6df23f4c 277 else if (strstr(str, "retire_page") != NULL)
cbb8f989 278 op = 3;
b076296b 279 else if (str[0] && str[1] && str[2] && str[3])
96ebb307 280 /* ascii string, but commands are not matched. */
281 return -EINVAL;
282
283 if (op != -1) {
cbb8f989 284 if (op == 3) {
546aa546
LT		 285 if (sscanf(str, "%*s 0x%llx", &address) != 1 &&
286 sscanf(str, "%*s %llu", &address) != 1)
6cb7a1d4 287 return -EINVAL;
cbb8f989
JC		 288
289 data->op = op;
290 data->inject.address = address;
291
292 return 0;
293 }
294
96ebb307 295 if (amdgpu_ras_find_block_id_by_name(block_name, &block_id))
296 return -EINVAL;
297
298 data->head.block = block_id;
e1063493
TZ		 299 /* only ue and ce errors are supported */
300 if (!memcmp("ue", err, 2))
301 data->head.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
302 else if (!memcmp("ce", err, 2))
303 data->head.type = AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE;
304 else
305 return -EINVAL;
306
96ebb307 307 data->op = op;
308
309 if (op == 2) {
2c22ed0b
TZ		 310 if (sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx 0x%x",
311 &sub_block, &address, &value, &instance_mask) != 4 &&
312 sscanf(str, "%*s %*s %*s %u %llu %llu %u",
313 &sub_block, &address, &value, &instance_mask) != 4 &&
314 sscanf(str, "%*s %*s %*s 0x%x 0x%llx 0x%llx",
546aa546
LT		 315 &sub_block, &address, &value) != 3 &&
316 sscanf(str, "%*s %*s %*s %u %llu %llu",
6cb7a1d4
LT		 317 &sub_block, &address, &value) != 3)
318 return -EINVAL;
44494f96 319 data->head.sub_block_index = sub_block;
96ebb307 320 data->inject.address = address;
321 data->inject.value = value;
2c22ed0b 322 data->inject.instance_mask = instance_mask;
96ebb307 323 }
324 } else {
73aa8e1a 325 if (size < sizeof(*data))
96ebb307 326 return -EINVAL;
327
328 if (copy_from_user(data, buf, sizeof(*data)))
329 return -EINVAL;
330 }
331
332 return 0;
333}
7c6e68c7 334
f464c5dd
TZ		 335static void amdgpu_ras_instance_mask_check(struct amdgpu_device *adev,
336 struct ras_debug_if *data)
337{
338 int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
339 uint32_t mask, inst_mask = data->inject.instance_mask;
340
341 /* no need to set instance mask if there is only one instance */
342 if (num_xcc <= 1 && inst_mask) {
343 data->inject.instance_mask = 0;
344 dev_dbg(adev->dev,
345 "RAS inject mask(0x%x) isn't supported and force it to 0.\n",
346 inst_mask);
347
348 return;
349 }
350
351 switch (data->head.block) {
352 case AMDGPU_RAS_BLOCK__GFX:
353 mask = GENMASK(num_xcc - 1, 0);
354 break;
355 case AMDGPU_RAS_BLOCK__SDMA:
356 mask = GENMASK(adev->sdma.num_instances - 1, 0);
357 break;
e3959cb5
SY		 358 case AMDGPU_RAS_BLOCK__VCN:
359 case AMDGPU_RAS_BLOCK__JPEG:
360 mask = GENMASK(adev->vcn.num_vcn_inst - 1, 0);
361 break;
f464c5dd 362 default:
e3959cb5 363 mask = inst_mask;
f464c5dd
TZ		 364 break;
365 }
366
367 /* remove invalid bits in instance mask */
368 data->inject.instance_mask &= mask;
369 if (inst_mask != data->inject.instance_mask)
370 dev_dbg(adev->dev,
371 "Adjust RAS inject mask 0x%x to 0x%x\n",
372 inst_mask, data->inject.instance_mask);
373}
374
74abc221
TSD		 375/**
376 * DOC: AMDGPU RAS debugfs control interface
36ea1bd2 377 *
737c375b 378 * The control interface accepts struct ras_debug_if which has two members.
36ea1bd2 379 *
380 * First member: ras_debug_if::head or ras_debug_if::inject.
96ebb307 381 *
382 * head is used to indicate which IP block will be under control.
36ea1bd2 383 *
384 * head has four members, they are block, type, sub_block_index, name.
385 * block: which IP will be under control.
386 * type: what kind of error will be enabled/disabled/injected.
387 * sub_block_index: some IPs have subcomponets. say, GFX, sDMA.
388 * name: the name of IP.
389 *
2c22ed0b 390 * inject has three more members than head, they are address, value and mask.
36ea1bd2 391 * As their names indicate, inject operation will write the
392 * value to the address.
393 *
ef177d11 394 * The second member: struct ras_debug_if::op.
c688a06b 395 * It has three kinds of operations.
879e723d
AZ		 396 *
397 * - 0: disable RAS on the block. Take ::head as its data.
398 * - 1: enable RAS on the block. Take ::head as its data.
399 * - 2: inject errors on the block. Take ::inject as its data.
36ea1bd2 400 *
96ebb307 401 * How to use the interface?
ef177d11 402 *
737c375b 403 * In a program
ef177d11 404 *
737c375b
LT		 405 * Copy the struct ras_debug_if in your code and initialize it.
406 * Write the struct to the control interface.
ef177d11 407 *
737c375b 408 * From shell
96ebb307 409 *
879e723d
AZ		 410 * .. code-block:: bash
411 *
737c375b
LT		 412 * echo "disable <block>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
413 * echo "enable <block> <error>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
2c22ed0b 414 * echo "inject <block> <error> <sub-block> <address> <value> <mask>" > /sys/kernel/debug/dri/<N>/ras/ras_ctrl
879e723d 415 *
737c375b 416 * Where N, is the card which you want to affect.
ef177d11 417 *
737c375b
LT		 418 * "disable" requires only the block.
419 * "enable" requires the block and error type.
420 * "inject" requires the block, error type, address, and value.
c666bbf0 421 *
737c375b 422 * The block is one of: umc, sdma, gfx, etc.
879e723d 423 * see ras_block_string[] for details
c666bbf0 424 *
737c375b
LT		 425 * The error type is one of: ue, ce, where,
426 * ue is multi-uncorrectable
427 * ce is single-correctable
c666bbf0 428 *
737c375b
LT		 429 * The sub-block is a the sub-block index, pass 0 if there is no sub-block.
430 * The address and value are hexadecimal numbers, leading 0x is optional.
2c22ed0b 431 * The mask means instance mask, is optional, default value is 0x1.
879e723d 432 *
737c375b 433 * For instance,
879e723d
AZ		 434 *
435 * .. code-block:: bash
96ebb307 436 *
44494f96 437 * echo inject umc ue 0x0 0x0 0x0 > /sys/kernel/debug/dri/0/ras/ras_ctrl
2c22ed0b 438 * echo inject umc ce 0 0 0 3 > /sys/kernel/debug/dri/0/ras/ras_ctrl
96ebb307 439 * echo disable umc > /sys/kernel/debug/dri/0/ras/ras_ctrl
440 *
737c375b 441 * How to check the result of the operation?
36ea1bd2 442 *
737c375b 443 * To check disable/enable, see "ras" features at,
36ea1bd2 444 * /sys/class/drm/card[0/1/2...]/device/ras/features
445 *
737c375b
LT		 446 * To check inject, see the corresponding error count at,
447 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx|sdma|umc|...]_err_count
36ea1bd2 448 *
879e723d 449 * .. note::
ef177d11 450 * Operations are only allowed on blocks which are supported.
737c375b 451 * Check the "ras" mask at /sys/module/amdgpu/parameters/ras_mask
ef177d11
AD		 452 * to see which blocks support RAS on a particular asic.
453 *
36ea1bd2 454 */
cf696091
LT		 455static ssize_t amdgpu_ras_debugfs_ctrl_write(struct file *f,
456 const char __user *buf,
457 size_t size, loff_t *pos)
36ea1bd2 458{
459 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
460 struct ras_debug_if data;
461 int ret = 0;
462
61380faa 463 if (!amdgpu_ras_get_error_query_ready(adev)) {
6952e99c
GC		 464 dev_warn(adev->dev, "RAS WARN: error injection "
465 "currently inaccessible\n");
43c4d576
JC		 466 return size;
467 }
468
96ebb307 469 ret = amdgpu_ras_debugfs_ctrl_parse_data(f, buf, size, pos, &data);
470 if (ret)
cf696091 471 return ret;
36ea1bd2 472
80b0cd0f 473 if (data.op == 3) {
cbb8f989 474 ret = amdgpu_reserve_page_direct(adev, data.inject.address);
80b0cd0f 475 if (!ret)
cbb8f989
JC		 476 return size;
477 else
478 return ret;
479 }
480
36ea1bd2 481 if (!amdgpu_ras_is_supported(adev, data.head.block))
482 return -EINVAL;
483
484 switch (data.op) {
485 case 0:
486 ret = amdgpu_ras_feature_enable(adev, &data.head, 0);
487 break;
488 case 1:
489 ret = amdgpu_ras_feature_enable(adev, &data.head, 1);
490 break;
491 case 2:
43aedbf4
SY		 492 if ((data.inject.address >= adev->gmc.mc_vram_size &&
493 adev->gmc.mc_vram_size) ||
7cdc2ee3 494 (data.inject.address >= RAS_UMC_INJECT_ADDR_LIMIT)) {
b0d4783a
GC		 495 dev_warn(adev->dev, "RAS WARN: input address "
496 "0x%llx is invalid.",
497 data.inject.address);
7cdc2ee3
TZ		 498 ret = -EINVAL;
499 break;
500 }
501
6e4be987
TZ		 502 /* umc ce/ue error injection for a bad page is not allowed */
503 if ((data.head.block == AMDGPU_RAS_BLOCK__UMC) &&
504 amdgpu_ras_check_bad_page(adev, data.inject.address)) {
c65b0805
LT		 505 dev_warn(adev->dev, "RAS WARN: inject: 0x%llx has "
506 "already been marked as bad!\n",
507 data.inject.address);
6e4be987
TZ		 508 break;
509 }
510
f464c5dd
TZ		 511 amdgpu_ras_instance_mask_check(adev, &data);
512
7cdc2ee3 513 /* data.inject.address is offset instead of absolute gpu address */
36ea1bd2 514 ret = amdgpu_ras_error_inject(adev, &data.inject);
515 break;
96ebb307 516 default:
517 ret = -EINVAL;
518 break;
374bf7bd 519 }
36ea1bd2 520
521 if (ret)
79c04621 522 return ret;
36ea1bd2 523
524 return size;
525}
526
084fe13b
AG		 527/**
528 * DOC: AMDGPU RAS debugfs EEPROM table reset interface
529 *
f77c7109 530 * Some boards contain an EEPROM which is used to persistently store a list of
ef177d11 531 * bad pages which experiences ECC errors in vram. This interface provides
f77c7109
AD		 532 * a way to reset the EEPROM, e.g., after testing error injection.
533 *
534 * Usage:
535 *
536 * .. code-block:: bash
537 *
538 * echo 1 > ../ras/ras_eeprom_reset
539 *
540 * will reset EEPROM table to 0 entries.
541 *
084fe13b 542 */
cf696091
LT		 543static ssize_t amdgpu_ras_debugfs_eeprom_write(struct file *f,
544 const char __user *buf,
545 size_t size, loff_t *pos)
084fe13b 546{
bf0b91b7
GC		 547 struct amdgpu_device *adev =
548 (struct amdgpu_device *)file_inode(f)->i_private;
084fe13b
AG		 549 int ret;
550
bf0b91b7 551 ret = amdgpu_ras_eeprom_reset_table(
cf696091 552 &(amdgpu_ras_get_context(adev)->eeprom_control));
084fe13b 553
63d4c081 554 if (!ret) {
cf696091
LT		 555 /* Something was written to EEPROM.
556 */
bf0b91b7
GC		 557 amdgpu_ras_get_context(adev)->flags = RAS_DEFAULT_FLAGS;
558 return size;
559 } else {
cf696091 560 return ret;
bf0b91b7 561 }
084fe13b
AG		 562}
563
36ea1bd2 564static const struct file_operations amdgpu_ras_debugfs_ctrl_ops = {
565 .owner = THIS_MODULE,
566 .read = NULL,
567 .write = amdgpu_ras_debugfs_ctrl_write,
568 .llseek = default_llseek
569};
570
084fe13b
AG		 571static const struct file_operations amdgpu_ras_debugfs_eeprom_ops = {
572 .owner = THIS_MODULE,
573 .read = NULL,
574 .write = amdgpu_ras_debugfs_eeprom_write,
575 .llseek = default_llseek
576};
577
f77c7109
AD		 578/**
579 * DOC: AMDGPU RAS sysfs Error Count Interface
580 *
ef177d11 581 * It allows the user to read the error count for each IP block on the gpu through
f77c7109
AD		 582 * /sys/class/drm/card[0/1/2...]/device/ras/[gfx/sdma/...]_err_count
583 *
584 * It outputs the multiple lines which report the uncorrected (ue) and corrected
585 * (ce) error counts.
586 *
587 * The format of one line is below,
588 *
589 * [ce|ue]: count
590 *
591 * Example:
592 *
593 * .. code-block:: bash
594 *
595 * ue: 0
596 * ce: 1
597 *
598 */
c030f2e4 599static ssize_t amdgpu_ras_sysfs_read(struct device *dev,
600 struct device_attribute *attr, char *buf)
601{
602 struct ras_manager *obj = container_of(attr, struct ras_manager, sysfs_attr);
603 struct ras_query_if info = {
604 .head = obj->head,
605 };
606
61380faa 607 if (!amdgpu_ras_get_error_query_ready(obj->adev))
36000c7a 608 return sysfs_emit(buf, "Query currently inaccessible\n");
43c4d576 609
761d86d3 610 if (amdgpu_ras_query_error_status(obj->adev, &info))
c030f2e4 611 return -EINVAL;
612
2a460963
CL		 613 if (obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
614 obj->adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
1f0d8e37 615 if (amdgpu_ras_reset_error_status(obj->adev, info.head.block))
2a460963 616 dev_warn(obj->adev->dev, "Failed to reset error counter and error status");
1f0d8e37
MJ		 617 }
618
36000c7a
TT		 619 return sysfs_emit(buf, "%s: %lu\n%s: %lu\n", "ue", info.ue_count,
620 "ce", info.ce_count);
c030f2e4 621}
622
623/* obj begin */
624
625#define get_obj(obj) do { (obj)->use++; } while (0)
626#define alive_obj(obj) ((obj)->use)
627
628static inline void put_obj(struct ras_manager *obj)
629{
f0872686 630 if (obj && (--obj->use == 0))
c030f2e4 631 list_del(&obj->node);
f0872686 632 if (obj && (obj->use < 0))
640ae42e 633 DRM_ERROR("RAS ERROR: Unbalance obj(%s) use\n", get_ras_block_str(&obj->head));
c030f2e4 634}
635
636/* make one obj and return it. */
637static struct ras_manager *amdgpu_ras_create_obj(struct amdgpu_device *adev,
638 struct ras_common_if *head)
639{
640 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
641 struct ras_manager *obj;
642
8ab0d6f0 643 if (!adev->ras_enabled || !con)
c030f2e4 644 return NULL;
645
646 if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
647 return NULL;
648
640ae42e
JC		 649 if (head->block == AMDGPU_RAS_BLOCK__MCA) {
650 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
651 return NULL;
652
653 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
654 } else
655 obj = &con->objs[head->block];
656
c030f2e4 657 /* already exist. return obj? */
658 if (alive_obj(obj))
659 return NULL;
660
661 obj->head = *head;
662 obj->adev = adev;
663 list_add(&obj->node, &con->head);
664 get_obj(obj);
665
666 return obj;
667}
668
669/* return an obj equal to head, or the first when head is NULL */
f2a79be1 670struct ras_manager *amdgpu_ras_find_obj(struct amdgpu_device *adev,
c030f2e4 671 struct ras_common_if *head)
672{
673 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
674 struct ras_manager *obj;
675 int i;
676
8ab0d6f0 677 if (!adev->ras_enabled || !con)
c030f2e4 678 return NULL;
679
680 if (head) {
681 if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
682 return NULL;
683
640ae42e
JC		 684 if (head->block == AMDGPU_RAS_BLOCK__MCA) {
685 if (head->sub_block_index >= AMDGPU_RAS_MCA_BLOCK__LAST)
686 return NULL;
687
688 obj = &con->objs[AMDGPU_RAS_BLOCK__LAST + head->sub_block_index];
689 } else
690 obj = &con->objs[head->block];
c030f2e4 691
640ae42e 692 if (alive_obj(obj))
c030f2e4 693 return obj;
c030f2e4 694 } else {
640ae42e 695 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT + AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
c030f2e4 696 obj = &con->objs[i];
640ae42e 697 if (alive_obj(obj))
c030f2e4 698 return obj;
c030f2e4 699 }
700 }
701
702 return NULL;
703}
704/* obj end */
705
706/* feature ctl begin */
707static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
e509965e 708 struct ras_common_if *head)
c030f2e4 709{
8ab0d6f0 710 return adev->ras_hw_enabled & BIT(head->block);
c030f2e4 711}
712
713static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
714 struct ras_common_if *head)
715{
716 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
717
718 return con->features & BIT(head->block);
719}
720
721/*
722 * if obj is not created, then create one.
723 * set feature enable flag.
724 */
725static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev,
726 struct ras_common_if *head, int enable)
727{
728 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
729 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
730
5caf466a 731 /* If hardware does not support ras, then do not create obj.
732 * But if hardware support ras, we can create the obj.
733 * Ras framework checks con->hw_supported to see if it need do
734 * corresponding initialization.
735 * IP checks con->support to see if it need disable ras.
736 */
c030f2e4 737 if (!amdgpu_ras_is_feature_allowed(adev, head))
738 return 0;
c030f2e4 739
740 if (enable) {
741 if (!obj) {
742 obj = amdgpu_ras_create_obj(adev, head);
743 if (!obj)
744 return -EINVAL;
745 } else {
746 /* In case we create obj somewhere else */
747 get_obj(obj);
748 }
749 con->features |= BIT(head->block);
750 } else {
751 if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
19d0dfda 752 con->features &= ~BIT(head->block);
c030f2e4 753 put_obj(obj);
754 }
755 }
756
757 return 0;
758}
759
c26cd999
SY		 760static int amdgpu_ras_check_feature_allowed(struct amdgpu_device *adev,
761 struct ras_common_if *head)
762{
763 if (amdgpu_ras_is_feature_allowed(adev, head) ||
764 amdgpu_ras_is_poison_mode_supported(adev))
765 return 1;
766 else
767 return 0;
768}
769
c030f2e4 770/* wrapper of psp_ras_enable_features */
771int amdgpu_ras_feature_enable(struct amdgpu_device *adev,
772 struct ras_common_if *head, bool enable)
773{
774 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
7fcffecf 775 union ta_ras_cmd_input *info;
c26cd999 776 int ret = 0;
c030f2e4 777
778 if (!con)
779 return -EINVAL;
780
26093ce1
SY		 781 if (head->block == AMDGPU_RAS_BLOCK__GFX) {
782 info = kzalloc(sizeof(union ta_ras_cmd_input), GFP_KERNEL);
783 if (!info)
784 return -ENOMEM;
785
786 if (!enable) {
787 info->disable_features = (struct ta_ras_disable_features_input) {
788 .block_id = amdgpu_ras_block_to_ta(head->block),
789 .error_type = amdgpu_ras_error_to_ta(head->type),
790 };
791 } else {
792 info->enable_features = (struct ta_ras_enable_features_input) {
793 .block_id = amdgpu_ras_block_to_ta(head->block),
794 .error_type = amdgpu_ras_error_to_ta(head->type),
795 };
796 }
c030f2e4 797 }
798
799 /* Do not enable if it is not allowed. */
c26cd999
SY		 800 if (enable && !amdgpu_ras_check_feature_allowed(adev, head))
801 goto out;
c030f2e4 802
950d6425 803 /* Only enable ras feature operation handle on host side */
26093ce1
SY		 804 if (head->block == AMDGPU_RAS_BLOCK__GFX &&
805 !amdgpu_sriov_vf(adev) &&
950d6425 806 !amdgpu_ras_intr_triggered()) {
7fcffecf 807 ret = psp_ras_enable_features(&adev->psp, info, enable);
bff77e86 808 if (ret) {
e4348849 809 dev_err(adev->dev, "ras %s %s failed poison:%d ret:%d\n",
011907fd 810 enable ? "enable":"disable",
640ae42e 811 get_ras_block_str(head),
e4348849 812 amdgpu_ras_is_poison_mode_supported(adev), ret);
7fcffecf 813 goto out;
bff77e86 814 }
c030f2e4 815 }
816
817 /* setup the obj */
818 __amdgpu_ras_feature_enable(adev, head, enable);
7fcffecf 819out:
26093ce1
SY		 820 if (head->block == AMDGPU_RAS_BLOCK__GFX)
821 kfree(info);
7fcffecf 822 return ret;
c030f2e4 823}
824
77de502b 825/* Only used in device probe stage and called only once. */
826int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev,
827 struct ras_common_if *head, bool enable)
828{
829 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
830 int ret;
831
832 if (!con)
833 return -EINVAL;
834
835 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
7af23ebe 836 if (enable) {
837 /* There is no harm to issue a ras TA cmd regardless of
838 * the currecnt ras state.
839 * If current state == target state, it will do nothing
840 * But sometimes it requests driver to reset and repost
841 * with error code -EAGAIN.
842 */
843 ret = amdgpu_ras_feature_enable(adev, head, 1);
844 /* With old ras TA, we might fail to enable ras.
845 * Log it and just setup the object.
846 * TODO need remove this WA in the future.
847 */
848 if (ret == -EINVAL) {
849 ret = __amdgpu_ras_feature_enable(adev, head, 1);
850 if (!ret)
6952e99c
GC		 851 dev_info(adev->dev,
852 "RAS INFO: %s setup object\n",
640ae42e 853 get_ras_block_str(head));
7af23ebe 854 }
855 } else {
856 /* setup the object then issue a ras TA disable cmd.*/
857 ret = __amdgpu_ras_feature_enable(adev, head, 1);
858 if (ret)
859 return ret;
77de502b 860
970fd197
SY		 861 /* gfx block ras dsiable cmd must send to ras-ta */
862 if (head->block == AMDGPU_RAS_BLOCK__GFX)
863 con->features |= BIT(head->block);
864
77de502b 865 ret = amdgpu_ras_feature_enable(adev, head, 0);
19d0dfda
SY		 866
867 /* clean gfx block ras features flag */
8ab0d6f0 868 if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
19d0dfda 869 con->features &= ~BIT(head->block);
7af23ebe 870 }
77de502b 871 } else
872 ret = amdgpu_ras_feature_enable(adev, head, enable);
873
874 return ret;
875}
876
c030f2e4 877static int amdgpu_ras_disable_all_features(struct amdgpu_device *adev,
878 bool bypass)
879{
880 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
881 struct ras_manager *obj, *tmp;
882
883 list_for_each_entry_safe(obj, tmp, &con->head, node) {
884 /* bypass psp.
885 * aka just release the obj and corresponding flags
886 */
887 if (bypass) {
888 if (__amdgpu_ras_feature_enable(adev, &obj->head, 0))
889 break;
890 } else {
891 if (amdgpu_ras_feature_enable(adev, &obj->head, 0))
892 break;
893 }
289d513b 894 }
c030f2e4 895
896 return con->features;
897}
898
899static int amdgpu_ras_enable_all_features(struct amdgpu_device *adev,
900 bool bypass)
901{
902 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
c030f2e4 903 int i;
640ae42e 904 const enum amdgpu_ras_error_type default_ras_type = AMDGPU_RAS_ERROR__NONE;
c030f2e4 905
640ae42e 906 for (i = 0; i < AMDGPU_RAS_BLOCK_COUNT; i++) {
c030f2e4 907 struct ras_common_if head = {
908 .block = i,
191051a1 909 .type = default_ras_type,
c030f2e4 910 .sub_block_index = 0,
911 };
640ae42e
JC		 912
913 if (i == AMDGPU_RAS_BLOCK__MCA)
914 continue;
915
916 if (bypass) {
917 /*
918 * bypass psp. vbios enable ras for us.
919 * so just create the obj
920 */
921 if (__amdgpu_ras_feature_enable(adev, &head, 1))
922 break;
923 } else {
924 if (amdgpu_ras_feature_enable(adev, &head, 1))
925 break;
926 }
927 }
928
929 for (i = 0; i < AMDGPU_RAS_MCA_BLOCK_COUNT; i++) {
930 struct ras_common_if head = {
931 .block = AMDGPU_RAS_BLOCK__MCA,
932 .type = default_ras_type,
933 .sub_block_index = i,
934 };
935
c030f2e4 936 if (bypass) {
937 /*
938 * bypass psp. vbios enable ras for us.
939 * so just create the obj
940 */
941 if (__amdgpu_ras_feature_enable(adev, &head, 1))
942 break;
943 } else {
944 if (amdgpu_ras_feature_enable(adev, &head, 1))
945 break;
946 }
289d513b 947 }
c030f2e4 948
949 return con->features;
950}
951/* feature ctl end */
952
e3d833f4 953static int amdgpu_ras_block_match_default(struct amdgpu_ras_block_object *block_obj,
954 enum amdgpu_ras_block block)
6492e1b0 955{
b6efdb02 956 if (!block_obj)
6492e1b0 957 return -EINVAL;
958
bdb3489c 959 if (block_obj->ras_comm.block == block)
6492e1b0 960 return 0;
640ae42e 961
6492e1b0 962 return -EINVAL;
963}
964
b6efdb02 965static struct amdgpu_ras_block_object *amdgpu_ras_get_ras_block(struct amdgpu_device *adev,
6492e1b0 966 enum amdgpu_ras_block block, uint32_t sub_block_index)
640ae42e 967{
d5e8ff5f 968 struct amdgpu_ras_block_list *node, *tmp;
969 struct amdgpu_ras_block_object *obj;
6492e1b0 970
971 if (block >= AMDGPU_RAS_BLOCK__LAST)
972 return NULL;
973
d5e8ff5f 974 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
975 if (!node->ras_obj) {
976 dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
977 continue;
978 }
979
980 obj = node->ras_obj;
6492e1b0 981 if (obj->ras_block_match) {
982 if (obj->ras_block_match(obj, block, sub_block_index) == 0)
983 return obj;
984 } else {
985 if (amdgpu_ras_block_match_default(obj, block) == 0)
986 return obj;
987 }
640ae42e 988 }
6492e1b0 989
990 return NULL;
640ae42e
JC		 991}
992
fdcb279d
SY		 993static void amdgpu_ras_get_ecc_info(struct amdgpu_device *adev, struct ras_err_data *err_data)
994{
995 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
996 int ret = 0;
997
998 /*
999 * choosing right query method according to
1000 * whether smu support query error information
1001 */
bc143d8b 1002 ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(ras->umc_ecc));
fdcb279d 1003 if (ret == -EOPNOTSUPP) {
efe17d5a 1004 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1005 adev->umc.ras->ras_block.hw_ops->query_ras_error_count)
1006 adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, err_data);
fdcb279d
SY		 1007
1008 /* umc query_ras_error_address is also responsible for clearing
1009 * error status
1010 */
efe17d5a 1011 if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
1012 adev->umc.ras->ras_block.hw_ops->query_ras_error_address)
1013 adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, err_data);
fdcb279d 1014 } else if (!ret) {
efe17d5a 1015 if (adev->umc.ras &&
1016 adev->umc.ras->ecc_info_query_ras_error_count)
1017 adev->umc.ras->ecc_info_query_ras_error_count(adev, err_data);
fdcb279d 1018
efe17d5a 1019 if (adev->umc.ras &&
1020 adev->umc.ras->ecc_info_query_ras_error_address)
1021 adev->umc.ras->ecc_info_query_ras_error_address(adev, err_data);
fdcb279d
SY		 1022 }
1023}
1024
c030f2e4 1025/* query/inject/cure begin */
761d86d3 1026int amdgpu_ras_query_error_status(struct amdgpu_device *adev,
4d9f771e 1027 struct ras_query_if *info)
c030f2e4 1028{
b6efdb02 1029 struct amdgpu_ras_block_object *block_obj = NULL;
c030f2e4 1030 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
6f102dba 1031 struct ras_err_data err_data = {0, 0, 0, NULL};
c030f2e4 1032
1033 if (!obj)
1034 return -EINVAL;
c030f2e4 1035
7389a5b8 1036 if (info->head.block == AMDGPU_RAS_BLOCK__UMC) {
fdcb279d 1037 amdgpu_ras_get_ecc_info(adev, &err_data);
7389a5b8 1038 } else {
1039 block_obj = amdgpu_ras_get_ras_block(adev, info->head.block, 0);
8b0fb0e9 1040 if (!block_obj || !block_obj->hw_ops) {
afa37315
LT		 1041 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1042 get_ras_block_str(&info->head));
8b0fb0e9 1043 return -EINVAL;
3e81ee9a 1044 }
761d86d3 1045
6c245386 1046 if (block_obj->hw_ops->query_ras_error_count)
1047 block_obj->hw_ops->query_ras_error_count(adev, &err_data);
7389a5b8 1048
1049 if ((info->head.block == AMDGPU_RAS_BLOCK__SDMA) ||
1050 (info->head.block == AMDGPU_RAS_BLOCK__GFX) ||
1051 (info->head.block == AMDGPU_RAS_BLOCK__MMHUB)) {
1052 if (block_obj->hw_ops->query_ras_error_status)
1053 block_obj->hw_ops->query_ras_error_status(adev);
1054 }
939e2258 1055 }
05a58345
TZ		 1056
1057 obj->err_data.ue_count += err_data.ue_count;
1058 obj->err_data.ce_count += err_data.ce_count;
1059
c030f2e4 1060 info->ue_count = obj->err_data.ue_count;
1061 info->ce_count = obj->err_data.ce_count;
1062
7c6e68c7 1063 if (err_data.ce_count) {
a30f1286
HZ		 1064 if (adev->smuio.funcs &&
1065 adev->smuio.funcs->get_socket_id &&
1066 adev->smuio.funcs->get_die_id) {
1067 dev_info(adev->dev, "socket: %d, die: %d "
1068 "%ld correctable hardware errors "
6952e99c
GC		 1069 "detected in %s block, no user "
1070 "action is needed.\n",
a30f1286
HZ		 1071 adev->smuio.funcs->get_socket_id(adev),
1072 adev->smuio.funcs->get_die_id(adev),
6952e99c 1073 obj->err_data.ce_count,
640ae42e 1074 get_ras_block_str(&info->head));
a30f1286
HZ		 1075 } else {
1076 dev_info(adev->dev, "%ld correctable hardware errors "
6952e99c
GC		 1077 "detected in %s block, no user "
1078 "action is needed.\n",
1079 obj->err_data.ce_count,
640ae42e 1080 get_ras_block_str(&info->head));
a30f1286 1081 }
7c6e68c7
AG		 1082 }
1083 if (err_data.ue_count) {
a30f1286
HZ		 1084 if (adev->smuio.funcs &&
1085 adev->smuio.funcs->get_socket_id &&
1086 adev->smuio.funcs->get_die_id) {
1087 dev_info(adev->dev, "socket: %d, die: %d "
1088 "%ld uncorrectable hardware errors "
6952e99c 1089 "detected in %s block\n",
a30f1286
HZ		 1090 adev->smuio.funcs->get_socket_id(adev),
1091 adev->smuio.funcs->get_die_id(adev),
6952e99c 1092 obj->err_data.ue_count,
640ae42e 1093 get_ras_block_str(&info->head));
a30f1286
HZ		 1094 } else {
1095 dev_info(adev->dev, "%ld uncorrectable hardware errors "
6952e99c
GC		 1096 "detected in %s block\n",
1097 obj->err_data.ue_count,
640ae42e 1098 get_ras_block_str(&info->head));
a30f1286 1099 }
7c6e68c7 1100 }
05a58345 1101
c030f2e4 1102 return 0;
1103}
1104
761d86d3
DL		 1105int amdgpu_ras_reset_error_status(struct amdgpu_device *adev,
1106 enum amdgpu_ras_block block)
1107{
b6efdb02 1108 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0);
8b0fb0e9 1109
761d86d3
DL		 1110 if (!amdgpu_ras_is_supported(adev, block))
1111 return -EINVAL;
1112
7389a5b8 1113 if (!block_obj || !block_obj->hw_ops) {
afa37315
LT		 1114 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1115 ras_block_str(block));
7389a5b8 1116 return -EINVAL;
761d86d3
DL		 1117 }
1118
7389a5b8 1119 if (block_obj->hw_ops->reset_ras_error_count)
1120 block_obj->hw_ops->reset_ras_error_count(adev);
5c23e9e0 1121
7389a5b8 1122 if ((block == AMDGPU_RAS_BLOCK__GFX) ||
1123 (block == AMDGPU_RAS_BLOCK__MMHUB)) {
8b0fb0e9 1124 if (block_obj->hw_ops->reset_ras_error_status)
1125 block_obj->hw_ops->reset_ras_error_status(adev);
761d86d3 1126 }
5c23e9e0 1127
761d86d3 1128 return 0;
5c23e9e0
JC		 1129}
1130
c030f2e4 1131/* wrapper of psp_ras_trigger_error */
1132int amdgpu_ras_error_inject(struct amdgpu_device *adev,
1133 struct ras_inject_if *info)
1134{
1135 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1136 struct ta_ras_trigger_error_input block_info = {
828cfa29 1137 .block_id = amdgpu_ras_block_to_ta(info->head.block),
1138 .inject_error_type = amdgpu_ras_error_to_ta(info->head.type),
c030f2e4 1139 .sub_block_index = info->head.sub_block_index,
1140 .address = info->address,
1141 .value = info->value,
1142 };
ab3b9de6
YL		 1143 int ret = -EINVAL;
1144 struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev,
1145 info->head.block,
1146 info->head.sub_block_index);
c030f2e4 1147
248c9635
TZ		 1148 /* inject on guest isn't allowed, return success directly */
1149 if (amdgpu_sriov_vf(adev))
1150 return 0;
1151
c030f2e4 1152 if (!obj)
1153 return -EINVAL;
1154
22d4ba53 1155 if (!block_obj || !block_obj->hw_ops) {
afa37315
LT		 1156 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1157 get_ras_block_str(&info->head));
22d4ba53 1158 return -EINVAL;
1159 }
1160
a6c44d25
JC		 1161 /* Calculate XGMI relative offset */
1162 if (adev->gmc.xgmi.num_physical_nodes > 1) {
19744f5f
HZ		 1163 block_info.address =
1164 amdgpu_xgmi_get_relative_phy_addr(adev,
1165 block_info.address);
a6c44d25
JC		 1166 }
1167
27c5f295
TZ		 1168 if (block_obj->hw_ops->ras_error_inject) {
1169 if (info->head.block == AMDGPU_RAS_BLOCK__GFX)
2c22ed0b 1170 ret = block_obj->hw_ops->ras_error_inject(adev, info, info->instance_mask);
27c5f295 1171 else /* Special ras_error_inject is defined (e.g: xgmi) */
2c22ed0b
TZ		 1172 ret = block_obj->hw_ops->ras_error_inject(adev, &block_info,
1173 info->instance_mask);
27c5f295
TZ		 1174 } else {
1175 /* default path */
1176 ret = psp_ras_trigger_error(&adev->psp, &block_info, info->instance_mask);
a5dd40ca
HZ		 1177 }
1178
011907fd
DL		 1179 if (ret)
1180 dev_err(adev->dev, "ras inject %s failed %d\n",
640ae42e 1181 get_ras_block_str(&info->head), ret);
c030f2e4 1182
1183 return ret;
1184}
1185
4d9f771e 1186/**
4a1c9a44
HZ		 1187 * amdgpu_ras_query_error_count_helper -- Get error counter for specific IP
1188 * @adev: pointer to AMD GPU device
1189 * @ce_count: pointer to an integer to be set to the count of correctible errors.
1190 * @ue_count: pointer to an integer to be set to the count of uncorrectible errors.
1191 * @query_info: pointer to ras_query_if
1192 *
1193 * Return 0 for query success or do nothing, otherwise return an error
1194 * on failures
1195 */
1196static int amdgpu_ras_query_error_count_helper(struct amdgpu_device *adev,
1197 unsigned long *ce_count,
1198 unsigned long *ue_count,
1199 struct ras_query_if *query_info)
1200{
1201 int ret;
1202
1203 if (!query_info)
1204 /* do nothing if query_info is not specified */
1205 return 0;
1206
1207 ret = amdgpu_ras_query_error_status(adev, query_info);
1208 if (ret)
1209 return ret;
1210
1211 *ce_count += query_info->ce_count;
1212 *ue_count += query_info->ue_count;
1213
1214 /* some hardware/IP supports read to clear
1215 * no need to explictly reset the err status after the query call */
1216 if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
1217 adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4)) {
1218 if (amdgpu_ras_reset_error_status(adev, query_info->head.block))
1219 dev_warn(adev->dev,
1220 "Failed to reset error counter and error status\n");
1221 }
1222
1223 return 0;
1224}
1225
1226/**
1227 * amdgpu_ras_query_error_count -- Get error counts of all IPs or specific IP
bbe04dec
IB		 1228 * @adev: pointer to AMD GPU device
1229 * @ce_count: pointer to an integer to be set to the count of correctible errors.
1230 * @ue_count: pointer to an integer to be set to the count of uncorrectible
4d9f771e 1231 * errors.
4a1c9a44
HZ		 1232 * @query_info: pointer to ras_query_if if the query request is only for
1233 * specific ip block; if info is NULL, then the qurey request is for
1234 * all the ip blocks that support query ras error counters/status
4d9f771e
LT		 1235 *
1236 * If set, @ce_count or @ue_count, count and return the corresponding
1237 * error counts in those integer pointers. Return 0 if the device
1238 * supports RAS. Return -EOPNOTSUPP if the device doesn't support RAS.
1239 */
1240int amdgpu_ras_query_error_count(struct amdgpu_device *adev,
1241 unsigned long *ce_count,
4a1c9a44
HZ		 1242 unsigned long *ue_count,
1243 struct ras_query_if *query_info)
c030f2e4 1244{
1245 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1246 struct ras_manager *obj;
a46751fb 1247 unsigned long ce, ue;
4a1c9a44 1248 int ret;
c030f2e4 1249
8ab0d6f0 1250 if (!adev->ras_enabled || !con)
4d9f771e
LT		 1251 return -EOPNOTSUPP;
1252
1253 /* Don't count since no reporting.
1254 */
1255 if (!ce_count && !ue_count)
1256 return 0;
c030f2e4 1257
a46751fb
LT		 1258 ce = 0;
1259 ue = 0;
4a1c9a44
HZ		 1260 if (!query_info) {
1261 /* query all the ip blocks that support ras query interface */
1262 list_for_each_entry(obj, &con->head, node) {
1263 struct ras_query_if info = {
1264 .head = obj->head,
1265 };
c030f2e4 1266
4a1c9a44 1267 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, &info);
2a460963 1268 }
4a1c9a44
HZ		 1269 } else {
1270 /* query specific ip block */
1271 ret = amdgpu_ras_query_error_count_helper(adev, &ce, &ue, query_info);
c030f2e4 1272 }
1273
4a1c9a44
HZ		 1274 if (ret)
1275 return ret;
1276
a46751fb
LT		 1277 if (ce_count)
1278 *ce_count = ce;
1279
1280 if (ue_count)
1281 *ue_count = ue;
4d9f771e
LT		 1282
1283 return 0;
c030f2e4 1284}
1285/* query/inject/cure end */
1286
1287
1288/* sysfs begin */
1289
466b1793 1290static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1291 struct ras_badpage **bps, unsigned int *count);
1292
1293static char *amdgpu_ras_badpage_flags_str(unsigned int flags)
1294{
1295 switch (flags) {
52dd95f2 1296 case AMDGPU_RAS_RETIRE_PAGE_RESERVED:
466b1793 1297 return "R";
52dd95f2 1298 case AMDGPU_RAS_RETIRE_PAGE_PENDING:
466b1793 1299 return "P";
52dd95f2 1300 case AMDGPU_RAS_RETIRE_PAGE_FAULT:
466b1793 1301 default:
1302 return "F";
aec576f9 1303 }
466b1793 1304}
1305
f77c7109
AD		 1306/**
1307 * DOC: AMDGPU RAS sysfs gpu_vram_bad_pages Interface
466b1793 1308 *
1309 * It allows user to read the bad pages of vram on the gpu through
1310 * /sys/class/drm/card[0/1/2...]/device/ras/gpu_vram_bad_pages
1311 *
1312 * It outputs multiple lines, and each line stands for one gpu page.
1313 *
1314 * The format of one line is below,
1315 * gpu pfn : gpu page size : flags
1316 *
1317 * gpu pfn and gpu page size are printed in hex format.
1318 * flags can be one of below character,
f77c7109 1319 *
466b1793 1320 * R: reserved, this gpu page is reserved and not able to use.
f77c7109 1321 *
466b1793 1322 * P: pending for reserve, this gpu page is marked as bad, will be reserved
f77c7109
AD		 1323 * in next window of page_reserve.
1324 *
466b1793 1325 * F: unable to reserve. this gpu page can't be reserved due to some reasons.
1326 *
f77c7109
AD		 1327 * Examples:
1328 *
1329 * .. code-block:: bash
1330 *
1331 * 0x00000001 : 0x00001000 : R
1332 * 0x00000002 : 0x00001000 : P
1333 *
466b1793 1334 */
1335
1336static ssize_t amdgpu_ras_sysfs_badpages_read(struct file *f,
1337 struct kobject *kobj, struct bin_attribute *attr,
1338 char *buf, loff_t ppos, size_t count)
1339{
1340 struct amdgpu_ras *con =
1341 container_of(attr, struct amdgpu_ras, badpages_attr);
1342 struct amdgpu_device *adev = con->adev;
1343 const unsigned int element_size =
1344 sizeof("0xabcdabcd : 0x12345678 : R\n") - 1;
d6ee400e
SA		 1345 unsigned int start = div64_ul(ppos + element_size - 1, element_size);
1346 unsigned int end = div64_ul(ppos + count - 1, element_size);
466b1793 1347 ssize_t s = 0;
1348 struct ras_badpage *bps = NULL;
1349 unsigned int bps_count = 0;
1350
1351 memset(buf, 0, count);
1352
1353 if (amdgpu_ras_badpages_read(adev, &bps, &bps_count))
1354 return 0;
1355
1356 for (; start < end && start < bps_count; start++)
1357 s += scnprintf(&buf[s], element_size + 1,
1358 "0x%08x : 0x%08x : %1s\n",
1359 bps[start].bp,
1360 bps[start].size,
1361 amdgpu_ras_badpage_flags_str(bps[start].flags));
1362
1363 kfree(bps);
1364
1365 return s;
1366}
1367
c030f2e4 1368static ssize_t amdgpu_ras_sysfs_features_read(struct device *dev,
1369 struct device_attribute *attr, char *buf)
1370{
1371 struct amdgpu_ras *con =
1372 container_of(attr, struct amdgpu_ras, features_attr);
c030f2e4 1373
2cffcb66 1374 return sysfs_emit(buf, "feature mask: 0x%x\n", con->features);
c030f2e4 1375}
1376
f848159b
GC		 1377static void amdgpu_ras_sysfs_remove_bad_page_node(struct amdgpu_device *adev)
1378{
1379 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1380
1381 sysfs_remove_file_from_group(&adev->dev->kobj,
1382 &con->badpages_attr.attr,
1383 RAS_FS_NAME);
1384}
1385
c030f2e4 1386static int amdgpu_ras_sysfs_remove_feature_node(struct amdgpu_device *adev)
1387{
1388 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1389 struct attribute *attrs[] = {
1390 &con->features_attr.attr,
1391 NULL
1392 };
1393 struct attribute_group group = {
eb0c3cd4 1394 .name = RAS_FS_NAME,
c030f2e4 1395 .attrs = attrs,
1396 };
1397
1398 sysfs_remove_group(&adev->dev->kobj, &group);
1399
1400 return 0;
1401}
1402
1403int amdgpu_ras_sysfs_create(struct amdgpu_device *adev,
9252d33d 1404 struct ras_common_if *head)
c030f2e4 1405{
9252d33d 1406 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
c030f2e4 1407
1408 if (!obj || obj->attr_inuse)
1409 return -EINVAL;
1410
1411 get_obj(obj);
1412
9252d33d 1413 snprintf(obj->fs_data.sysfs_name, sizeof(obj->fs_data.sysfs_name),
1414 "%s_err_count", head->name);
c030f2e4 1415
1416 obj->sysfs_attr = (struct device_attribute){
1417 .attr = {
1418 .name = obj->fs_data.sysfs_name,
1419 .mode = S_IRUGO,
1420 },
1421 .show = amdgpu_ras_sysfs_read,
1422 };
163def43 1423 sysfs_attr_init(&obj->sysfs_attr.attr);
c030f2e4 1424
1425 if (sysfs_add_file_to_group(&adev->dev->kobj,
1426 &obj->sysfs_attr.attr,
eb0c3cd4 1427 RAS_FS_NAME)) {
c030f2e4 1428 put_obj(obj);
1429 return -EINVAL;
1430 }
1431
1432 obj->attr_inuse = 1;
1433
1434 return 0;
1435}
1436
1437int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev,
1438 struct ras_common_if *head)
1439{
1440 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
1441
1442 if (!obj || !obj->attr_inuse)
1443 return -EINVAL;
1444
1445 sysfs_remove_file_from_group(&adev->dev->kobj,
1446 &obj->sysfs_attr.attr,
eb0c3cd4 1447 RAS_FS_NAME);
c030f2e4 1448 obj->attr_inuse = 0;
1449 put_obj(obj);
1450
1451 return 0;
1452}
1453
1454static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev)
1455{
1456 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1457 struct ras_manager *obj, *tmp;
1458
1459 list_for_each_entry_safe(obj, tmp, &con->head, node) {
1460 amdgpu_ras_sysfs_remove(adev, &obj->head);
1461 }
1462
f848159b
GC		 1463 if (amdgpu_bad_page_threshold != 0)
1464 amdgpu_ras_sysfs_remove_bad_page_node(adev);
1465
c030f2e4 1466 amdgpu_ras_sysfs_remove_feature_node(adev);
1467
1468 return 0;
1469}
1470/* sysfs end */
1471
ef177d11
AD		 1472/**
1473 * DOC: AMDGPU RAS Reboot Behavior for Unrecoverable Errors
1474 *
1475 * Normally when there is an uncorrectable error, the driver will reset
1476 * the GPU to recover. However, in the event of an unrecoverable error,
1477 * the driver provides an interface to reboot the system automatically
1478 * in that event.
1479 *
1480 * The following file in debugfs provides that interface:
1481 * /sys/kernel/debug/dri/[0/1/2...]/ras/auto_reboot
1482 *
1483 * Usage:
1484 *
1485 * .. code-block:: bash
1486 *
1487 * echo true > .../ras/auto_reboot
1488 *
1489 */
c030f2e4 1490/* debugfs begin */
ea1b8c9b 1491static struct dentry *amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device *adev)
36ea1bd2 1492{
1493 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
740f42a2 1494 struct amdgpu_ras_eeprom_control *eeprom = &con->eeprom_control;
ef0d7d20
LT		 1495 struct drm_minor *minor = adev_to_drm(adev)->primary;
1496 struct dentry *dir;
36ea1bd2 1497
88293c03
ND		 1498 dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);
1499 debugfs_create_file("ras_ctrl", S_IWUGO | S_IRUGO, dir, adev,
1500 &amdgpu_ras_debugfs_ctrl_ops);
1501 debugfs_create_file("ras_eeprom_reset", S_IWUGO | S_IRUGO, dir, adev,
1502 &amdgpu_ras_debugfs_eeprom_ops);
7fb64071
LT		 1503 debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,
1504 &con->bad_page_cnt_threshold);
740f42a2 1505 debugfs_create_u32("ras_num_recs", 0444, dir, &eeprom->ras_num_recs);
ef0d7d20
LT		 1506 debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled);
1507 debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled);
c65b0805
LT		 1508 debugfs_create_file("ras_eeprom_size", S_IRUGO, dir, adev,
1509 &amdgpu_ras_debugfs_eeprom_size_ops);
1510 con->de_ras_eeprom_table = debugfs_create_file("ras_eeprom_table",
1511 S_IRUGO, dir, adev,
1512 &amdgpu_ras_debugfs_eeprom_table_ops);
1513 amdgpu_ras_debugfs_set_ret_size(&con->eeprom_control);
c688a06b
GC		 1514
1515 /*
1516 * After one uncorrectable error happens, usually GPU recovery will
1517 * be scheduled. But due to the known problem in GPU recovery failing
1518 * to bring GPU back, below interface provides one direct way to
1519 * user to reboot system automatically in such case within
1520 * ERREVENT_ATHUB_INTERRUPT generated. Normal GPU recovery routine
1521 * will never be called.
1522 */
88293c03 1523 debugfs_create_bool("auto_reboot", S_IWUGO | S_IRUGO, dir, &con->reboot);
66459e1d
GC		 1524
1525 /*
1526 * User could set this not to clean up hardware's error count register
1527 * of RAS IPs during ras recovery.
1528 */
88293c03
ND		 1529 debugfs_create_bool("disable_ras_err_cnt_harvest", 0644, dir,
1530 &con->disable_ras_err_cnt_harvest);
1531 return dir;
36ea1bd2 1532}
1533
cedf7884 1534static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev,
88293c03
ND		 1535 struct ras_fs_if *head,
1536 struct dentry *dir)
c030f2e4 1537{
c030f2e4 1538 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &head->head);
c030f2e4 1539
88293c03 1540 if (!obj || !dir)
450f30ea 1541 return;
c030f2e4 1542
1543 get_obj(obj);
1544
1545 memcpy(obj->fs_data.debugfs_name,
1546 head->debugfs_name,
1547 sizeof(obj->fs_data.debugfs_name));
1548
88293c03
ND		 1549 debugfs_create_file(obj->fs_data.debugfs_name, S_IWUGO | S_IRUGO, dir,
1550 obj, &amdgpu_ras_debugfs_ops);
c030f2e4 1551}
1552
f9317014
TZ		 1553void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev)
1554{
1555 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
88293c03 1556 struct dentry *dir;
c1509f3f 1557 struct ras_manager *obj;
f9317014
TZ		 1558 struct ras_fs_if fs_info;
1559
1560 /*
1561 * it won't be called in resume path, no need to check
1562 * suspend and gpu reset status
1563 */
cedf7884 1564 if (!IS_ENABLED(CONFIG_DEBUG_FS) || !con)
f9317014
TZ		 1565 return;
1566
88293c03 1567 dir = amdgpu_ras_debugfs_create_ctrl_node(adev);
f9317014 1568
c1509f3f 1569 list_for_each_entry(obj, &con->head, node) {
f9317014
TZ		 1570 if (amdgpu_ras_is_supported(adev, obj->head.block) &&
1571 (obj->attr_inuse == 1)) {
1572 sprintf(fs_info.debugfs_name, "%s_err_inject",
640ae42e 1573 get_ras_block_str(&obj->head));
f9317014 1574 fs_info.head = obj->head;
88293c03 1575 amdgpu_ras_debugfs_create(adev, &fs_info, dir);
f9317014
TZ		 1576 }
1577 }
1578}
1579
c030f2e4 1580/* debugfs end */
1581
1582/* ras fs */
c3d4d45d
GC		 1583static BIN_ATTR(gpu_vram_bad_pages, S_IRUGO,
1584 amdgpu_ras_sysfs_badpages_read, NULL, 0);
1585static DEVICE_ATTR(features, S_IRUGO,
1586 amdgpu_ras_sysfs_features_read, NULL);
c030f2e4 1587static int amdgpu_ras_fs_init(struct amdgpu_device *adev)
1588{
c3d4d45d
GC		 1589 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1590 struct attribute_group group = {
1591 .name = RAS_FS_NAME,
1592 };
1593 struct attribute *attrs[] = {
1594 &con->features_attr.attr,
1595 NULL
1596 };
1597 struct bin_attribute *bin_attrs[] = {
1598 NULL,
1599 NULL,
1600 };
a069a9eb 1601 int r;
c030f2e4 1602
c3d4d45d
GC		 1603 /* add features entry */
1604 con->features_attr = dev_attr_features;
1605 group.attrs = attrs;
1606 sysfs_attr_init(attrs[0]);
1607
1608 if (amdgpu_bad_page_threshold != 0) {
1609 /* add bad_page_features entry */
1610 bin_attr_gpu_vram_bad_pages.private = NULL;
1611 con->badpages_attr = bin_attr_gpu_vram_bad_pages;
1612 bin_attrs[0] = &con->badpages_attr;
1613 group.bin_attrs = bin_attrs;
1614 sysfs_bin_attr_init(bin_attrs[0]);
1615 }
1616
a069a9eb
AD		 1617 r = sysfs_create_group(&adev->dev->kobj, &group);
1618 if (r)
1619 dev_err(adev->dev, "Failed to create RAS sysfs group!");
f848159b 1620
c030f2e4 1621 return 0;
1622}
1623
1624static int amdgpu_ras_fs_fini(struct amdgpu_device *adev)
1625{
88293c03
ND		 1626 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1627 struct ras_manager *con_obj, *ip_obj, *tmp;
1628
1629 if (IS_ENABLED(CONFIG_DEBUG_FS)) {
1630 list_for_each_entry_safe(con_obj, tmp, &con->head, node) {
1631 ip_obj = amdgpu_ras_find_obj(adev, &con_obj->head);
1632 if (ip_obj)
1633 put_obj(ip_obj);
1634 }
1635 }
1636
c030f2e4 1637 amdgpu_ras_sysfs_remove_all(adev);
1638 return 0;
1639}
1640/* ras fs end */
1641
1642/* ih begin */
b3c76814
TZ		 1643
1644/* For the hardware that cannot enable bif ring for both ras_controller_irq
1645 * and ras_err_evnet_athub_irq ih cookies, the driver has to poll status
1646 * register to check whether the interrupt is triggered or not, and properly
1647 * ack the interrupt if it is there
1648 */
1649void amdgpu_ras_interrupt_fatal_error_handler(struct amdgpu_device *adev)
1650{
950d6425 1651 /* Fatal error events are handled on host side */
8eba7205 1652 if (amdgpu_sriov_vf(adev))
b3c76814
TZ		 1653 return;
1654
1655 if (adev->nbio.ras &&
1656 adev->nbio.ras->handle_ras_controller_intr_no_bifring)
1657 adev->nbio.ras->handle_ras_controller_intr_no_bifring(adev);
1658
1659 if (adev->nbio.ras &&
1660 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring)
1661 adev->nbio.ras->handle_ras_err_event_athub_intr_no_bifring(adev);
1662}
1663
66f87949
TZ		 1664static void amdgpu_ras_interrupt_poison_consumption_handler(struct ras_manager *obj,
1665 struct amdgpu_iv_entry *entry)
1666{
b63ac5d3 1667 bool poison_stat = false;
66f87949 1668 struct amdgpu_device *adev = obj->adev;
66f87949
TZ		 1669 struct amdgpu_ras_block_object *block_obj =
1670 amdgpu_ras_get_ras_block(adev, obj->head.block, 0);
1671
ac7b25d9 1672 if (!block_obj)
b63ac5d3 1673 return;
66f87949 1674
b63ac5d3
TZ		 1675 /* both query_poison_status and handle_poison_consumption are optional,
1676 * but at least one of them should be implemented if we need poison
1677 * consumption handler
1678 */
ac7b25d9 1679 if (block_obj->hw_ops && block_obj->hw_ops->query_poison_status) {
b63ac5d3
TZ		 1680 poison_stat = block_obj->hw_ops->query_poison_status(adev);
1681 if (!poison_stat) {
1682 /* Not poison consumption interrupt, no need to handle it */
1683 dev_info(adev->dev, "No RAS poison status in %s poison IH.\n",
1684 block_obj->ras_comm.name);
1685
1686 return;
66f87949
TZ		 1687 }
1688 }
1689
38298ce6 1690 amdgpu_umc_poison_handler(adev, false);
b63ac5d3 1691
ac7b25d9 1692 if (block_obj->hw_ops && block_obj->hw_ops->handle_poison_consumption)
b63ac5d3
TZ		 1693 poison_stat = block_obj->hw_ops->handle_poison_consumption(adev);
1694
1695 /* gpu reset is fallback for failed and default cases */
1696 if (poison_stat) {
1697 dev_info(adev->dev, "GPU reset for %s RAS poison consumption is issued!\n",
1698 block_obj->ras_comm.name);
66f87949 1699 amdgpu_ras_reset_gpu(adev);
ac7b25d9
YC		 1700 } else {
1701 amdgpu_gfx_poison_consumption_handler(adev, entry);
b63ac5d3 1702 }
66f87949
TZ		 1703}
1704
50a7d025
TZ		 1705static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj,
1706 struct amdgpu_iv_entry *entry)
1707{
1708 dev_info(obj->adev->dev,
1709 "Poison is created, no user action is needed.\n");
1710}
1711
1712static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj,
1713 struct amdgpu_iv_entry *entry)
1714{
1715 struct ras_ih_data *data = &obj->ih_data;
1716 struct ras_err_data err_data = {0, 0, 0, NULL};
1717 int ret;
1718
1719 if (!data->cb)
1720 return;
1721
1722 /* Let IP handle its data, maybe we need get the output
1723 * from the callback to update the error type/count, etc
1724 */
1725 ret = data->cb(obj->adev, &err_data, entry);
1726 /* ue will trigger an interrupt, and in that case
1727 * we need do a reset to recovery the whole system.
1728 * But leave IP do that recovery, here we just dispatch
1729 * the error.
1730 */
1731 if (ret == AMDGPU_RAS_SUCCESS) {
1732 /* these counts could be left as 0 if
1733 * some blocks do not count error number
1734 */
1735 obj->err_data.ue_count += err_data.ue_count;
1736 obj->err_data.ce_count += err_data.ce_count;
1737 }
1738}
1739
c030f2e4 1740static void amdgpu_ras_interrupt_handler(struct ras_manager *obj)
1741{
1742 struct ras_ih_data *data = &obj->ih_data;
1743 struct amdgpu_iv_entry entry;
c030f2e4 1744
1745 while (data->rptr != data->wptr) {
1746 rmb();
1747 memcpy(&entry, &data->ring[data->rptr],
1748 data->element_size);
1749
1750 wmb();
1751 data->rptr = (data->aligned_element_size +
1752 data->rptr) % data->ring_size;
1753
50a7d025
TZ		 1754 if (amdgpu_ras_is_poison_mode_supported(obj->adev)) {
1755 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
1756 amdgpu_ras_interrupt_poison_creation_handler(obj, &entry);
66f87949
TZ		 1757 else
1758 amdgpu_ras_interrupt_poison_consumption_handler(obj, &entry);
50a7d025
TZ		 1759 } else {
1760 if (obj->head.block == AMDGPU_RAS_BLOCK__UMC)
1761 amdgpu_ras_interrupt_umc_handler(obj, &entry);
1762 else
1763 dev_warn(obj->adev->dev,
1764 "No RAS interrupt handler for non-UMC block with poison disabled.\n");
c030f2e4 1765 }
1766 }
1767}
1768
1769static void amdgpu_ras_interrupt_process_handler(struct work_struct *work)
1770{
1771 struct ras_ih_data *data =
1772 container_of(work, struct ras_ih_data, ih_work);
1773 struct ras_manager *obj =
1774 container_of(data, struct ras_manager, ih_data);
1775
1776 amdgpu_ras_interrupt_handler(obj);
1777}
1778
1779int amdgpu_ras_interrupt_dispatch(struct amdgpu_device *adev,
1780 struct ras_dispatch_if *info)
1781{
1782 struct ras_manager *obj = amdgpu_ras_find_obj(adev, &info->head);
1783 struct ras_ih_data *data = &obj->ih_data;
1784
1785 if (!obj)
1786 return -EINVAL;
1787
1788 if (data->inuse == 0)
1789 return 0;
1790
1791 /* Might be overflow... */
1792 memcpy(&data->ring[data->wptr], info->entry,
1793 data->element_size);
1794
1795 wmb();
1796 data->wptr = (data->aligned_element_size +
1797 data->wptr) % data->ring_size;
1798
1799 schedule_work(&data->ih_work);
1800
1801 return 0;
1802}
1803
1804int amdgpu_ras_interrupt_remove_handler(struct amdgpu_device *adev,
9252d33d 1805 struct ras_common_if *head)
c030f2e4 1806{
9252d33d 1807 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
c030f2e4 1808 struct ras_ih_data *data;
1809
1810 if (!obj)
1811 return -EINVAL;
1812
1813 data = &obj->ih_data;
1814 if (data->inuse == 0)
1815 return 0;
1816
1817 cancel_work_sync(&data->ih_work);
1818
1819 kfree(data->ring);
1820 memset(data, 0, sizeof(*data));
1821 put_obj(obj);
1822
1823 return 0;
1824}
1825
1826int amdgpu_ras_interrupt_add_handler(struct amdgpu_device *adev,
9252d33d 1827 struct ras_common_if *head)
c030f2e4 1828{
9252d33d 1829 struct ras_manager *obj = amdgpu_ras_find_obj(adev, head);
c030f2e4 1830 struct ras_ih_data *data;
9252d33d 1831 struct amdgpu_ras_block_object *ras_obj;
c030f2e4 1832
1833 if (!obj) {
1834 /* in case we registe the IH before enable ras feature */
9252d33d 1835 obj = amdgpu_ras_create_obj(adev, head);
c030f2e4 1836 if (!obj)
1837 return -EINVAL;
1838 } else
1839 get_obj(obj);
1840
9252d33d 1841 ras_obj = container_of(head, struct amdgpu_ras_block_object, ras_comm);
1842
c030f2e4 1843 data = &obj->ih_data;
1844 /* add the callback.etc */
1845 *data = (struct ras_ih_data) {
1846 .inuse = 0,
9252d33d 1847 .cb = ras_obj->ras_cb,
c030f2e4 1848 .element_size = sizeof(struct amdgpu_iv_entry),
1849 .rptr = 0,
1850 .wptr = 0,
1851 };
1852
1853 INIT_WORK(&data->ih_work, amdgpu_ras_interrupt_process_handler);
1854
1855 data->aligned_element_size = ALIGN(data->element_size, 8);
1856 /* the ring can store 64 iv entries. */
1857 data->ring_size = 64 * data->aligned_element_size;
1858 data->ring = kmalloc(data->ring_size, GFP_KERNEL);
1859 if (!data->ring) {
1860 put_obj(obj);
1861 return -ENOMEM;
1862 }
1863
1864 /* IH is ready */
1865 data->inuse = 1;
1866
1867 return 0;
1868}
1869
1870static int amdgpu_ras_interrupt_remove_all(struct amdgpu_device *adev)
1871{
1872 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1873 struct ras_manager *obj, *tmp;
1874
1875 list_for_each_entry_safe(obj, tmp, &con->head, node) {
9252d33d 1876 amdgpu_ras_interrupt_remove_handler(adev, &obj->head);
c030f2e4 1877 }
1878
1879 return 0;
1880}
1881/* ih end */
1882
313c8fd3
GC		 1883/* traversal all IPs except NBIO to query error counter */
1884static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev)
1885{
1886 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1887 struct ras_manager *obj;
1888
8ab0d6f0 1889 if (!adev->ras_enabled || !con)
313c8fd3
GC		 1890 return;
1891
1892 list_for_each_entry(obj, &con->head, node) {
1893 struct ras_query_if info = {
1894 .head = obj->head,
1895 };
1896
1897 /*
1898 * PCIE_BIF IP has one different isr by ras controller
1899 * interrupt, the specific ras counter query will be
1900 * done in that isr. So skip such block from common
1901 * sync flood interrupt isr calling.
1902 */
1903 if (info.head.block == AMDGPU_RAS_BLOCK__PCIE_BIF)
1904 continue;
1905
cf63b702
SY		 1906 /*
1907 * this is a workaround for aldebaran, skip send msg to
1908 * smu to get ecc_info table due to smu handle get ecc
1909 * info table failed temporarily.
1910 * should be removed until smu fix handle ecc_info table.
1911 */
1912 if ((info.head.block == AMDGPU_RAS_BLOCK__UMC) &&
1913 (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 2)))
1914 continue;
1915
761d86d3 1916 amdgpu_ras_query_error_status(adev, &info);
2a460963
CL		 1917
1918 if (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 2) &&
6da15a23
CL		 1919 adev->ip_versions[MP0_HWIP][0] != IP_VERSION(11, 0, 4) &&
1920 adev->ip_versions[MP0_HWIP][0] != IP_VERSION(13, 0, 0)) {
2a460963
CL		 1921 if (amdgpu_ras_reset_error_status(adev, info.head.block))
1922 dev_warn(adev->dev, "Failed to reset error counter and error status");
1923 }
313c8fd3
GC		 1924 }
1925}
1926
3f975d0f 1927/* Parse RdRspStatus and WrRspStatus */
cd92df93
LJ		 1928static void amdgpu_ras_error_status_query(struct amdgpu_device *adev,
1929 struct ras_query_if *info)
3f975d0f 1930{
8eb53bb2 1931 struct amdgpu_ras_block_object *block_obj;
3f975d0f
SY		 1932 /*
1933 * Only two block need to query read/write
1934 * RspStatus at current state
1935 */
5e67bba3 1936 if ((info->head.block != AMDGPU_RAS_BLOCK__GFX) &&
1937 (info->head.block != AMDGPU_RAS_BLOCK__MMHUB))
b6efdb02 1938 return;
1939
1940 block_obj = amdgpu_ras_get_ras_block(adev,
1941 info->head.block,
1942 info->head.sub_block_index);
5e67bba3 1943
5e67bba3 1944 if (!block_obj || !block_obj->hw_ops) {
afa37315
LT		 1945 dev_dbg_once(adev->dev, "%s doesn't config RAS function\n",
1946 get_ras_block_str(&info->head));
b6efdb02 1947 return;
3f975d0f 1948 }
5e67bba3 1949
1950 if (block_obj->hw_ops->query_ras_error_status)
ab3b9de6 1951 block_obj->hw_ops->query_ras_error_status(adev);
5e67bba3 1952
3f975d0f
SY		 1953}
1954
1955static void amdgpu_ras_query_err_status(struct amdgpu_device *adev)
1956{
1957 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1958 struct ras_manager *obj;
1959
8ab0d6f0 1960 if (!adev->ras_enabled || !con)
3f975d0f
SY		 1961 return;
1962
1963 list_for_each_entry(obj, &con->head, node) {
1964 struct ras_query_if info = {
1965 .head = obj->head,
1966 };
1967
1968 amdgpu_ras_error_status_query(adev, &info);
1969 }
1970}
1971
c030f2e4 1972/* recovery begin */
466b1793 1973
1974/* return 0 on success.
1975 * caller need free bps.
1976 */
1977static int amdgpu_ras_badpages_read(struct amdgpu_device *adev,
1978 struct ras_badpage **bps, unsigned int *count)
1979{
1980 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1981 struct ras_err_handler_data *data;
1982 int i = 0;
732f2a30 1983 int ret = 0, status;
466b1793 1984
1985 if (!con || !con->eh_data || !bps || !count)
1986 return -EINVAL;
1987
1988 mutex_lock(&con->recovery_lock);
1989 data = con->eh_data;
1990 if (!data || data->count == 0) {
1991 *bps = NULL;
46cf2fec 1992 ret = -EINVAL;
466b1793 1993 goto out;
1994 }
1995
1996 *bps = kmalloc(sizeof(struct ras_badpage) * data->count, GFP_KERNEL);
1997 if (!*bps) {
1998 ret = -ENOMEM;
1999 goto out;
2000 }
2001
2002 for (; i < data->count; i++) {
2003 (*bps)[i] = (struct ras_badpage){
9dc23a63 2004 .bp = data->bps[i].retired_page,
466b1793 2005 .size = AMDGPU_GPU_PAGE_SIZE,
52dd95f2 2006 .flags = AMDGPU_RAS_RETIRE_PAGE_RESERVED,
466b1793 2007 };
ec6aae97 2008 status = amdgpu_vram_mgr_query_page_status(&adev->mman.vram_mgr,
676deb38 2009 data->bps[i].retired_page);
732f2a30 2010 if (status == -EBUSY)
52dd95f2 2011 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_PENDING;
732f2a30 2012 else if (status == -ENOENT)
52dd95f2 2013 (*bps)[i].flags = AMDGPU_RAS_RETIRE_PAGE_FAULT;
466b1793 2014 }
2015
2016 *count = data->count;
2017out:
2018 mutex_unlock(&con->recovery_lock);
2019 return ret;
2020}
2021
c030f2e4 2022static void amdgpu_ras_do_recovery(struct work_struct *work)
2023{
2024 struct amdgpu_ras *ras =
2025 container_of(work, struct amdgpu_ras, recovery_work);
b3dbd6d3
JC		 2026 struct amdgpu_device *remote_adev = NULL;
2027 struct amdgpu_device *adev = ras->adev;
2028 struct list_head device_list, *device_list_handle = NULL;
b3dbd6d3 2029
f75e94d8 2030 if (!ras->disable_ras_err_cnt_harvest) {
d95e8e97
DL		 2031 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2032
f75e94d8
GC		 2033 /* Build list of devices to query RAS related errors */
2034 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) {
2035 device_list_handle = &hive->device_list;
2036 } else {
2037 INIT_LIST_HEAD(&device_list);
2038 list_add_tail(&adev->gmc.xgmi.head, &device_list);
2039 device_list_handle = &device_list;
2040 }
c030f2e4 2041
f75e94d8 2042 list_for_each_entry(remote_adev,
3f975d0f
SY		 2043 device_list_handle, gmc.xgmi.head) {
2044 amdgpu_ras_query_err_status(remote_adev);
f75e94d8 2045 amdgpu_ras_log_on_err_counter(remote_adev);
3f975d0f 2046 }
d95e8e97
DL		 2047
2048 amdgpu_put_xgmi_hive(hive);
b3dbd6d3 2049 }
313c8fd3 2050
f1549c09
LG		 2051 if (amdgpu_device_should_recover_gpu(ras->adev)) {
2052 struct amdgpu_reset_context reset_context;
2053 memset(&reset_context, 0, sizeof(reset_context));
2054
2055 reset_context.method = AMD_RESET_METHOD_NONE;
2056 reset_context.reset_req_dev = adev;
1a11a65d
YC		 2057
2058 /* Perform full reset in fatal error mode */
2059 if (!amdgpu_ras_is_poison_mode_supported(ras->adev))
2060 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
6c47a79b 2061 else {
1a11a65d 2062 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
f1549c09 2063
6c47a79b
YC		 2064 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) {
2065 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE2_RESET;
2066 reset_context.method = AMD_RESET_METHOD_MODE2;
2067 }
2c7cd280
YC		 2068
2069 /* Fatal error occurs in poison mode, mode1 reset is used to
2070 * recover gpu.
2071 */
2072 if (ras->gpu_reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) {
2073 ras->gpu_reset_flags &= ~AMDGPU_RAS_GPU_RESET_MODE1_RESET;
2074 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
2075 }
6c47a79b
YC		 2076 }
2077
f1549c09
LG		 2078 amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context);
2079 }
c030f2e4 2080 atomic_set(&ras->in_recovery, 0);
2081}
2082
c030f2e4 2083/* alloc/realloc bps array */
2084static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev,
2085 struct ras_err_handler_data *data, int pages)
2086{
2087 unsigned int old_space = data->count + data->space_left;
2088 unsigned int new_space = old_space + pages;
9dc23a63
TZ		 2089 unsigned int align_space = ALIGN(new_space, 512);
2090 void *bps = kmalloc(align_space * sizeof(*data->bps), GFP_KERNEL);
9dc23a63 2091
676deb38 2092 if (!bps) {
c030f2e4 2093 return -ENOMEM;
9dc23a63 2094 }
c030f2e4 2095
2096 if (data->bps) {
9dc23a63 2097 memcpy(bps, data->bps,
c030f2e4 2098 data->count * sizeof(*data->bps));
2099 kfree(data->bps);
2100 }
2101
9dc23a63 2102 data->bps = bps;
c030f2e4 2103 data->space_left += align_space - old_space;
2104 return 0;
2105}
2106
2107/* it deal with vram only. */
2108int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev,
9dc23a63 2109 struct eeprom_table_record *bps, int pages)
c030f2e4 2110{
2111 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
73aa8e1a 2112 struct ras_err_handler_data *data;
c030f2e4 2113 int ret = 0;
676deb38 2114 uint32_t i;
c030f2e4 2115
73aa8e1a 2116 if (!con || !con->eh_data || !bps || pages <= 0)
c030f2e4 2117 return 0;
2118
2119 mutex_lock(&con->recovery_lock);
73aa8e1a 2120 data = con->eh_data;
c030f2e4 2121 if (!data)
2122 goto out;
2123
676deb38
DL		 2124 for (i = 0; i < pages; i++) {
2125 if (amdgpu_ras_check_bad_page_unlock(con,
2126 bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT))
2127 continue;
2128
2129 if (!data->space_left &&
2130 amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
c030f2e4 2131 ret = -ENOMEM;
2132 goto out;
2133 }
2134
ec6aae97 2135 amdgpu_vram_mgr_reserve_range(&adev->mman.vram_mgr,
676deb38
DL		 2136 bps[i].retired_page << AMDGPU_GPU_PAGE_SHIFT,
2137 AMDGPU_GPU_PAGE_SIZE);
9dc23a63 2138
676deb38
DL		 2139 memcpy(&data->bps[data->count], &bps[i], sizeof(*data->bps));
2140 data->count++;
2141 data->space_left--;
2142 }
c030f2e4 2143out:
2144 mutex_unlock(&con->recovery_lock);
2145
2146 return ret;
2147}
2148
78ad00c9
TZ		 2149/*
2150 * write error record array to eeprom, the function should be
2151 * protected by recovery_lock
4d33e0f1 2152 * new_cnt: new added UE count, excluding reserved bad pages, can be NULL
78ad00c9 2153 */
4d33e0f1
TZ		 2154int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev,
2155 unsigned long *new_cnt)
78ad00c9
TZ		 2156{
2157 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2158 struct ras_err_handler_data *data;
8a3e801f 2159 struct amdgpu_ras_eeprom_control *control;
78ad00c9
TZ		 2160 int save_count;
2161
4d33e0f1
TZ		 2162 if (!con || !con->eh_data) {
2163 if (new_cnt)
2164 *new_cnt = 0;
2165
78ad00c9 2166 return 0;
4d33e0f1 2167 }
78ad00c9 2168
d9a69fe5 2169 mutex_lock(&con->recovery_lock);
8a3e801f 2170 control = &con->eeprom_control;
78ad00c9 2171 data = con->eh_data;
0686627b 2172 save_count = data->count - control->ras_num_recs;
d9a69fe5 2173 mutex_unlock(&con->recovery_lock);
4d33e0f1
TZ		 2174
2175 if (new_cnt)
2176 *new_cnt = save_count / adev->umc.retire_unit;
2177
78ad00c9 2178 /* only new entries are saved */
b1628425 2179 if (save_count > 0) {
63d4c081
LT		 2180 if (amdgpu_ras_eeprom_append(control,
2181 &data->bps[control->ras_num_recs],
2182 save_count)) {
6952e99c 2183 dev_err(adev->dev, "Failed to save EEPROM table data!");
78ad00c9
TZ		 2184 return -EIO;
2185 }
2186
b1628425
GC		 2187 dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count);
2188 }
2189
78ad00c9
TZ		 2190 return 0;
2191}
2192
2193/*
2194 * read error record array in eeprom and reserve enough space for
2195 * storing new bad pages
2196 */
2197static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev)
2198{
2199 struct amdgpu_ras_eeprom_control *control =
6457205c 2200 &adev->psp.ras_context.ras->eeprom_control;
e4e6a589
LT		 2201 struct eeprom_table_record *bps;
2202 int ret;
78ad00c9
TZ		 2203
2204 /* no bad page record, skip eeprom access */
0686627b 2205 if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0)
e4e6a589 2206 return 0;
78ad00c9 2207
0686627b 2208 bps = kcalloc(control->ras_num_recs, sizeof(*bps), GFP_KERNEL);
78ad00c9
TZ		 2209 if (!bps)
2210 return -ENOMEM;
2211
0686627b 2212 ret = amdgpu_ras_eeprom_read(control, bps, control->ras_num_recs);
e4e6a589 2213 if (ret)
6952e99c 2214 dev_err(adev->dev, "Failed to load EEPROM table records!");
e4e6a589 2215 else
0686627b 2216 ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs);
78ad00c9 2217
78ad00c9
TZ		 2218 kfree(bps);
2219 return ret;
2220}
2221
676deb38
DL		 2222static bool amdgpu_ras_check_bad_page_unlock(struct amdgpu_ras *con,
2223 uint64_t addr)
2224{
2225 struct ras_err_handler_data *data = con->eh_data;
2226 int i;
2227
2228 addr >>= AMDGPU_GPU_PAGE_SHIFT;
2229 for (i = 0; i < data->count; i++)
2230 if (addr == data->bps[i].retired_page)
2231 return true;
2232
2233 return false;
2234}
2235
6e4be987
TZ		 2236/*
2237 * check if an address belongs to bad page
2238 *
2239 * Note: this check is only for umc block
2240 */
2241static bool amdgpu_ras_check_bad_page(struct amdgpu_device *adev,
2242 uint64_t addr)
2243{
2244 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
6e4be987
TZ		 2245 bool ret = false;
2246
2247 if (!con || !con->eh_data)
2248 return ret;
2249
2250 mutex_lock(&con->recovery_lock);
676deb38 2251 ret = amdgpu_ras_check_bad_page_unlock(con, addr);
6e4be987
TZ		 2252 mutex_unlock(&con->recovery_lock);
2253 return ret;
2254}
2255
e5c04edf 2256static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev,
e4e6a589 2257 uint32_t max_count)
c84d4670 2258{
e5c04edf 2259 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
c84d4670
GC		 2260
2261 /*
2262 * Justification of value bad_page_cnt_threshold in ras structure
2263 *
f3cbe70e
TZ		 2264 * Generally, 0 <= amdgpu_bad_page_threshold <= max record length
2265 * in eeprom or amdgpu_bad_page_threshold == -2, introduce two
2266 * scenarios accordingly.
c84d4670
GC		 2267 *
2268 * Bad page retirement enablement:
f3cbe70e 2269 * - If amdgpu_bad_page_threshold = -2,
c84d4670
GC		 2270 * bad_page_cnt_threshold = typical value by formula.
2271 *
2272 * - When the value from user is 0 < amdgpu_bad_page_threshold <
2273 * max record length in eeprom, use it directly.
2274 *
2275 * Bad page retirement disablement:
2276 * - If amdgpu_bad_page_threshold = 0, bad page retirement
2277 * functionality is disabled, and bad_page_cnt_threshold will
2278 * take no effect.
2279 */
2280
e4e6a589
LT		 2281 if (amdgpu_bad_page_threshold < 0) {
2282 u64 val = adev->gmc.mc_vram_size;
c84d4670 2283
e4e6a589 2284 do_div(val, RAS_BAD_PAGE_COVER);
e5c04edf 2285 con->bad_page_cnt_threshold = min(lower_32_bits(val),
e4e6a589 2286 max_count);
e5c04edf 2287 } else {
e4e6a589
LT		 2288 con->bad_page_cnt_threshold = min_t(int, max_count,
2289 amdgpu_bad_page_threshold);
c84d4670
GC		 2290 }
2291}
2292
1a6fc071 2293int amdgpu_ras_recovery_init(struct amdgpu_device *adev)
c030f2e4 2294{
2295 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4d1337d2 2296 struct ras_err_handler_data **data;
e4e6a589 2297 u32 max_eeprom_records_count = 0;
b82e65a9 2298 bool exc_err_limit = false;
78ad00c9 2299 int ret;
c030f2e4 2300
e0e146d5 2301 if (!con || amdgpu_sriov_vf(adev))
1d9d2ca8
LT		 2302 return 0;
2303
2304 /* Allow access to RAS EEPROM via debugfs, when the ASIC
2305 * supports RAS and debugfs is enabled, but when
2306 * adev->ras_enabled is unset, i.e. when "ras_enable"
2307 * module parameter is set to 0.
2308 */
2309 con->adev = adev;
2310
2311 if (!adev->ras_enabled)
4d1337d2
AG		 2312 return 0;
2313
1d9d2ca8 2314 data = &con->eh_data;
1a6fc071
TZ		 2315 *data = kmalloc(sizeof(**data), GFP_KERNEL | __GFP_ZERO);
2316 if (!*data) {
2317 ret = -ENOMEM;
2318 goto out;
2319 }
c030f2e4 2320
2321 mutex_init(&con->recovery_lock);
2322 INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery);
2323 atomic_set(&con->in_recovery, 0);
69691c82 2324 con->eeprom_control.bad_channel_bitmap = 0;
c030f2e4 2325
7f599fed 2326 max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control);
e4e6a589 2327 amdgpu_ras_validate_threshold(adev, max_eeprom_records_count);
c84d4670 2328
e5086659 2329 /* Todo: During test the SMU might fail to read the eeprom through I2C
2330 * when the GPU is pending on XGMI reset during probe time
2331 * (Mostly after second bus reset), skip it now
2332 */
2333 if (adev->gmc.xgmi.pending_reset)
2334 return 0;
b82e65a9
GC		 2335 ret = amdgpu_ras_eeprom_init(&con->eeprom_control, &exc_err_limit);
2336 /*
2337 * This calling fails when exc_err_limit is true or
2338 * ret != 0.
2339 */
2340 if (exc_err_limit || ret)
1a6fc071 2341 goto free;
78ad00c9 2342
0686627b 2343 if (con->eeprom_control.ras_num_recs) {
78ad00c9
TZ		 2344 ret = amdgpu_ras_load_bad_pages(adev);
2345 if (ret)
1a6fc071 2346 goto free;
513befa6 2347
bc143d8b 2348 amdgpu_dpm_send_hbm_bad_pages_num(adev, con->eeprom_control.ras_num_recs);
69691c82
SY		 2349
2350 if (con->update_channel_flag == true) {
2351 amdgpu_dpm_send_hbm_bad_channel_flag(adev, con->eeprom_control.bad_channel_bitmap);
2352 con->update_channel_flag = false;
2353 }
78ad00c9 2354 }
c030f2e4 2355
12b2cab7
MJ		 2356#ifdef CONFIG_X86_MCE_AMD
2357 if ((adev->asic_type == CHIP_ALDEBARAN) &&
2358 (adev->gmc.xgmi.connected_to_cpu))
91a1a52d 2359 amdgpu_register_bad_pages_mca_notifier(adev);
12b2cab7 2360#endif
c030f2e4 2361 return 0;
1a6fc071 2362
1a6fc071 2363free:
1a6fc071 2364 kfree((*data)->bps);
1a6fc071 2365 kfree(*data);
1995b3a3 2366 con->eh_data = NULL;
1a6fc071 2367out:
cf696091 2368 dev_warn(adev->dev, "Failed to initialize ras recovery! (%d)\n", ret);
1a6fc071 2369
b82e65a9
GC		 2370 /*
2371 * Except error threshold exceeding case, other failure cases in this
2372 * function would not fail amdgpu driver init.
2373 */
2374 if (!exc_err_limit)
2375 ret = 0;
2376 else
2377 ret = -EINVAL;
2378
1a6fc071 2379 return ret;
c030f2e4 2380}
2381
2382static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev)
2383{
2384 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2385 struct ras_err_handler_data *data = con->eh_data;
2386
1a6fc071
TZ		 2387 /* recovery_init failed to init it, fini is useless */
2388 if (!data)
2389 return 0;
2390
c030f2e4 2391 cancel_work_sync(&con->recovery_work);
c030f2e4 2392
2393 mutex_lock(&con->recovery_lock);
2394 con->eh_data = NULL;
2395 kfree(data->bps);
2396 kfree(data);
2397 mutex_unlock(&con->recovery_lock);
2398
2399 return 0;
2400}
2401/* recovery end */
2402
084e2640 2403static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev)
5436ab94 2404{
82835055
YC		 2405 if (amdgpu_sriov_vf(adev)) {
2406 switch (adev->ip_versions[MP0_HWIP][0]) {
2407 case IP_VERSION(13, 0, 2):
2408 return true;
2409 default:
2410 return false;
2411 }
2412 }
2413
073285ef
YC		 2414 if (adev->asic_type == CHIP_IP_DISCOVERY) {
2415 switch (adev->ip_versions[MP0_HWIP][0]) {
2416 case IP_VERSION(13, 0, 0):
2417 case IP_VERSION(13, 0, 10):
2418 return true;
2419 default:
2420 return false;
2421 }
2422 }
2423
084e2640
LT		 2424 return adev->asic_type == CHIP_VEGA10 ||
2425 adev->asic_type == CHIP_VEGA20 ||
2426 adev->asic_type == CHIP_ARCTURUS ||
75f06251 2427 adev->asic_type == CHIP_ALDEBARAN ||
084e2640 2428 adev->asic_type == CHIP_SIENNA_CICHLID;
5436ab94
SY		 2429}
2430
f50160cf
SY		 2431/*
2432 * this is workaround for vega20 workstation sku,
2433 * force enable gfx ras, ignore vbios gfx ras flag
2434 * due to GC EDC can not write
2435 */
e509965e 2436static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
f50160cf
SY		 2437{
2438 struct atom_context *ctx = adev->mode_info.atom_context;
2439
2440 if (!ctx)
2441 return;
2442
2443 if (strnstr(ctx->vbios_version, "D16406",
e11d5e0d
SY		 2444 sizeof(ctx->vbios_version)) ||
2445 strnstr(ctx->vbios_version, "D36002",
2446 sizeof(ctx->vbios_version)))
8ab0d6f0 2447 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX);
f50160cf
SY		 2448}
2449
5caf466a 2450/*
2451 * check hardware's ras ability which will be saved in hw_supported.
2452 * if hardware does not support ras, we can skip some ras initializtion and
2453 * forbid some ras operations from IP.
2454 * if software itself, say boot parameter, limit the ras ability. We still
2455 * need allow IP do some limited operations, like disable. In such case,
2456 * we have to initialize ras as normal. but need check if operation is
2457 * allowed or not in each function.
2458 */
e509965e 2459static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
c030f2e4 2460{
8ab0d6f0 2461 adev->ras_hw_enabled = adev->ras_enabled = 0;
c030f2e4 2462
38298ce6 2463 if (!amdgpu_ras_asic_supported(adev))
5caf466a 2464 return;
b404ae82 2465
38298ce6 2466 if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
75f06251
HZ		 2467 if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
2468 dev_info(adev->dev, "MEM ECC is active.\n");
8ab0d6f0 2469 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__UMC |
e509965e 2470 1 << AMDGPU_RAS_BLOCK__DF);
75f06251
HZ		 2471 } else {
2472 dev_info(adev->dev, "MEM ECC is not presented.\n");
2473 }
88474cca 2474
75f06251
HZ		 2475 if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
2476 dev_info(adev->dev, "SRAM ECC is active.\n");
3189501e 2477 if (!amdgpu_sriov_vf(adev))
950d6425
SY		 2478 adev->ras_hw_enabled |= ~(1 << AMDGPU_RAS_BLOCK__UMC |
2479 1 << AMDGPU_RAS_BLOCK__DF);
3189501e 2480 else
950d6425
SY		 2481 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__PCIE_BIF |
2482 1 << AMDGPU_RAS_BLOCK__SDMA |
2483 1 << AMDGPU_RAS_BLOCK__GFX);
3189501e
TZ		 2484
2485 /* VCN/JPEG RAS can be supported on both bare metal and
2486 * SRIOV environment
2487 */
2488 if (adev->ip_versions[VCN_HWIP][0] == IP_VERSION(2, 6, 0) ||
2489 adev->ip_versions[VCN_HWIP][0] == IP_VERSION(4, 0, 0))
2490 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__VCN |
2491 1 << AMDGPU_RAS_BLOCK__JPEG);
2492 else
2493 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__VCN |
2494 1 << AMDGPU_RAS_BLOCK__JPEG);
58bc2a9c
SY		 2495
2496 /*
2497 * XGMI RAS is not supported if xgmi num physical nodes
2498 * is zero
2499 */
2500 if (!adev->gmc.xgmi.num_physical_nodes)
2501 adev->ras_hw_enabled &= ~(1 << AMDGPU_RAS_BLOCK__XGMI_WAFL);
75f06251
HZ		 2502 } else {
2503 dev_info(adev->dev, "SRAM ECC is not presented.\n");
2504 }
2505 } else {
2506 /* driver only manages a few IP blocks RAS feature
2507 * when GPU is connected cpu through XGMI */
8ab0d6f0 2508 adev->ras_hw_enabled |= (1 << AMDGPU_RAS_BLOCK__GFX |
e509965e
LT		 2509 1 << AMDGPU_RAS_BLOCK__SDMA |
2510 1 << AMDGPU_RAS_BLOCK__MMHUB);
75f06251 2511 }
88474cca 2512
e509965e 2513 amdgpu_ras_get_quirks(adev);
f50160cf 2514
88474cca 2515 /* hw_supported needs to be aligned with RAS block mask. */
8ab0d6f0 2516 adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
b404ae82 2517
8ab0d6f0
LT		 2518 adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
2519 adev->ras_hw_enabled & amdgpu_ras_mask;
c030f2e4 2520}
2521
05adfd80
LT		 2522static void amdgpu_ras_counte_dw(struct work_struct *work)
2523{
2524 struct amdgpu_ras *con = container_of(work, struct amdgpu_ras,
2525 ras_counte_delay_work.work);
2526 struct amdgpu_device *adev = con->adev;
a3fbb0d8 2527 struct drm_device *dev = adev_to_drm(adev);
05adfd80
LT		 2528 unsigned long ce_count, ue_count;
2529 int res;
2530
2531 res = pm_runtime_get_sync(dev->dev);
2532 if (res < 0)
2533 goto Out;
2534
2535 /* Cache new values.
2536 */
4a1c9a44 2537 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, NULL) == 0) {
4d9f771e
LT		 2538 atomic_set(&con->ras_ce_count, ce_count);
2539 atomic_set(&con->ras_ue_count, ue_count);
2540 }
05adfd80
LT		 2541
2542 pm_runtime_mark_last_busy(dev->dev);
2543Out:
2544 pm_runtime_put_autosuspend(dev->dev);
2545}
2546
2dd9032b
TZ		 2547static void amdgpu_ras_query_poison_mode(struct amdgpu_device *adev)
2548{
2549 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2550 bool df_poison, umc_poison;
2551
2552 /* poison setting is useless on SRIOV guest */
2553 if (amdgpu_sriov_vf(adev) || !con)
2554 return;
2555
2556 /* Init poison supported flag, the default value is false */
2557 if (adev->gmc.xgmi.connected_to_cpu) {
2558 /* enabled by default when GPU is connected to CPU */
2559 con->poison_supported = true;
2560 } else if (adev->df.funcs &&
2561 adev->df.funcs->query_ras_poison_mode &&
2562 adev->umc.ras &&
2563 adev->umc.ras->query_ras_poison_mode) {
2564 df_poison =
2565 adev->df.funcs->query_ras_poison_mode(adev);
2566 umc_poison =
2567 adev->umc.ras->query_ras_poison_mode(adev);
2568
2569 /* Only poison is set in both DF and UMC, we can support it */
2570 if (df_poison && umc_poison)
2571 con->poison_supported = true;
2572 else if (df_poison != umc_poison)
2573 dev_warn(adev->dev,
2574 "Poison setting is inconsistent in DF/UMC(%d:%d)!\n",
2575 df_poison, umc_poison);
2576 }
2577}
2578
c030f2e4 2579int amdgpu_ras_init(struct amdgpu_device *adev)
2580{
2581 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4e644fff 2582 int r;
c030f2e4 2583
b404ae82 2584 if (con)
c030f2e4 2585 return 0;
2586
2587 con = kmalloc(sizeof(struct amdgpu_ras) +
640ae42e
JC		 2588 sizeof(struct ras_manager) * AMDGPU_RAS_BLOCK_COUNT +
2589 sizeof(struct ras_manager) * AMDGPU_RAS_MCA_BLOCK_COUNT,
c030f2e4 2590 GFP_KERNEL|__GFP_ZERO);
2591 if (!con)
2592 return -ENOMEM;
2593
05adfd80
LT		 2594 con->adev = adev;
2595 INIT_DELAYED_WORK(&con->ras_counte_delay_work, amdgpu_ras_counte_dw);
2596 atomic_set(&con->ras_ce_count, 0);
2597 atomic_set(&con->ras_ue_count, 0);
2598
c030f2e4 2599 con->objs = (struct ras_manager *)(con + 1);
2600
2601 amdgpu_ras_set_context(adev, con);
2602
e509965e
LT		 2603 amdgpu_ras_check_supported(adev);
2604
7ddd9770 2605 if (!adev->ras_enabled || adev->asic_type == CHIP_VEGA10) {
970fd197
SY		 2606 /* set gfx block ras context feature for VEGA20 Gaming
2607 * send ras disable cmd to ras ta during ras late init.
2608 */
8ab0d6f0 2609 if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
970fd197
SY		 2610 con->features |= BIT(AMDGPU_RAS_BLOCK__GFX);
2611
2612 return 0;
2613 }
2614
5e91160a 2615 r = 0;
5436ab94 2616 goto release_con;
fb2a3607
HZ		 2617 }
2618
69691c82 2619 con->update_channel_flag = false;
c030f2e4 2620 con->features = 0;
2621 INIT_LIST_HEAD(&con->head);
108c6a63 2622 /* Might need get this flag from vbios. */
2623 con->flags = RAS_DEFAULT_FLAGS;
c030f2e4 2624
6e36f231
HZ		 2625 /* initialize nbio ras function ahead of any other
2626 * ras functions so hardware fatal error interrupt
2627 * can be enabled as early as possible */
fdc94d3a
HZ		 2628 switch (adev->ip_versions[NBIO_HWIP][0]) {
2629 case IP_VERSION(7, 4, 0):
2630 case IP_VERSION(7, 4, 1):
2631 case IP_VERSION(7, 4, 4):
2632 if (!adev->gmc.xgmi.connected_to_cpu)
2e54fe5d 2633 adev->nbio.ras = &nbio_v7_4_ras;
6e36f231 2634 break;
9af357bc
HZ		 2635 case IP_VERSION(4, 3, 0):
2636 if (adev->ras_hw_enabled & (1 << AMDGPU_RAS_BLOCK__DF))
2637 /* unlike other generation of nbio ras,
2638 * nbio v4_3 only support fatal error interrupt
2639 * to inform software that DF is freezed due to
2640 * system fatal error event. driver should not
2641 * enable nbio ras in such case. Instead,
2642 * check DF RAS */
2643 adev->nbio.ras = &nbio_v4_3_ras;
2644 break;
6e36f231
HZ		 2645 default:
2646 /* nbio ras is not available */
2647 break;
2648 }
2649
fdc94d3a
HZ		 2650 /* nbio ras block needs to be enabled ahead of other ras blocks
2651 * to handle fatal error */
2652 r = amdgpu_nbio_ras_sw_init(adev);
2653 if (r)
2654 return r;
2655
2e54fe5d 2656 if (adev->nbio.ras &&
2657 adev->nbio.ras->init_ras_controller_interrupt) {
2658 r = adev->nbio.ras->init_ras_controller_interrupt(adev);
4e644fff 2659 if (r)
5436ab94 2660 goto release_con;
4e644fff
HZ		 2661 }
2662
2e54fe5d 2663 if (adev->nbio.ras &&
2664 adev->nbio.ras->init_ras_err_event_athub_interrupt) {
2665 r = adev->nbio.ras->init_ras_err_event_athub_interrupt(adev);
4e644fff 2666 if (r)
5436ab94 2667 goto release_con;
4e644fff
HZ		 2668 }
2669
2dd9032b 2670 amdgpu_ras_query_poison_mode(adev);
e4348849 2671
5e91160a
GC		 2672 if (amdgpu_ras_fs_init(adev)) {
2673 r = -EINVAL;
5436ab94 2674 goto release_con;
5e91160a 2675 }
c030f2e4 2676
6952e99c 2677 dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
e509965e 2678 "hardware ability[%x] ras_mask[%x]\n",
8ab0d6f0 2679 adev->ras_hw_enabled, adev->ras_enabled);
e509965e 2680
c030f2e4 2681 return 0;
5436ab94 2682release_con:
c030f2e4 2683 amdgpu_ras_set_context(adev, NULL);
2684 kfree(con);
2685
5e91160a 2686 return r;
c030f2e4 2687}
2688
8f6368a9 2689int amdgpu_persistent_edc_harvesting_supported(struct amdgpu_device *adev)
134d16d5 2690{
8107e499
HZ		 2691 if (adev->gmc.xgmi.connected_to_cpu ||
2692 adev->gmc.is_app_apu)
134d16d5
JC		 2693 return 1;
2694 return 0;
2695}
2696
2697static int amdgpu_persistent_edc_harvesting(struct amdgpu_device *adev,
2698 struct ras_common_if *ras_block)
2699{
2700 struct ras_query_if info = {
2701 .head = *ras_block,
2702 };
2703
2704 if (!amdgpu_persistent_edc_harvesting_supported(adev))
2705 return 0;
2706
2707 if (amdgpu_ras_query_error_status(adev, &info) != 0)
2708 DRM_WARN("RAS init harvest failure");
2709
2710 if (amdgpu_ras_reset_error_status(adev, ras_block->block) != 0)
2711 DRM_WARN("RAS init harvest reset failure");
2712
2713 return 0;
2714}
2715
e4348849
TZ		 2716bool amdgpu_ras_is_poison_mode_supported(struct amdgpu_device *adev)
2717{
2718 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2719
2720 if (!con)
2721 return false;
2722
2723 return con->poison_supported;
2724}
2725
b293e891 2726/* helper function to handle common stuff in ip late init phase */
563285c8 2727int amdgpu_ras_block_late_init(struct amdgpu_device *adev,
2728 struct ras_common_if *ras_block)
b293e891 2729{
29c9b6cd 2730 struct amdgpu_ras_block_object *ras_obj = NULL;
05adfd80 2731 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
4a1c9a44 2732 struct ras_query_if *query_info;
05adfd80 2733 unsigned long ue_count, ce_count;
b293e891
HZ		 2734 int r;
2735
2736 /* disable RAS feature per IP block if it is not supported */
2737 if (!amdgpu_ras_is_supported(adev, ras_block->block)) {
2738 amdgpu_ras_feature_enable_on_boot(adev, ras_block, 0);
2739 return 0;
2740 }
2741
2742 r = amdgpu_ras_feature_enable_on_boot(adev, ras_block, 1);
2743 if (r) {
9080a18f 2744 if (adev->in_suspend || amdgpu_in_reset(adev)) {
b293e891
HZ		 2745 /* in resume phase, if fail to enable ras,
2746 * clean up all ras fs nodes, and disable ras */
2747 goto cleanup;
2748 } else
2749 return r;
2750 }
2751
134d16d5
JC		 2752 /* check for errors on warm reset edc persisant supported ASIC */
2753 amdgpu_persistent_edc_harvesting(adev, ras_block);
2754
b293e891 2755 /* in resume phase, no need to create ras fs node */
53b3f8f4 2756 if (adev->in_suspend || amdgpu_in_reset(adev))
b293e891
HZ		 2757 return 0;
2758
563285c8 2759 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
36780606
TZ		 2760 if (ras_obj->ras_cb || (ras_obj->hw_ops &&
2761 (ras_obj->hw_ops->query_poison_status ||
2762 ras_obj->hw_ops->handle_poison_consumption))) {
9252d33d 2763 r = amdgpu_ras_interrupt_add_handler(adev, ras_block);
b293e891 2764 if (r)
779596ce 2765 goto cleanup;
b293e891
HZ		 2766 }
2767
9252d33d 2768 r = amdgpu_ras_sysfs_create(adev, ras_block);
b293e891 2769 if (r)
779596ce 2770 goto interrupt;
b293e891 2771
05adfd80
LT		 2772 /* Those are the cached values at init.
2773 */
4a1c9a44
HZ		 2774 query_info = kzalloc(sizeof(struct ras_query_if), GFP_KERNEL);
2775 if (!query_info)
2776 return -ENOMEM;
2777 memcpy(&query_info->head, ras_block, sizeof(struct ras_common_if));
2778
2779 if (amdgpu_ras_query_error_count(adev, &ce_count, &ue_count, query_info) == 0) {
4d9f771e
LT		 2780 atomic_set(&con->ras_ce_count, ce_count);
2781 atomic_set(&con->ras_ue_count, ue_count);
2782 }
05adfd80 2783
4a1c9a44 2784 kfree(query_info);
b293e891 2785 return 0;
779596ce
TR		 2786
2787interrupt:
563285c8 2788 if (ras_obj->ras_cb)
9252d33d 2789 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
779596ce 2790cleanup:
b293e891
HZ		 2791 amdgpu_ras_feature_enable(adev, ras_block, 0);
2792 return r;
2793}
2794
d41ff22a 2795static int amdgpu_ras_block_late_init_default(struct amdgpu_device *adev,
418abce2 2796 struct ras_common_if *ras_block)
2797{
2798 return amdgpu_ras_block_late_init(adev, ras_block);
2799}
2800
b293e891 2801/* helper function to remove ras fs node and interrupt handler */
bdb3489c 2802void amdgpu_ras_block_late_fini(struct amdgpu_device *adev,
2803 struct ras_common_if *ras_block)
2804{
563285c8 2805 struct amdgpu_ras_block_object *ras_obj;
bdb3489c 2806 if (!ras_block)
2807 return;
2808
563285c8 2809 amdgpu_ras_sysfs_remove(adev, ras_block);
bdb3489c 2810
563285c8 2811 ras_obj = container_of(ras_block, struct amdgpu_ras_block_object, ras_comm);
2812 if (ras_obj->ras_cb)
2813 amdgpu_ras_interrupt_remove_handler(adev, ras_block);
bdb3489c 2814}
2815
80e0c2cb 2816static void amdgpu_ras_block_late_fini_default(struct amdgpu_device *adev,
2817 struct ras_common_if *ras_block)
2818{
2819 return amdgpu_ras_block_late_fini(adev, ras_block);
2820}
2821
a564808e 2822/* do some init work after IP late init as dependence.
511fdbc3 2823 * and it runs in resume/gpu reset/booting up cases.
a564808e 2824 */
511fdbc3 2825void amdgpu_ras_resume(struct amdgpu_device *adev)
108c6a63 2826{
2827 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2828 struct ras_manager *obj, *tmp;
2829
8ab0d6f0 2830 if (!adev->ras_enabled || !con) {
970fd197
SY		 2831 /* clean ras context for VEGA20 Gaming after send ras disable cmd */
2832 amdgpu_release_ras_context(adev);
2833
108c6a63 2834 return;
970fd197 2835 }
108c6a63 2836
108c6a63 2837 if (con->flags & AMDGPU_RAS_FLAG_INIT_BY_VBIOS) {
191051a1 2838 /* Set up all other IPs which are not implemented. There is a
2839 * tricky thing that IP's actual ras error type should be
2840 * MULTI_UNCORRECTABLE, but as driver does not handle it, so
2841 * ERROR_NONE make sense anyway.
2842 */
2843 amdgpu_ras_enable_all_features(adev, 1);
2844
2845 /* We enable ras on all hw_supported block, but as boot
2846 * parameter might disable some of them and one or more IP has
2847 * not implemented yet. So we disable them on behalf.
2848 */
108c6a63 2849 list_for_each_entry_safe(obj, tmp, &con->head, node) {
2850 if (!amdgpu_ras_is_supported(adev, obj->head.block)) {
2851 amdgpu_ras_feature_enable(adev, &obj->head, 0);
2852 /* there should be no any reference. */
2853 WARN_ON(alive_obj(obj));
2854 }
191051a1 2855 }
108c6a63 2856 }
2857}
2858
511fdbc3 2859void amdgpu_ras_suspend(struct amdgpu_device *adev)
2860{
2861 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2862
8ab0d6f0 2863 if (!adev->ras_enabled || !con)
511fdbc3 2864 return;
2865
2866 amdgpu_ras_disable_all_features(adev, 0);
2867 /* Make sure all ras objects are disabled. */
2868 if (con->features)
2869 amdgpu_ras_disable_all_features(adev, 1);
2870}
2871
867e24ca 2872int amdgpu_ras_late_init(struct amdgpu_device *adev)
2873{
2874 struct amdgpu_ras_block_list *node, *tmp;
2875 struct amdgpu_ras_block_object *obj;
2876 int r;
2877
950d6425
SY		 2878 /* Guest side doesn't need init ras feature */
2879 if (amdgpu_sriov_vf(adev))
2880 return 0;
2881
867e24ca 2882 list_for_each_entry_safe(node, tmp, &adev->ras_list, node) {
2883 if (!node->ras_obj) {
2884 dev_warn(adev->dev, "Warning: abnormal ras list node.\n");
2885 continue;
2886 }
418abce2 2887
867e24ca 2888 obj = node->ras_obj;
2889 if (obj->ras_late_init) {
2890 r = obj->ras_late_init(adev, &obj->ras_comm);
2891 if (r) {
2892 dev_err(adev->dev, "%s failed to execute ras_late_init! ret:%d\n",
2893 obj->ras_comm.name, r);
2894 return r;
2895 }
418abce2 2896 } else
2897 amdgpu_ras_block_late_init_default(adev, &obj->ras_comm);
867e24ca 2898 }
2899
2900 return 0;
2901}
2902
c030f2e4 2903/* do some fini work before IP fini as dependence */
2904int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
2905{
2906 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2907
8ab0d6f0 2908 if (!adev->ras_enabled || !con)
c030f2e4 2909 return 0;
2910
72c8c97b 2911
c030f2e4 2912 /* Need disable ras on all IPs here before ip [hw/sw]fini */
642c0401
YC		 2913 if (con->features)
2914 amdgpu_ras_disable_all_features(adev, 0);
c030f2e4 2915 amdgpu_ras_recovery_fini(adev);
2916 return 0;
2917}
2918
2919int amdgpu_ras_fini(struct amdgpu_device *adev)
2920{
d5e8ff5f 2921 struct amdgpu_ras_block_list *ras_node, *tmp;
1f211a82 2922 struct amdgpu_ras_block_object *obj = NULL;
c030f2e4 2923 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2924
8ab0d6f0 2925 if (!adev->ras_enabled || !con)
c030f2e4 2926 return 0;
2927
1f211a82 2928 list_for_each_entry_safe(ras_node, tmp, &adev->ras_list, node) {
2929 if (ras_node->ras_obj) {
2930 obj = ras_node->ras_obj;
2931 if (amdgpu_ras_is_supported(adev, obj->ras_comm.block) &&
2932 obj->ras_fini)
2933 obj->ras_fini(adev, &obj->ras_comm);
80e0c2cb 2934 else
2935 amdgpu_ras_block_late_fini_default(adev, &obj->ras_comm);
1f211a82 2936 }
2937
2938 /* Clear ras blocks from ras_list and free ras block list node */
2939 list_del(&ras_node->node);
2940 kfree(ras_node);
2941 }
2942
c030f2e4 2943 amdgpu_ras_fs_fini(adev);
2944 amdgpu_ras_interrupt_remove_all(adev);
2945
2946 WARN(con->features, "Feature mask is not cleared");
2947
2948 if (con->features)
2949 amdgpu_ras_disable_all_features(adev, 1);
2950
05adfd80
LT		 2951 cancel_delayed_work_sync(&con->ras_counte_delay_work);
2952
c030f2e4 2953 amdgpu_ras_set_context(adev, NULL);
2954 kfree(con);
2955
2956 return 0;
2957}
7c6e68c7
AG		 2958
2959void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
2960{
e509965e 2961 amdgpu_ras_check_supported(adev);
8ab0d6f0 2962 if (!adev->ras_hw_enabled)
ed606f8a
AG		 2963 return;
2964
7c6e68c7 2965 if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
2c7cd280
YC		 2966 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
2967
6952e99c
GC		 2968 dev_info(adev->dev, "uncorrectable hardware error"
2969 "(ERREVENT_ATHUB_INTERRUPT) detected!\n");
d5ea093e 2970
2c7cd280 2971 ras->gpu_reset_flags |= AMDGPU_RAS_GPU_RESET_MODE1_RESET;
61934624 2972 amdgpu_ras_reset_gpu(adev);
7c6e68c7
AG		 2973 }
2974}
bb5c7235
WS		 2975
2976bool amdgpu_ras_need_emergency_restart(struct amdgpu_device *adev)
2977{
2978 if (adev->asic_type == CHIP_VEGA20 &&
2979 adev->pm.fw_version <= 0x283400) {
2980 return !(amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) &&
2981 amdgpu_ras_intr_triggered();
2982 }
2983
2984 return false;
2985}
970fd197
SY		 2986
2987void amdgpu_release_ras_context(struct amdgpu_device *adev)
2988{
2989 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2990
2991 if (!con)
2992 return;
2993
8ab0d6f0 2994 if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
970fd197
SY		 2995 con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
2996 amdgpu_ras_set_context(adev, NULL);
2997 kfree(con);
2998 }
2999}
12b2cab7
MJ		 3000
3001#ifdef CONFIG_X86_MCE_AMD
3002static struct amdgpu_device *find_adev(uint32_t node_id)
3003{
12b2cab7
MJ		 3004 int i;
3005 struct amdgpu_device *adev = NULL;
3006
91a1a52d
MJ		 3007 for (i = 0; i < mce_adev_list.num_gpu; i++) {
3008 adev = mce_adev_list.devs[i];
12b2cab7 3009
91a1a52d 3010 if (adev && adev->gmc.xgmi.connected_to_cpu &&
12b2cab7
MJ		 3011 adev->gmc.xgmi.physical_node_id == node_id)
3012 break;
3013 adev = NULL;
3014 }
3015
12b2cab7
MJ		 3016 return adev;
3017}
3018
3019#define GET_MCA_IPID_GPUID(m) (((m) >> 44) & 0xF)
3020#define GET_UMC_INST(m) (((m) >> 21) & 0x7)
3021#define GET_CHAN_INDEX(m) ((((m) >> 12) & 0x3) | (((m) >> 18) & 0x4))
3022#define GPU_ID_OFFSET 8
3023
3024static int amdgpu_bad_page_notifier(struct notifier_block *nb,
3025 unsigned long val, void *data)
3026{
3027 struct mce *m = (struct mce *)data;
3028 struct amdgpu_device *adev = NULL;
3029 uint32_t gpu_id = 0;
cd4c99f1 3030 uint32_t umc_inst = 0, ch_inst = 0;
12b2cab7
MJ		 3031
3032 /*
3033 * If the error was generated in UMC_V2, which belongs to GPU UMCs,
3034 * and error occurred in DramECC (Extended error code = 0) then only
3035 * process the error, else bail out.
3036 */
91f75eb4 3037 if (!m || !((smca_get_bank_type(m->extcpu, m->bank) == SMCA_UMC_V2) &&
12b2cab7
MJ		 3038 (XEC(m->status, 0x3f) == 0x0)))
3039 return NOTIFY_DONE;
3040
3041 /*
3042 * If it is correctable error, return.
3043 */
3044 if (mce_is_correctable(m))
3045 return NOTIFY_OK;
3046
3047 /*
3048 * GPU Id is offset by GPU_ID_OFFSET in MCA_IPID_UMC register.
3049 */
3050 gpu_id = GET_MCA_IPID_GPUID(m->ipid) - GPU_ID_OFFSET;
3051
3052 adev = find_adev(gpu_id);
3053 if (!adev) {
3054 DRM_WARN("%s: Unable to find adev for gpu_id: %d\n", __func__,
3055 gpu_id);
3056 return NOTIFY_DONE;
3057 }
3058
3059 /*
3060 * If it is uncorrectable error, then find out UMC instance and
3061 * channel index.
3062 */
3063 umc_inst = GET_UMC_INST(m->ipid);
3064 ch_inst = GET_CHAN_INDEX(m->ipid);
3065
3066 dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d",
3067 umc_inst, ch_inst);
3068
24b82292
TZ		 3069 if (!amdgpu_umc_page_retirement_mca(adev, m->addr, ch_inst, umc_inst))
3070 return NOTIFY_OK;
3071 else
6c0ca748 3072 return NOTIFY_DONE;
12b2cab7
MJ		 3073}
3074
3075static struct notifier_block amdgpu_bad_page_nb = {
3076 .notifier_call = amdgpu_bad_page_notifier,
3077 .priority = MCE_PRIO_UC,
3078};
3079
91a1a52d 3080static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev)
12b2cab7 3081{
91a1a52d
MJ		 3082 /*
3083 * Add the adev to the mce_adev_list.
3084 * During mode2 reset, amdgpu device is temporarily
3085 * removed from the mgpu_info list which can cause
3086 * page retirement to fail.
3087 * Use this list instead of mgpu_info to find the amdgpu
3088 * device on which the UMC error was reported.
3089 */
3090 mce_adev_list.devs[mce_adev_list.num_gpu++] = adev;
3091
12b2cab7
MJ		 3092 /*
3093 * Register the x86 notifier only once
3094 * with MCE subsystem.
3095 */
3096 if (notifier_registered == false) {
3097 mce_register_decode_chain(&amdgpu_bad_page_nb);
3098 notifier_registered = true;
3099 }
3100}
3101#endif
7cab2124 3102
b6efdb02 3103struct amdgpu_ras *amdgpu_ras_get_context(struct amdgpu_device *adev)
7cab2124 3104{
3105 if (!adev)
3106 return NULL;
3107
3108 return adev->psp.ras_context.ras;
3109}
3110
b6efdb02 3111int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con)
7cab2124 3112{
3113 if (!adev)
69f91d32 3114 return -EINVAL;
7cab2124 3115
3116 adev->psp.ras_context.ras = ras_con;
3117 return 0;
3118}
3119
3120/* check if ras is supported on block, say, sdma, gfx */
3121int amdgpu_ras_is_supported(struct amdgpu_device *adev,
3122 unsigned int block)
3123{
8f453c51 3124 int ret = 0;
7cab2124 3125 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3126
3127 if (block >= AMDGPU_RAS_BLOCK_COUNT)
3128 return 0;
8f453c51
YC		 3129
3130 ret = ras && (adev->ras_enabled & (1 << block));
3131
3132 /* For the special asic with mem ecc enabled but sram ecc
3133 * not enabled, even if the ras block is not supported on
3134 * .ras_enabled, if the asic supports poison mode and the
3135 * ras block has ras configuration, it can be considered
3136 * that the ras block supports ras function.
3137 */
3138 if (!ret &&
3139 amdgpu_ras_is_poison_mode_supported(adev) &&
3140 amdgpu_ras_get_ras_block(adev, block, 0))
3141 ret = 1;
3142
3143 return ret;
7cab2124 3144}
3145
3146int amdgpu_ras_reset_gpu(struct amdgpu_device *adev)
3147{
3148 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
3149
3150 if (atomic_cmpxchg(&ras->in_recovery, 0, 1) == 0)
25a2b22e 3151 amdgpu_reset_domain_schedule(ras->adev->reset_domain, &ras->recovery_work);
7cab2124 3152 return 0;
3153}
3154
3155
6492e1b0 3156/* Register each ip ras block into amdgpu ras */
3157int amdgpu_ras_register_ras_block(struct amdgpu_device *adev,
b6efdb02 3158 struct amdgpu_ras_block_object *ras_block_obj)
6492e1b0 3159{
d5e8ff5f 3160 struct amdgpu_ras_block_list *ras_node;
6492e1b0 3161 if (!adev || !ras_block_obj)
3162 return -EINVAL;
3163
d5e8ff5f 3164 ras_node = kzalloc(sizeof(*ras_node), GFP_KERNEL);
3165 if (!ras_node)
3166 return -ENOMEM;
3167
3168 INIT_LIST_HEAD(&ras_node->node);
3169 ras_node->ras_obj = ras_block_obj;
3170 list_add_tail(&ras_node->node, &adev->ras_list);
6492e1b0 3171
3172 return 0;
3173}
322a7e00
HZ		 3174
3175void amdgpu_ras_get_error_type_name(uint32_t err_type, char *err_type_name)
3176{
3177 if (!err_type_name)
3178 return;
3179
3180 switch (err_type) {
3181 case AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE:
3182 sprintf(err_type_name, "correctable");
3183 break;
3184 case AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE:
3185 sprintf(err_type_name, "uncorrectable");
3186 break;
3187 default:
3188 sprintf(err_type_name, "unknown");
3189 break;
3190 }
3191}
3192
3193bool amdgpu_ras_inst_get_memory_id_field(struct amdgpu_device *adev,
3194 const struct amdgpu_ras_err_status_reg_entry *reg_entry,
3195 uint32_t instance,
3196 uint32_t *memory_id)
3197{
3198 uint32_t err_status_lo_data, err_status_lo_offset;
3199
3200 if (!reg_entry)
3201 return false;
3202
3203 err_status_lo_offset =
3204 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
3205 reg_entry->seg_lo, reg_entry->reg_lo);
3206 err_status_lo_data = RREG32(err_status_lo_offset);
3207
3208 if ((reg_entry->flags & AMDGPU_RAS_ERR_STATUS_VALID) &&
3209 !REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, ERR_STATUS_VALID_FLAG))
3210 return false;
3211
3212 *memory_id = REG_GET_FIELD(err_status_lo_data, ERR_STATUS_LO, MEMORY_ID);
3213
3214 return true;
3215}
3216
3217bool amdgpu_ras_inst_get_err_cnt_field(struct amdgpu_device *adev,
3218 const struct amdgpu_ras_err_status_reg_entry *reg_entry,
3219 uint32_t instance,
3220 unsigned long *err_cnt)
3221{
3222 uint32_t err_status_hi_data, err_status_hi_offset;
3223
3224 if (!reg_entry)
3225 return false;
3226
3227 err_status_hi_offset =
3228 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_entry->hwip, instance,
3229 reg_entry->seg_hi, reg_entry->reg_hi);
3230 err_status_hi_data = RREG32(err_status_hi_offset);
3231
3232 if ((reg_entry->flags & AMDGPU_RAS_ERR_INFO_VALID) &&
3233 !REG_GET_FIELD(err_status_hi_data, ERR_STATUS_HI, ERR_INFO_VALID_FLAG))
9b337b7d
HZ		 3234 /* keep the check here in case we need to refer to the result later */
3235 dev_dbg(adev->dev, "Invalid err_info field\n");
322a7e00
HZ		 3236
3237 /* read err count */
3238 *err_cnt = REG_GET_FIELD(err_status_hi_data, ERR_STATUS, ERR_CNT);
3239
3240 return true;
3241}
3242
3243void amdgpu_ras_inst_query_ras_error_count(struct amdgpu_device *adev,
3244 const struct amdgpu_ras_err_status_reg_entry *reg_list,
3245 uint32_t reg_list_size,
3246 const struct amdgpu_ras_memory_id_entry *mem_list,
3247 uint32_t mem_list_size,
3248 uint32_t instance,
3249 uint32_t err_type,
3250 unsigned long *err_count)
3251{
3252 uint32_t memory_id;
3253 unsigned long err_cnt;
3254 char err_type_name[16];
3255 uint32_t i, j;
3256
3257 for (i = 0; i < reg_list_size; i++) {
9b337b7d
HZ		 3258 /* query memory_id from err_status_lo */
3259 if (!amdgpu_ras_inst_get_memory_id_field(adev, &reg_list[i],
3260 instance, &memory_id))
3261 continue;
3262
322a7e00
HZ		 3263 /* query err_cnt from err_status_hi */
3264 if (!amdgpu_ras_inst_get_err_cnt_field(adev, &reg_list[i],
3265 instance, &err_cnt) ||
3266 !err_cnt)
3267 continue;
3268
322a7e00
HZ		 3269 *err_count += err_cnt;
3270
3271 /* log the errors */
3272 amdgpu_ras_get_error_type_name(err_type, err_type_name);
3273 if (!mem_list) {
3274 /* memory_list is not supported */
3275 dev_info(adev->dev,
3276 "%ld %s hardware errors detected in %s, instance: %d, memory_id: %d\n",
3277 err_cnt, err_type_name,
3278 reg_list[i].block_name,
3279 instance, memory_id);
3280 } else {
3281 for (j = 0; j < mem_list_size; j++) {
3282 if (memory_id == mem_list[j].memory_id) {
3283 dev_info(adev->dev,
3284 "%ld %s hardware errors detected in %s, instance: %d, memory block: %s\n",
3285 err_cnt, err_type_name,
3286 reg_list[i].block_name,
3287 instance, mem_list[j].name);
3288 break;
3289 }
3290 }
3291 }
3292 }
3293}
e53a3250
HZ		 3294
3295void amdgpu_ras_inst_reset_ras_error_count(struct amdgpu_device *adev,
3296 const struct amdgpu_ras_err_status_reg_entry *reg_list,
3297 uint32_t reg_list_size,
3298 uint32_t instance)
3299{
3300 uint32_t err_status_lo_offset, err_status_hi_offset;
3301 uint32_t i;
3302
3303 for (i = 0; i < reg_list_size; i++) {
3304 err_status_lo_offset =
3305 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
3306 reg_list[i].seg_lo, reg_list[i].reg_lo);
3307 err_status_hi_offset =
3308 AMDGPU_RAS_REG_ENTRY_OFFSET(reg_list[i].hwip, instance,
3309 reg_list[i].seg_hi, reg_list[i].reg_hi);
3310 WREG32(err_status_lo_offset, 0);
3311 WREG32(err_status_hi_offset, 0);
3312 }
3313}
Linux 6.x block layer and io_uring tree(s)