projects / linux-2.6-block.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
drm/amdgpu: fix the hibernation suspend with s0ix
[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / amdgpu_device.c
... / ...
CommitLineData
1/*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28#include <linux/power_supply.h>
29#include <linux/kthread.h>
30#include <linux/module.h>
31#include <linux/console.h>
32#include <linux/slab.h>
33
34#include <drm/drm_atomic_helper.h>
35#include <drm/drm_probe_helper.h>
36#include <drm/amdgpu_drm.h>
37#include <linux/vgaarb.h>
38#include <linux/vga_switcheroo.h>
39#include <linux/efi.h>
40#include "amdgpu.h"
41#include "amdgpu_trace.h"
42#include "amdgpu_i2c.h"
43#include "atom.h"
44#include "amdgpu_atombios.h"
45#include "amdgpu_atomfirmware.h"
46#include "amd_pcie.h"
47#ifdef CONFIG_DRM_AMDGPU_SI
48#include "si.h"
49#endif
50#ifdef CONFIG_DRM_AMDGPU_CIK
51#include "cik.h"
52#endif
53#include "vi.h"
54#include "soc15.h"
55#include "nv.h"
56#include "bif/bif_4_1_d.h"
57#include <linux/pci.h>
58#include <linux/firmware.h>
59#include "amdgpu_vf_error.h"
60
61#include "amdgpu_amdkfd.h"
62#include "amdgpu_pm.h"
63
64#include "amdgpu_xgmi.h"
65#include "amdgpu_ras.h"
66#include "amdgpu_pmu.h"
67#include "amdgpu_fru_eeprom.h"
68
69#include <linux/suspend.h>
70#include <drm/task_barrier.h>
71#include <linux/pm_runtime.h>
72
73MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
74MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
75MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
76MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
77MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
78MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
79MODULE_FIRMWARE("amdgpu/renoir_gpu_info.bin");
80MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin");
81MODULE_FIRMWARE("amdgpu/navi14_gpu_info.bin");
82MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
83MODULE_FIRMWARE("amdgpu/vangogh_gpu_info.bin");
84
85#define AMDGPU_RESUME_MS 2000
86
87const char *amdgpu_asic_name[] = {
88 "TAHITI",
89 "PITCAIRN",
90 "VERDE",
91 "OLAND",
92 "HAINAN",
93 "BONAIRE",
94 "KAVERI",
95 "KABINI",
96 "HAWAII",
97 "MULLINS",
98 "TOPAZ",
99 "TONGA",
100 "FIJI",
101 "CARRIZO",
102 "STONEY",
103 "POLARIS10",
104 "POLARIS11",
105 "POLARIS12",
106 "VEGAM",
107 "VEGA10",
108 "VEGA12",
109 "VEGA20",
110 "RAVEN",
111 "ARCTURUS",
112 "RENOIR",
113 "ALDEBARAN",
114 "NAVI10",
115 "NAVI14",
116 "NAVI12",
117 "SIENNA_CICHLID",
118 "NAVY_FLOUNDER",
119 "VANGOGH",
120 "DIMGREY_CAVEFISH",
121 "LAST",
122};
123
124/**
125 * DOC: pcie_replay_count
126 *
127 * The amdgpu driver provides a sysfs API for reporting the total number
128 * of PCIe replays (NAKs)
129 * The file pcie_replay_count is used for this and returns the total
130 * number of replays as a sum of the NAKs generated and NAKs received
131 */
132
133static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
134 struct device_attribute *attr, char *buf)
135{
136 struct drm_device *ddev = dev_get_drvdata(dev);
137 struct amdgpu_device *adev = drm_to_adev(ddev);
138 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
139
140 return snprintf(buf, PAGE_SIZE, "%llu\n", cnt);
141}
142
143static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
144 amdgpu_device_get_pcie_replay_count, NULL);
145
146static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
147
148/**
149 * DOC: product_name
150 *
151 * The amdgpu driver provides a sysfs API for reporting the product name
152 * for the device
153 * The file serial_number is used for this and returns the product name
154 * as returned from the FRU.
155 * NOTE: This is only available for certain server cards
156 */
157
158static ssize_t amdgpu_device_get_product_name(struct device *dev,
159 struct device_attribute *attr, char *buf)
160{
161 struct drm_device *ddev = dev_get_drvdata(dev);
162 struct amdgpu_device *adev = drm_to_adev(ddev);
163
164 return snprintf(buf, PAGE_SIZE, "%s\n", adev->product_name);
165}
166
167static DEVICE_ATTR(product_name, S_IRUGO,
168 amdgpu_device_get_product_name, NULL);
169
170/**
171 * DOC: product_number
172 *
173 * The amdgpu driver provides a sysfs API for reporting the part number
174 * for the device
175 * The file serial_number is used for this and returns the part number
176 * as returned from the FRU.
177 * NOTE: This is only available for certain server cards
178 */
179
180static ssize_t amdgpu_device_get_product_number(struct device *dev,
181 struct device_attribute *attr, char *buf)
182{
183 struct drm_device *ddev = dev_get_drvdata(dev);
184 struct amdgpu_device *adev = drm_to_adev(ddev);
185
186 return snprintf(buf, PAGE_SIZE, "%s\n", adev->product_number);
187}
188
189static DEVICE_ATTR(product_number, S_IRUGO,
190 amdgpu_device_get_product_number, NULL);
191
192/**
193 * DOC: serial_number
194 *
195 * The amdgpu driver provides a sysfs API for reporting the serial number
196 * for the device
197 * The file serial_number is used for this and returns the serial number
198 * as returned from the FRU.
199 * NOTE: This is only available for certain server cards
200 */
201
202static ssize_t amdgpu_device_get_serial_number(struct device *dev,
203 struct device_attribute *attr, char *buf)
204{
205 struct drm_device *ddev = dev_get_drvdata(dev);
206 struct amdgpu_device *adev = drm_to_adev(ddev);
207
208 return snprintf(buf, PAGE_SIZE, "%s\n", adev->serial);
209}
210
211static DEVICE_ATTR(serial_number, S_IRUGO,
212 amdgpu_device_get_serial_number, NULL);
213
214/**
215 * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
216 *
217 * @dev: drm_device pointer
218 *
219 * Returns true if the device is a dGPU with ATPX power control,
220 * otherwise return false.
221 */
222bool amdgpu_device_supports_px(struct drm_device *dev)
223{
224 struct amdgpu_device *adev = drm_to_adev(dev);
225
226 if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
227 return true;
228 return false;
229}
230
231/**
232 * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
233 *
234 * @dev: drm_device pointer
235 *
236 * Returns true if the device is a dGPU with ACPI power control,
237 * otherwise return false.
238 */
239bool amdgpu_device_supports_boco(struct drm_device *dev)
240{
241 struct amdgpu_device *adev = drm_to_adev(dev);
242
243 if (adev->has_pr3 ||
244 ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
245 return true;
246 return false;
247}
248
249/**
250 * amdgpu_device_supports_baco - Does the device support BACO
251 *
252 * @dev: drm_device pointer
253 *
254 * Returns true if the device supporte BACO,
255 * otherwise return false.
256 */
257bool amdgpu_device_supports_baco(struct drm_device *dev)
258{
259 struct amdgpu_device *adev = drm_to_adev(dev);
260
261 return amdgpu_asic_supports_baco(adev);
262}
263
264/*
265 * VRAM access helper functions
266 */
267
268/**
269 * amdgpu_device_vram_access - read/write a buffer in vram
270 *
271 * @adev: amdgpu_device pointer
272 * @pos: offset of the buffer in vram
273 * @buf: virtual address of the buffer in system memory
274 * @size: read/write size, sizeof(@buf) must > @size
275 * @write: true - write to vram, otherwise - read from vram
276 */
277void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
278 uint32_t *buf, size_t size, bool write)
279{
280 unsigned long flags;
281 uint32_t hi = ~0;
282 uint64_t last;
283
284
285#ifdef CONFIG_64BIT
286 last = min(pos + size, adev->gmc.visible_vram_size);
287 if (last > pos) {
288 void __iomem *addr = adev->mman.aper_base_kaddr + pos;
289 size_t count = last - pos;
290
291 if (write) {
292 memcpy_toio(addr, buf, count);
293 mb();
294 amdgpu_asic_flush_hdp(adev, NULL);
295 } else {
296 amdgpu_asic_invalidate_hdp(adev, NULL);
297 mb();
298 memcpy_fromio(buf, addr, count);
299 }
300
301 if (count == size)
302 return;
303
304 pos += count;
305 buf += count / 4;
306 size -= count;
307 }
308#endif
309
310 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
311 for (last = pos + size; pos < last; pos += 4) {
312 uint32_t tmp = pos >> 31;
313
314 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
315 if (tmp != hi) {
316 WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
317 hi = tmp;
318 }
319 if (write)
320 WREG32_NO_KIQ(mmMM_DATA, *buf++);
321 else
322 *buf++ = RREG32_NO_KIQ(mmMM_DATA);
323 }
324 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
325}
326
327/*
328 * register access helper functions.
329 */
330
331/* Check if hw access should be skipped because of hotplug or device error */
332bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
333{
334 if (adev->in_pci_err_recovery)
335 return true;
336
337#ifdef CONFIG_LOCKDEP
338 /*
339 * This is a bit complicated to understand, so worth a comment. What we assert
340 * here is that the GPU reset is not running on another thread in parallel.
341 *
342 * For this we trylock the read side of the reset semaphore, if that succeeds
343 * we know that the reset is not running in paralell.
344 *
345 * If the trylock fails we assert that we are either already holding the read
346 * side of the lock or are the reset thread itself and hold the write side of
347 * the lock.
348 */
349 if (in_task()) {
350 if (down_read_trylock(&adev->reset_sem))
351 up_read(&adev->reset_sem);
352 else
353 lockdep_assert_held(&adev->reset_sem);
354 }
355#endif
356 return false;
357}
358
359/**
360 * amdgpu_device_rreg - read a memory mapped IO or indirect register
361 *
362 * @adev: amdgpu_device pointer
363 * @reg: dword aligned register offset
364 * @acc_flags: access flags which require special behavior
365 *
366 * Returns the 32 bit value from the offset specified.
367 */
368uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
369 uint32_t reg, uint32_t acc_flags)
370{
371 uint32_t ret;
372
373 if (amdgpu_device_skip_hw_access(adev))
374 return 0;
375
376 if ((reg * 4) < adev->rmmio_size) {
377 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
378 amdgpu_sriov_runtime(adev) &&
379 down_read_trylock(&adev->reset_sem)) {
380 ret = amdgpu_kiq_rreg(adev, reg);
381 up_read(&adev->reset_sem);
382 } else {
383 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
384 }
385 } else {
386 ret = adev->pcie_rreg(adev, reg * 4);
387 }
388
389 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
390
391 return ret;
392}
393
394/*
395 * MMIO register read with bytes helper functions
396 * @offset:bytes offset from MMIO start
397 *
398*/
399
400/**
401 * amdgpu_mm_rreg8 - read a memory mapped IO register
402 *
403 * @adev: amdgpu_device pointer
404 * @offset: byte aligned register offset
405 *
406 * Returns the 8 bit value from the offset specified.
407 */
408uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
409{
410 if (amdgpu_device_skip_hw_access(adev))
411 return 0;
412
413 if (offset < adev->rmmio_size)
414 return (readb(adev->rmmio + offset));
415 BUG();
416}
417
418/*
419 * MMIO register write with bytes helper functions
420 * @offset:bytes offset from MMIO start
421 * @value: the value want to be written to the register
422 *
423*/
424/**
425 * amdgpu_mm_wreg8 - read a memory mapped IO register
426 *
427 * @adev: amdgpu_device pointer
428 * @offset: byte aligned register offset
429 * @value: 8 bit value to write
430 *
431 * Writes the value specified to the offset specified.
432 */
433void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
434{
435 if (amdgpu_device_skip_hw_access(adev))
436 return;
437
438 if (offset < adev->rmmio_size)
439 writeb(value, adev->rmmio + offset);
440 else
441 BUG();
442}
443
444/**
445 * amdgpu_device_wreg - write to a memory mapped IO or indirect register
446 *
447 * @adev: amdgpu_device pointer
448 * @reg: dword aligned register offset
449 * @v: 32 bit value to write to the register
450 * @acc_flags: access flags which require special behavior
451 *
452 * Writes the value specified to the offset specified.
453 */
454void amdgpu_device_wreg(struct amdgpu_device *adev,
455 uint32_t reg, uint32_t v,
456 uint32_t acc_flags)
457{
458 if (amdgpu_device_skip_hw_access(adev))
459 return;
460
461 if ((reg * 4) < adev->rmmio_size) {
462 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
463 amdgpu_sriov_runtime(adev) &&
464 down_read_trylock(&adev->reset_sem)) {
465 amdgpu_kiq_wreg(adev, reg, v);
466 up_read(&adev->reset_sem);
467 } else {
468 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
469 }
470 } else {
471 adev->pcie_wreg(adev, reg * 4, v);
472 }
473
474 trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
475}
476
477/*
478 * amdgpu_mm_wreg_mmio_rlc - write register either with mmio or with RLC path if in range
479 *
480 * this function is invoked only the debugfs register access
481 * */
482void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
483 uint32_t reg, uint32_t v)
484{
485 if (amdgpu_device_skip_hw_access(adev))
486 return;
487
488 if (amdgpu_sriov_fullaccess(adev) &&
489 adev->gfx.rlc.funcs &&
490 adev->gfx.rlc.funcs->is_rlcg_access_range) {
491 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
492 return adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v);
493 } else {
494 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
495 }
496}
497
498/**
499 * amdgpu_mm_rdoorbell - read a doorbell dword
500 *
501 * @adev: amdgpu_device pointer
502 * @index: doorbell index
503 *
504 * Returns the value in the doorbell aperture at the
505 * requested doorbell index (CIK).
506 */
507u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
508{
509 if (amdgpu_device_skip_hw_access(adev))
510 return 0;
511
512 if (index < adev->doorbell.num_doorbells) {
513 return readl(adev->doorbell.ptr + index);
514 } else {
515 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
516 return 0;
517 }
518}
519
520/**
521 * amdgpu_mm_wdoorbell - write a doorbell dword
522 *
523 * @adev: amdgpu_device pointer
524 * @index: doorbell index
525 * @v: value to write
526 *
527 * Writes @v to the doorbell aperture at the
528 * requested doorbell index (CIK).
529 */
530void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
531{
532 if (amdgpu_device_skip_hw_access(adev))
533 return;
534
535 if (index < adev->doorbell.num_doorbells) {
536 writel(v, adev->doorbell.ptr + index);
537 } else {
538 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
539 }
540}
541
542/**
543 * amdgpu_mm_rdoorbell64 - read a doorbell Qword
544 *
545 * @adev: amdgpu_device pointer
546 * @index: doorbell index
547 *
548 * Returns the value in the doorbell aperture at the
549 * requested doorbell index (VEGA10+).
550 */
551u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
552{
553 if (amdgpu_device_skip_hw_access(adev))
554 return 0;
555
556 if (index < adev->doorbell.num_doorbells) {
557 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
558 } else {
559 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
560 return 0;
561 }
562}
563
564/**
565 * amdgpu_mm_wdoorbell64 - write a doorbell Qword
566 *
567 * @adev: amdgpu_device pointer
568 * @index: doorbell index
569 * @v: value to write
570 *
571 * Writes @v to the doorbell aperture at the
572 * requested doorbell index (VEGA10+).
573 */
574void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
575{
576 if (amdgpu_device_skip_hw_access(adev))
577 return;
578
579 if (index < adev->doorbell.num_doorbells) {
580 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
581 } else {
582 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
583 }
584}
585
586/**
587 * amdgpu_device_indirect_rreg - read an indirect register
588 *
589 * @adev: amdgpu_device pointer
590 * @pcie_index: mmio register offset
591 * @pcie_data: mmio register offset
592 * @reg_addr: indirect register address to read from
593 *
594 * Returns the value of indirect register @reg_addr
595 */
596u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
597 u32 pcie_index, u32 pcie_data,
598 u32 reg_addr)
599{
600 unsigned long flags;
601 u32 r;
602 void __iomem *pcie_index_offset;
603 void __iomem *pcie_data_offset;
604
605 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
606 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
607 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
608
609 writel(reg_addr, pcie_index_offset);
610 readl(pcie_index_offset);
611 r = readl(pcie_data_offset);
612 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
613
614 return r;
615}
616
617/**
618 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
619 *
620 * @adev: amdgpu_device pointer
621 * @pcie_index: mmio register offset
622 * @pcie_data: mmio register offset
623 * @reg_addr: indirect register address to read from
624 *
625 * Returns the value of indirect register @reg_addr
626 */
627u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
628 u32 pcie_index, u32 pcie_data,
629 u32 reg_addr)
630{
631 unsigned long flags;
632 u64 r;
633 void __iomem *pcie_index_offset;
634 void __iomem *pcie_data_offset;
635
636 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
637 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
638 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
639
640 /* read low 32 bits */
641 writel(reg_addr, pcie_index_offset);
642 readl(pcie_index_offset);
643 r = readl(pcie_data_offset);
644 /* read high 32 bits */
645 writel(reg_addr + 4, pcie_index_offset);
646 readl(pcie_index_offset);
647 r |= ((u64)readl(pcie_data_offset) << 32);
648 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
649
650 return r;
651}
652
653/**
654 * amdgpu_device_indirect_wreg - write an indirect register address
655 *
656 * @adev: amdgpu_device pointer
657 * @pcie_index: mmio register offset
658 * @pcie_data: mmio register offset
659 * @reg_addr: indirect register offset
660 * @reg_data: indirect register data
661 *
662 */
663void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
664 u32 pcie_index, u32 pcie_data,
665 u32 reg_addr, u32 reg_data)
666{
667 unsigned long flags;
668 void __iomem *pcie_index_offset;
669 void __iomem *pcie_data_offset;
670
671 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
672 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
673 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
674
675 writel(reg_addr, pcie_index_offset);
676 readl(pcie_index_offset);
677 writel(reg_data, pcie_data_offset);
678 readl(pcie_data_offset);
679 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
680}
681
682/**
683 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
684 *
685 * @adev: amdgpu_device pointer
686 * @pcie_index: mmio register offset
687 * @pcie_data: mmio register offset
688 * @reg_addr: indirect register offset
689 * @reg_data: indirect register data
690 *
691 */
692void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
693 u32 pcie_index, u32 pcie_data,
694 u32 reg_addr, u64 reg_data)
695{
696 unsigned long flags;
697 void __iomem *pcie_index_offset;
698 void __iomem *pcie_data_offset;
699
700 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
701 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
702 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
703
704 /* write low 32 bits */
705 writel(reg_addr, pcie_index_offset);
706 readl(pcie_index_offset);
707 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
708 readl(pcie_data_offset);
709 /* write high 32 bits */
710 writel(reg_addr + 4, pcie_index_offset);
711 readl(pcie_index_offset);
712 writel((u32)(reg_data >> 32), pcie_data_offset);
713 readl(pcie_data_offset);
714 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
715}
716
717/**
718 * amdgpu_invalid_rreg - dummy reg read function
719 *
720 * @adev: amdgpu_device pointer
721 * @reg: offset of register
722 *
723 * Dummy register read function. Used for register blocks
724 * that certain asics don't have (all asics).
725 * Returns the value in the register.
726 */
727static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
728{
729 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
730 BUG();
731 return 0;
732}
733
734/**
735 * amdgpu_invalid_wreg - dummy reg write function
736 *
737 * @adev: amdgpu_device pointer
738 * @reg: offset of register
739 * @v: value to write to the register
740 *
741 * Dummy register read function. Used for register blocks
742 * that certain asics don't have (all asics).
743 */
744static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
745{
746 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
747 reg, v);
748 BUG();
749}
750
751/**
752 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
753 *
754 * @adev: amdgpu_device pointer
755 * @reg: offset of register
756 *
757 * Dummy register read function. Used for register blocks
758 * that certain asics don't have (all asics).
759 * Returns the value in the register.
760 */
761static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
762{
763 DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
764 BUG();
765 return 0;
766}
767
768/**
769 * amdgpu_invalid_wreg64 - dummy reg write function
770 *
771 * @adev: amdgpu_device pointer
772 * @reg: offset of register
773 * @v: value to write to the register
774 *
775 * Dummy register read function. Used for register blocks
776 * that certain asics don't have (all asics).
777 */
778static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
779{
780 DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
781 reg, v);
782 BUG();
783}
784
785/**
786 * amdgpu_block_invalid_rreg - dummy reg read function
787 *
788 * @adev: amdgpu_device pointer
789 * @block: offset of instance
790 * @reg: offset of register
791 *
792 * Dummy register read function. Used for register blocks
793 * that certain asics don't have (all asics).
794 * Returns the value in the register.
795 */
796static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
797 uint32_t block, uint32_t reg)
798{
799 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
800 reg, block);
801 BUG();
802 return 0;
803}
804
805/**
806 * amdgpu_block_invalid_wreg - dummy reg write function
807 *
808 * @adev: amdgpu_device pointer
809 * @block: offset of instance
810 * @reg: offset of register
811 * @v: value to write to the register
812 *
813 * Dummy register read function. Used for register blocks
814 * that certain asics don't have (all asics).
815 */
816static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
817 uint32_t block,
818 uint32_t reg, uint32_t v)
819{
820 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
821 reg, block, v);
822 BUG();
823}
824
825/**
826 * amdgpu_device_asic_init - Wrapper for atom asic_init
827 *
828 * @adev: amdgpu_device pointer
829 *
830 * Does any asic specific work and then calls atom asic init.
831 */
832static int amdgpu_device_asic_init(struct amdgpu_device *adev)
833{
834 amdgpu_asic_pre_asic_init(adev);
835
836 return amdgpu_atom_asic_init(adev->mode_info.atom_context);
837}
838
839/**
840 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
841 *
842 * @adev: amdgpu_device pointer
843 *
844 * Allocates a scratch page of VRAM for use by various things in the
845 * driver.
846 */
847static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
848{
849 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
850 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
851 &adev->vram_scratch.robj,
852 &adev->vram_scratch.gpu_addr,
853 (void **)&adev->vram_scratch.ptr);
854}
855
856/**
857 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
858 *
859 * @adev: amdgpu_device pointer
860 *
861 * Frees the VRAM scratch page.
862 */
863static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
864{
865 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
866}
867
868/**
869 * amdgpu_device_program_register_sequence - program an array of registers.
870 *
871 * @adev: amdgpu_device pointer
872 * @registers: pointer to the register array
873 * @array_size: size of the register array
874 *
875 * Programs an array or registers with and and or masks.
876 * This is a helper for setting golden registers.
877 */
878void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
879 const u32 *registers,
880 const u32 array_size)
881{
882 u32 tmp, reg, and_mask, or_mask;
883 int i;
884
885 if (array_size % 3)
886 return;
887
888 for (i = 0; i < array_size; i +=3) {
889 reg = registers[i + 0];
890 and_mask = registers[i + 1];
891 or_mask = registers[i + 2];
892
893 if (and_mask == 0xffffffff) {
894 tmp = or_mask;
895 } else {
896 tmp = RREG32(reg);
897 tmp &= ~and_mask;
898 if (adev->family >= AMDGPU_FAMILY_AI)
899 tmp |= (or_mask & and_mask);
900 else
901 tmp |= or_mask;
902 }
903 WREG32(reg, tmp);
904 }
905}
906
907/**
908 * amdgpu_device_pci_config_reset - reset the GPU
909 *
910 * @adev: amdgpu_device pointer
911 *
912 * Resets the GPU using the pci config reset sequence.
913 * Only applicable to asics prior to vega10.
914 */
915void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
916{
917 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
918}
919
920/**
921 * amdgpu_device_pci_reset - reset the GPU using generic PCI means
922 *
923 * @adev: amdgpu_device pointer
924 *
925 * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
926 */
927int amdgpu_device_pci_reset(struct amdgpu_device *adev)
928{
929 return pci_reset_function(adev->pdev);
930}
931
932/*
933 * GPU doorbell aperture helpers function.
934 */
935/**
936 * amdgpu_device_doorbell_init - Init doorbell driver information.
937 *
938 * @adev: amdgpu_device pointer
939 *
940 * Init doorbell driver information (CIK)
941 * Returns 0 on success, error on failure.
942 */
943static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
944{
945
946 /* No doorbell on SI hardware generation */
947 if (adev->asic_type < CHIP_BONAIRE) {
948 adev->doorbell.base = 0;
949 adev->doorbell.size = 0;
950 adev->doorbell.num_doorbells = 0;
951 adev->doorbell.ptr = NULL;
952 return 0;
953 }
954
955 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
956 return -EINVAL;
957
958 amdgpu_asic_init_doorbell_index(adev);
959
960 /* doorbell bar mapping */
961 adev->doorbell.base = pci_resource_start(adev->pdev, 2);
962 adev->doorbell.size = pci_resource_len(adev->pdev, 2);
963
964 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
965 adev->doorbell_index.max_assignment+1);
966 if (adev->doorbell.num_doorbells == 0)
967 return -EINVAL;
968
969 /* For Vega, reserve and map two pages on doorbell BAR since SDMA
970 * paging queue doorbell use the second page. The
971 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
972 * doorbells are in the first page. So with paging queue enabled,
973 * the max num_doorbells should + 1 page (0x400 in dword)
974 */
975 if (adev->asic_type >= CHIP_VEGA10)
976 adev->doorbell.num_doorbells += 0x400;
977
978 adev->doorbell.ptr = ioremap(adev->doorbell.base,
979 adev->doorbell.num_doorbells *
980 sizeof(u32));
981 if (adev->doorbell.ptr == NULL)
982 return -ENOMEM;
983
984 return 0;
985}
986
987/**
988 * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
989 *
990 * @adev: amdgpu_device pointer
991 *
992 * Tear down doorbell driver information (CIK)
993 */
994static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
995{
996 iounmap(adev->doorbell.ptr);
997 adev->doorbell.ptr = NULL;
998}
999
1000
1001
1002/*
1003 * amdgpu_device_wb_*()
1004 * Writeback is the method by which the GPU updates special pages in memory
1005 * with the status of certain GPU events (fences, ring pointers,etc.).
1006 */
1007
1008/**
1009 * amdgpu_device_wb_fini - Disable Writeback and free memory
1010 *
1011 * @adev: amdgpu_device pointer
1012 *
1013 * Disables Writeback and frees the Writeback memory (all asics).
1014 * Used at driver shutdown.
1015 */
1016static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1017{
1018 if (adev->wb.wb_obj) {
1019 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1020 &adev->wb.gpu_addr,
1021 (void **)&adev->wb.wb);
1022 adev->wb.wb_obj = NULL;
1023 }
1024}
1025
1026/**
1027 * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
1028 *
1029 * @adev: amdgpu_device pointer
1030 *
1031 * Initializes writeback and allocates writeback memory (all asics).
1032 * Used at driver startup.
1033 * Returns 0 on success or an -error on failure.
1034 */
1035static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1036{
1037 int r;
1038
1039 if (adev->wb.wb_obj == NULL) {
1040 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1041 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1042 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1043 &adev->wb.wb_obj, &adev->wb.gpu_addr,
1044 (void **)&adev->wb.wb);
1045 if (r) {
1046 dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1047 return r;
1048 }
1049
1050 adev->wb.num_wb = AMDGPU_MAX_WB;
1051 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1052
1053 /* clear wb memory */
1054 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1055 }
1056
1057 return 0;
1058}
1059
1060/**
1061 * amdgpu_device_wb_get - Allocate a wb entry
1062 *
1063 * @adev: amdgpu_device pointer
1064 * @wb: wb index
1065 *
1066 * Allocate a wb slot for use by the driver (all asics).
1067 * Returns 0 on success or -EINVAL on failure.
1068 */
1069int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1070{
1071 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1072
1073 if (offset < adev->wb.num_wb) {
1074 __set_bit(offset, adev->wb.used);
1075 *wb = offset << 3; /* convert to dw offset */
1076 return 0;
1077 } else {
1078 return -EINVAL;
1079 }
1080}
1081
1082/**
1083 * amdgpu_device_wb_free - Free a wb entry
1084 *
1085 * @adev: amdgpu_device pointer
1086 * @wb: wb index
1087 *
1088 * Free a wb slot allocated for use by the driver (all asics)
1089 */
1090void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1091{
1092 wb >>= 3;
1093 if (wb < adev->wb.num_wb)
1094 __clear_bit(wb, adev->wb.used);
1095}
1096
1097/**
1098 * amdgpu_device_resize_fb_bar - try to resize FB BAR
1099 *
1100 * @adev: amdgpu_device pointer
1101 *
1102 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1103 * to fail, but if any of the BARs is not accessible after the size we abort
1104 * driver loading by returning -ENODEV.
1105 */
1106int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1107{
1108 int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1109 struct pci_bus *root;
1110 struct resource *res;
1111 unsigned i;
1112 u16 cmd;
1113 int r;
1114
1115 /* Bypass for VF */
1116 if (amdgpu_sriov_vf(adev))
1117 return 0;
1118
1119 /* skip if the bios has already enabled large BAR */
1120 if (adev->gmc.real_vram_size &&
1121 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1122 return 0;
1123
1124 /* Check if the root BUS has 64bit memory resources */
1125 root = adev->pdev->bus;
1126 while (root->parent)
1127 root = root->parent;
1128
1129 pci_bus_for_each_resource(root, res, i) {
1130 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1131 res->start > 0x100000000ull)
1132 break;
1133 }
1134
1135 /* Trying to resize is pointless without a root hub window above 4GB */
1136 if (!res)
1137 return 0;
1138
1139 /* Limit the BAR size to what is available */
1140 rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1141 rbar_size);
1142
1143 /* Disable memory decoding while we change the BAR addresses and size */
1144 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1145 pci_write_config_word(adev->pdev, PCI_COMMAND,
1146 cmd & ~PCI_COMMAND_MEMORY);
1147
1148 /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1149 amdgpu_device_doorbell_fini(adev);
1150 if (adev->asic_type >= CHIP_BONAIRE)
1151 pci_release_resource(adev->pdev, 2);
1152
1153 pci_release_resource(adev->pdev, 0);
1154
1155 r = pci_resize_resource(adev->pdev, 0, rbar_size);
1156 if (r == -ENOSPC)
1157 DRM_INFO("Not enough PCI address space for a large BAR.");
1158 else if (r && r != -ENOTSUPP)
1159 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1160
1161 pci_assign_unassigned_bus_resources(adev->pdev->bus);
1162
1163 /* When the doorbell or fb BAR isn't available we have no chance of
1164 * using the device.
1165 */
1166 r = amdgpu_device_doorbell_init(adev);
1167 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1168 return -ENODEV;
1169
1170 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1171
1172 return 0;
1173}
1174
1175/*
1176 * GPU helpers function.
1177 */
1178/**
1179 * amdgpu_device_need_post - check if the hw need post or not
1180 *
1181 * @adev: amdgpu_device pointer
1182 *
1183 * Check if the asic has been initialized (all asics) at driver startup
1184 * or post is needed if hw reset is performed.
1185 * Returns true if need or false if not.
1186 */
1187bool amdgpu_device_need_post(struct amdgpu_device *adev)
1188{
1189 uint32_t reg;
1190
1191 if (amdgpu_sriov_vf(adev))
1192 return false;
1193
1194 if (amdgpu_passthrough(adev)) {
1195 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1196 * some old smc fw still need driver do vPost otherwise gpu hang, while
1197 * those smc fw version above 22.15 doesn't have this flaw, so we force
1198 * vpost executed for smc version below 22.15
1199 */
1200 if (adev->asic_type == CHIP_FIJI) {
1201 int err;
1202 uint32_t fw_ver;
1203 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1204 /* force vPost if error occured */
1205 if (err)
1206 return true;
1207
1208 fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1209 if (fw_ver < 0x00160e00)
1210 return true;
1211 }
1212 }
1213
1214 /* Don't post if we need to reset whole hive on init */
1215 if (adev->gmc.xgmi.pending_reset)
1216 return false;
1217
1218 if (adev->has_hw_reset) {
1219 adev->has_hw_reset = false;
1220 return true;
1221 }
1222
1223 /* bios scratch used on CIK+ */
1224 if (adev->asic_type >= CHIP_BONAIRE)
1225 return amdgpu_atombios_scratch_need_asic_init(adev);
1226
1227 /* check MEM_SIZE for older asics */
1228 reg = amdgpu_asic_get_config_memsize(adev);
1229
1230 if ((reg != 0) && (reg != 0xffffffff))
1231 return false;
1232
1233 return true;
1234}
1235
1236/* if we get transitioned to only one device, take VGA back */
1237/**
1238 * amdgpu_device_vga_set_decode - enable/disable vga decode
1239 *
1240 * @cookie: amdgpu_device pointer
1241 * @state: enable/disable vga decode
1242 *
1243 * Enable/disable vga decode (all asics).
1244 * Returns VGA resource flags.
1245 */
1246static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
1247{
1248 struct amdgpu_device *adev = cookie;
1249 amdgpu_asic_set_vga_state(adev, state);
1250 if (state)
1251 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1252 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1253 else
1254 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1255}
1256
1257/**
1258 * amdgpu_device_check_block_size - validate the vm block size
1259 *
1260 * @adev: amdgpu_device pointer
1261 *
1262 * Validates the vm block size specified via module parameter.
1263 * The vm block size defines number of bits in page table versus page directory,
1264 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1265 * page table and the remaining bits are in the page directory.
1266 */
1267static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1268{
1269 /* defines number of bits in page table versus page directory,
1270 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1271 * page table and the remaining bits are in the page directory */
1272 if (amdgpu_vm_block_size == -1)
1273 return;
1274
1275 if (amdgpu_vm_block_size < 9) {
1276 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1277 amdgpu_vm_block_size);
1278 amdgpu_vm_block_size = -1;
1279 }
1280}
1281
1282/**
1283 * amdgpu_device_check_vm_size - validate the vm size
1284 *
1285 * @adev: amdgpu_device pointer
1286 *
1287 * Validates the vm size in GB specified via module parameter.
1288 * The VM size is the size of the GPU virtual memory space in GB.
1289 */
1290static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1291{
1292 /* no need to check the default value */
1293 if (amdgpu_vm_size == -1)
1294 return;
1295
1296 if (amdgpu_vm_size < 1) {
1297 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1298 amdgpu_vm_size);
1299 amdgpu_vm_size = -1;
1300 }
1301}
1302
1303static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1304{
1305 struct sysinfo si;
1306 bool is_os_64 = (sizeof(void *) == 8);
1307 uint64_t total_memory;
1308 uint64_t dram_size_seven_GB = 0x1B8000000;
1309 uint64_t dram_size_three_GB = 0xB8000000;
1310
1311 if (amdgpu_smu_memory_pool_size == 0)
1312 return;
1313
1314 if (!is_os_64) {
1315 DRM_WARN("Not 64-bit OS, feature not supported\n");
1316 goto def_value;
1317 }
1318 si_meminfo(&si);
1319 total_memory = (uint64_t)si.totalram * si.mem_unit;
1320
1321 if ((amdgpu_smu_memory_pool_size == 1) ||
1322 (amdgpu_smu_memory_pool_size == 2)) {
1323 if (total_memory < dram_size_three_GB)
1324 goto def_value1;
1325 } else if ((amdgpu_smu_memory_pool_size == 4) ||
1326 (amdgpu_smu_memory_pool_size == 8)) {
1327 if (total_memory < dram_size_seven_GB)
1328 goto def_value1;
1329 } else {
1330 DRM_WARN("Smu memory pool size not supported\n");
1331 goto def_value;
1332 }
1333 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1334
1335 return;
1336
1337def_value1:
1338 DRM_WARN("No enough system memory\n");
1339def_value:
1340 adev->pm.smu_prv_buffer_size = 0;
1341}
1342
1343/**
1344 * amdgpu_device_check_arguments - validate module params
1345 *
1346 * @adev: amdgpu_device pointer
1347 *
1348 * Validates certain module parameters and updates
1349 * the associated values used by the driver (all asics).
1350 */
1351static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1352{
1353 if (amdgpu_sched_jobs < 4) {
1354 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1355 amdgpu_sched_jobs);
1356 amdgpu_sched_jobs = 4;
1357 } else if (!is_power_of_2(amdgpu_sched_jobs)){
1358 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1359 amdgpu_sched_jobs);
1360 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1361 }
1362
1363 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1364 /* gart size must be greater or equal to 32M */
1365 dev_warn(adev->dev, "gart size (%d) too small\n",
1366 amdgpu_gart_size);
1367 amdgpu_gart_size = -1;
1368 }
1369
1370 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1371 /* gtt size must be greater or equal to 32M */
1372 dev_warn(adev->dev, "gtt size (%d) too small\n",
1373 amdgpu_gtt_size);
1374 amdgpu_gtt_size = -1;
1375 }
1376
1377 /* valid range is between 4 and 9 inclusive */
1378 if (amdgpu_vm_fragment_size != -1 &&
1379 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1380 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1381 amdgpu_vm_fragment_size = -1;
1382 }
1383
1384 if (amdgpu_sched_hw_submission < 2) {
1385 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1386 amdgpu_sched_hw_submission);
1387 amdgpu_sched_hw_submission = 2;
1388 } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1389 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1390 amdgpu_sched_hw_submission);
1391 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1392 }
1393
1394 amdgpu_device_check_smu_prv_buffer_size(adev);
1395
1396 amdgpu_device_check_vm_size(adev);
1397
1398 amdgpu_device_check_block_size(adev);
1399
1400 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1401
1402 amdgpu_gmc_tmz_set(adev);
1403
1404 amdgpu_gmc_noretry_set(adev);
1405
1406 return 0;
1407}
1408
1409/**
1410 * amdgpu_switcheroo_set_state - set switcheroo state
1411 *
1412 * @pdev: pci dev pointer
1413 * @state: vga_switcheroo state
1414 *
1415 * Callback for the switcheroo driver. Suspends or resumes the
1416 * the asics before or after it is powered up using ACPI methods.
1417 */
1418static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1419 enum vga_switcheroo_state state)
1420{
1421 struct drm_device *dev = pci_get_drvdata(pdev);
1422 int r;
1423
1424 if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1425 return;
1426
1427 if (state == VGA_SWITCHEROO_ON) {
1428 pr_info("switched on\n");
1429 /* don't suspend or resume card normally */
1430 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1431
1432 pci_set_power_state(pdev, PCI_D0);
1433 amdgpu_device_load_pci_state(pdev);
1434 r = pci_enable_device(pdev);
1435 if (r)
1436 DRM_WARN("pci_enable_device failed (%d)\n", r);
1437 amdgpu_device_resume(dev, true);
1438
1439 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1440 } else {
1441 pr_info("switched off\n");
1442 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1443 amdgpu_device_suspend(dev, true);
1444 amdgpu_device_cache_pci_state(pdev);
1445 /* Shut down the device */
1446 pci_disable_device(pdev);
1447 pci_set_power_state(pdev, PCI_D3cold);
1448 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1449 }
1450}
1451
1452/**
1453 * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1454 *
1455 * @pdev: pci dev pointer
1456 *
1457 * Callback for the switcheroo driver. Check of the switcheroo
1458 * state can be changed.
1459 * Returns true if the state can be changed, false if not.
1460 */
1461static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1462{
1463 struct drm_device *dev = pci_get_drvdata(pdev);
1464
1465 /*
1466 * FIXME: open_count is protected by drm_global_mutex but that would lead to
1467 * locking inversion with the driver load path. And the access here is
1468 * completely racy anyway. So don't bother with locking for now.
1469 */
1470 return atomic_read(&dev->open_count) == 0;
1471}
1472
1473static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1474 .set_gpu_state = amdgpu_switcheroo_set_state,
1475 .reprobe = NULL,
1476 .can_switch = amdgpu_switcheroo_can_switch,
1477};
1478
1479/**
1480 * amdgpu_device_ip_set_clockgating_state - set the CG state
1481 *
1482 * @dev: amdgpu_device pointer
1483 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1484 * @state: clockgating state (gate or ungate)
1485 *
1486 * Sets the requested clockgating state for all instances of
1487 * the hardware IP specified.
1488 * Returns the error code from the last instance.
1489 */
1490int amdgpu_device_ip_set_clockgating_state(void *dev,
1491 enum amd_ip_block_type block_type,
1492 enum amd_clockgating_state state)
1493{
1494 struct amdgpu_device *adev = dev;
1495 int i, r = 0;
1496
1497 for (i = 0; i < adev->num_ip_blocks; i++) {
1498 if (!adev->ip_blocks[i].status.valid)
1499 continue;
1500 if (adev->ip_blocks[i].version->type != block_type)
1501 continue;
1502 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1503 continue;
1504 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1505 (void *)adev, state);
1506 if (r)
1507 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1508 adev->ip_blocks[i].version->funcs->name, r);
1509 }
1510 return r;
1511}
1512
1513/**
1514 * amdgpu_device_ip_set_powergating_state - set the PG state
1515 *
1516 * @dev: amdgpu_device pointer
1517 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1518 * @state: powergating state (gate or ungate)
1519 *
1520 * Sets the requested powergating state for all instances of
1521 * the hardware IP specified.
1522 * Returns the error code from the last instance.
1523 */
1524int amdgpu_device_ip_set_powergating_state(void *dev,
1525 enum amd_ip_block_type block_type,
1526 enum amd_powergating_state state)
1527{
1528 struct amdgpu_device *adev = dev;
1529 int i, r = 0;
1530
1531 for (i = 0; i < adev->num_ip_blocks; i++) {
1532 if (!adev->ip_blocks[i].status.valid)
1533 continue;
1534 if (adev->ip_blocks[i].version->type != block_type)
1535 continue;
1536 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1537 continue;
1538 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1539 (void *)adev, state);
1540 if (r)
1541 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1542 adev->ip_blocks[i].version->funcs->name, r);
1543 }
1544 return r;
1545}
1546
1547/**
1548 * amdgpu_device_ip_get_clockgating_state - get the CG state
1549 *
1550 * @adev: amdgpu_device pointer
1551 * @flags: clockgating feature flags
1552 *
1553 * Walks the list of IPs on the device and updates the clockgating
1554 * flags for each IP.
1555 * Updates @flags with the feature flags for each hardware IP where
1556 * clockgating is enabled.
1557 */
1558void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1559 u32 *flags)
1560{
1561 int i;
1562
1563 for (i = 0; i < adev->num_ip_blocks; i++) {
1564 if (!adev->ip_blocks[i].status.valid)
1565 continue;
1566 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1567 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1568 }
1569}
1570
1571/**
1572 * amdgpu_device_ip_wait_for_idle - wait for idle
1573 *
1574 * @adev: amdgpu_device pointer
1575 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1576 *
1577 * Waits for the request hardware IP to be idle.
1578 * Returns 0 for success or a negative error code on failure.
1579 */
1580int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1581 enum amd_ip_block_type block_type)
1582{
1583 int i, r;
1584
1585 for (i = 0; i < adev->num_ip_blocks; i++) {
1586 if (!adev->ip_blocks[i].status.valid)
1587 continue;
1588 if (adev->ip_blocks[i].version->type == block_type) {
1589 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1590 if (r)
1591 return r;
1592 break;
1593 }
1594 }
1595 return 0;
1596
1597}
1598
1599/**
1600 * amdgpu_device_ip_is_idle - is the hardware IP idle
1601 *
1602 * @adev: amdgpu_device pointer
1603 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1604 *
1605 * Check if the hardware IP is idle or not.
1606 * Returns true if it the IP is idle, false if not.
1607 */
1608bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1609 enum amd_ip_block_type block_type)
1610{
1611 int i;
1612
1613 for (i = 0; i < adev->num_ip_blocks; i++) {
1614 if (!adev->ip_blocks[i].status.valid)
1615 continue;
1616 if (adev->ip_blocks[i].version->type == block_type)
1617 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1618 }
1619 return true;
1620
1621}
1622
1623/**
1624 * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1625 *
1626 * @adev: amdgpu_device pointer
1627 * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1628 *
1629 * Returns a pointer to the hardware IP block structure
1630 * if it exists for the asic, otherwise NULL.
1631 */
1632struct amdgpu_ip_block *
1633amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1634 enum amd_ip_block_type type)
1635{
1636 int i;
1637
1638 for (i = 0; i < adev->num_ip_blocks; i++)
1639 if (adev->ip_blocks[i].version->type == type)
1640 return &adev->ip_blocks[i];
1641
1642 return NULL;
1643}
1644
1645/**
1646 * amdgpu_device_ip_block_version_cmp
1647 *
1648 * @adev: amdgpu_device pointer
1649 * @type: enum amd_ip_block_type
1650 * @major: major version
1651 * @minor: minor version
1652 *
1653 * return 0 if equal or greater
1654 * return 1 if smaller or the ip_block doesn't exist
1655 */
1656int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1657 enum amd_ip_block_type type,
1658 u32 major, u32 minor)
1659{
1660 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1661
1662 if (ip_block && ((ip_block->version->major > major) ||
1663 ((ip_block->version->major == major) &&
1664 (ip_block->version->minor >= minor))))
1665 return 0;
1666
1667 return 1;
1668}
1669
1670/**
1671 * amdgpu_device_ip_block_add
1672 *
1673 * @adev: amdgpu_device pointer
1674 * @ip_block_version: pointer to the IP to add
1675 *
1676 * Adds the IP block driver information to the collection of IPs
1677 * on the asic.
1678 */
1679int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1680 const struct amdgpu_ip_block_version *ip_block_version)
1681{
1682 if (!ip_block_version)
1683 return -EINVAL;
1684
1685 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1686 ip_block_version->funcs->name);
1687
1688 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1689
1690 return 0;
1691}
1692
1693/**
1694 * amdgpu_device_enable_virtual_display - enable virtual display feature
1695 *
1696 * @adev: amdgpu_device pointer
1697 *
1698 * Enabled the virtual display feature if the user has enabled it via
1699 * the module parameter virtual_display. This feature provides a virtual
1700 * display hardware on headless boards or in virtualized environments.
1701 * This function parses and validates the configuration string specified by
1702 * the user and configues the virtual display configuration (number of
1703 * virtual connectors, crtcs, etc.) specified.
1704 */
1705static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1706{
1707 adev->enable_virtual_display = false;
1708
1709 if (amdgpu_virtual_display) {
1710 const char *pci_address_name = pci_name(adev->pdev);
1711 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1712
1713 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1714 pciaddstr_tmp = pciaddstr;
1715 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1716 pciaddname = strsep(&pciaddname_tmp, ",");
1717 if (!strcmp("all", pciaddname)
1718 || !strcmp(pci_address_name, pciaddname)) {
1719 long num_crtc;
1720 int res = -1;
1721
1722 adev->enable_virtual_display = true;
1723
1724 if (pciaddname_tmp)
1725 res = kstrtol(pciaddname_tmp, 10,
1726 &num_crtc);
1727
1728 if (!res) {
1729 if (num_crtc < 1)
1730 num_crtc = 1;
1731 if (num_crtc > 6)
1732 num_crtc = 6;
1733 adev->mode_info.num_crtc = num_crtc;
1734 } else {
1735 adev->mode_info.num_crtc = 1;
1736 }
1737 break;
1738 }
1739 }
1740
1741 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1742 amdgpu_virtual_display, pci_address_name,
1743 adev->enable_virtual_display, adev->mode_info.num_crtc);
1744
1745 kfree(pciaddstr);
1746 }
1747}
1748
1749/**
1750 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1751 *
1752 * @adev: amdgpu_device pointer
1753 *
1754 * Parses the asic configuration parameters specified in the gpu info
1755 * firmware and makes them availale to the driver for use in configuring
1756 * the asic.
1757 * Returns 0 on success, -EINVAL on failure.
1758 */
1759static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1760{
1761 const char *chip_name;
1762 char fw_name[40];
1763 int err;
1764 const struct gpu_info_firmware_header_v1_0 *hdr;
1765
1766 adev->firmware.gpu_info_fw = NULL;
1767
1768 if (adev->mman.discovery_bin) {
1769 amdgpu_discovery_get_gfx_info(adev);
1770
1771 /*
1772 * FIXME: The bounding box is still needed by Navi12, so
1773 * temporarily read it from gpu_info firmware. Should be droped
1774 * when DAL no longer needs it.
1775 */
1776 if (adev->asic_type != CHIP_NAVI12)
1777 return 0;
1778 }
1779
1780 switch (adev->asic_type) {
1781#ifdef CONFIG_DRM_AMDGPU_SI
1782 case CHIP_VERDE:
1783 case CHIP_TAHITI:
1784 case CHIP_PITCAIRN:
1785 case CHIP_OLAND:
1786 case CHIP_HAINAN:
1787#endif
1788#ifdef CONFIG_DRM_AMDGPU_CIK
1789 case CHIP_BONAIRE:
1790 case CHIP_HAWAII:
1791 case CHIP_KAVERI:
1792 case CHIP_KABINI:
1793 case CHIP_MULLINS:
1794#endif
1795 case CHIP_TOPAZ:
1796 case CHIP_TONGA:
1797 case CHIP_FIJI:
1798 case CHIP_POLARIS10:
1799 case CHIP_POLARIS11:
1800 case CHIP_POLARIS12:
1801 case CHIP_VEGAM:
1802 case CHIP_CARRIZO:
1803 case CHIP_STONEY:
1804 case CHIP_VEGA20:
1805 case CHIP_ALDEBARAN:
1806 case CHIP_SIENNA_CICHLID:
1807 case CHIP_NAVY_FLOUNDER:
1808 case CHIP_DIMGREY_CAVEFISH:
1809 default:
1810 return 0;
1811 case CHIP_VEGA10:
1812 chip_name = "vega10";
1813 break;
1814 case CHIP_VEGA12:
1815 chip_name = "vega12";
1816 break;
1817 case CHIP_RAVEN:
1818 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1819 chip_name = "raven2";
1820 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1821 chip_name = "picasso";
1822 else
1823 chip_name = "raven";
1824 break;
1825 case CHIP_ARCTURUS:
1826 chip_name = "arcturus";
1827 break;
1828 case CHIP_RENOIR:
1829 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1830 chip_name = "renoir";
1831 else
1832 chip_name = "green_sardine";
1833 break;
1834 case CHIP_NAVI10:
1835 chip_name = "navi10";
1836 break;
1837 case CHIP_NAVI14:
1838 chip_name = "navi14";
1839 break;
1840 case CHIP_NAVI12:
1841 chip_name = "navi12";
1842 break;
1843 case CHIP_VANGOGH:
1844 chip_name = "vangogh";
1845 break;
1846 }
1847
1848 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1849 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1850 if (err) {
1851 dev_err(adev->dev,
1852 "Failed to load gpu_info firmware \"%s\"\n",
1853 fw_name);
1854 goto out;
1855 }
1856 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1857 if (err) {
1858 dev_err(adev->dev,
1859 "Failed to validate gpu_info firmware \"%s\"\n",
1860 fw_name);
1861 goto out;
1862 }
1863
1864 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1865 amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1866
1867 switch (hdr->version_major) {
1868 case 1:
1869 {
1870 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1871 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1872 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1873
1874 /*
1875 * Should be droped when DAL no longer needs it.
1876 */
1877 if (adev->asic_type == CHIP_NAVI12)
1878 goto parse_soc_bounding_box;
1879
1880 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1881 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1882 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1883 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1884 adev->gfx.config.max_texture_channel_caches =
1885 le32_to_cpu(gpu_info_fw->gc_num_tccs);
1886 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1887 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1888 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1889 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1890 adev->gfx.config.double_offchip_lds_buf =
1891 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1892 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1893 adev->gfx.cu_info.max_waves_per_simd =
1894 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1895 adev->gfx.cu_info.max_scratch_slots_per_cu =
1896 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1897 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1898 if (hdr->version_minor >= 1) {
1899 const struct gpu_info_firmware_v1_1 *gpu_info_fw =
1900 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
1901 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1902 adev->gfx.config.num_sc_per_sh =
1903 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
1904 adev->gfx.config.num_packer_per_sc =
1905 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
1906 }
1907
1908parse_soc_bounding_box:
1909 /*
1910 * soc bounding box info is not integrated in disocovery table,
1911 * we always need to parse it from gpu info firmware if needed.
1912 */
1913 if (hdr->version_minor == 2) {
1914 const struct gpu_info_firmware_v1_2 *gpu_info_fw =
1915 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
1916 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1917 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
1918 }
1919 break;
1920 }
1921 default:
1922 dev_err(adev->dev,
1923 "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1924 err = -EINVAL;
1925 goto out;
1926 }
1927out:
1928 return err;
1929}
1930
1931/**
1932 * amdgpu_device_ip_early_init - run early init for hardware IPs
1933 *
1934 * @adev: amdgpu_device pointer
1935 *
1936 * Early initialization pass for hardware IPs. The hardware IPs that make
1937 * up each asic are discovered each IP's early_init callback is run. This
1938 * is the first stage in initializing the asic.
1939 * Returns 0 on success, negative error code on failure.
1940 */
1941static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
1942{
1943 int i, r;
1944
1945 amdgpu_device_enable_virtual_display(adev);
1946
1947 if (amdgpu_sriov_vf(adev)) {
1948 r = amdgpu_virt_request_full_gpu(adev, true);
1949 if (r)
1950 return r;
1951 }
1952
1953 switch (adev->asic_type) {
1954#ifdef CONFIG_DRM_AMDGPU_SI
1955 case CHIP_VERDE:
1956 case CHIP_TAHITI:
1957 case CHIP_PITCAIRN:
1958 case CHIP_OLAND:
1959 case CHIP_HAINAN:
1960 adev->family = AMDGPU_FAMILY_SI;
1961 r = si_set_ip_blocks(adev);
1962 if (r)
1963 return r;
1964 break;
1965#endif
1966#ifdef CONFIG_DRM_AMDGPU_CIK
1967 case CHIP_BONAIRE:
1968 case CHIP_HAWAII:
1969 case CHIP_KAVERI:
1970 case CHIP_KABINI:
1971 case CHIP_MULLINS:
1972 if (adev->flags & AMD_IS_APU)
1973 adev->family = AMDGPU_FAMILY_KV;
1974 else
1975 adev->family = AMDGPU_FAMILY_CI;
1976
1977 r = cik_set_ip_blocks(adev);
1978 if (r)
1979 return r;
1980 break;
1981#endif
1982 case CHIP_TOPAZ:
1983 case CHIP_TONGA:
1984 case CHIP_FIJI:
1985 case CHIP_POLARIS10:
1986 case CHIP_POLARIS11:
1987 case CHIP_POLARIS12:
1988 case CHIP_VEGAM:
1989 case CHIP_CARRIZO:
1990 case CHIP_STONEY:
1991 if (adev->flags & AMD_IS_APU)
1992 adev->family = AMDGPU_FAMILY_CZ;
1993 else
1994 adev->family = AMDGPU_FAMILY_VI;
1995
1996 r = vi_set_ip_blocks(adev);
1997 if (r)
1998 return r;
1999 break;
2000 case CHIP_VEGA10:
2001 case CHIP_VEGA12:
2002 case CHIP_VEGA20:
2003 case CHIP_RAVEN:
2004 case CHIP_ARCTURUS:
2005 case CHIP_RENOIR:
2006 case CHIP_ALDEBARAN:
2007 if (adev->flags & AMD_IS_APU)
2008 adev->family = AMDGPU_FAMILY_RV;
2009 else
2010 adev->family = AMDGPU_FAMILY_AI;
2011
2012 r = soc15_set_ip_blocks(adev);
2013 if (r)
2014 return r;
2015 break;
2016 case CHIP_NAVI10:
2017 case CHIP_NAVI14:
2018 case CHIP_NAVI12:
2019 case CHIP_SIENNA_CICHLID:
2020 case CHIP_NAVY_FLOUNDER:
2021 case CHIP_DIMGREY_CAVEFISH:
2022 case CHIP_VANGOGH:
2023 if (adev->asic_type == CHIP_VANGOGH)
2024 adev->family = AMDGPU_FAMILY_VGH;
2025 else
2026 adev->family = AMDGPU_FAMILY_NV;
2027
2028 r = nv_set_ip_blocks(adev);
2029 if (r)
2030 return r;
2031 break;
2032 default:
2033 /* FIXME: not supported yet */
2034 return -EINVAL;
2035 }
2036
2037 amdgpu_amdkfd_device_probe(adev);
2038
2039 adev->pm.pp_feature = amdgpu_pp_feature_mask;
2040 if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2041 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2042 if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2043 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2044
2045 for (i = 0; i < adev->num_ip_blocks; i++) {
2046 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2047 DRM_ERROR("disabled ip block: %d <%s>\n",
2048 i, adev->ip_blocks[i].version->funcs->name);
2049 adev->ip_blocks[i].status.valid = false;
2050 } else {
2051 if (adev->ip_blocks[i].version->funcs->early_init) {
2052 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2053 if (r == -ENOENT) {
2054 adev->ip_blocks[i].status.valid = false;
2055 } else if (r) {
2056 DRM_ERROR("early_init of IP block <%s> failed %d\n",
2057 adev->ip_blocks[i].version->funcs->name, r);
2058 return r;
2059 } else {
2060 adev->ip_blocks[i].status.valid = true;
2061 }
2062 } else {
2063 adev->ip_blocks[i].status.valid = true;
2064 }
2065 }
2066 /* get the vbios after the asic_funcs are set up */
2067 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2068 r = amdgpu_device_parse_gpu_info_fw(adev);
2069 if (r)
2070 return r;
2071
2072 /* Read BIOS */
2073 if (!amdgpu_get_bios(adev))
2074 return -EINVAL;
2075
2076 r = amdgpu_atombios_init(adev);
2077 if (r) {
2078 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2079 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2080 return r;
2081 }
2082 }
2083 }
2084
2085 adev->cg_flags &= amdgpu_cg_mask;
2086 adev->pg_flags &= amdgpu_pg_mask;
2087
2088 return 0;
2089}
2090
2091static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2092{
2093 int i, r;
2094
2095 for (i = 0; i < adev->num_ip_blocks; i++) {
2096 if (!adev->ip_blocks[i].status.sw)
2097 continue;
2098 if (adev->ip_blocks[i].status.hw)
2099 continue;
2100 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2101 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2102 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2103 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2104 if (r) {
2105 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2106 adev->ip_blocks[i].version->funcs->name, r);
2107 return r;
2108 }
2109 adev->ip_blocks[i].status.hw = true;
2110 }
2111 }
2112
2113 return 0;
2114}
2115
2116static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2117{
2118 int i, r;
2119
2120 for (i = 0; i < adev->num_ip_blocks; i++) {
2121 if (!adev->ip_blocks[i].status.sw)
2122 continue;
2123 if (adev->ip_blocks[i].status.hw)
2124 continue;
2125 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2126 if (r) {
2127 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2128 adev->ip_blocks[i].version->funcs->name, r);
2129 return r;
2130 }
2131 adev->ip_blocks[i].status.hw = true;
2132 }
2133
2134 return 0;
2135}
2136
2137static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2138{
2139 int r = 0;
2140 int i;
2141 uint32_t smu_version;
2142
2143 if (adev->asic_type >= CHIP_VEGA10) {
2144 for (i = 0; i < adev->num_ip_blocks; i++) {
2145 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2146 continue;
2147
2148 if (!adev->ip_blocks[i].status.sw)
2149 continue;
2150
2151 /* no need to do the fw loading again if already done*/
2152 if (adev->ip_blocks[i].status.hw == true)
2153 break;
2154
2155 if (amdgpu_in_reset(adev) || adev->in_suspend) {
2156 r = adev->ip_blocks[i].version->funcs->resume(adev);
2157 if (r) {
2158 DRM_ERROR("resume of IP block <%s> failed %d\n",
2159 adev->ip_blocks[i].version->funcs->name, r);
2160 return r;
2161 }
2162 } else {
2163 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2164 if (r) {
2165 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2166 adev->ip_blocks[i].version->funcs->name, r);
2167 return r;
2168 }
2169 }
2170
2171 adev->ip_blocks[i].status.hw = true;
2172 break;
2173 }
2174 }
2175
2176 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2177 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2178
2179 return r;
2180}
2181
2182/**
2183 * amdgpu_device_ip_init - run init for hardware IPs
2184 *
2185 * @adev: amdgpu_device pointer
2186 *
2187 * Main initialization pass for hardware IPs. The list of all the hardware
2188 * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2189 * are run. sw_init initializes the software state associated with each IP
2190 * and hw_init initializes the hardware associated with each IP.
2191 * Returns 0 on success, negative error code on failure.
2192 */
2193static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2194{
2195 int i, r;
2196
2197 r = amdgpu_ras_init(adev);
2198 if (r)
2199 return r;
2200
2201 for (i = 0; i < adev->num_ip_blocks; i++) {
2202 if (!adev->ip_blocks[i].status.valid)
2203 continue;
2204 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2205 if (r) {
2206 DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2207 adev->ip_blocks[i].version->funcs->name, r);
2208 goto init_failed;
2209 }
2210 adev->ip_blocks[i].status.sw = true;
2211
2212 /* need to do gmc hw init early so we can allocate gpu mem */
2213 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2214 r = amdgpu_device_vram_scratch_init(adev);
2215 if (r) {
2216 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
2217 goto init_failed;
2218 }
2219 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2220 if (r) {
2221 DRM_ERROR("hw_init %d failed %d\n", i, r);
2222 goto init_failed;
2223 }
2224 r = amdgpu_device_wb_init(adev);
2225 if (r) {
2226 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2227 goto init_failed;
2228 }
2229 adev->ip_blocks[i].status.hw = true;
2230
2231 /* right after GMC hw init, we create CSA */
2232 if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
2233 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2234 AMDGPU_GEM_DOMAIN_VRAM,
2235 AMDGPU_CSA_SIZE);
2236 if (r) {
2237 DRM_ERROR("allocate CSA failed %d\n", r);
2238 goto init_failed;
2239 }
2240 }
2241 }
2242 }
2243
2244 if (amdgpu_sriov_vf(adev))
2245 amdgpu_virt_init_data_exchange(adev);
2246
2247 r = amdgpu_ib_pool_init(adev);
2248 if (r) {
2249 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2250 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2251 goto init_failed;
2252 }
2253
2254 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2255 if (r)
2256 goto init_failed;
2257
2258 r = amdgpu_device_ip_hw_init_phase1(adev);
2259 if (r)
2260 goto init_failed;
2261
2262 r = amdgpu_device_fw_loading(adev);
2263 if (r)
2264 goto init_failed;
2265
2266 r = amdgpu_device_ip_hw_init_phase2(adev);
2267 if (r)
2268 goto init_failed;
2269
2270 /*
2271 * retired pages will be loaded from eeprom and reserved here,
2272 * it should be called after amdgpu_device_ip_hw_init_phase2 since
2273 * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2274 * for I2C communication which only true at this point.
2275 *
2276 * amdgpu_ras_recovery_init may fail, but the upper only cares the
2277 * failure from bad gpu situation and stop amdgpu init process
2278 * accordingly. For other failed cases, it will still release all
2279 * the resource and print error message, rather than returning one
2280 * negative value to upper level.
2281 *
2282 * Note: theoretically, this should be called before all vram allocations
2283 * to protect retired page from abusing
2284 */
2285 r = amdgpu_ras_recovery_init(adev);
2286 if (r)
2287 goto init_failed;
2288
2289 if (adev->gmc.xgmi.num_physical_nodes > 1)
2290 amdgpu_xgmi_add_device(adev);
2291
2292 /* Don't init kfd if whole hive need to be reset during init */
2293 if (!adev->gmc.xgmi.pending_reset)
2294 amdgpu_amdkfd_device_init(adev);
2295
2296 amdgpu_fru_get_product_info(adev);
2297
2298init_failed:
2299 if (amdgpu_sriov_vf(adev))
2300 amdgpu_virt_release_full_gpu(adev, true);
2301
2302 return r;
2303}
2304
2305/**
2306 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2307 *
2308 * @adev: amdgpu_device pointer
2309 *
2310 * Writes a reset magic value to the gart pointer in VRAM. The driver calls
2311 * this function before a GPU reset. If the value is retained after a
2312 * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents.
2313 */
2314static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2315{
2316 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2317}
2318
2319/**
2320 * amdgpu_device_check_vram_lost - check if vram is valid
2321 *
2322 * @adev: amdgpu_device pointer
2323 *
2324 * Checks the reset magic value written to the gart pointer in VRAM.
2325 * The driver calls this after a GPU reset to see if the contents of
2326 * VRAM is lost or now.
2327 * returns true if vram is lost, false if not.
2328 */
2329static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2330{
2331 if (memcmp(adev->gart.ptr, adev->reset_magic,
2332 AMDGPU_RESET_MAGIC_NUM))
2333 return true;
2334
2335 if (!amdgpu_in_reset(adev))
2336 return false;
2337
2338 /*
2339 * For all ASICs with baco/mode1 reset, the VRAM is
2340 * always assumed to be lost.
2341 */
2342 switch (amdgpu_asic_reset_method(adev)) {
2343 case AMD_RESET_METHOD_BACO:
2344 case AMD_RESET_METHOD_MODE1:
2345 return true;
2346 default:
2347 return false;
2348 }
2349}
2350
2351/**
2352 * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2353 *
2354 * @adev: amdgpu_device pointer
2355 * @state: clockgating state (gate or ungate)
2356 *
2357 * The list of all the hardware IPs that make up the asic is walked and the
2358 * set_clockgating_state callbacks are run.
2359 * Late initialization pass enabling clockgating for hardware IPs.
2360 * Fini or suspend, pass disabling clockgating for hardware IPs.
2361 * Returns 0 on success, negative error code on failure.
2362 */
2363
2364static int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2365 enum amd_clockgating_state state)
2366{
2367 int i, j, r;
2368
2369 if (amdgpu_emu_mode == 1)
2370 return 0;
2371
2372 for (j = 0; j < adev->num_ip_blocks; j++) {
2373 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2374 if (!adev->ip_blocks[i].status.late_initialized)
2375 continue;
2376 /* skip CG for VCE/UVD, it's handled specially */
2377 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2378 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2379 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2380 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2381 adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2382 /* enable clockgating to save power */
2383 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2384 state);
2385 if (r) {
2386 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2387 adev->ip_blocks[i].version->funcs->name, r);
2388 return r;
2389 }
2390 }
2391 }
2392
2393 return 0;
2394}
2395
2396static int amdgpu_device_set_pg_state(struct amdgpu_device *adev, enum amd_powergating_state state)
2397{
2398 int i, j, r;
2399
2400 if (amdgpu_emu_mode == 1)
2401 return 0;
2402
2403 for (j = 0; j < adev->num_ip_blocks; j++) {
2404 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2405 if (!adev->ip_blocks[i].status.late_initialized)
2406 continue;
2407 /* skip CG for VCE/UVD, it's handled specially */
2408 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2409 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2410 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2411 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2412 adev->ip_blocks[i].version->funcs->set_powergating_state) {
2413 /* enable powergating to save power */
2414 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2415 state);
2416 if (r) {
2417 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2418 adev->ip_blocks[i].version->funcs->name, r);
2419 return r;
2420 }
2421 }
2422 }
2423 return 0;
2424}
2425
2426static int amdgpu_device_enable_mgpu_fan_boost(void)
2427{
2428 struct amdgpu_gpu_instance *gpu_ins;
2429 struct amdgpu_device *adev;
2430 int i, ret = 0;
2431
2432 mutex_lock(&mgpu_info.mutex);
2433
2434 /*
2435 * MGPU fan boost feature should be enabled
2436 * only when there are two or more dGPUs in
2437 * the system
2438 */
2439 if (mgpu_info.num_dgpu < 2)
2440 goto out;
2441
2442 for (i = 0; i < mgpu_info.num_dgpu; i++) {
2443 gpu_ins = &(mgpu_info.gpu_ins[i]);
2444 adev = gpu_ins->adev;
2445 if (!(adev->flags & AMD_IS_APU) &&
2446 !gpu_ins->mgpu_fan_enabled) {
2447 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2448 if (ret)
2449 break;
2450
2451 gpu_ins->mgpu_fan_enabled = 1;
2452 }
2453 }
2454
2455out:
2456 mutex_unlock(&mgpu_info.mutex);
2457
2458 return ret;
2459}
2460
2461/**
2462 * amdgpu_device_ip_late_init - run late init for hardware IPs
2463 *
2464 * @adev: amdgpu_device pointer
2465 *
2466 * Late initialization pass for hardware IPs. The list of all the hardware
2467 * IPs that make up the asic is walked and the late_init callbacks are run.
2468 * late_init covers any special initialization that an IP requires
2469 * after all of the have been initialized or something that needs to happen
2470 * late in the init process.
2471 * Returns 0 on success, negative error code on failure.
2472 */
2473static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2474{
2475 struct amdgpu_gpu_instance *gpu_instance;
2476 int i = 0, r;
2477
2478 for (i = 0; i < adev->num_ip_blocks; i++) {
2479 if (!adev->ip_blocks[i].status.hw)
2480 continue;
2481 if (adev->ip_blocks[i].version->funcs->late_init) {
2482 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2483 if (r) {
2484 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2485 adev->ip_blocks[i].version->funcs->name, r);
2486 return r;
2487 }
2488 }
2489 adev->ip_blocks[i].status.late_initialized = true;
2490 }
2491
2492 amdgpu_ras_set_error_query_ready(adev, true);
2493
2494 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2495 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2496
2497 amdgpu_device_fill_reset_magic(adev);
2498
2499 r = amdgpu_device_enable_mgpu_fan_boost();
2500 if (r)
2501 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2502
2503 /* For XGMI + passthrough configuration on arcturus, enable light SBR */
2504 if (adev->asic_type == CHIP_ARCTURUS &&
2505 amdgpu_passthrough(adev) &&
2506 adev->gmc.xgmi.num_physical_nodes > 1)
2507 smu_set_light_sbr(&adev->smu, true);
2508
2509 if (adev->gmc.xgmi.num_physical_nodes > 1) {
2510 mutex_lock(&mgpu_info.mutex);
2511
2512 /*
2513 * Reset device p-state to low as this was booted with high.
2514 *
2515 * This should be performed only after all devices from the same
2516 * hive get initialized.
2517 *
2518 * However, it's unknown how many device in the hive in advance.
2519 * As this is counted one by one during devices initializations.
2520 *
2521 * So, we wait for all XGMI interlinked devices initialized.
2522 * This may bring some delays as those devices may come from
2523 * different hives. But that should be OK.
2524 */
2525 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2526 for (i = 0; i < mgpu_info.num_gpu; i++) {
2527 gpu_instance = &(mgpu_info.gpu_ins[i]);
2528 if (gpu_instance->adev->flags & AMD_IS_APU)
2529 continue;
2530
2531 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2532 AMDGPU_XGMI_PSTATE_MIN);
2533 if (r) {
2534 DRM_ERROR("pstate setting failed (%d).\n", r);
2535 break;
2536 }
2537 }
2538 }
2539
2540 mutex_unlock(&mgpu_info.mutex);
2541 }
2542
2543 return 0;
2544}
2545
2546/**
2547 * amdgpu_device_ip_fini - run fini for hardware IPs
2548 *
2549 * @adev: amdgpu_device pointer
2550 *
2551 * Main teardown pass for hardware IPs. The list of all the hardware
2552 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2553 * are run. hw_fini tears down the hardware associated with each IP
2554 * and sw_fini tears down any software state associated with each IP.
2555 * Returns 0 on success, negative error code on failure.
2556 */
2557static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2558{
2559 int i, r;
2560
2561 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2562 amdgpu_virt_release_ras_err_handler_data(adev);
2563
2564 amdgpu_ras_pre_fini(adev);
2565
2566 if (adev->gmc.xgmi.num_physical_nodes > 1)
2567 amdgpu_xgmi_remove_device(adev);
2568
2569 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2570 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2571
2572 amdgpu_amdkfd_device_fini(adev);
2573
2574 /* need to disable SMC first */
2575 for (i = 0; i < adev->num_ip_blocks; i++) {
2576 if (!adev->ip_blocks[i].status.hw)
2577 continue;
2578 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2579 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2580 /* XXX handle errors */
2581 if (r) {
2582 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2583 adev->ip_blocks[i].version->funcs->name, r);
2584 }
2585 adev->ip_blocks[i].status.hw = false;
2586 break;
2587 }
2588 }
2589
2590 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2591 if (!adev->ip_blocks[i].status.hw)
2592 continue;
2593
2594 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2595 /* XXX handle errors */
2596 if (r) {
2597 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2598 adev->ip_blocks[i].version->funcs->name, r);
2599 }
2600
2601 adev->ip_blocks[i].status.hw = false;
2602 }
2603
2604
2605 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2606 if (!adev->ip_blocks[i].status.sw)
2607 continue;
2608
2609 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2610 amdgpu_ucode_free_bo(adev);
2611 amdgpu_free_static_csa(&adev->virt.csa_obj);
2612 amdgpu_device_wb_fini(adev);
2613 amdgpu_device_vram_scratch_fini(adev);
2614 amdgpu_ib_pool_fini(adev);
2615 }
2616
2617 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2618 /* XXX handle errors */
2619 if (r) {
2620 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2621 adev->ip_blocks[i].version->funcs->name, r);
2622 }
2623 adev->ip_blocks[i].status.sw = false;
2624 adev->ip_blocks[i].status.valid = false;
2625 }
2626
2627 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2628 if (!adev->ip_blocks[i].status.late_initialized)
2629 continue;
2630 if (adev->ip_blocks[i].version->funcs->late_fini)
2631 adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2632 adev->ip_blocks[i].status.late_initialized = false;
2633 }
2634
2635 amdgpu_ras_fini(adev);
2636
2637 if (amdgpu_sriov_vf(adev))
2638 if (amdgpu_virt_release_full_gpu(adev, false))
2639 DRM_ERROR("failed to release exclusive mode on fini\n");
2640
2641 return 0;
2642}
2643
2644/**
2645 * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2646 *
2647 * @work: work_struct.
2648 */
2649static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2650{
2651 struct amdgpu_device *adev =
2652 container_of(work, struct amdgpu_device, delayed_init_work.work);
2653 int r;
2654
2655 r = amdgpu_ib_ring_tests(adev);
2656 if (r)
2657 DRM_ERROR("ib ring test failed (%d).\n", r);
2658}
2659
2660static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2661{
2662 struct amdgpu_device *adev =
2663 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2664
2665 mutex_lock(&adev->gfx.gfx_off_mutex);
2666 if (!adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) {
2667 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2668 adev->gfx.gfx_off_state = true;
2669 }
2670 mutex_unlock(&adev->gfx.gfx_off_mutex);
2671}
2672
2673/**
2674 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2675 *
2676 * @adev: amdgpu_device pointer
2677 *
2678 * Main suspend function for hardware IPs. The list of all the hardware
2679 * IPs that make up the asic is walked, clockgating is disabled and the
2680 * suspend callbacks are run. suspend puts the hardware and software state
2681 * in each IP into a state suitable for suspend.
2682 * Returns 0 on success, negative error code on failure.
2683 */
2684static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2685{
2686 int i, r;
2687
2688 if (adev->in_poweroff_reboot_com || adev->in_hibernate ||
2689 !amdgpu_acpi_is_s0ix_supported(adev) || amdgpu_in_reset(adev)) {
2690 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2691 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2692 }
2693
2694 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2695 if (!adev->ip_blocks[i].status.valid)
2696 continue;
2697
2698 /* displays are handled separately */
2699 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
2700 continue;
2701
2702 /* XXX handle errors */
2703 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2704 /* XXX handle errors */
2705 if (r) {
2706 DRM_ERROR("suspend of IP block <%s> failed %d\n",
2707 adev->ip_blocks[i].version->funcs->name, r);
2708 return r;
2709 }
2710
2711 adev->ip_blocks[i].status.hw = false;
2712 }
2713
2714 return 0;
2715}
2716
2717/**
2718 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2719 *
2720 * @adev: amdgpu_device pointer
2721 *
2722 * Main suspend function for hardware IPs. The list of all the hardware
2723 * IPs that make up the asic is walked, clockgating is disabled and the
2724 * suspend callbacks are run. suspend puts the hardware and software state
2725 * in each IP into a state suitable for suspend.
2726 * Returns 0 on success, negative error code on failure.
2727 */
2728static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2729{
2730 int i, r;
2731
2732 for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2733 if (!adev->ip_blocks[i].status.valid)
2734 continue;
2735 /* displays are handled in phase1 */
2736 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
2737 continue;
2738 /* PSP lost connection when err_event_athub occurs */
2739 if (amdgpu_ras_intr_triggered() &&
2740 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2741 adev->ip_blocks[i].status.hw = false;
2742 continue;
2743 }
2744
2745 /* skip unnecessary suspend if we do not initialize them yet */
2746 if (adev->gmc.xgmi.pending_reset &&
2747 !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2748 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
2749 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2750 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
2751 adev->ip_blocks[i].status.hw = false;
2752 continue;
2753 }
2754 /* XXX handle errors */
2755 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2756 /* XXX handle errors */
2757 if (r) {
2758 DRM_ERROR("suspend of IP block <%s> failed %d\n",
2759 adev->ip_blocks[i].version->funcs->name, r);
2760 }
2761 adev->ip_blocks[i].status.hw = false;
2762 /* handle putting the SMC in the appropriate state */
2763 if(!amdgpu_sriov_vf(adev)){
2764 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2765 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
2766 if (r) {
2767 DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
2768 adev->mp1_state, r);
2769 return r;
2770 }
2771 }
2772 }
2773 }
2774
2775 return 0;
2776}
2777
2778/**
2779 * amdgpu_device_ip_suspend - run suspend for hardware IPs
2780 *
2781 * @adev: amdgpu_device pointer
2782 *
2783 * Main suspend function for hardware IPs. The list of all the hardware
2784 * IPs that make up the asic is walked, clockgating is disabled and the
2785 * suspend callbacks are run. suspend puts the hardware and software state
2786 * in each IP into a state suitable for suspend.
2787 * Returns 0 on success, negative error code on failure.
2788 */
2789int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
2790{
2791 int r;
2792
2793 if (amdgpu_sriov_vf(adev)) {
2794 amdgpu_virt_fini_data_exchange(adev);
2795 amdgpu_virt_request_full_gpu(adev, false);
2796 }
2797
2798 r = amdgpu_device_ip_suspend_phase1(adev);
2799 if (r)
2800 return r;
2801 r = amdgpu_device_ip_suspend_phase2(adev);
2802
2803 if (amdgpu_sriov_vf(adev))
2804 amdgpu_virt_release_full_gpu(adev, false);
2805
2806 return r;
2807}
2808
2809static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
2810{
2811 int i, r;
2812
2813 static enum amd_ip_block_type ip_order[] = {
2814 AMD_IP_BLOCK_TYPE_GMC,
2815 AMD_IP_BLOCK_TYPE_COMMON,
2816 AMD_IP_BLOCK_TYPE_PSP,
2817 AMD_IP_BLOCK_TYPE_IH,
2818 };
2819
2820 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2821 int j;
2822 struct amdgpu_ip_block *block;
2823
2824 block = &adev->ip_blocks[i];
2825 block->status.hw = false;
2826
2827 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
2828
2829 if (block->version->type != ip_order[j] ||
2830 !block->status.valid)
2831 continue;
2832
2833 r = block->version->funcs->hw_init(adev);
2834 DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2835 if (r)
2836 return r;
2837 block->status.hw = true;
2838 }
2839 }
2840
2841 return 0;
2842}
2843
2844static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
2845{
2846 int i, r;
2847
2848 static enum amd_ip_block_type ip_order[] = {
2849 AMD_IP_BLOCK_TYPE_SMC,
2850 AMD_IP_BLOCK_TYPE_DCE,
2851 AMD_IP_BLOCK_TYPE_GFX,
2852 AMD_IP_BLOCK_TYPE_SDMA,
2853 AMD_IP_BLOCK_TYPE_UVD,
2854 AMD_IP_BLOCK_TYPE_VCE,
2855 AMD_IP_BLOCK_TYPE_VCN
2856 };
2857
2858 for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2859 int j;
2860 struct amdgpu_ip_block *block;
2861
2862 for (j = 0; j < adev->num_ip_blocks; j++) {
2863 block = &adev->ip_blocks[j];
2864
2865 if (block->version->type != ip_order[i] ||
2866 !block->status.valid ||
2867 block->status.hw)
2868 continue;
2869
2870 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
2871 r = block->version->funcs->resume(adev);
2872 else
2873 r = block->version->funcs->hw_init(adev);
2874
2875 DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2876 if (r)
2877 return r;
2878 block->status.hw = true;
2879 }
2880 }
2881
2882 return 0;
2883}
2884
2885/**
2886 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
2887 *
2888 * @adev: amdgpu_device pointer
2889 *
2890 * First resume function for hardware IPs. The list of all the hardware
2891 * IPs that make up the asic is walked and the resume callbacks are run for
2892 * COMMON, GMC, and IH. resume puts the hardware into a functional state
2893 * after a suspend and updates the software state as necessary. This
2894 * function is also used for restoring the GPU after a GPU reset.
2895 * Returns 0 on success, negative error code on failure.
2896 */
2897static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
2898{
2899 int i, r;
2900
2901 for (i = 0; i < adev->num_ip_blocks; i++) {
2902 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
2903 continue;
2904 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2905 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2906 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2907
2908 r = adev->ip_blocks[i].version->funcs->resume(adev);
2909 if (r) {
2910 DRM_ERROR("resume of IP block <%s> failed %d\n",
2911 adev->ip_blocks[i].version->funcs->name, r);
2912 return r;
2913 }
2914 adev->ip_blocks[i].status.hw = true;
2915 }
2916 }
2917
2918 return 0;
2919}
2920
2921/**
2922 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
2923 *
2924 * @adev: amdgpu_device pointer
2925 *
2926 * First resume function for hardware IPs. The list of all the hardware
2927 * IPs that make up the asic is walked and the resume callbacks are run for
2928 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a
2929 * functional state after a suspend and updates the software state as
2930 * necessary. This function is also used for restoring the GPU after a GPU
2931 * reset.
2932 * Returns 0 on success, negative error code on failure.
2933 */
2934static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
2935{
2936 int i, r;
2937
2938 for (i = 0; i < adev->num_ip_blocks; i++) {
2939 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
2940 continue;
2941 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2942 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2943 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
2944 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
2945 continue;
2946 r = adev->ip_blocks[i].version->funcs->resume(adev);
2947 if (r) {
2948 DRM_ERROR("resume of IP block <%s> failed %d\n",
2949 adev->ip_blocks[i].version->funcs->name, r);
2950 return r;
2951 }
2952 adev->ip_blocks[i].status.hw = true;
2953 }
2954
2955 return 0;
2956}
2957
2958/**
2959 * amdgpu_device_ip_resume - run resume for hardware IPs
2960 *
2961 * @adev: amdgpu_device pointer
2962 *
2963 * Main resume function for hardware IPs. The hardware IPs
2964 * are split into two resume functions because they are
2965 * are also used in in recovering from a GPU reset and some additional
2966 * steps need to be take between them. In this case (S3/S4) they are
2967 * run sequentially.
2968 * Returns 0 on success, negative error code on failure.
2969 */
2970static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
2971{
2972 int r;
2973
2974 r = amdgpu_device_ip_resume_phase1(adev);
2975 if (r)
2976 return r;
2977
2978 r = amdgpu_device_fw_loading(adev);
2979 if (r)
2980 return r;
2981
2982 r = amdgpu_device_ip_resume_phase2(adev);
2983
2984 return r;
2985}
2986
2987/**
2988 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
2989 *
2990 * @adev: amdgpu_device pointer
2991 *
2992 * Query the VBIOS data tables to determine if the board supports SR-IOV.
2993 */
2994static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
2995{
2996 if (amdgpu_sriov_vf(adev)) {
2997 if (adev->is_atom_fw) {
2998 if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
2999 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3000 } else {
3001 if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3002 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3003 }
3004
3005 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3006 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3007 }
3008}
3009
3010/**
3011 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3012 *
3013 * @asic_type: AMD asic type
3014 *
3015 * Check if there is DC (new modesetting infrastructre) support for an asic.
3016 * returns true if DC has support, false if not.
3017 */
3018bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3019{
3020 switch (asic_type) {
3021#if defined(CONFIG_DRM_AMD_DC)
3022#if defined(CONFIG_DRM_AMD_DC_SI)
3023 case CHIP_TAHITI:
3024 case CHIP_PITCAIRN:
3025 case CHIP_VERDE:
3026 case CHIP_OLAND:
3027#endif
3028 case CHIP_BONAIRE:
3029 case CHIP_KAVERI:
3030 case CHIP_KABINI:
3031 case CHIP_MULLINS:
3032 /*
3033 * We have systems in the wild with these ASICs that require
3034 * LVDS and VGA support which is not supported with DC.
3035 *
3036 * Fallback to the non-DC driver here by default so as not to
3037 * cause regressions.
3038 */
3039 return amdgpu_dc > 0;
3040 case CHIP_HAWAII:
3041 case CHIP_CARRIZO:
3042 case CHIP_STONEY:
3043 case CHIP_POLARIS10:
3044 case CHIP_POLARIS11:
3045 case CHIP_POLARIS12:
3046 case CHIP_VEGAM:
3047 case CHIP_TONGA:
3048 case CHIP_FIJI:
3049 case CHIP_VEGA10:
3050 case CHIP_VEGA12:
3051 case CHIP_VEGA20:
3052#if defined(CONFIG_DRM_AMD_DC_DCN)
3053 case CHIP_RAVEN:
3054 case CHIP_NAVI10:
3055 case CHIP_NAVI14:
3056 case CHIP_NAVI12:
3057 case CHIP_RENOIR:
3058 case CHIP_SIENNA_CICHLID:
3059 case CHIP_NAVY_FLOUNDER:
3060 case CHIP_DIMGREY_CAVEFISH:
3061 case CHIP_VANGOGH:
3062#endif
3063 return amdgpu_dc != 0;
3064#endif
3065 default:
3066 if (amdgpu_dc > 0)
3067 DRM_INFO_ONCE("Display Core has been requested via kernel parameter "
3068 "but isn't supported by ASIC, ignoring\n");
3069 return false;
3070 }
3071}
3072
3073/**
3074 * amdgpu_device_has_dc_support - check if dc is supported
3075 *
3076 * @adev: amdgpu_device pointer
3077 *
3078 * Returns true for supported, false for not supported
3079 */
3080bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3081{
3082 if (amdgpu_sriov_vf(adev) || adev->enable_virtual_display)
3083 return false;
3084
3085 return amdgpu_device_asic_has_dc_support(adev->asic_type);
3086}
3087
3088
3089static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3090{
3091 struct amdgpu_device *adev =
3092 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3093 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3094
3095 /* It's a bug to not have a hive within this function */
3096 if (WARN_ON(!hive))
3097 return;
3098
3099 /*
3100 * Use task barrier to synchronize all xgmi reset works across the
3101 * hive. task_barrier_enter and task_barrier_exit will block
3102 * until all the threads running the xgmi reset works reach
3103 * those points. task_barrier_full will do both blocks.
3104 */
3105 if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3106
3107 task_barrier_enter(&hive->tb);
3108 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3109
3110 if (adev->asic_reset_res)
3111 goto fail;
3112
3113 task_barrier_exit(&hive->tb);
3114 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3115
3116 if (adev->asic_reset_res)
3117 goto fail;
3118
3119 if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count)
3120 adev->mmhub.funcs->reset_ras_error_count(adev);
3121 } else {
3122
3123 task_barrier_full(&hive->tb);
3124 adev->asic_reset_res = amdgpu_asic_reset(adev);
3125 }
3126
3127fail:
3128 if (adev->asic_reset_res)
3129 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3130 adev->asic_reset_res, adev_to_drm(adev)->unique);
3131 amdgpu_put_xgmi_hive(hive);
3132}
3133
3134static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3135{
3136 char *input = amdgpu_lockup_timeout;
3137 char *timeout_setting = NULL;
3138 int index = 0;
3139 long timeout;
3140 int ret = 0;
3141
3142 /*
3143 * By default timeout for non compute jobs is 10000.
3144 * And there is no timeout enforced on compute jobs.
3145 * In SR-IOV or passthrough mode, timeout for compute
3146 * jobs are 60000 by default.
3147 */
3148 adev->gfx_timeout = msecs_to_jiffies(10000);
3149 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3150 if (amdgpu_sriov_vf(adev))
3151 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3152 msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3153 else if (amdgpu_passthrough(adev))
3154 adev->compute_timeout = msecs_to_jiffies(60000);
3155 else
3156 adev->compute_timeout = MAX_SCHEDULE_TIMEOUT;
3157
3158 if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3159 while ((timeout_setting = strsep(&input, ",")) &&
3160 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3161 ret = kstrtol(timeout_setting, 0, &timeout);
3162 if (ret)
3163 return ret;
3164
3165 if (timeout == 0) {
3166 index++;
3167 continue;
3168 } else if (timeout < 0) {
3169 timeout = MAX_SCHEDULE_TIMEOUT;
3170 } else {
3171 timeout = msecs_to_jiffies(timeout);
3172 }
3173
3174 switch (index++) {
3175 case 0:
3176 adev->gfx_timeout = timeout;
3177 break;
3178 case 1:
3179 adev->compute_timeout = timeout;
3180 break;
3181 case 2:
3182 adev->sdma_timeout = timeout;
3183 break;
3184 case 3:
3185 adev->video_timeout = timeout;
3186 break;
3187 default:
3188 break;
3189 }
3190 }
3191 /*
3192 * There is only one value specified and
3193 * it should apply to all non-compute jobs.
3194 */
3195 if (index == 1) {
3196 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3197 if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3198 adev->compute_timeout = adev->gfx_timeout;
3199 }
3200 }
3201
3202 return ret;
3203}
3204
3205static const struct attribute *amdgpu_dev_attributes[] = {
3206 &dev_attr_product_name.attr,
3207 &dev_attr_product_number.attr,
3208 &dev_attr_serial_number.attr,
3209 &dev_attr_pcie_replay_count.attr,
3210 NULL
3211};
3212
3213
3214/**
3215 * amdgpu_device_init - initialize the driver
3216 *
3217 * @adev: amdgpu_device pointer
3218 * @flags: driver flags
3219 *
3220 * Initializes the driver info and hw (all asics).
3221 * Returns 0 for success or an error on failure.
3222 * Called at driver startup.
3223 */
3224int amdgpu_device_init(struct amdgpu_device *adev,
3225 uint32_t flags)
3226{
3227 struct drm_device *ddev = adev_to_drm(adev);
3228 struct pci_dev *pdev = adev->pdev;
3229 int r, i;
3230 bool px = false;
3231 u32 max_MBps;
3232
3233 adev->shutdown = false;
3234 adev->flags = flags;
3235
3236 if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3237 adev->asic_type = amdgpu_force_asic_type;
3238 else
3239 adev->asic_type = flags & AMD_ASIC_MASK;
3240
3241 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3242 if (amdgpu_emu_mode == 1)
3243 adev->usec_timeout *= 10;
3244 adev->gmc.gart_size = 512 * 1024 * 1024;
3245 adev->accel_working = false;
3246 adev->num_rings = 0;
3247 adev->mman.buffer_funcs = NULL;
3248 adev->mman.buffer_funcs_ring = NULL;
3249 adev->vm_manager.vm_pte_funcs = NULL;
3250 adev->vm_manager.vm_pte_num_scheds = 0;
3251 adev->gmc.gmc_funcs = NULL;
3252 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3253 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3254
3255 adev->smc_rreg = &amdgpu_invalid_rreg;
3256 adev->smc_wreg = &amdgpu_invalid_wreg;
3257 adev->pcie_rreg = &amdgpu_invalid_rreg;
3258 adev->pcie_wreg = &amdgpu_invalid_wreg;
3259 adev->pciep_rreg = &amdgpu_invalid_rreg;
3260 adev->pciep_wreg = &amdgpu_invalid_wreg;
3261 adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3262 adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3263 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3264 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3265 adev->didt_rreg = &amdgpu_invalid_rreg;
3266 adev->didt_wreg = &amdgpu_invalid_wreg;
3267 adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3268 adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3269 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3270 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3271
3272 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3273 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3274 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3275
3276 /* mutex initialization are all done here so we
3277 * can recall function without having locking issues */
3278 mutex_init(&adev->firmware.mutex);
3279 mutex_init(&adev->pm.mutex);
3280 mutex_init(&adev->gfx.gpu_clock_mutex);
3281 mutex_init(&adev->srbm_mutex);
3282 mutex_init(&adev->gfx.pipe_reserve_mutex);
3283 mutex_init(&adev->gfx.gfx_off_mutex);
3284 mutex_init(&adev->grbm_idx_mutex);
3285 mutex_init(&adev->mn_lock);
3286 mutex_init(&adev->virt.vf_errors.lock);
3287 hash_init(adev->mn_hash);
3288 atomic_set(&adev->in_gpu_reset, 0);
3289 init_rwsem(&adev->reset_sem);
3290 mutex_init(&adev->psp.mutex);
3291 mutex_init(&adev->notifier_lock);
3292
3293 r = amdgpu_device_check_arguments(adev);
3294 if (r)
3295 return r;
3296
3297 spin_lock_init(&adev->mmio_idx_lock);
3298 spin_lock_init(&adev->smc_idx_lock);
3299 spin_lock_init(&adev->pcie_idx_lock);
3300 spin_lock_init(&adev->uvd_ctx_idx_lock);
3301 spin_lock_init(&adev->didt_idx_lock);
3302 spin_lock_init(&adev->gc_cac_idx_lock);
3303 spin_lock_init(&adev->se_cac_idx_lock);
3304 spin_lock_init(&adev->audio_endpt_idx_lock);
3305 spin_lock_init(&adev->mm_stats.lock);
3306
3307 INIT_LIST_HEAD(&adev->shadow_list);
3308 mutex_init(&adev->shadow_list_lock);
3309
3310 INIT_LIST_HEAD(&adev->reset_list);
3311
3312 INIT_DELAYED_WORK(&adev->delayed_init_work,
3313 amdgpu_device_delayed_init_work_handler);
3314 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3315 amdgpu_device_delay_enable_gfx_off);
3316
3317 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3318
3319 adev->gfx.gfx_off_req_count = 1;
3320 adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3321
3322 atomic_set(&adev->throttling_logging_enabled, 1);
3323 /*
3324 * If throttling continues, logging will be performed every minute
3325 * to avoid log flooding. "-1" is subtracted since the thermal
3326 * throttling interrupt comes every second. Thus, the total logging
3327 * interval is 59 seconds(retelimited printk interval) + 1(waiting
3328 * for throttling interrupt) = 60 seconds.
3329 */
3330 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3331 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3332
3333 /* Registers mapping */
3334 /* TODO: block userspace mapping of io register */
3335 if (adev->asic_type >= CHIP_BONAIRE) {
3336 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3337 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3338 } else {
3339 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3340 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3341 }
3342
3343 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3344 if (adev->rmmio == NULL) {
3345 return -ENOMEM;
3346 }
3347 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3348 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
3349
3350 /* enable PCIE atomic ops */
3351 r = pci_enable_atomic_ops_to_root(adev->pdev,
3352 PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3353 PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3354 if (r) {
3355 adev->have_atomics_support = false;
3356 DRM_INFO("PCIE atomic ops is not supported\n");
3357 } else {
3358 adev->have_atomics_support = true;
3359 }
3360
3361 amdgpu_device_get_pcie_info(adev);
3362
3363 if (amdgpu_mcbp)
3364 DRM_INFO("MCBP is enabled\n");
3365
3366 if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10)
3367 adev->enable_mes = true;
3368
3369 /* detect hw virtualization here */
3370 amdgpu_detect_virtualization(adev);
3371
3372 r = amdgpu_device_get_job_timeout_settings(adev);
3373 if (r) {
3374 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3375 goto failed_unmap;
3376 }
3377
3378 /* early init functions */
3379 r = amdgpu_device_ip_early_init(adev);
3380 if (r)
3381 goto failed_unmap;
3382
3383 /* doorbell bar mapping and doorbell index init*/
3384 amdgpu_device_doorbell_init(adev);
3385
3386 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
3387 /* this will fail for cards that aren't VGA class devices, just
3388 * ignore it */
3389 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3390 vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
3391
3392 if (amdgpu_device_supports_px(ddev)) {
3393 px = true;
3394 vga_switcheroo_register_client(adev->pdev,
3395 &amdgpu_switcheroo_ops, px);
3396 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
3397 }
3398
3399 if (amdgpu_emu_mode == 1) {
3400 /* post the asic on emulation mode */
3401 emu_soc_asic_init(adev);
3402 goto fence_driver_init;
3403 }
3404
3405 /* detect if we are with an SRIOV vbios */
3406 amdgpu_device_detect_sriov_bios(adev);
3407
3408 /* check if we need to reset the asic
3409 * E.g., driver was not cleanly unloaded previously, etc.
3410 */
3411 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3412 if (adev->gmc.xgmi.num_physical_nodes) {
3413 dev_info(adev->dev, "Pending hive reset.\n");
3414 adev->gmc.xgmi.pending_reset = true;
3415 /* Only need to init necessary block for SMU to handle the reset */
3416 for (i = 0; i < adev->num_ip_blocks; i++) {
3417 if (!adev->ip_blocks[i].status.valid)
3418 continue;
3419 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3420 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3421 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3422 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3423 DRM_DEBUG("IP %s disabled for hw_init.\n",
3424 adev->ip_blocks[i].version->funcs->name);
3425 adev->ip_blocks[i].status.hw = true;
3426 }
3427 }
3428 } else {
3429 r = amdgpu_asic_reset(adev);
3430 if (r) {
3431 dev_err(adev->dev, "asic reset on init failed\n");
3432 goto failed;
3433 }
3434 }
3435 }
3436
3437 pci_enable_pcie_error_reporting(adev->pdev);
3438
3439 /* Post card if necessary */
3440 if (amdgpu_device_need_post(adev)) {
3441 if (!adev->bios) {
3442 dev_err(adev->dev, "no vBIOS found\n");
3443 r = -EINVAL;
3444 goto failed;
3445 }
3446 DRM_INFO("GPU posting now...\n");
3447 r = amdgpu_device_asic_init(adev);
3448 if (r) {
3449 dev_err(adev->dev, "gpu post error!\n");
3450 goto failed;
3451 }
3452 }
3453
3454 if (adev->is_atom_fw) {
3455 /* Initialize clocks */
3456 r = amdgpu_atomfirmware_get_clock_info(adev);
3457 if (r) {
3458 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
3459 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3460 goto failed;
3461 }
3462 } else {
3463 /* Initialize clocks */
3464 r = amdgpu_atombios_get_clock_info(adev);
3465 if (r) {
3466 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
3467 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3468 goto failed;
3469 }
3470 /* init i2c buses */
3471 if (!amdgpu_device_has_dc_support(adev))
3472 amdgpu_atombios_i2c_init(adev);
3473 }
3474
3475fence_driver_init:
3476 /* Fence driver */
3477 r = amdgpu_fence_driver_init(adev);
3478 if (r) {
3479 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
3480 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
3481 goto failed;
3482 }
3483
3484 /* init the mode config */
3485 drm_mode_config_init(adev_to_drm(adev));
3486
3487 r = amdgpu_device_ip_init(adev);
3488 if (r) {
3489 /* failed in exclusive mode due to timeout */
3490 if (amdgpu_sriov_vf(adev) &&
3491 !amdgpu_sriov_runtime(adev) &&
3492 amdgpu_virt_mmio_blocked(adev) &&
3493 !amdgpu_virt_wait_reset(adev)) {
3494 dev_err(adev->dev, "VF exclusive mode timeout\n");
3495 /* Don't send request since VF is inactive. */
3496 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
3497 adev->virt.ops = NULL;
3498 r = -EAGAIN;
3499 goto release_ras_con;
3500 }
3501 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
3502 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
3503 goto release_ras_con;
3504 }
3505
3506 dev_info(adev->dev,
3507 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
3508 adev->gfx.config.max_shader_engines,
3509 adev->gfx.config.max_sh_per_se,
3510 adev->gfx.config.max_cu_per_sh,
3511 adev->gfx.cu_info.number);
3512
3513 adev->accel_working = true;
3514
3515 amdgpu_vm_check_compute_bug(adev);
3516
3517 /* Initialize the buffer migration limit. */
3518 if (amdgpu_moverate >= 0)
3519 max_MBps = amdgpu_moverate;
3520 else
3521 max_MBps = 8; /* Allow 8 MB/s. */
3522 /* Get a log2 for easy divisions. */
3523 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
3524
3525 amdgpu_fbdev_init(adev);
3526
3527 r = amdgpu_pm_sysfs_init(adev);
3528 if (r) {
3529 adev->pm_sysfs_en = false;
3530 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
3531 } else
3532 adev->pm_sysfs_en = true;
3533
3534 r = amdgpu_ucode_sysfs_init(adev);
3535 if (r) {
3536 adev->ucode_sysfs_en = false;
3537 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
3538 } else
3539 adev->ucode_sysfs_en = true;
3540
3541 if ((amdgpu_testing & 1)) {
3542 if (adev->accel_working)
3543 amdgpu_test_moves(adev);
3544 else
3545 DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
3546 }
3547 if (amdgpu_benchmarking) {
3548 if (adev->accel_working)
3549 amdgpu_benchmark(adev, amdgpu_benchmarking);
3550 else
3551 DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
3552 }
3553
3554 /*
3555 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
3556 * Otherwise the mgpu fan boost feature will be skipped due to the
3557 * gpu instance is counted less.
3558 */
3559 amdgpu_register_gpu_instance(adev);
3560
3561 /* enable clockgating, etc. after ib tests, etc. since some blocks require
3562 * explicit gating rather than handling it automatically.
3563 */
3564 if (!adev->gmc.xgmi.pending_reset) {
3565 r = amdgpu_device_ip_late_init(adev);
3566 if (r) {
3567 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
3568 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
3569 goto release_ras_con;
3570 }
3571 /* must succeed. */
3572 amdgpu_ras_resume(adev);
3573 queue_delayed_work(system_wq, &adev->delayed_init_work,
3574 msecs_to_jiffies(AMDGPU_RESUME_MS));
3575 }
3576
3577 if (amdgpu_sriov_vf(adev))
3578 flush_delayed_work(&adev->delayed_init_work);
3579
3580 r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
3581 if (r)
3582 dev_err(adev->dev, "Could not create amdgpu device attr\n");
3583
3584 if (IS_ENABLED(CONFIG_PERF_EVENTS))
3585 r = amdgpu_pmu_init(adev);
3586 if (r)
3587 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
3588
3589 /* Have stored pci confspace at hand for restore in sudden PCI error */
3590 if (amdgpu_device_cache_pci_state(adev->pdev))
3591 pci_restore_state(pdev);
3592
3593 if (adev->gmc.xgmi.pending_reset)
3594 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
3595 msecs_to_jiffies(AMDGPU_RESUME_MS));
3596
3597 return 0;
3598
3599release_ras_con:
3600 amdgpu_release_ras_context(adev);
3601
3602failed:
3603 amdgpu_vf_error_trans_all(adev);
3604 if (px)
3605 vga_switcheroo_fini_domain_pm_ops(adev->dev);
3606
3607failed_unmap:
3608 iounmap(adev->rmmio);
3609 adev->rmmio = NULL;
3610
3611 return r;
3612}
3613
3614/**
3615 * amdgpu_device_fini - tear down the driver
3616 *
3617 * @adev: amdgpu_device pointer
3618 *
3619 * Tear down the driver info (all asics).
3620 * Called at driver shutdown.
3621 */
3622void amdgpu_device_fini(struct amdgpu_device *adev)
3623{
3624 dev_info(adev->dev, "amdgpu: finishing device.\n");
3625 flush_delayed_work(&adev->delayed_init_work);
3626 ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
3627 adev->shutdown = true;
3628
3629 kfree(adev->pci_state);
3630
3631 /* make sure IB test finished before entering exclusive mode
3632 * to avoid preemption on IB test
3633 * */
3634 if (amdgpu_sriov_vf(adev)) {
3635 amdgpu_virt_request_full_gpu(adev, false);
3636 amdgpu_virt_fini_data_exchange(adev);
3637 }
3638
3639 /* disable all interrupts */
3640 amdgpu_irq_disable_all(adev);
3641 if (adev->mode_info.mode_config_initialized){
3642 if (!amdgpu_device_has_dc_support(adev))
3643 drm_helper_force_disable_all(adev_to_drm(adev));
3644 else
3645 drm_atomic_helper_shutdown(adev_to_drm(adev));
3646 }
3647 amdgpu_fence_driver_fini(adev);
3648 if (adev->pm_sysfs_en)
3649 amdgpu_pm_sysfs_fini(adev);
3650 amdgpu_fbdev_fini(adev);
3651 amdgpu_device_ip_fini(adev);
3652 release_firmware(adev->firmware.gpu_info_fw);
3653 adev->firmware.gpu_info_fw = NULL;
3654 adev->accel_working = false;
3655 /* free i2c buses */
3656 if (!amdgpu_device_has_dc_support(adev))
3657 amdgpu_i2c_fini(adev);
3658
3659 if (amdgpu_emu_mode != 1)
3660 amdgpu_atombios_fini(adev);
3661
3662 kfree(adev->bios);
3663 adev->bios = NULL;
3664 if (amdgpu_device_supports_px(adev_to_drm(adev))) {
3665 vga_switcheroo_unregister_client(adev->pdev);
3666 vga_switcheroo_fini_domain_pm_ops(adev->dev);
3667 }
3668 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3669 vga_client_register(adev->pdev, NULL, NULL, NULL);
3670 iounmap(adev->rmmio);
3671 adev->rmmio = NULL;
3672 amdgpu_device_doorbell_fini(adev);
3673
3674 if (adev->ucode_sysfs_en)
3675 amdgpu_ucode_sysfs_fini(adev);
3676
3677 sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
3678 if (IS_ENABLED(CONFIG_PERF_EVENTS))
3679 amdgpu_pmu_fini(adev);
3680 if (adev->mman.discovery_bin)
3681 amdgpu_discovery_fini(adev);
3682}
3683
3684
3685/*
3686 * Suspend & resume.
3687 */
3688/**
3689 * amdgpu_device_suspend - initiate device suspend
3690 *
3691 * @dev: drm dev pointer
3692 * @fbcon : notify the fbdev of suspend
3693 *
3694 * Puts the hw in the suspend state (all asics).
3695 * Returns 0 for success or an error on failure.
3696 * Called at driver suspend.
3697 */
3698int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
3699{
3700 struct amdgpu_device *adev;
3701 struct drm_crtc *crtc;
3702 struct drm_connector *connector;
3703 struct drm_connector_list_iter iter;
3704 int r;
3705
3706 adev = drm_to_adev(dev);
3707
3708 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3709 return 0;
3710
3711 adev->in_suspend = true;
3712 drm_kms_helper_poll_disable(dev);
3713
3714 if (fbcon)
3715 amdgpu_fbdev_set_suspend(adev, 1);
3716
3717 cancel_delayed_work_sync(&adev->delayed_init_work);
3718
3719 if (!amdgpu_device_has_dc_support(adev)) {
3720 /* turn off display hw */
3721 drm_modeset_lock_all(dev);
3722 drm_connector_list_iter_begin(dev, &iter);
3723 drm_for_each_connector_iter(connector, &iter)
3724 drm_helper_connector_dpms(connector,
3725 DRM_MODE_DPMS_OFF);
3726 drm_connector_list_iter_end(&iter);
3727 drm_modeset_unlock_all(dev);
3728 /* unpin the front buffers and cursors */
3729 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3730 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
3731 struct drm_framebuffer *fb = crtc->primary->fb;
3732 struct amdgpu_bo *robj;
3733
3734 if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
3735 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
3736 r = amdgpu_bo_reserve(aobj, true);
3737 if (r == 0) {
3738 amdgpu_bo_unpin(aobj);
3739 amdgpu_bo_unreserve(aobj);
3740 }
3741 }
3742
3743 if (fb == NULL || fb->obj[0] == NULL) {
3744 continue;
3745 }
3746 robj = gem_to_amdgpu_bo(fb->obj[0]);
3747 /* don't unpin kernel fb objects */
3748 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
3749 r = amdgpu_bo_reserve(robj, true);
3750 if (r == 0) {
3751 amdgpu_bo_unpin(robj);
3752 amdgpu_bo_unreserve(robj);
3753 }
3754 }
3755 }
3756 }
3757
3758 amdgpu_ras_suspend(adev);
3759
3760 r = amdgpu_device_ip_suspend_phase1(adev);
3761
3762 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
3763
3764 /* evict vram memory */
3765 amdgpu_bo_evict_vram(adev);
3766
3767 amdgpu_fence_driver_suspend(adev);
3768
3769 /*
3770 * TODO: Need figure out the each GNB IP idle off dependency and then
3771 * improve the AMDGPU suspend/resume sequence for system-wide Sx entry/exit.
3772 */
3773 if (adev->in_poweroff_reboot_com || adev->in_hibernate ||
3774 !amdgpu_acpi_is_s0ix_supported(adev) || amdgpu_in_reset(adev))
3775 r = amdgpu_device_ip_suspend_phase2(adev);
3776 else
3777 amdgpu_gfx_state_change_set(adev, sGpuChangeState_D3Entry);
3778 /* evict remaining vram memory
3779 * This second call to evict vram is to evict the gart page table
3780 * using the CPU.
3781 */
3782 amdgpu_bo_evict_vram(adev);
3783
3784 return 0;
3785}
3786
3787/**
3788 * amdgpu_device_resume - initiate device resume
3789 *
3790 * @dev: drm dev pointer
3791 * @fbcon : notify the fbdev of resume
3792 *
3793 * Bring the hw back to operating state (all asics).
3794 * Returns 0 for success or an error on failure.
3795 * Called at driver resume.
3796 */
3797int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
3798{
3799 struct drm_connector *connector;
3800 struct drm_connector_list_iter iter;
3801 struct amdgpu_device *adev = drm_to_adev(dev);
3802 struct drm_crtc *crtc;
3803 int r = 0;
3804
3805 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3806 return 0;
3807
3808 if (amdgpu_acpi_is_s0ix_supported(adev))
3809 amdgpu_gfx_state_change_set(adev, sGpuChangeState_D0Entry);
3810
3811 /* post card */
3812 if (amdgpu_device_need_post(adev)) {
3813 r = amdgpu_device_asic_init(adev);
3814 if (r)
3815 dev_err(adev->dev, "amdgpu asic init failed\n");
3816 }
3817
3818 r = amdgpu_device_ip_resume(adev);
3819 if (r) {
3820 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
3821 return r;
3822 }
3823 amdgpu_fence_driver_resume(adev);
3824
3825
3826 r = amdgpu_device_ip_late_init(adev);
3827 if (r)
3828 return r;
3829
3830 queue_delayed_work(system_wq, &adev->delayed_init_work,
3831 msecs_to_jiffies(AMDGPU_RESUME_MS));
3832
3833 if (!amdgpu_device_has_dc_support(adev)) {
3834 /* pin cursors */
3835 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3836 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
3837
3838 if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) {
3839 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
3840 r = amdgpu_bo_reserve(aobj, true);
3841 if (r == 0) {
3842 r = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM);
3843 if (r != 0)
3844 dev_err(adev->dev, "Failed to pin cursor BO (%d)\n", r);
3845 amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj);
3846 amdgpu_bo_unreserve(aobj);
3847 }
3848 }
3849 }
3850 }
3851 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
3852 if (r)
3853 return r;
3854
3855 /* Make sure IB tests flushed */
3856 flush_delayed_work(&adev->delayed_init_work);
3857
3858 /* blat the mode back in */
3859 if (fbcon) {
3860 if (!amdgpu_device_has_dc_support(adev)) {
3861 /* pre DCE11 */
3862 drm_helper_resume_force_mode(dev);
3863
3864 /* turn on display hw */
3865 drm_modeset_lock_all(dev);
3866
3867 drm_connector_list_iter_begin(dev, &iter);
3868 drm_for_each_connector_iter(connector, &iter)
3869 drm_helper_connector_dpms(connector,
3870 DRM_MODE_DPMS_ON);
3871 drm_connector_list_iter_end(&iter);
3872
3873 drm_modeset_unlock_all(dev);
3874 }
3875 amdgpu_fbdev_set_suspend(adev, 0);
3876 }
3877
3878 drm_kms_helper_poll_enable(dev);
3879
3880 amdgpu_ras_resume(adev);
3881
3882 /*
3883 * Most of the connector probing functions try to acquire runtime pm
3884 * refs to ensure that the GPU is powered on when connector polling is
3885 * performed. Since we're calling this from a runtime PM callback,
3886 * trying to acquire rpm refs will cause us to deadlock.
3887 *
3888 * Since we're guaranteed to be holding the rpm lock, it's safe to
3889 * temporarily disable the rpm helpers so this doesn't deadlock us.
3890 */
3891#ifdef CONFIG_PM
3892 dev->dev->power.disable_depth++;
3893#endif
3894 if (!amdgpu_device_has_dc_support(adev))
3895 drm_helper_hpd_irq_event(dev);
3896 else
3897 drm_kms_helper_hotplug_event(dev);
3898#ifdef CONFIG_PM
3899 dev->dev->power.disable_depth--;
3900#endif
3901 adev->in_suspend = false;
3902
3903 return 0;
3904}
3905
3906/**
3907 * amdgpu_device_ip_check_soft_reset - did soft reset succeed
3908 *
3909 * @adev: amdgpu_device pointer
3910 *
3911 * The list of all the hardware IPs that make up the asic is walked and
3912 * the check_soft_reset callbacks are run. check_soft_reset determines
3913 * if the asic is still hung or not.
3914 * Returns true if any of the IPs are still in a hung state, false if not.
3915 */
3916static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
3917{
3918 int i;
3919 bool asic_hang = false;
3920
3921 if (amdgpu_sriov_vf(adev))
3922 return true;
3923
3924 if (amdgpu_asic_need_full_reset(adev))
3925 return true;
3926
3927 for (i = 0; i < adev->num_ip_blocks; i++) {
3928 if (!adev->ip_blocks[i].status.valid)
3929 continue;
3930 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
3931 adev->ip_blocks[i].status.hang =
3932 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
3933 if (adev->ip_blocks[i].status.hang) {
3934 dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
3935 asic_hang = true;
3936 }
3937 }
3938 return asic_hang;
3939}
3940
3941/**
3942 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
3943 *
3944 * @adev: amdgpu_device pointer
3945 *
3946 * The list of all the hardware IPs that make up the asic is walked and the
3947 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset
3948 * handles any IP specific hardware or software state changes that are
3949 * necessary for a soft reset to succeed.
3950 * Returns 0 on success, negative error code on failure.
3951 */
3952static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
3953{
3954 int i, r = 0;
3955
3956 for (i = 0; i < adev->num_ip_blocks; i++) {
3957 if (!adev->ip_blocks[i].status.valid)
3958 continue;
3959 if (adev->ip_blocks[i].status.hang &&
3960 adev->ip_blocks[i].version->funcs->pre_soft_reset) {
3961 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
3962 if (r)
3963 return r;
3964 }
3965 }
3966
3967 return 0;
3968}
3969
3970/**
3971 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
3972 *
3973 * @adev: amdgpu_device pointer
3974 *
3975 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu
3976 * reset is necessary to recover.
3977 * Returns true if a full asic reset is required, false if not.
3978 */
3979static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
3980{
3981 int i;
3982
3983 if (amdgpu_asic_need_full_reset(adev))
3984 return true;
3985
3986 for (i = 0; i < adev->num_ip_blocks; i++) {
3987 if (!adev->ip_blocks[i].status.valid)
3988 continue;
3989 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
3990 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
3991 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
3992 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
3993 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3994 if (adev->ip_blocks[i].status.hang) {
3995 dev_info(adev->dev, "Some block need full reset!\n");
3996 return true;
3997 }
3998 }
3999 }
4000 return false;
4001}
4002
4003/**
4004 * amdgpu_device_ip_soft_reset - do a soft reset
4005 *
4006 * @adev: amdgpu_device pointer
4007 *
4008 * The list of all the hardware IPs that make up the asic is walked and the
4009 * soft_reset callbacks are run if the block is hung. soft_reset handles any
4010 * IP specific hardware or software state changes that are necessary to soft
4011 * reset the IP.
4012 * Returns 0 on success, negative error code on failure.
4013 */
4014static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4015{
4016 int i, r = 0;
4017
4018 for (i = 0; i < adev->num_ip_blocks; i++) {
4019 if (!adev->ip_blocks[i].status.valid)
4020 continue;
4021 if (adev->ip_blocks[i].status.hang &&
4022 adev->ip_blocks[i].version->funcs->soft_reset) {
4023 r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4024 if (r)
4025 return r;
4026 }
4027 }
4028
4029 return 0;
4030}
4031
4032/**
4033 * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4034 *
4035 * @adev: amdgpu_device pointer
4036 *
4037 * The list of all the hardware IPs that make up the asic is walked and the
4038 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset
4039 * handles any IP specific hardware or software state changes that are
4040 * necessary after the IP has been soft reset.
4041 * Returns 0 on success, negative error code on failure.
4042 */
4043static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4044{
4045 int i, r = 0;
4046
4047 for (i = 0; i < adev->num_ip_blocks; i++) {
4048 if (!adev->ip_blocks[i].status.valid)
4049 continue;
4050 if (adev->ip_blocks[i].status.hang &&
4051 adev->ip_blocks[i].version->funcs->post_soft_reset)
4052 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4053 if (r)
4054 return r;
4055 }
4056
4057 return 0;
4058}
4059
4060/**
4061 * amdgpu_device_recover_vram - Recover some VRAM contents
4062 *
4063 * @adev: amdgpu_device pointer
4064 *
4065 * Restores the contents of VRAM buffers from the shadows in GTT. Used to
4066 * restore things like GPUVM page tables after a GPU reset where
4067 * the contents of VRAM might be lost.
4068 *
4069 * Returns:
4070 * 0 on success, negative error code on failure.
4071 */
4072static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4073{
4074 struct dma_fence *fence = NULL, *next = NULL;
4075 struct amdgpu_bo *shadow;
4076 long r = 1, tmo;
4077
4078 if (amdgpu_sriov_runtime(adev))
4079 tmo = msecs_to_jiffies(8000);
4080 else
4081 tmo = msecs_to_jiffies(100);
4082
4083 dev_info(adev->dev, "recover vram bo from shadow start\n");
4084 mutex_lock(&adev->shadow_list_lock);
4085 list_for_each_entry(shadow, &adev->shadow_list, shadow_list) {
4086
4087 /* No need to recover an evicted BO */
4088 if (shadow->tbo.mem.mem_type != TTM_PL_TT ||
4089 shadow->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET ||
4090 shadow->parent->tbo.mem.mem_type != TTM_PL_VRAM)
4091 continue;
4092
4093 r = amdgpu_bo_restore_shadow(shadow, &next);
4094 if (r)
4095 break;
4096
4097 if (fence) {
4098 tmo = dma_fence_wait_timeout(fence, false, tmo);
4099 dma_fence_put(fence);
4100 fence = next;
4101 if (tmo == 0) {
4102 r = -ETIMEDOUT;
4103 break;
4104 } else if (tmo < 0) {
4105 r = tmo;
4106 break;
4107 }
4108 } else {
4109 fence = next;
4110 }
4111 }
4112 mutex_unlock(&adev->shadow_list_lock);
4113
4114 if (fence)
4115 tmo = dma_fence_wait_timeout(fence, false, tmo);
4116 dma_fence_put(fence);
4117
4118 if (r < 0 || tmo <= 0) {
4119 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4120 return -EIO;
4121 }
4122
4123 dev_info(adev->dev, "recover vram bo from shadow done\n");
4124 return 0;
4125}
4126
4127
4128/**
4129 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4130 *
4131 * @adev: amdgpu_device pointer
4132 * @from_hypervisor: request from hypervisor
4133 *
4134 * do VF FLR and reinitialize Asic
4135 * return 0 means succeeded otherwise failed
4136 */
4137static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4138 bool from_hypervisor)
4139{
4140 int r;
4141
4142 if (from_hypervisor)
4143 r = amdgpu_virt_request_full_gpu(adev, true);
4144 else
4145 r = amdgpu_virt_reset_gpu(adev);
4146 if (r)
4147 return r;
4148
4149 amdgpu_amdkfd_pre_reset(adev);
4150
4151 /* Resume IP prior to SMC */
4152 r = amdgpu_device_ip_reinit_early_sriov(adev);
4153 if (r)
4154 goto error;
4155
4156 amdgpu_virt_init_data_exchange(adev);
4157 /* we need recover gart prior to run SMC/CP/SDMA resume */
4158 amdgpu_gtt_mgr_recover(ttm_manager_type(&adev->mman.bdev, TTM_PL_TT));
4159
4160 r = amdgpu_device_fw_loading(adev);
4161 if (r)
4162 return r;
4163
4164 /* now we are okay to resume SMC/CP/SDMA */
4165 r = amdgpu_device_ip_reinit_late_sriov(adev);
4166 if (r)
4167 goto error;
4168
4169 amdgpu_irq_gpu_reset_resume_helper(adev);
4170 r = amdgpu_ib_ring_tests(adev);
4171 amdgpu_amdkfd_post_reset(adev);
4172
4173error:
4174 amdgpu_virt_release_full_gpu(adev, true);
4175 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4176 amdgpu_inc_vram_lost(adev);
4177 r = amdgpu_device_recover_vram(adev);
4178 }
4179
4180 return r;
4181}
4182
4183/**
4184 * amdgpu_device_has_job_running - check if there is any job in mirror list
4185 *
4186 * @adev: amdgpu_device pointer
4187 *
4188 * check if there is any job in mirror list
4189 */
4190bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4191{
4192 int i;
4193 struct drm_sched_job *job;
4194
4195 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4196 struct amdgpu_ring *ring = adev->rings[i];
4197
4198 if (!ring || !ring->sched.thread)
4199 continue;
4200
4201 spin_lock(&ring->sched.job_list_lock);
4202 job = list_first_entry_or_null(&ring->sched.pending_list,
4203 struct drm_sched_job, list);
4204 spin_unlock(&ring->sched.job_list_lock);
4205 if (job)
4206 return true;
4207 }
4208 return false;
4209}
4210
4211/**
4212 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4213 *
4214 * @adev: amdgpu_device pointer
4215 *
4216 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4217 * a hung GPU.
4218 */
4219bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4220{
4221 if (!amdgpu_device_ip_check_soft_reset(adev)) {
4222 dev_info(adev->dev, "Timeout, but no hardware hang detected.\n");
4223 return false;
4224 }
4225
4226 if (amdgpu_gpu_recovery == 0)
4227 goto disabled;
4228
4229 if (amdgpu_sriov_vf(adev))
4230 return true;
4231
4232 if (amdgpu_gpu_recovery == -1) {
4233 switch (adev->asic_type) {
4234 case CHIP_BONAIRE:
4235 case CHIP_HAWAII:
4236 case CHIP_TOPAZ:
4237 case CHIP_TONGA:
4238 case CHIP_FIJI:
4239 case CHIP_POLARIS10:
4240 case CHIP_POLARIS11:
4241 case CHIP_POLARIS12:
4242 case CHIP_VEGAM:
4243 case CHIP_VEGA20:
4244 case CHIP_VEGA10:
4245 case CHIP_VEGA12:
4246 case CHIP_RAVEN:
4247 case CHIP_ARCTURUS:
4248 case CHIP_RENOIR:
4249 case CHIP_NAVI10:
4250 case CHIP_NAVI14:
4251 case CHIP_NAVI12:
4252 case CHIP_SIENNA_CICHLID:
4253 case CHIP_NAVY_FLOUNDER:
4254 case CHIP_DIMGREY_CAVEFISH:
4255 case CHIP_VANGOGH:
4256 break;
4257 default:
4258 goto disabled;
4259 }
4260 }
4261
4262 return true;
4263
4264disabled:
4265 dev_info(adev->dev, "GPU recovery disabled.\n");
4266 return false;
4267}
4268
4269int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4270{
4271 u32 i;
4272 int ret = 0;
4273
4274 amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4275
4276 dev_info(adev->dev, "GPU mode1 reset\n");
4277
4278 /* disable BM */
4279 pci_clear_master(adev->pdev);
4280
4281 amdgpu_device_cache_pci_state(adev->pdev);
4282
4283 if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4284 dev_info(adev->dev, "GPU smu mode1 reset\n");
4285 ret = amdgpu_dpm_mode1_reset(adev);
4286 } else {
4287 dev_info(adev->dev, "GPU psp mode1 reset\n");
4288 ret = psp_gpu_reset(adev);
4289 }
4290
4291 if (ret)
4292 dev_err(adev->dev, "GPU mode1 reset failed\n");
4293
4294 amdgpu_device_load_pci_state(adev->pdev);
4295
4296 /* wait for asic to come out of reset */
4297 for (i = 0; i < adev->usec_timeout; i++) {
4298 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4299
4300 if (memsize != 0xffffffff)
4301 break;
4302 udelay(1);
4303 }
4304
4305 amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4306 return ret;
4307}
4308
4309int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4310 struct amdgpu_job *job,
4311 bool *need_full_reset_arg)
4312{
4313 int i, r = 0;
4314 bool need_full_reset = *need_full_reset_arg;
4315
4316 /* no need to dump if device is not in good state during probe period */
4317 if (!adev->gmc.xgmi.pending_reset)
4318 amdgpu_debugfs_wait_dump(adev);
4319
4320 if (amdgpu_sriov_vf(adev)) {
4321 /* stop the data exchange thread */
4322 amdgpu_virt_fini_data_exchange(adev);
4323 }
4324
4325 /* block all schedulers and reset given job's ring */
4326 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4327 struct amdgpu_ring *ring = adev->rings[i];
4328
4329 if (!ring || !ring->sched.thread)
4330 continue;
4331
4332 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
4333 amdgpu_fence_driver_force_completion(ring);
4334 }
4335
4336 if(job)
4337 drm_sched_increase_karma(&job->base);
4338
4339 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
4340 if (!amdgpu_sriov_vf(adev)) {
4341
4342 if (!need_full_reset)
4343 need_full_reset = amdgpu_device_ip_need_full_reset(adev);
4344
4345 if (!need_full_reset) {
4346 amdgpu_device_ip_pre_soft_reset(adev);
4347 r = amdgpu_device_ip_soft_reset(adev);
4348 amdgpu_device_ip_post_soft_reset(adev);
4349 if (r || amdgpu_device_ip_check_soft_reset(adev)) {
4350 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
4351 need_full_reset = true;
4352 }
4353 }
4354
4355 if (need_full_reset)
4356 r = amdgpu_device_ip_suspend(adev);
4357
4358 *need_full_reset_arg = need_full_reset;
4359 }
4360
4361 return r;
4362}
4363
4364int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive,
4365 struct list_head *device_list_handle,
4366 bool *need_full_reset_arg,
4367 bool skip_hw_reset)
4368{
4369 struct amdgpu_device *tmp_adev = NULL;
4370 bool need_full_reset = *need_full_reset_arg, vram_lost = false;
4371 int r = 0;
4372
4373 /*
4374 * ASIC reset has to be done on all XGMI hive nodes ASAP
4375 * to allow proper links negotiation in FW (within 1 sec)
4376 */
4377 if (!skip_hw_reset && need_full_reset) {
4378 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4379 /* For XGMI run all resets in parallel to speed up the process */
4380 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4381 tmp_adev->gmc.xgmi.pending_reset = false;
4382 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
4383 r = -EALREADY;
4384 } else
4385 r = amdgpu_asic_reset(tmp_adev);
4386
4387 if (r) {
4388 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
4389 r, adev_to_drm(tmp_adev)->unique);
4390 break;
4391 }
4392 }
4393
4394 /* For XGMI wait for all resets to complete before proceed */
4395 if (!r) {
4396 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4397 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4398 flush_work(&tmp_adev->xgmi_reset_work);
4399 r = tmp_adev->asic_reset_res;
4400 if (r)
4401 break;
4402 }
4403 }
4404 }
4405 }
4406
4407 if (!r && amdgpu_ras_intr_triggered()) {
4408 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4409 if (tmp_adev->mmhub.funcs &&
4410 tmp_adev->mmhub.funcs->reset_ras_error_count)
4411 tmp_adev->mmhub.funcs->reset_ras_error_count(tmp_adev);
4412 }
4413
4414 amdgpu_ras_intr_cleared();
4415 }
4416
4417 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4418 if (need_full_reset) {
4419 /* post card */
4420 r = amdgpu_device_asic_init(tmp_adev);
4421 if (r) {
4422 dev_warn(tmp_adev->dev, "asic atom init failed!");
4423 } else {
4424 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
4425 r = amdgpu_device_ip_resume_phase1(tmp_adev);
4426 if (r)
4427 goto out;
4428
4429 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
4430 if (vram_lost) {
4431 DRM_INFO("VRAM is lost due to GPU reset!\n");
4432 amdgpu_inc_vram_lost(tmp_adev);
4433 }
4434
4435 r = amdgpu_gtt_mgr_recover(ttm_manager_type(&tmp_adev->mman.bdev, TTM_PL_TT));
4436 if (r)
4437 goto out;
4438
4439 r = amdgpu_device_fw_loading(tmp_adev);
4440 if (r)
4441 return r;
4442
4443 r = amdgpu_device_ip_resume_phase2(tmp_adev);
4444 if (r)
4445 goto out;
4446
4447 if (vram_lost)
4448 amdgpu_device_fill_reset_magic(tmp_adev);
4449
4450 /*
4451 * Add this ASIC as tracked as reset was already
4452 * complete successfully.
4453 */
4454 amdgpu_register_gpu_instance(tmp_adev);
4455
4456 if (!hive && tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4457 amdgpu_xgmi_add_device(tmp_adev);
4458
4459 r = amdgpu_device_ip_late_init(tmp_adev);
4460 if (r)
4461 goto out;
4462
4463 amdgpu_fbdev_set_suspend(tmp_adev, 0);
4464
4465 /*
4466 * The GPU enters bad state once faulty pages
4467 * by ECC has reached the threshold, and ras
4468 * recovery is scheduled next. So add one check
4469 * here to break recovery if it indeed exceeds
4470 * bad page threshold, and remind user to
4471 * retire this GPU or setting one bigger
4472 * bad_page_threshold value to fix this once
4473 * probing driver again.
4474 */
4475 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
4476 /* must succeed. */
4477 amdgpu_ras_resume(tmp_adev);
4478 } else {
4479 r = -EINVAL;
4480 goto out;
4481 }
4482
4483 /* Update PSP FW topology after reset */
4484 if (hive && tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4485 r = amdgpu_xgmi_update_topology(hive, tmp_adev);
4486 }
4487 }
4488
4489out:
4490 if (!r) {
4491 amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
4492 r = amdgpu_ib_ring_tests(tmp_adev);
4493 if (r) {
4494 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
4495 r = amdgpu_device_ip_suspend(tmp_adev);
4496 need_full_reset = true;
4497 r = -EAGAIN;
4498 goto end;
4499 }
4500 }
4501
4502 if (!r)
4503 r = amdgpu_device_recover_vram(tmp_adev);
4504 else
4505 tmp_adev->asic_reset_res = r;
4506 }
4507
4508end:
4509 *need_full_reset_arg = need_full_reset;
4510 return r;
4511}
4512
4513static bool amdgpu_device_lock_adev(struct amdgpu_device *adev,
4514 struct amdgpu_hive_info *hive)
4515{
4516 if (atomic_cmpxchg(&adev->in_gpu_reset, 0, 1) != 0)
4517 return false;
4518
4519 if (hive) {
4520 down_write_nest_lock(&adev->reset_sem, &hive->hive_lock);
4521 } else {
4522 down_write(&adev->reset_sem);
4523 }
4524
4525 switch (amdgpu_asic_reset_method(adev)) {
4526 case AMD_RESET_METHOD_MODE1:
4527 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
4528 break;
4529 case AMD_RESET_METHOD_MODE2:
4530 adev->mp1_state = PP_MP1_STATE_RESET;
4531 break;
4532 default:
4533 adev->mp1_state = PP_MP1_STATE_NONE;
4534 break;
4535 }
4536
4537 return true;
4538}
4539
4540static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
4541{
4542 amdgpu_vf_error_trans_all(adev);
4543 adev->mp1_state = PP_MP1_STATE_NONE;
4544 atomic_set(&adev->in_gpu_reset, 0);
4545 up_write(&adev->reset_sem);
4546}
4547
4548/*
4549 * to lockup a list of amdgpu devices in a hive safely, if not a hive
4550 * with multiple nodes, it will be similar as amdgpu_device_lock_adev.
4551 *
4552 * unlock won't require roll back.
4553 */
4554static int amdgpu_device_lock_hive_adev(struct amdgpu_device *adev, struct amdgpu_hive_info *hive)
4555{
4556 struct amdgpu_device *tmp_adev = NULL;
4557
4558 if (adev->gmc.xgmi.num_physical_nodes > 1) {
4559 if (!hive) {
4560 dev_err(adev->dev, "Hive is NULL while device has multiple xgmi nodes");
4561 return -ENODEV;
4562 }
4563 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
4564 if (!amdgpu_device_lock_adev(tmp_adev, hive))
4565 goto roll_back;
4566 }
4567 } else if (!amdgpu_device_lock_adev(adev, hive))
4568 return -EAGAIN;
4569
4570 return 0;
4571roll_back:
4572 if (!list_is_first(&tmp_adev->gmc.xgmi.head, &hive->device_list)) {
4573 /*
4574 * if the lockup iteration break in the middle of a hive,
4575 * it may means there may has a race issue,
4576 * or a hive device locked up independently.
4577 * we may be in trouble and may not, so will try to roll back
4578 * the lock and give out a warnning.
4579 */
4580 dev_warn(tmp_adev->dev, "Hive lock iteration broke in the middle. Rolling back to unlock");
4581 list_for_each_entry_continue_reverse(tmp_adev, &hive->device_list, gmc.xgmi.head) {
4582 amdgpu_device_unlock_adev(tmp_adev);
4583 }
4584 }
4585 return -EAGAIN;
4586}
4587
4588static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
4589{
4590 struct pci_dev *p = NULL;
4591
4592 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4593 adev->pdev->bus->number, 1);
4594 if (p) {
4595 pm_runtime_enable(&(p->dev));
4596 pm_runtime_resume(&(p->dev));
4597 }
4598}
4599
4600static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
4601{
4602 enum amd_reset_method reset_method;
4603 struct pci_dev *p = NULL;
4604 u64 expires;
4605
4606 /*
4607 * For now, only BACO and mode1 reset are confirmed
4608 * to suffer the audio issue without proper suspended.
4609 */
4610 reset_method = amdgpu_asic_reset_method(adev);
4611 if ((reset_method != AMD_RESET_METHOD_BACO) &&
4612 (reset_method != AMD_RESET_METHOD_MODE1))
4613 return -EINVAL;
4614
4615 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4616 adev->pdev->bus->number, 1);
4617 if (!p)
4618 return -ENODEV;
4619
4620 expires = pm_runtime_autosuspend_expiration(&(p->dev));
4621 if (!expires)
4622 /*
4623 * If we cannot get the audio device autosuspend delay,
4624 * a fixed 4S interval will be used. Considering 3S is
4625 * the audio controller default autosuspend delay setting.
4626 * 4S used here is guaranteed to cover that.
4627 */
4628 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
4629
4630 while (!pm_runtime_status_suspended(&(p->dev))) {
4631 if (!pm_runtime_suspend(&(p->dev)))
4632 break;
4633
4634 if (expires < ktime_get_mono_fast_ns()) {
4635 dev_warn(adev->dev, "failed to suspend display audio\n");
4636 /* TODO: abort the succeeding gpu reset? */
4637 return -ETIMEDOUT;
4638 }
4639 }
4640
4641 pm_runtime_disable(&(p->dev));
4642
4643 return 0;
4644}
4645
4646/**
4647 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
4648 *
4649 * @adev: amdgpu_device pointer
4650 * @job: which job trigger hang
4651 *
4652 * Attempt to reset the GPU if it has hung (all asics).
4653 * Attempt to do soft-reset or full-reset and reinitialize Asic
4654 * Returns 0 for success or an error on failure.
4655 */
4656
4657int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
4658 struct amdgpu_job *job)
4659{
4660 struct list_head device_list, *device_list_handle = NULL;
4661 bool need_full_reset = false;
4662 bool job_signaled = false;
4663 struct amdgpu_hive_info *hive = NULL;
4664 struct amdgpu_device *tmp_adev = NULL;
4665 int i, r = 0;
4666 bool need_emergency_restart = false;
4667 bool audio_suspended = false;
4668
4669 /*
4670 * Special case: RAS triggered and full reset isn't supported
4671 */
4672 need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
4673
4674 /*
4675 * Flush RAM to disk so that after reboot
4676 * the user can read log and see why the system rebooted.
4677 */
4678 if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {
4679 DRM_WARN("Emergency reboot.");
4680
4681 ksys_sync_helper();
4682 emergency_restart();
4683 }
4684
4685 dev_info(adev->dev, "GPU %s begin!\n",
4686 need_emergency_restart ? "jobs stop":"reset");
4687
4688 /*
4689 * Here we trylock to avoid chain of resets executing from
4690 * either trigger by jobs on different adevs in XGMI hive or jobs on
4691 * different schedulers for same device while this TO handler is running.
4692 * We always reset all schedulers for device and all devices for XGMI
4693 * hive so that should take care of them too.
4694 */
4695 hive = amdgpu_get_xgmi_hive(adev);
4696 if (hive) {
4697 if (atomic_cmpxchg(&hive->in_reset, 0, 1) != 0) {
4698 DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress",
4699 job ? job->base.id : -1, hive->hive_id);
4700 amdgpu_put_xgmi_hive(hive);
4701 if (job)
4702 drm_sched_increase_karma(&job->base);
4703 return 0;
4704 }
4705 mutex_lock(&hive->hive_lock);
4706 }
4707
4708 /*
4709 * lock the device before we try to operate the linked list
4710 * if didn't get the device lock, don't touch the linked list since
4711 * others may iterating it.
4712 */
4713 r = amdgpu_device_lock_hive_adev(adev, hive);
4714 if (r) {
4715 dev_info(adev->dev, "Bailing on TDR for s_job:%llx, as another already in progress",
4716 job ? job->base.id : -1);
4717
4718 /* even we skipped this reset, still need to set the job to guilty */
4719 if (job)
4720 drm_sched_increase_karma(&job->base);
4721 goto skip_recovery;
4722 }
4723
4724 /*
4725 * Build list of devices to reset.
4726 * In case we are in XGMI hive mode, resort the device list
4727 * to put adev in the 1st position.
4728 */
4729 INIT_LIST_HEAD(&device_list);
4730 if (adev->gmc.xgmi.num_physical_nodes > 1) {
4731 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
4732 list_add_tail(&tmp_adev->reset_list, &device_list);
4733 if (!list_is_first(&adev->reset_list, &device_list))
4734 list_rotate_to_front(&adev->reset_list, &device_list);
4735 device_list_handle = &device_list;
4736 } else {
4737 list_add_tail(&adev->reset_list, &device_list);
4738 device_list_handle = &device_list;
4739 }
4740
4741 /* block all schedulers and reset given job's ring */
4742 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4743 /*
4744 * Try to put the audio codec into suspend state
4745 * before gpu reset started.
4746 *
4747 * Due to the power domain of the graphics device
4748 * is shared with AZ power domain. Without this,
4749 * we may change the audio hardware from behind
4750 * the audio driver's back. That will trigger
4751 * some audio codec errors.
4752 */
4753 if (!amdgpu_device_suspend_display_audio(tmp_adev))
4754 audio_suspended = true;
4755
4756 amdgpu_ras_set_error_query_ready(tmp_adev, false);
4757
4758 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
4759
4760 if (!amdgpu_sriov_vf(tmp_adev))
4761 amdgpu_amdkfd_pre_reset(tmp_adev);
4762
4763 /*
4764 * Mark these ASICs to be reseted as untracked first
4765 * And add them back after reset completed
4766 */
4767 amdgpu_unregister_gpu_instance(tmp_adev);
4768
4769 amdgpu_fbdev_set_suspend(tmp_adev, 1);
4770
4771 /* disable ras on ALL IPs */
4772 if (!need_emergency_restart &&
4773 amdgpu_device_ip_need_full_reset(tmp_adev))
4774 amdgpu_ras_suspend(tmp_adev);
4775
4776 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4777 struct amdgpu_ring *ring = tmp_adev->rings[i];
4778
4779 if (!ring || !ring->sched.thread)
4780 continue;
4781
4782 drm_sched_stop(&ring->sched, job ? &job->base : NULL);
4783
4784 if (need_emergency_restart)
4785 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
4786 }
4787 atomic_inc(&tmp_adev->gpu_reset_counter);
4788 }
4789
4790 if (need_emergency_restart)
4791 goto skip_sched_resume;
4792
4793 /*
4794 * Must check guilty signal here since after this point all old
4795 * HW fences are force signaled.
4796 *
4797 * job->base holds a reference to parent fence
4798 */
4799 if (job && job->base.s_fence->parent &&
4800 dma_fence_is_signaled(job->base.s_fence->parent)) {
4801 job_signaled = true;
4802 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
4803 goto skip_hw_reset;
4804 }
4805
4806retry: /* Rest of adevs pre asic reset from XGMI hive. */
4807 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4808 r = amdgpu_device_pre_asic_reset(tmp_adev,
4809 (tmp_adev == adev) ? job : NULL,
4810 &need_full_reset);
4811 /*TODO Should we stop ?*/
4812 if (r) {
4813 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
4814 r, adev_to_drm(tmp_adev)->unique);
4815 tmp_adev->asic_reset_res = r;
4816 }
4817 }
4818
4819 /* Actual ASIC resets if needed.*/
4820 /* TODO Implement XGMI hive reset logic for SRIOV */
4821 if (amdgpu_sriov_vf(adev)) {
4822 r = amdgpu_device_reset_sriov(adev, job ? false : true);
4823 if (r)
4824 adev->asic_reset_res = r;
4825 } else {
4826 r = amdgpu_do_asic_reset(hive, device_list_handle, &need_full_reset, false);
4827 if (r && r == -EAGAIN)
4828 goto retry;
4829 }
4830
4831skip_hw_reset:
4832
4833 /* Post ASIC reset for all devs .*/
4834 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4835
4836 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4837 struct amdgpu_ring *ring = tmp_adev->rings[i];
4838
4839 if (!ring || !ring->sched.thread)
4840 continue;
4841
4842 /* No point to resubmit jobs if we didn't HW reset*/
4843 if (!tmp_adev->asic_reset_res && !job_signaled)
4844 drm_sched_resubmit_jobs(&ring->sched);
4845
4846 drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);
4847 }
4848
4849 if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) {
4850 drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
4851 }
4852
4853 tmp_adev->asic_reset_res = 0;
4854
4855 if (r) {
4856 /* bad news, how to tell it to userspace ? */
4857 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
4858 amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
4859 } else {
4860 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
4861 }
4862 }
4863
4864skip_sched_resume:
4865 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4866 /* unlock kfd: SRIOV would do it separately */
4867 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
4868 amdgpu_amdkfd_post_reset(tmp_adev);
4869
4870 /* kfd_post_reset will do nothing if kfd device is not initialized,
4871 * need to bring up kfd here if it's not be initialized before
4872 */
4873 if (!adev->kfd.init_complete)
4874 amdgpu_amdkfd_device_init(adev);
4875
4876 if (audio_suspended)
4877 amdgpu_device_resume_display_audio(tmp_adev);
4878 amdgpu_device_unlock_adev(tmp_adev);
4879 }
4880
4881skip_recovery:
4882 if (hive) {
4883 atomic_set(&hive->in_reset, 0);
4884 mutex_unlock(&hive->hive_lock);
4885 amdgpu_put_xgmi_hive(hive);
4886 }
4887
4888 if (r && r != -EAGAIN)
4889 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
4890 return r;
4891}
4892
4893/**
4894 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
4895 *
4896 * @adev: amdgpu_device pointer
4897 *
4898 * Fetchs and stores in the driver the PCIE capabilities (gen speed
4899 * and lanes) of the slot the device is in. Handles APUs and
4900 * virtualized environments where PCIE config space may not be available.
4901 */
4902static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
4903{
4904 struct pci_dev *pdev;
4905 enum pci_bus_speed speed_cap, platform_speed_cap;
4906 enum pcie_link_width platform_link_width;
4907
4908 if (amdgpu_pcie_gen_cap)
4909 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
4910
4911 if (amdgpu_pcie_lane_cap)
4912 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
4913
4914 /* covers APUs as well */
4915 if (pci_is_root_bus(adev->pdev->bus)) {
4916 if (adev->pm.pcie_gen_mask == 0)
4917 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
4918 if (adev->pm.pcie_mlw_mask == 0)
4919 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
4920 return;
4921 }
4922
4923 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
4924 return;
4925
4926 pcie_bandwidth_available(adev->pdev, NULL,
4927 &platform_speed_cap, &platform_link_width);
4928
4929 if (adev->pm.pcie_gen_mask == 0) {
4930 /* asic caps */
4931 pdev = adev->pdev;
4932 speed_cap = pcie_get_speed_cap(pdev);
4933 if (speed_cap == PCI_SPEED_UNKNOWN) {
4934 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4935 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4936 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
4937 } else {
4938 if (speed_cap == PCIE_SPEED_32_0GT)
4939 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4940 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4941 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
4942 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
4943 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
4944 else if (speed_cap == PCIE_SPEED_16_0GT)
4945 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4946 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4947 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
4948 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
4949 else if (speed_cap == PCIE_SPEED_8_0GT)
4950 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4951 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4952 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
4953 else if (speed_cap == PCIE_SPEED_5_0GT)
4954 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4955 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
4956 else
4957 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
4958 }
4959 /* platform caps */
4960 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
4961 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4962 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
4963 } else {
4964 if (platform_speed_cap == PCIE_SPEED_32_0GT)
4965 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4966 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4967 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
4968 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
4969 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
4970 else if (platform_speed_cap == PCIE_SPEED_16_0GT)
4971 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4972 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4973 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
4974 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
4975 else if (platform_speed_cap == PCIE_SPEED_8_0GT)
4976 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4977 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
4978 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
4979 else if (platform_speed_cap == PCIE_SPEED_5_0GT)
4980 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
4981 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
4982 else
4983 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
4984
4985 }
4986 }
4987 if (adev->pm.pcie_mlw_mask == 0) {
4988 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
4989 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
4990 } else {
4991 switch (platform_link_width) {
4992 case PCIE_LNK_X32:
4993 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
4994 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
4995 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
4996 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
4997 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
4998 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
4999 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5000 break;
5001 case PCIE_LNK_X16:
5002 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5003 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5004 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5005 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5006 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5007 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5008 break;
5009 case PCIE_LNK_X12:
5010 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5011 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5012 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5013 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5014 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5015 break;
5016 case PCIE_LNK_X8:
5017 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5018 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5019 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5020 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5021 break;
5022 case PCIE_LNK_X4:
5023 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5024 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5025 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5026 break;
5027 case PCIE_LNK_X2:
5028 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5029 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5030 break;
5031 case PCIE_LNK_X1:
5032 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5033 break;
5034 default:
5035 break;
5036 }
5037 }
5038 }
5039}
5040
5041int amdgpu_device_baco_enter(struct drm_device *dev)
5042{
5043 struct amdgpu_device *adev = drm_to_adev(dev);
5044 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5045
5046 if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5047 return -ENOTSUPP;
5048
5049 if (ras && ras->supported && adev->nbio.funcs->enable_doorbell_interrupt)
5050 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5051
5052 return amdgpu_dpm_baco_enter(adev);
5053}
5054
5055int amdgpu_device_baco_exit(struct drm_device *dev)
5056{
5057 struct amdgpu_device *adev = drm_to_adev(dev);
5058 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5059 int ret = 0;
5060
5061 if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5062 return -ENOTSUPP;
5063
5064 ret = amdgpu_dpm_baco_exit(adev);
5065 if (ret)
5066 return ret;
5067
5068 if (ras && ras->supported && adev->nbio.funcs->enable_doorbell_interrupt)
5069 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5070
5071 return 0;
5072}
5073
5074static void amdgpu_cancel_all_tdr(struct amdgpu_device *adev)
5075{
5076 int i;
5077
5078 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5079 struct amdgpu_ring *ring = adev->rings[i];
5080
5081 if (!ring || !ring->sched.thread)
5082 continue;
5083
5084 cancel_delayed_work_sync(&ring->sched.work_tdr);
5085 }
5086}
5087
5088/**
5089 * amdgpu_pci_error_detected - Called when a PCI error is detected.
5090 * @pdev: PCI device struct
5091 * @state: PCI channel state
5092 *
5093 * Description: Called when a PCI error is detected.
5094 *
5095 * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5096 */
5097pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5098{
5099 struct drm_device *dev = pci_get_drvdata(pdev);
5100 struct amdgpu_device *adev = drm_to_adev(dev);
5101 int i;
5102
5103 DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5104
5105 if (adev->gmc.xgmi.num_physical_nodes > 1) {
5106 DRM_WARN("No support for XGMI hive yet...");
5107 return PCI_ERS_RESULT_DISCONNECT;
5108 }
5109
5110 switch (state) {
5111 case pci_channel_io_normal:
5112 return PCI_ERS_RESULT_CAN_RECOVER;
5113 /* Fatal error, prepare for slot reset */
5114 case pci_channel_io_frozen:
5115 /*
5116 * Cancel and wait for all TDRs in progress if failing to
5117 * set adev->in_gpu_reset in amdgpu_device_lock_adev
5118 *
5119 * Locking adev->reset_sem will prevent any external access
5120 * to GPU during PCI error recovery
5121 */
5122 while (!amdgpu_device_lock_adev(adev, NULL))
5123 amdgpu_cancel_all_tdr(adev);
5124
5125 /*
5126 * Block any work scheduling as we do for regular GPU reset
5127 * for the duration of the recovery
5128 */
5129 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5130 struct amdgpu_ring *ring = adev->rings[i];
5131
5132 if (!ring || !ring->sched.thread)
5133 continue;
5134
5135 drm_sched_stop(&ring->sched, NULL);
5136 }
5137 atomic_inc(&adev->gpu_reset_counter);
5138 return PCI_ERS_RESULT_NEED_RESET;
5139 case pci_channel_io_perm_failure:
5140 /* Permanent error, prepare for device removal */
5141 return PCI_ERS_RESULT_DISCONNECT;
5142 }
5143
5144 return PCI_ERS_RESULT_NEED_RESET;
5145}
5146
5147/**
5148 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5149 * @pdev: pointer to PCI device
5150 */
5151pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5152{
5153
5154 DRM_INFO("PCI error: mmio enabled callback!!\n");
5155
5156 /* TODO - dump whatever for debugging purposes */
5157
5158 /* This called only if amdgpu_pci_error_detected returns
5159 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5160 * works, no need to reset slot.
5161 */
5162
5163 return PCI_ERS_RESULT_RECOVERED;
5164}
5165
5166/**
5167 * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5168 * @pdev: PCI device struct
5169 *
5170 * Description: This routine is called by the pci error recovery
5171 * code after the PCI slot has been reset, just before we
5172 * should resume normal operations.
5173 */
5174pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5175{
5176 struct drm_device *dev = pci_get_drvdata(pdev);
5177 struct amdgpu_device *adev = drm_to_adev(dev);
5178 int r, i;
5179 bool need_full_reset = true;
5180 u32 memsize;
5181 struct list_head device_list;
5182
5183 DRM_INFO("PCI error: slot reset callback!!\n");
5184
5185 INIT_LIST_HEAD(&device_list);
5186 list_add_tail(&adev->reset_list, &device_list);
5187
5188 /* wait for asic to come out of reset */
5189 msleep(500);
5190
5191 /* Restore PCI confspace */
5192 amdgpu_device_load_pci_state(pdev);
5193
5194 /* confirm ASIC came out of reset */
5195 for (i = 0; i < adev->usec_timeout; i++) {
5196 memsize = amdgpu_asic_get_config_memsize(adev);
5197
5198 if (memsize != 0xffffffff)
5199 break;
5200 udelay(1);
5201 }
5202 if (memsize == 0xffffffff) {
5203 r = -ETIME;
5204 goto out;
5205 }
5206
5207 adev->in_pci_err_recovery = true;
5208 r = amdgpu_device_pre_asic_reset(adev, NULL, &need_full_reset);
5209 adev->in_pci_err_recovery = false;
5210 if (r)
5211 goto out;
5212
5213 r = amdgpu_do_asic_reset(NULL, &device_list, &need_full_reset, true);
5214
5215out:
5216 if (!r) {
5217 if (amdgpu_device_cache_pci_state(adev->pdev))
5218 pci_restore_state(adev->pdev);
5219
5220 DRM_INFO("PCIe error recovery succeeded\n");
5221 } else {
5222 DRM_ERROR("PCIe error recovery failed, err:%d", r);
5223 amdgpu_device_unlock_adev(adev);
5224 }
5225
5226 return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
5227}
5228
5229/**
5230 * amdgpu_pci_resume() - resume normal ops after PCI reset
5231 * @pdev: pointer to PCI device
5232 *
5233 * Called when the error recovery driver tells us that its
5234 * OK to resume normal operation.
5235 */
5236void amdgpu_pci_resume(struct pci_dev *pdev)
5237{
5238 struct drm_device *dev = pci_get_drvdata(pdev);
5239 struct amdgpu_device *adev = drm_to_adev(dev);
5240 int i;
5241
5242
5243 DRM_INFO("PCI error: resume callback!!\n");
5244
5245 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5246 struct amdgpu_ring *ring = adev->rings[i];
5247
5248 if (!ring || !ring->sched.thread)
5249 continue;
5250
5251
5252 drm_sched_resubmit_jobs(&ring->sched);
5253 drm_sched_start(&ring->sched, true);
5254 }
5255
5256 amdgpu_device_unlock_adev(adev);
5257}
5258
5259bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
5260{
5261 struct drm_device *dev = pci_get_drvdata(pdev);
5262 struct amdgpu_device *adev = drm_to_adev(dev);
5263 int r;
5264
5265 r = pci_save_state(pdev);
5266 if (!r) {
5267 kfree(adev->pci_state);
5268
5269 adev->pci_state = pci_store_saved_state(pdev);
5270
5271 if (!adev->pci_state) {
5272 DRM_ERROR("Failed to store PCI saved state");
5273 return false;
5274 }
5275 } else {
5276 DRM_WARN("Failed to save PCI state, err:%d\n", r);
5277 return false;
5278 }
5279
5280 return true;
5281}
5282
5283bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
5284{
5285 struct drm_device *dev = pci_get_drvdata(pdev);
5286 struct amdgpu_device *adev = drm_to_adev(dev);
5287 int r;
5288
5289 if (!adev->pci_state)
5290 return false;
5291
5292 r = pci_load_saved_state(pdev, adev->pci_state);
5293
5294 if (!r) {
5295 pci_restore_state(pdev);
5296 } else {
5297 DRM_WARN("Failed to load PCI state, err:%d\n", r);
5298 return false;
5299 }
5300
5301 return true;
5302}
5303
5304
Linux 6.x block layer and io_uring tree(s)