drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/power_supply.h>
  29 #include <linux/kthread.h>
  30 #include <linux/module.h>
  31 #include <linux/console.h>
  32 #include <linux/slab.h>
  33 #include <linux/iommu.h>
  34 #include <linux/pci.h>
  35 #include <linux/devcoredump.h>
  36 #include <generated/utsrelease.h>
  37 #include <linux/pci-p2pdma.h>
  38
  39 #include <drm/drm_aperture.h>
  40 #include <drm/drm_atomic_helper.h>
  41 #include <drm/drm_crtc_helper.h>
  42 #include <drm/drm_fb_helper.h>
  43 #include <drm/drm_probe_helper.h>
  44 #include <drm/amdgpu_drm.h>
  45 #include <linux/vgaarb.h>
  46 #include <linux/vga_switcheroo.h>
  47 #include <linux/efi.h>
  48 #include "amdgpu.h"
  49 #include "amdgpu_trace.h"
  50 #include "amdgpu_i2c.h"
  51 #include "atom.h"
  52 #include "amdgpu_atombios.h"
  53 #include "amdgpu_atomfirmware.h"
  54 #include "amd_pcie.h"
  55 #ifdef CONFIG_DRM_AMDGPU_SI
  56 #include "si.h"
  57 #endif
  58 #ifdef CONFIG_DRM_AMDGPU_CIK
  59 #include "cik.h"
  60 #endif
  61 #include "vi.h"
  62 #include "soc15.h"
  63 #include "nv.h"
  64 #include "bif/bif_4_1_d.h"
  65 #include <linux/firmware.h>
  66 #include "amdgpu_vf_error.h"
  67
  68 #include "amdgpu_amdkfd.h"
  69 #include "amdgpu_pm.h"
  70
  71 #include "amdgpu_xgmi.h"
  72 #include "amdgpu_ras.h"
  73 #include "amdgpu_pmu.h"
  74 #include "amdgpu_fru_eeprom.h"
  75 #include "amdgpu_reset.h"
  76
  77 #include <linux/suspend.h>
  78 #include <drm/task_barrier.h>
  79 #include <linux/pm_runtime.h>
  80
  81 #include <drm/drm_drv.h>
  82
  83 #if IS_ENABLED(CONFIG_X86)
  84 #include <asm/intel-family.h>
  85 #endif
  86
  87 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  88 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
  89 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  90 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
  91 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
  92 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
  93 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
  94
  95 #define AMDGPU_RESUME_MS                2000
  96 #define AMDGPU_MAX_RETRY_LIMIT          2
  97 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
  98
  99 static const struct drm_driver amdgpu_kms_driver;
 100
 101 const char *amdgpu_asic_name[] = {
 102         "TAHITI",
 103         "PITCAIRN",
 104         "VERDE",
 105         "OLAND",
 106         "HAINAN",
 107         "BONAIRE",
 108         "KAVERI",
 109         "KABINI",
 110         "HAWAII",
 111         "MULLINS",
 112         "TOPAZ",
 113         "TONGA",
 114         "FIJI",
 115         "CARRIZO",
 116         "STONEY",
 117         "POLARIS10",
 118         "POLARIS11",
 119         "POLARIS12",
 120         "VEGAM",
 121         "VEGA10",
 122         "VEGA12",
 123         "VEGA20",
 124         "RAVEN",
 125         "ARCTURUS",
 126         "RENOIR",
 127         "ALDEBARAN",
 128         "NAVI10",
 129         "CYAN_SKILLFISH",
 130         "NAVI14",
 131         "NAVI12",
 132         "SIENNA_CICHLID",
 133         "NAVY_FLOUNDER",
 134         "VANGOGH",
 135         "DIMGREY_CAVEFISH",
 136         "BEIGE_GOBY",
 137         "YELLOW_CARP",
 138         "IP DISCOVERY",
 139         "LAST",
 140 };
 141
 142 /**
 143  * DOC: pcie_replay_count
 144  *
 145  * The amdgpu driver provides a sysfs API for reporting the total number
 146  * of PCIe replays (NAKs)
 147  * The file pcie_replay_count is used for this and returns the total
 148  * number of replays as a sum of the NAKs generated and NAKs received
 149  */
 150
 151 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
 152                 struct device_attribute *attr, char *buf)
 153 {
 154         struct drm_device *ddev = dev_get_drvdata(dev);
 155         struct amdgpu_device *adev = drm_to_adev(ddev);
 156         uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
 157
 158         return sysfs_emit(buf, "%llu\n", cnt);
 159 }
 160
 161 static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
 162                 amdgpu_device_get_pcie_replay_count, NULL);
 163
 164 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 165
 166 /**
 167  * DOC: product_name
 168  *
 169  * The amdgpu driver provides a sysfs API for reporting the product name
 170  * for the device
 171  * The file product_name is used for this and returns the product name
 172  * as returned from the FRU.
 173  * NOTE: This is only available for certain server cards
 174  */
 175
 176 static ssize_t amdgpu_device_get_product_name(struct device *dev,
 177                 struct device_attribute *attr, char *buf)
 178 {
 179         struct drm_device *ddev = dev_get_drvdata(dev);
 180         struct amdgpu_device *adev = drm_to_adev(ddev);
 181
 182         return sysfs_emit(buf, "%s\n", adev->product_name);
 183 }
 184
 185 static DEVICE_ATTR(product_name, S_IRUGO,
 186                 amdgpu_device_get_product_name, NULL);
 187
 188 /**
 189  * DOC: product_number
 190  *
 191  * The amdgpu driver provides a sysfs API for reporting the part number
 192  * for the device
 193  * The file product_number is used for this and returns the part number
 194  * as returned from the FRU.
 195  * NOTE: This is only available for certain server cards
 196  */
 197
 198 static ssize_t amdgpu_device_get_product_number(struct device *dev,
 199                 struct device_attribute *attr, char *buf)
 200 {
 201         struct drm_device *ddev = dev_get_drvdata(dev);
 202         struct amdgpu_device *adev = drm_to_adev(ddev);
 203
 204         return sysfs_emit(buf, "%s\n", adev->product_number);
 205 }
 206
 207 static DEVICE_ATTR(product_number, S_IRUGO,
 208                 amdgpu_device_get_product_number, NULL);
 209
 210 /**
 211  * DOC: serial_number
 212  *
 213  * The amdgpu driver provides a sysfs API for reporting the serial number
 214  * for the device
 215  * The file serial_number is used for this and returns the serial number
 216  * as returned from the FRU.
 217  * NOTE: This is only available for certain server cards
 218  */
 219
 220 static ssize_t amdgpu_device_get_serial_number(struct device *dev,
 221                 struct device_attribute *attr, char *buf)
 222 {
 223         struct drm_device *ddev = dev_get_drvdata(dev);
 224         struct amdgpu_device *adev = drm_to_adev(ddev);
 225
 226         return sysfs_emit(buf, "%s\n", adev->serial);
 227 }
 228
 229 static DEVICE_ATTR(serial_number, S_IRUGO,
 230                 amdgpu_device_get_serial_number, NULL);
 231
 232 /**
 233  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
 234  *
 235  * @dev: drm_device pointer
 236  *
 237  * Returns true if the device is a dGPU with ATPX power control,
 238  * otherwise return false.
 239  */
 240 bool amdgpu_device_supports_px(struct drm_device *dev)
 241 {
 242         struct amdgpu_device *adev = drm_to_adev(dev);
 243
 244         if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
 245                 return true;
 246         return false;
 247 }
 248
 249 /**
 250  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
 251  *
 252  * @dev: drm_device pointer
 253  *
 254  * Returns true if the device is a dGPU with ACPI power control,
 255  * otherwise return false.
 256  */
 257 bool amdgpu_device_supports_boco(struct drm_device *dev)
 258 {
 259         struct amdgpu_device *adev = drm_to_adev(dev);
 260
 261         if (adev->has_pr3 ||
 262             ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
 263                 return true;
 264         return false;
 265 }
 266
 267 /**
 268  * amdgpu_device_supports_baco - Does the device support BACO
 269  *
 270  * @dev: drm_device pointer
 271  *
 272  * Returns true if the device supporte BACO,
 273  * otherwise return false.
 274  */
 275 bool amdgpu_device_supports_baco(struct drm_device *dev)
 276 {
 277         struct amdgpu_device *adev = drm_to_adev(dev);
 278
 279         return amdgpu_asic_supports_baco(adev);
 280 }
 281
 282 /**
 283  * amdgpu_device_supports_smart_shift - Is the device dGPU with
 284  * smart shift support
 285  *
 286  * @dev: drm_device pointer
 287  *
 288  * Returns true if the device is a dGPU with Smart Shift support,
 289  * otherwise returns false.
 290  */
 291 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
 292 {
 293         return (amdgpu_device_supports_boco(dev) &&
 294                 amdgpu_acpi_is_power_shift_control_supported());
 295 }
 296
 297 /*
 298  * VRAM access helper functions
 299  */
 300
 301 /**
 302  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
 303  *
 304  * @adev: amdgpu_device pointer
 305  * @pos: offset of the buffer in vram
 306  * @buf: virtual address of the buffer in system memory
 307  * @size: read/write size, sizeof(@buf) must > @size
 308  * @write: true - write to vram, otherwise - read from vram
 309  */
 310 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
 311                              void *buf, size_t size, bool write)
 312 {
 313         unsigned long flags;
 314         uint32_t hi = ~0, tmp = 0;
 315         uint32_t *data = buf;
 316         uint64_t last;
 317         int idx;
 318
 319         if (!drm_dev_enter(adev_to_drm(adev), &idx))
 320                 return;
 321
 322         BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
 323
 324         spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 325         for (last = pos + size; pos < last; pos += 4) {
 326                 tmp = pos >> 31;
 327
 328                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
 329                 if (tmp != hi) {
 330                         WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
 331                         hi = tmp;
 332                 }
 333                 if (write)
 334                         WREG32_NO_KIQ(mmMM_DATA, *data++);
 335                 else
 336                         *data++ = RREG32_NO_KIQ(mmMM_DATA);
 337         }
 338
 339         spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 340         drm_dev_exit(idx);
 341 }
 342
 343 /**
 344  * amdgpu_device_aper_access - access vram by vram aperature
 345  *
 346  * @adev: amdgpu_device pointer
 347  * @pos: offset of the buffer in vram
 348  * @buf: virtual address of the buffer in system memory
 349  * @size: read/write size, sizeof(@buf) must > @size
 350  * @write: true - write to vram, otherwise - read from vram
 351  *
 352  * The return value means how many bytes have been transferred.
 353  */
 354 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
 355                                  void *buf, size_t size, bool write)
 356 {
 357 #ifdef CONFIG_64BIT
 358         void __iomem *addr;
 359         size_t count = 0;
 360         uint64_t last;
 361
 362         if (!adev->mman.aper_base_kaddr)
 363                 return 0;
 364
 365         last = min(pos + size, adev->gmc.visible_vram_size);
 366         if (last > pos) {
 367                 addr = adev->mman.aper_base_kaddr + pos;
 368                 count = last - pos;
 369
 370                 if (write) {
 371                         memcpy_toio(addr, buf, count);
 372                         mb();
 373                         amdgpu_device_flush_hdp(adev, NULL);
 374                 } else {
 375                         amdgpu_device_invalidate_hdp(adev, NULL);
 376                         mb();
 377                         memcpy_fromio(buf, addr, count);
 378                 }
 379
 380         }
 381
 382         return count;
 383 #else
 384         return 0;
 385 #endif
 386 }
 387
 388 /**
 389  * amdgpu_device_vram_access - read/write a buffer in vram
 390  *
 391  * @adev: amdgpu_device pointer
 392  * @pos: offset of the buffer in vram
 393  * @buf: virtual address of the buffer in system memory
 394  * @size: read/write size, sizeof(@buf) must > @size
 395  * @write: true - write to vram, otherwise - read from vram
 396  */
 397 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
 398                                void *buf, size_t size, bool write)
 399 {
 400         size_t count;
 401
 402         /* try to using vram apreature to access vram first */
 403         count = amdgpu_device_aper_access(adev, pos, buf, size, write);
 404         size -= count;
 405         if (size) {
 406                 /* using MM to access rest vram */
 407                 pos += count;
 408                 buf += count;
 409                 amdgpu_device_mm_access(adev, pos, buf, size, write);
 410         }
 411 }
 412
 413 /*
 414  * register access helper functions.
 415  */
 416
 417 /* Check if hw access should be skipped because of hotplug or device error */
 418 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
 419 {
 420         if (adev->no_hw_access)
 421                 return true;
 422
 423 #ifdef CONFIG_LOCKDEP
 424         /*
 425          * This is a bit complicated to understand, so worth a comment. What we assert
 426          * here is that the GPU reset is not running on another thread in parallel.
 427          *
 428          * For this we trylock the read side of the reset semaphore, if that succeeds
 429          * we know that the reset is not running in paralell.
 430          *
 431          * If the trylock fails we assert that we are either already holding the read
 432          * side of the lock or are the reset thread itself and hold the write side of
 433          * the lock.
 434          */
 435         if (in_task()) {
 436                 if (down_read_trylock(&adev->reset_domain->sem))
 437                         up_read(&adev->reset_domain->sem);
 438                 else
 439                         lockdep_assert_held(&adev->reset_domain->sem);
 440         }
 441 #endif
 442         return false;
 443 }
 444
 445 /**
 446  * amdgpu_device_rreg - read a memory mapped IO or indirect register
 447  *
 448  * @adev: amdgpu_device pointer
 449  * @reg: dword aligned register offset
 450  * @acc_flags: access flags which require special behavior
 451  *
 452  * Returns the 32 bit value from the offset specified.
 453  */
 454 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 455                             uint32_t reg, uint32_t acc_flags)
 456 {
 457         uint32_t ret;
 458
 459         if (amdgpu_device_skip_hw_access(adev))
 460                 return 0;
 461
 462         if ((reg * 4) < adev->rmmio_size) {
 463                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 464                     amdgpu_sriov_runtime(adev) &&
 465                     down_read_trylock(&adev->reset_domain->sem)) {
 466                         ret = amdgpu_kiq_rreg(adev, reg);
 467                         up_read(&adev->reset_domain->sem);
 468                 } else {
 469                         ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 470                 }
 471         } else {
 472                 ret = adev->pcie_rreg(adev, reg * 4);
 473         }
 474
 475         trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
 476
 477         return ret;
 478 }
 479
 480 /*
 481  * MMIO register read with bytes helper functions
 482  * @offset:bytes offset from MMIO start
 483  *
 484 */
 485
 486 /**
 487  * amdgpu_mm_rreg8 - read a memory mapped IO register
 488  *
 489  * @adev: amdgpu_device pointer
 490  * @offset: byte aligned register offset
 491  *
 492  * Returns the 8 bit value from the offset specified.
 493  */
 494 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
 495 {
 496         if (amdgpu_device_skip_hw_access(adev))
 497                 return 0;
 498
 499         if (offset < adev->rmmio_size)
 500                 return (readb(adev->rmmio + offset));
 501         BUG();
 502 }
 503
 504 /*
 505  * MMIO register write with bytes helper functions
 506  * @offset:bytes offset from MMIO start
 507  * @value: the value want to be written to the register
 508  *
 509 */
 510 /**
 511  * amdgpu_mm_wreg8 - read a memory mapped IO register
 512  *
 513  * @adev: amdgpu_device pointer
 514  * @offset: byte aligned register offset
 515  * @value: 8 bit value to write
 516  *
 517  * Writes the value specified to the offset specified.
 518  */
 519 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
 520 {
 521         if (amdgpu_device_skip_hw_access(adev))
 522                 return;
 523
 524         if (offset < adev->rmmio_size)
 525                 writeb(value, adev->rmmio + offset);
 526         else
 527                 BUG();
 528 }
 529
 530 /**
 531  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
 532  *
 533  * @adev: amdgpu_device pointer
 534  * @reg: dword aligned register offset
 535  * @v: 32 bit value to write to the register
 536  * @acc_flags: access flags which require special behavior
 537  *
 538  * Writes the value specified to the offset specified.
 539  */
 540 void amdgpu_device_wreg(struct amdgpu_device *adev,
 541                         uint32_t reg, uint32_t v,
 542                         uint32_t acc_flags)
 543 {
 544         if (amdgpu_device_skip_hw_access(adev))
 545                 return;
 546
 547         if ((reg * 4) < adev->rmmio_size) {
 548                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 549                     amdgpu_sriov_runtime(adev) &&
 550                     down_read_trylock(&adev->reset_domain->sem)) {
 551                         amdgpu_kiq_wreg(adev, reg, v);
 552                         up_read(&adev->reset_domain->sem);
 553                 } else {
 554                         writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 555                 }
 556         } else {
 557                 adev->pcie_wreg(adev, reg * 4, v);
 558         }
 559
 560         trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
 561 }
 562
 563 /**
 564  * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
 565  *
 566  * @adev: amdgpu_device pointer
 567  * @reg: mmio/rlc register
 568  * @v: value to write
 569  *
 570  * this function is invoked only for the debugfs register access
 571  */
 572 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
 573                              uint32_t reg, uint32_t v)
 574 {
 575         if (amdgpu_device_skip_hw_access(adev))
 576                 return;
 577
 578         if (amdgpu_sriov_fullaccess(adev) &&
 579             adev->gfx.rlc.funcs &&
 580             adev->gfx.rlc.funcs->is_rlcg_access_range) {
 581                 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
 582                         return amdgpu_sriov_wreg(adev, reg, v, 0, 0);
 583         } else if ((reg * 4) >= adev->rmmio_size) {
 584                 adev->pcie_wreg(adev, reg * 4, v);
 585         } else {
 586                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 587         }
 588 }
 589
 590 /**
 591  * amdgpu_mm_rdoorbell - read a doorbell dword
 592  *
 593  * @adev: amdgpu_device pointer
 594  * @index: doorbell index
 595  *
 596  * Returns the value in the doorbell aperture at the
 597  * requested doorbell index (CIK).
 598  */
 599 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 600 {
 601         if (amdgpu_device_skip_hw_access(adev))
 602                 return 0;
 603
 604         if (index < adev->doorbell.num_doorbells) {
 605                 return readl(adev->doorbell.ptr + index);
 606         } else {
 607                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 608                 return 0;
 609         }
 610 }
 611
 612 /**
 613  * amdgpu_mm_wdoorbell - write a doorbell dword
 614  *
 615  * @adev: amdgpu_device pointer
 616  * @index: doorbell index
 617  * @v: value to write
 618  *
 619  * Writes @v to the doorbell aperture at the
 620  * requested doorbell index (CIK).
 621  */
 622 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 623 {
 624         if (amdgpu_device_skip_hw_access(adev))
 625                 return;
 626
 627         if (index < adev->doorbell.num_doorbells) {
 628                 writel(v, adev->doorbell.ptr + index);
 629         } else {
 630                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 631         }
 632 }
 633
 634 /**
 635  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 636  *
 637  * @adev: amdgpu_device pointer
 638  * @index: doorbell index
 639  *
 640  * Returns the value in the doorbell aperture at the
 641  * requested doorbell index (VEGA10+).
 642  */
 643 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 644 {
 645         if (amdgpu_device_skip_hw_access(adev))
 646                 return 0;
 647
 648         if (index < adev->doorbell.num_doorbells) {
 649                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 650         } else {
 651                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 652                 return 0;
 653         }
 654 }
 655
 656 /**
 657  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 658  *
 659  * @adev: amdgpu_device pointer
 660  * @index: doorbell index
 661  * @v: value to write
 662  *
 663  * Writes @v to the doorbell aperture at the
 664  * requested doorbell index (VEGA10+).
 665  */
 666 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 667 {
 668         if (amdgpu_device_skip_hw_access(adev))
 669                 return;
 670
 671         if (index < adev->doorbell.num_doorbells) {
 672                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 673         } else {
 674                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 675         }
 676 }
 677
 678 /**
 679  * amdgpu_device_indirect_rreg - read an indirect register
 680  *
 681  * @adev: amdgpu_device pointer
 682  * @pcie_index: mmio register offset
 683  * @pcie_data: mmio register offset
 684  * @reg_addr: indirect register address to read from
 685  *
 686  * Returns the value of indirect register @reg_addr
 687  */
 688 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
 689                                 u32 pcie_index, u32 pcie_data,
 690                                 u32 reg_addr)
 691 {
 692         unsigned long flags;
 693         u32 r;
 694         void __iomem *pcie_index_offset;
 695         void __iomem *pcie_data_offset;
 696
 697         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 698         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 699         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 700
 701         writel(reg_addr, pcie_index_offset);
 702         readl(pcie_index_offset);
 703         r = readl(pcie_data_offset);
 704         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 705
 706         return r;
 707 }
 708
 709 /**
 710  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
 711  *
 712  * @adev: amdgpu_device pointer
 713  * @pcie_index: mmio register offset
 714  * @pcie_data: mmio register offset
 715  * @reg_addr: indirect register address to read from
 716  *
 717  * Returns the value of indirect register @reg_addr
 718  */
 719 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
 720                                   u32 pcie_index, u32 pcie_data,
 721                                   u32 reg_addr)
 722 {
 723         unsigned long flags;
 724         u64 r;
 725         void __iomem *pcie_index_offset;
 726         void __iomem *pcie_data_offset;
 727
 728         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 729         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 730         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 731
 732         /* read low 32 bits */
 733         writel(reg_addr, pcie_index_offset);
 734         readl(pcie_index_offset);
 735         r = readl(pcie_data_offset);
 736         /* read high 32 bits */
 737         writel(reg_addr + 4, pcie_index_offset);
 738         readl(pcie_index_offset);
 739         r |= ((u64)readl(pcie_data_offset) << 32);
 740         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 741
 742         return r;
 743 }
 744
 745 /**
 746  * amdgpu_device_indirect_wreg - write an indirect register address
 747  *
 748  * @adev: amdgpu_device pointer
 749  * @pcie_index: mmio register offset
 750  * @pcie_data: mmio register offset
 751  * @reg_addr: indirect register offset
 752  * @reg_data: indirect register data
 753  *
 754  */
 755 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
 756                                  u32 pcie_index, u32 pcie_data,
 757                                  u32 reg_addr, u32 reg_data)
 758 {
 759         unsigned long flags;
 760         void __iomem *pcie_index_offset;
 761         void __iomem *pcie_data_offset;
 762
 763         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 764         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 765         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 766
 767         writel(reg_addr, pcie_index_offset);
 768         readl(pcie_index_offset);
 769         writel(reg_data, pcie_data_offset);
 770         readl(pcie_data_offset);
 771         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 772 }
 773
 774 /**
 775  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
 776  *
 777  * @adev: amdgpu_device pointer
 778  * @pcie_index: mmio register offset
 779  * @pcie_data: mmio register offset
 780  * @reg_addr: indirect register offset
 781  * @reg_data: indirect register data
 782  *
 783  */
 784 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
 785                                    u32 pcie_index, u32 pcie_data,
 786                                    u32 reg_addr, u64 reg_data)
 787 {
 788         unsigned long flags;
 789         void __iomem *pcie_index_offset;
 790         void __iomem *pcie_data_offset;
 791
 792         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 793         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 794         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 795
 796         /* write low 32 bits */
 797         writel(reg_addr, pcie_index_offset);
 798         readl(pcie_index_offset);
 799         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
 800         readl(pcie_data_offset);
 801         /* write high 32 bits */
 802         writel(reg_addr + 4, pcie_index_offset);
 803         readl(pcie_index_offset);
 804         writel((u32)(reg_data >> 32), pcie_data_offset);
 805         readl(pcie_data_offset);
 806         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 807 }
 808
 809 /**
 810  * amdgpu_invalid_rreg - dummy reg read function
 811  *
 812  * @adev: amdgpu_device pointer
 813  * @reg: offset of register
 814  *
 815  * Dummy register read function.  Used for register blocks
 816  * that certain asics don't have (all asics).
 817  * Returns the value in the register.
 818  */
 819 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 820 {
 821         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 822         BUG();
 823         return 0;
 824 }
 825
 826 /**
 827  * amdgpu_invalid_wreg - dummy reg write function
 828  *
 829  * @adev: amdgpu_device pointer
 830  * @reg: offset of register
 831  * @v: value to write to the register
 832  *
 833  * Dummy register read function.  Used for register blocks
 834  * that certain asics don't have (all asics).
 835  */
 836 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 837 {
 838         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 839                   reg, v);
 840         BUG();
 841 }
 842
 843 /**
 844  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
 845  *
 846  * @adev: amdgpu_device pointer
 847  * @reg: offset of register
 848  *
 849  * Dummy register read function.  Used for register blocks
 850  * that certain asics don't have (all asics).
 851  * Returns the value in the register.
 852  */
 853 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
 854 {
 855         DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
 856         BUG();
 857         return 0;
 858 }
 859
 860 /**
 861  * amdgpu_invalid_wreg64 - dummy reg write function
 862  *
 863  * @adev: amdgpu_device pointer
 864  * @reg: offset of register
 865  * @v: value to write to the register
 866  *
 867  * Dummy register read function.  Used for register blocks
 868  * that certain asics don't have (all asics).
 869  */
 870 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
 871 {
 872         DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
 873                   reg, v);
 874         BUG();
 875 }
 876
 877 /**
 878  * amdgpu_block_invalid_rreg - dummy reg read function
 879  *
 880  * @adev: amdgpu_device pointer
 881  * @block: offset of instance
 882  * @reg: offset of register
 883  *
 884  * Dummy register read function.  Used for register blocks
 885  * that certain asics don't have (all asics).
 886  * Returns the value in the register.
 887  */
 888 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 889                                           uint32_t block, uint32_t reg)
 890 {
 891         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 892                   reg, block);
 893         BUG();
 894         return 0;
 895 }
 896
 897 /**
 898  * amdgpu_block_invalid_wreg - dummy reg write function
 899  *
 900  * @adev: amdgpu_device pointer
 901  * @block: offset of instance
 902  * @reg: offset of register
 903  * @v: value to write to the register
 904  *
 905  * Dummy register read function.  Used for register blocks
 906  * that certain asics don't have (all asics).
 907  */
 908 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 909                                       uint32_t block,
 910                                       uint32_t reg, uint32_t v)
 911 {
 912         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 913                   reg, block, v);
 914         BUG();
 915 }
 916
 917 /**
 918  * amdgpu_device_asic_init - Wrapper for atom asic_init
 919  *
 920  * @adev: amdgpu_device pointer
 921  *
 922  * Does any asic specific work and then calls atom asic init.
 923  */
 924 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
 925 {
 926         amdgpu_asic_pre_asic_init(adev);
 927
 928         if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0))
 929                 return amdgpu_atomfirmware_asic_init(adev, true);
 930         else
 931                 return amdgpu_atom_asic_init(adev->mode_info.atom_context);
 932 }
 933
 934 /**
 935  * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
 936  *
 937  * @adev: amdgpu_device pointer
 938  *
 939  * Allocates a scratch page of VRAM for use by various things in the
 940  * driver.
 941  */
 942 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
 943 {
 944         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
 945                                        AMDGPU_GEM_DOMAIN_VRAM |
 946                                        AMDGPU_GEM_DOMAIN_GTT,
 947                                        &adev->mem_scratch.robj,
 948                                        &adev->mem_scratch.gpu_addr,
 949                                        (void **)&adev->mem_scratch.ptr);
 950 }
 951
 952 /**
 953  * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
 954  *
 955  * @adev: amdgpu_device pointer
 956  *
 957  * Frees the VRAM scratch page.
 958  */
 959 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
 960 {
 961         amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
 962 }
 963
 964 /**
 965  * amdgpu_device_program_register_sequence - program an array of registers.
 966  *
 967  * @adev: amdgpu_device pointer
 968  * @registers: pointer to the register array
 969  * @array_size: size of the register array
 970  *
 971  * Programs an array or registers with and and or masks.
 972  * This is a helper for setting golden registers.
 973  */
 974 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
 975                                              const u32 *registers,
 976                                              const u32 array_size)
 977 {
 978         u32 tmp, reg, and_mask, or_mask;
 979         int i;
 980
 981         if (array_size % 3)
 982                 return;
 983
 984         for (i = 0; i < array_size; i +=3) {
 985                 reg = registers[i + 0];
 986                 and_mask = registers[i + 1];
 987                 or_mask = registers[i + 2];
 988
 989                 if (and_mask == 0xffffffff) {
 990                         tmp = or_mask;
 991                 } else {
 992                         tmp = RREG32(reg);
 993                         tmp &= ~and_mask;
 994                         if (adev->family >= AMDGPU_FAMILY_AI)
 995                                 tmp |= (or_mask & and_mask);
 996                         else
 997                                 tmp |= or_mask;
 998                 }
 999                 WREG32(reg, tmp);
1000         }
1001 }
1002
1003 /**
1004  * amdgpu_device_pci_config_reset - reset the GPU
1005  *
1006  * @adev: amdgpu_device pointer
1007  *
1008  * Resets the GPU using the pci config reset sequence.
1009  * Only applicable to asics prior to vega10.
1010  */
1011 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1012 {
1013         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1014 }
1015
1016 /**
1017  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1018  *
1019  * @adev: amdgpu_device pointer
1020  *
1021  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1022  */
1023 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1024 {
1025         return pci_reset_function(adev->pdev);
1026 }
1027
1028 /*
1029  * GPU doorbell aperture helpers function.
1030  */
1031 /**
1032  * amdgpu_device_doorbell_init - Init doorbell driver information.
1033  *
1034  * @adev: amdgpu_device pointer
1035  *
1036  * Init doorbell driver information (CIK)
1037  * Returns 0 on success, error on failure.
1038  */
1039 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
1040 {
1041
1042         /* No doorbell on SI hardware generation */
1043         if (adev->asic_type < CHIP_BONAIRE) {
1044                 adev->doorbell.base = 0;
1045                 adev->doorbell.size = 0;
1046                 adev->doorbell.num_doorbells = 0;
1047                 adev->doorbell.ptr = NULL;
1048                 return 0;
1049         }
1050
1051         if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
1052                 return -EINVAL;
1053
1054         amdgpu_asic_init_doorbell_index(adev);
1055
1056         /* doorbell bar mapping */
1057         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
1058         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
1059
1060         if (adev->enable_mes) {
1061                 adev->doorbell.num_doorbells =
1062                         adev->doorbell.size / sizeof(u32);
1063         } else {
1064                 adev->doorbell.num_doorbells =
1065                         min_t(u32, adev->doorbell.size / sizeof(u32),
1066                               adev->doorbell_index.max_assignment+1);
1067                 if (adev->doorbell.num_doorbells == 0)
1068                         return -EINVAL;
1069
1070                 /* For Vega, reserve and map two pages on doorbell BAR since SDMA
1071                  * paging queue doorbell use the second page. The
1072                  * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
1073                  * doorbells are in the first page. So with paging queue enabled,
1074                  * the max num_doorbells should + 1 page (0x400 in dword)
1075                  */
1076                 if (adev->asic_type >= CHIP_VEGA10)
1077                         adev->doorbell.num_doorbells += 0x400;
1078         }
1079
1080         adev->doorbell.ptr = ioremap(adev->doorbell.base,
1081                                      adev->doorbell.num_doorbells *
1082                                      sizeof(u32));
1083         if (adev->doorbell.ptr == NULL)
1084                 return -ENOMEM;
1085
1086         return 0;
1087 }
1088
1089 /**
1090  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
1091  *
1092  * @adev: amdgpu_device pointer
1093  *
1094  * Tear down doorbell driver information (CIK)
1095  */
1096 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
1097 {
1098         iounmap(adev->doorbell.ptr);
1099         adev->doorbell.ptr = NULL;
1100 }
1101
1102
1103
1104 /*
1105  * amdgpu_device_wb_*()
1106  * Writeback is the method by which the GPU updates special pages in memory
1107  * with the status of certain GPU events (fences, ring pointers,etc.).
1108  */
1109
1110 /**
1111  * amdgpu_device_wb_fini - Disable Writeback and free memory
1112  *
1113  * @adev: amdgpu_device pointer
1114  *
1115  * Disables Writeback and frees the Writeback memory (all asics).
1116  * Used at driver shutdown.
1117  */
1118 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1119 {
1120         if (adev->wb.wb_obj) {
1121                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1122                                       &adev->wb.gpu_addr,
1123                                       (void **)&adev->wb.wb);
1124                 adev->wb.wb_obj = NULL;
1125         }
1126 }
1127
1128 /**
1129  * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1130  *
1131  * @adev: amdgpu_device pointer
1132  *
1133  * Initializes writeback and allocates writeback memory (all asics).
1134  * Used at driver startup.
1135  * Returns 0 on success or an -error on failure.
1136  */
1137 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1138 {
1139         int r;
1140
1141         if (adev->wb.wb_obj == NULL) {
1142                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1143                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1144                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1145                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
1146                                             (void **)&adev->wb.wb);
1147                 if (r) {
1148                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1149                         return r;
1150                 }
1151
1152                 adev->wb.num_wb = AMDGPU_MAX_WB;
1153                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1154
1155                 /* clear wb memory */
1156                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1157         }
1158
1159         return 0;
1160 }
1161
1162 /**
1163  * amdgpu_device_wb_get - Allocate a wb entry
1164  *
1165  * @adev: amdgpu_device pointer
1166  * @wb: wb index
1167  *
1168  * Allocate a wb slot for use by the driver (all asics).
1169  * Returns 0 on success or -EINVAL on failure.
1170  */
1171 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1172 {
1173         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1174
1175         if (offset < adev->wb.num_wb) {
1176                 __set_bit(offset, adev->wb.used);
1177                 *wb = offset << 3; /* convert to dw offset */
1178                 return 0;
1179         } else {
1180                 return -EINVAL;
1181         }
1182 }
1183
1184 /**
1185  * amdgpu_device_wb_free - Free a wb entry
1186  *
1187  * @adev: amdgpu_device pointer
1188  * @wb: wb index
1189  *
1190  * Free a wb slot allocated for use by the driver (all asics)
1191  */
1192 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1193 {
1194         wb >>= 3;
1195         if (wb < adev->wb.num_wb)
1196                 __clear_bit(wb, adev->wb.used);
1197 }
1198
1199 /**
1200  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1201  *
1202  * @adev: amdgpu_device pointer
1203  *
1204  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1205  * to fail, but if any of the BARs is not accessible after the size we abort
1206  * driver loading by returning -ENODEV.
1207  */
1208 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1209 {
1210         int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1211         struct pci_bus *root;
1212         struct resource *res;
1213         unsigned i;
1214         u16 cmd;
1215         int r;
1216
1217         /* Bypass for VF */
1218         if (amdgpu_sriov_vf(adev))
1219                 return 0;
1220
1221         /* skip if the bios has already enabled large BAR */
1222         if (adev->gmc.real_vram_size &&
1223             (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1224                 return 0;
1225
1226         /* Check if the root BUS has 64bit memory resources */
1227         root = adev->pdev->bus;
1228         while (root->parent)
1229                 root = root->parent;
1230
1231         pci_bus_for_each_resource(root, res, i) {
1232                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1233                     res->start > 0x100000000ull)
1234                         break;
1235         }
1236
1237         /* Trying to resize is pointless without a root hub window above 4GB */
1238         if (!res)
1239                 return 0;
1240
1241         /* Limit the BAR size to what is available */
1242         rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1243                         rbar_size);
1244
1245         /* Disable memory decoding while we change the BAR addresses and size */
1246         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1247         pci_write_config_word(adev->pdev, PCI_COMMAND,
1248                               cmd & ~PCI_COMMAND_MEMORY);
1249
1250         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1251         amdgpu_device_doorbell_fini(adev);
1252         if (adev->asic_type >= CHIP_BONAIRE)
1253                 pci_release_resource(adev->pdev, 2);
1254
1255         pci_release_resource(adev->pdev, 0);
1256
1257         r = pci_resize_resource(adev->pdev, 0, rbar_size);
1258         if (r == -ENOSPC)
1259                 DRM_INFO("Not enough PCI address space for a large BAR.");
1260         else if (r && r != -ENOTSUPP)
1261                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1262
1263         pci_assign_unassigned_bus_resources(adev->pdev->bus);
1264
1265         /* When the doorbell or fb BAR isn't available we have no chance of
1266          * using the device.
1267          */
1268         r = amdgpu_device_doorbell_init(adev);
1269         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1270                 return -ENODEV;
1271
1272         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1273
1274         return 0;
1275 }
1276
1277 /*
1278  * GPU helpers function.
1279  */
1280 /**
1281  * amdgpu_device_need_post - check if the hw need post or not
1282  *
1283  * @adev: amdgpu_device pointer
1284  *
1285  * Check if the asic has been initialized (all asics) at driver startup
1286  * or post is needed if  hw reset is performed.
1287  * Returns true if need or false if not.
1288  */
1289 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1290 {
1291         uint32_t reg;
1292
1293         if (amdgpu_sriov_vf(adev))
1294                 return false;
1295
1296         if (amdgpu_passthrough(adev)) {
1297                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1298                  * some old smc fw still need driver do vPost otherwise gpu hang, while
1299                  * those smc fw version above 22.15 doesn't have this flaw, so we force
1300                  * vpost executed for smc version below 22.15
1301                  */
1302                 if (adev->asic_type == CHIP_FIJI) {
1303                         int err;
1304                         uint32_t fw_ver;
1305                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1306                         /* force vPost if error occured */
1307                         if (err)
1308                                 return true;
1309
1310                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1311                         if (fw_ver < 0x00160e00)
1312                                 return true;
1313                 }
1314         }
1315
1316         /* Don't post if we need to reset whole hive on init */
1317         if (adev->gmc.xgmi.pending_reset)
1318                 return false;
1319
1320         if (adev->has_hw_reset) {
1321                 adev->has_hw_reset = false;
1322                 return true;
1323         }
1324
1325         /* bios scratch used on CIK+ */
1326         if (adev->asic_type >= CHIP_BONAIRE)
1327                 return amdgpu_atombios_scratch_need_asic_init(adev);
1328
1329         /* check MEM_SIZE for older asics */
1330         reg = amdgpu_asic_get_config_memsize(adev);
1331
1332         if ((reg != 0) && (reg != 0xffffffff))
1333                 return false;
1334
1335         return true;
1336 }
1337
1338 /**
1339  * amdgpu_device_should_use_aspm - check if the device should program ASPM
1340  *
1341  * @adev: amdgpu_device pointer
1342  *
1343  * Confirm whether the module parameter and pcie bridge agree that ASPM should
1344  * be set for this device.
1345  *
1346  * Returns true if it should be used or false if not.
1347  */
1348 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1349 {
1350         switch (amdgpu_aspm) {
1351         case -1:
1352                 break;
1353         case 0:
1354                 return false;
1355         case 1:
1356                 return true;
1357         default:
1358                 return false;
1359         }
1360         return pcie_aspm_enabled(adev->pdev);
1361 }
1362
1363 bool amdgpu_device_aspm_support_quirk(void)
1364 {
1365 #if IS_ENABLED(CONFIG_X86)
1366         struct cpuinfo_x86 *c = &cpu_data(0);
1367
1368         return !(c->x86 == 6 && c->x86_model == INTEL_FAM6_ALDERLAKE);
1369 #else
1370         return true;
1371 #endif
1372 }
1373
1374 /* if we get transitioned to only one device, take VGA back */
1375 /**
1376  * amdgpu_device_vga_set_decode - enable/disable vga decode
1377  *
1378  * @pdev: PCI device pointer
1379  * @state: enable/disable vga decode
1380  *
1381  * Enable/disable vga decode (all asics).
1382  * Returns VGA resource flags.
1383  */
1384 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1385                 bool state)
1386 {
1387         struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1388         amdgpu_asic_set_vga_state(adev, state);
1389         if (state)
1390                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1391                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1392         else
1393                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1394 }
1395
1396 /**
1397  * amdgpu_device_check_block_size - validate the vm block size
1398  *
1399  * @adev: amdgpu_device pointer
1400  *
1401  * Validates the vm block size specified via module parameter.
1402  * The vm block size defines number of bits in page table versus page directory,
1403  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1404  * page table and the remaining bits are in the page directory.
1405  */
1406 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1407 {
1408         /* defines number of bits in page table versus page directory,
1409          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1410          * page table and the remaining bits are in the page directory */
1411         if (amdgpu_vm_block_size == -1)
1412                 return;
1413
1414         if (amdgpu_vm_block_size < 9) {
1415                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1416                          amdgpu_vm_block_size);
1417                 amdgpu_vm_block_size = -1;
1418         }
1419 }
1420
1421 /**
1422  * amdgpu_device_check_vm_size - validate the vm size
1423  *
1424  * @adev: amdgpu_device pointer
1425  *
1426  * Validates the vm size in GB specified via module parameter.
1427  * The VM size is the size of the GPU virtual memory space in GB.
1428  */
1429 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1430 {
1431         /* no need to check the default value */
1432         if (amdgpu_vm_size == -1)
1433                 return;
1434
1435         if (amdgpu_vm_size < 1) {
1436                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1437                          amdgpu_vm_size);
1438                 amdgpu_vm_size = -1;
1439         }
1440 }
1441
1442 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1443 {
1444         struct sysinfo si;
1445         bool is_os_64 = (sizeof(void *) == 8);
1446         uint64_t total_memory;
1447         uint64_t dram_size_seven_GB = 0x1B8000000;
1448         uint64_t dram_size_three_GB = 0xB8000000;
1449
1450         if (amdgpu_smu_memory_pool_size == 0)
1451                 return;
1452
1453         if (!is_os_64) {
1454                 DRM_WARN("Not 64-bit OS, feature not supported\n");
1455                 goto def_value;
1456         }
1457         si_meminfo(&si);
1458         total_memory = (uint64_t)si.totalram * si.mem_unit;
1459
1460         if ((amdgpu_smu_memory_pool_size == 1) ||
1461                 (amdgpu_smu_memory_pool_size == 2)) {
1462                 if (total_memory < dram_size_three_GB)
1463                         goto def_value1;
1464         } else if ((amdgpu_smu_memory_pool_size == 4) ||
1465                 (amdgpu_smu_memory_pool_size == 8)) {
1466                 if (total_memory < dram_size_seven_GB)
1467                         goto def_value1;
1468         } else {
1469                 DRM_WARN("Smu memory pool size not supported\n");
1470                 goto def_value;
1471         }
1472         adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1473
1474         return;
1475
1476 def_value1:
1477         DRM_WARN("No enough system memory\n");
1478 def_value:
1479         adev->pm.smu_prv_buffer_size = 0;
1480 }
1481
1482 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1483 {
1484         if (!(adev->flags & AMD_IS_APU) ||
1485             adev->asic_type < CHIP_RAVEN)
1486                 return 0;
1487
1488         switch (adev->asic_type) {
1489         case CHIP_RAVEN:
1490                 if (adev->pdev->device == 0x15dd)
1491                         adev->apu_flags |= AMD_APU_IS_RAVEN;
1492                 if (adev->pdev->device == 0x15d8)
1493                         adev->apu_flags |= AMD_APU_IS_PICASSO;
1494                 break;
1495         case CHIP_RENOIR:
1496                 if ((adev->pdev->device == 0x1636) ||
1497                     (adev->pdev->device == 0x164c))
1498                         adev->apu_flags |= AMD_APU_IS_RENOIR;
1499                 else
1500                         adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1501                 break;
1502         case CHIP_VANGOGH:
1503                 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1504                 break;
1505         case CHIP_YELLOW_CARP:
1506                 break;
1507         case CHIP_CYAN_SKILLFISH:
1508                 if ((adev->pdev->device == 0x13FE) ||
1509                     (adev->pdev->device == 0x143F))
1510                         adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1511                 break;
1512         default:
1513                 break;
1514         }
1515
1516         return 0;
1517 }
1518
1519 /**
1520  * amdgpu_device_check_arguments - validate module params
1521  *
1522  * @adev: amdgpu_device pointer
1523  *
1524  * Validates certain module parameters and updates
1525  * the associated values used by the driver (all asics).
1526  */
1527 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1528 {
1529         if (amdgpu_sched_jobs < 4) {
1530                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1531                          amdgpu_sched_jobs);
1532                 amdgpu_sched_jobs = 4;
1533         } else if (!is_power_of_2(amdgpu_sched_jobs)){
1534                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1535                          amdgpu_sched_jobs);
1536                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1537         }
1538
1539         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1540                 /* gart size must be greater or equal to 32M */
1541                 dev_warn(adev->dev, "gart size (%d) too small\n",
1542                          amdgpu_gart_size);
1543                 amdgpu_gart_size = -1;
1544         }
1545
1546         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1547                 /* gtt size must be greater or equal to 32M */
1548                 dev_warn(adev->dev, "gtt size (%d) too small\n",
1549                                  amdgpu_gtt_size);
1550                 amdgpu_gtt_size = -1;
1551         }
1552
1553         /* valid range is between 4 and 9 inclusive */
1554         if (amdgpu_vm_fragment_size != -1 &&
1555             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1556                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1557                 amdgpu_vm_fragment_size = -1;
1558         }
1559
1560         if (amdgpu_sched_hw_submission < 2) {
1561                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1562                          amdgpu_sched_hw_submission);
1563                 amdgpu_sched_hw_submission = 2;
1564         } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1565                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1566                          amdgpu_sched_hw_submission);
1567                 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1568         }
1569
1570         if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1571                 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1572                 amdgpu_reset_method = -1;
1573         }
1574
1575         amdgpu_device_check_smu_prv_buffer_size(adev);
1576
1577         amdgpu_device_check_vm_size(adev);
1578
1579         amdgpu_device_check_block_size(adev);
1580
1581         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1582
1583         return 0;
1584 }
1585
1586 /**
1587  * amdgpu_switcheroo_set_state - set switcheroo state
1588  *
1589  * @pdev: pci dev pointer
1590  * @state: vga_switcheroo state
1591  *
1592  * Callback for the switcheroo driver.  Suspends or resumes
1593  * the asics before or after it is powered up using ACPI methods.
1594  */
1595 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1596                                         enum vga_switcheroo_state state)
1597 {
1598         struct drm_device *dev = pci_get_drvdata(pdev);
1599         int r;
1600
1601         if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1602                 return;
1603
1604         if (state == VGA_SWITCHEROO_ON) {
1605                 pr_info("switched on\n");
1606                 /* don't suspend or resume card normally */
1607                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1608
1609                 pci_set_power_state(pdev, PCI_D0);
1610                 amdgpu_device_load_pci_state(pdev);
1611                 r = pci_enable_device(pdev);
1612                 if (r)
1613                         DRM_WARN("pci_enable_device failed (%d)\n", r);
1614                 amdgpu_device_resume(dev, true);
1615
1616                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1617         } else {
1618                 pr_info("switched off\n");
1619                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1620                 amdgpu_device_suspend(dev, true);
1621                 amdgpu_device_cache_pci_state(pdev);
1622                 /* Shut down the device */
1623                 pci_disable_device(pdev);
1624                 pci_set_power_state(pdev, PCI_D3cold);
1625                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1626         }
1627 }
1628
1629 /**
1630  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1631  *
1632  * @pdev: pci dev pointer
1633  *
1634  * Callback for the switcheroo driver.  Check of the switcheroo
1635  * state can be changed.
1636  * Returns true if the state can be changed, false if not.
1637  */
1638 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1639 {
1640         struct drm_device *dev = pci_get_drvdata(pdev);
1641
1642         /*
1643         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1644         * locking inversion with the driver load path. And the access here is
1645         * completely racy anyway. So don't bother with locking for now.
1646         */
1647         return atomic_read(&dev->open_count) == 0;
1648 }
1649
1650 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1651         .set_gpu_state = amdgpu_switcheroo_set_state,
1652         .reprobe = NULL,
1653         .can_switch = amdgpu_switcheroo_can_switch,
1654 };
1655
1656 /**
1657  * amdgpu_device_ip_set_clockgating_state - set the CG state
1658  *
1659  * @dev: amdgpu_device pointer
1660  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1661  * @state: clockgating state (gate or ungate)
1662  *
1663  * Sets the requested clockgating state for all instances of
1664  * the hardware IP specified.
1665  * Returns the error code from the last instance.
1666  */
1667 int amdgpu_device_ip_set_clockgating_state(void *dev,
1668                                            enum amd_ip_block_type block_type,
1669                                            enum amd_clockgating_state state)
1670 {
1671         struct amdgpu_device *adev = dev;
1672         int i, r = 0;
1673
1674         for (i = 0; i < adev->num_ip_blocks; i++) {
1675                 if (!adev->ip_blocks[i].status.valid)
1676                         continue;
1677                 if (adev->ip_blocks[i].version->type != block_type)
1678                         continue;
1679                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1680                         continue;
1681                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1682                         (void *)adev, state);
1683                 if (r)
1684                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1685                                   adev->ip_blocks[i].version->funcs->name, r);
1686         }
1687         return r;
1688 }
1689
1690 /**
1691  * amdgpu_device_ip_set_powergating_state - set the PG state
1692  *
1693  * @dev: amdgpu_device pointer
1694  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1695  * @state: powergating state (gate or ungate)
1696  *
1697  * Sets the requested powergating state for all instances of
1698  * the hardware IP specified.
1699  * Returns the error code from the last instance.
1700  */
1701 int amdgpu_device_ip_set_powergating_state(void *dev,
1702                                            enum amd_ip_block_type block_type,
1703                                            enum amd_powergating_state state)
1704 {
1705         struct amdgpu_device *adev = dev;
1706         int i, r = 0;
1707
1708         for (i = 0; i < adev->num_ip_blocks; i++) {
1709                 if (!adev->ip_blocks[i].status.valid)
1710                         continue;
1711                 if (adev->ip_blocks[i].version->type != block_type)
1712                         continue;
1713                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1714                         continue;
1715                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1716                         (void *)adev, state);
1717                 if (r)
1718                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1719                                   adev->ip_blocks[i].version->funcs->name, r);
1720         }
1721         return r;
1722 }
1723
1724 /**
1725  * amdgpu_device_ip_get_clockgating_state - get the CG state
1726  *
1727  * @adev: amdgpu_device pointer
1728  * @flags: clockgating feature flags
1729  *
1730  * Walks the list of IPs on the device and updates the clockgating
1731  * flags for each IP.
1732  * Updates @flags with the feature flags for each hardware IP where
1733  * clockgating is enabled.
1734  */
1735 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1736                                             u64 *flags)
1737 {
1738         int i;
1739
1740         for (i = 0; i < adev->num_ip_blocks; i++) {
1741                 if (!adev->ip_blocks[i].status.valid)
1742                         continue;
1743                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1744                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1745         }
1746 }
1747
1748 /**
1749  * amdgpu_device_ip_wait_for_idle - wait for idle
1750  *
1751  * @adev: amdgpu_device pointer
1752  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1753  *
1754  * Waits for the request hardware IP to be idle.
1755  * Returns 0 for success or a negative error code on failure.
1756  */
1757 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1758                                    enum amd_ip_block_type block_type)
1759 {
1760         int i, r;
1761
1762         for (i = 0; i < adev->num_ip_blocks; i++) {
1763                 if (!adev->ip_blocks[i].status.valid)
1764                         continue;
1765                 if (adev->ip_blocks[i].version->type == block_type) {
1766                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1767                         if (r)
1768                                 return r;
1769                         break;
1770                 }
1771         }
1772         return 0;
1773
1774 }
1775
1776 /**
1777  * amdgpu_device_ip_is_idle - is the hardware IP idle
1778  *
1779  * @adev: amdgpu_device pointer
1780  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1781  *
1782  * Check if the hardware IP is idle or not.
1783  * Returns true if it the IP is idle, false if not.
1784  */
1785 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1786                               enum amd_ip_block_type block_type)
1787 {
1788         int i;
1789
1790         for (i = 0; i < adev->num_ip_blocks; i++) {
1791                 if (!adev->ip_blocks[i].status.valid)
1792                         continue;
1793                 if (adev->ip_blocks[i].version->type == block_type)
1794                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1795         }
1796         return true;
1797
1798 }
1799
1800 /**
1801  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1802  *
1803  * @adev: amdgpu_device pointer
1804  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1805  *
1806  * Returns a pointer to the hardware IP block structure
1807  * if it exists for the asic, otherwise NULL.
1808  */
1809 struct amdgpu_ip_block *
1810 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1811                               enum amd_ip_block_type type)
1812 {
1813         int i;
1814
1815         for (i = 0; i < adev->num_ip_blocks; i++)
1816                 if (adev->ip_blocks[i].version->type == type)
1817                         return &adev->ip_blocks[i];
1818
1819         return NULL;
1820 }
1821
1822 /**
1823  * amdgpu_device_ip_block_version_cmp
1824  *
1825  * @adev: amdgpu_device pointer
1826  * @type: enum amd_ip_block_type
1827  * @major: major version
1828  * @minor: minor version
1829  *
1830  * return 0 if equal or greater
1831  * return 1 if smaller or the ip_block doesn't exist
1832  */
1833 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1834                                        enum amd_ip_block_type type,
1835                                        u32 major, u32 minor)
1836 {
1837         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1838
1839         if (ip_block && ((ip_block->version->major > major) ||
1840                         ((ip_block->version->major == major) &&
1841                         (ip_block->version->minor >= minor))))
1842                 return 0;
1843
1844         return 1;
1845 }
1846
1847 /**
1848  * amdgpu_device_ip_block_add
1849  *
1850  * @adev: amdgpu_device pointer
1851  * @ip_block_version: pointer to the IP to add
1852  *
1853  * Adds the IP block driver information to the collection of IPs
1854  * on the asic.
1855  */
1856 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1857                                const struct amdgpu_ip_block_version *ip_block_version)
1858 {
1859         if (!ip_block_version)
1860                 return -EINVAL;
1861
1862         switch (ip_block_version->type) {
1863         case AMD_IP_BLOCK_TYPE_VCN:
1864                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
1865                         return 0;
1866                 break;
1867         case AMD_IP_BLOCK_TYPE_JPEG:
1868                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
1869                         return 0;
1870                 break;
1871         default:
1872                 break;
1873         }
1874
1875         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1876                   ip_block_version->funcs->name);
1877
1878         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1879
1880         return 0;
1881 }
1882
1883 /**
1884  * amdgpu_device_enable_virtual_display - enable virtual display feature
1885  *
1886  * @adev: amdgpu_device pointer
1887  *
1888  * Enabled the virtual display feature if the user has enabled it via
1889  * the module parameter virtual_display.  This feature provides a virtual
1890  * display hardware on headless boards or in virtualized environments.
1891  * This function parses and validates the configuration string specified by
1892  * the user and configues the virtual display configuration (number of
1893  * virtual connectors, crtcs, etc.) specified.
1894  */
1895 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1896 {
1897         adev->enable_virtual_display = false;
1898
1899         if (amdgpu_virtual_display) {
1900                 const char *pci_address_name = pci_name(adev->pdev);
1901                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1902
1903                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1904                 pciaddstr_tmp = pciaddstr;
1905                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1906                         pciaddname = strsep(&pciaddname_tmp, ",");
1907                         if (!strcmp("all", pciaddname)
1908                             || !strcmp(pci_address_name, pciaddname)) {
1909                                 long num_crtc;
1910                                 int res = -1;
1911
1912                                 adev->enable_virtual_display = true;
1913
1914                                 if (pciaddname_tmp)
1915                                         res = kstrtol(pciaddname_tmp, 10,
1916                                                       &num_crtc);
1917
1918                                 if (!res) {
1919                                         if (num_crtc < 1)
1920                                                 num_crtc = 1;
1921                                         if (num_crtc > 6)
1922                                                 num_crtc = 6;
1923                                         adev->mode_info.num_crtc = num_crtc;
1924                                 } else {
1925                                         adev->mode_info.num_crtc = 1;
1926                                 }
1927                                 break;
1928                         }
1929                 }
1930
1931                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1932                          amdgpu_virtual_display, pci_address_name,
1933                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1934
1935                 kfree(pciaddstr);
1936         }
1937 }
1938
1939 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
1940 {
1941         if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
1942                 adev->mode_info.num_crtc = 1;
1943                 adev->enable_virtual_display = true;
1944                 DRM_INFO("virtual_display:%d, num_crtc:%d\n",
1945                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1946         }
1947 }
1948
1949 /**
1950  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1951  *
1952  * @adev: amdgpu_device pointer
1953  *
1954  * Parses the asic configuration parameters specified in the gpu info
1955  * firmware and makes them availale to the driver for use in configuring
1956  * the asic.
1957  * Returns 0 on success, -EINVAL on failure.
1958  */
1959 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1960 {
1961         const char *chip_name;
1962         char fw_name[40];
1963         int err;
1964         const struct gpu_info_firmware_header_v1_0 *hdr;
1965
1966         adev->firmware.gpu_info_fw = NULL;
1967
1968         if (adev->mman.discovery_bin) {
1969                 /*
1970                  * FIXME: The bounding box is still needed by Navi12, so
1971                  * temporarily read it from gpu_info firmware. Should be dropped
1972                  * when DAL no longer needs it.
1973                  */
1974                 if (adev->asic_type != CHIP_NAVI12)
1975                         return 0;
1976         }
1977
1978         switch (adev->asic_type) {
1979         default:
1980                 return 0;
1981         case CHIP_VEGA10:
1982                 chip_name = "vega10";
1983                 break;
1984         case CHIP_VEGA12:
1985                 chip_name = "vega12";
1986                 break;
1987         case CHIP_RAVEN:
1988                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1989                         chip_name = "raven2";
1990                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1991                         chip_name = "picasso";
1992                 else
1993                         chip_name = "raven";
1994                 break;
1995         case CHIP_ARCTURUS:
1996                 chip_name = "arcturus";
1997                 break;
1998         case CHIP_NAVI12:
1999                 chip_name = "navi12";
2000                 break;
2001         }
2002
2003         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
2004         err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, fw_name);
2005         if (err) {
2006                 dev_err(adev->dev,
2007                         "Failed to get gpu_info firmware \"%s\"\n",
2008                         fw_name);
2009                 goto out;
2010         }
2011
2012         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2013         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2014
2015         switch (hdr->version_major) {
2016         case 1:
2017         {
2018                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2019                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2020                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2021
2022                 /*
2023                  * Should be droped when DAL no longer needs it.
2024                  */
2025                 if (adev->asic_type == CHIP_NAVI12)
2026                         goto parse_soc_bounding_box;
2027
2028                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2029                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2030                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2031                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2032                 adev->gfx.config.max_texture_channel_caches =
2033                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
2034                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2035                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2036                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2037                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2038                 adev->gfx.config.double_offchip_lds_buf =
2039                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2040                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2041                 adev->gfx.cu_info.max_waves_per_simd =
2042                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2043                 adev->gfx.cu_info.max_scratch_slots_per_cu =
2044                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2045                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2046                 if (hdr->version_minor >= 1) {
2047                         const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2048                                 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2049                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2050                         adev->gfx.config.num_sc_per_sh =
2051                                 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2052                         adev->gfx.config.num_packer_per_sc =
2053                                 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2054                 }
2055
2056 parse_soc_bounding_box:
2057                 /*
2058                  * soc bounding box info is not integrated in disocovery table,
2059                  * we always need to parse it from gpu info firmware if needed.
2060                  */
2061                 if (hdr->version_minor == 2) {
2062                         const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2063                                 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2064                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2065                         adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2066                 }
2067                 break;
2068         }
2069         default:
2070                 dev_err(adev->dev,
2071                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2072                 err = -EINVAL;
2073                 goto out;
2074         }
2075 out:
2076         return err;
2077 }
2078
2079 /**
2080  * amdgpu_device_ip_early_init - run early init for hardware IPs
2081  *
2082  * @adev: amdgpu_device pointer
2083  *
2084  * Early initialization pass for hardware IPs.  The hardware IPs that make
2085  * up each asic are discovered each IP's early_init callback is run.  This
2086  * is the first stage in initializing the asic.
2087  * Returns 0 on success, negative error code on failure.
2088  */
2089 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2090 {
2091         struct drm_device *dev = adev_to_drm(adev);
2092         struct pci_dev *parent;
2093         int i, r;
2094         bool total;
2095
2096         amdgpu_device_enable_virtual_display(adev);
2097
2098         if (amdgpu_sriov_vf(adev)) {
2099                 r = amdgpu_virt_request_full_gpu(adev, true);
2100                 if (r)
2101                         return r;
2102         }
2103
2104         switch (adev->asic_type) {
2105 #ifdef CONFIG_DRM_AMDGPU_SI
2106         case CHIP_VERDE:
2107         case CHIP_TAHITI:
2108         case CHIP_PITCAIRN:
2109         case CHIP_OLAND:
2110         case CHIP_HAINAN:
2111                 adev->family = AMDGPU_FAMILY_SI;
2112                 r = si_set_ip_blocks(adev);
2113                 if (r)
2114                         return r;
2115                 break;
2116 #endif
2117 #ifdef CONFIG_DRM_AMDGPU_CIK
2118         case CHIP_BONAIRE:
2119         case CHIP_HAWAII:
2120         case CHIP_KAVERI:
2121         case CHIP_KABINI:
2122         case CHIP_MULLINS:
2123                 if (adev->flags & AMD_IS_APU)
2124                         adev->family = AMDGPU_FAMILY_KV;
2125                 else
2126                         adev->family = AMDGPU_FAMILY_CI;
2127
2128                 r = cik_set_ip_blocks(adev);
2129                 if (r)
2130                         return r;
2131                 break;
2132 #endif
2133         case CHIP_TOPAZ:
2134         case CHIP_TONGA:
2135         case CHIP_FIJI:
2136         case CHIP_POLARIS10:
2137         case CHIP_POLARIS11:
2138         case CHIP_POLARIS12:
2139         case CHIP_VEGAM:
2140         case CHIP_CARRIZO:
2141         case CHIP_STONEY:
2142                 if (adev->flags & AMD_IS_APU)
2143                         adev->family = AMDGPU_FAMILY_CZ;
2144                 else
2145                         adev->family = AMDGPU_FAMILY_VI;
2146
2147                 r = vi_set_ip_blocks(adev);
2148                 if (r)
2149                         return r;
2150                 break;
2151         default:
2152                 r = amdgpu_discovery_set_ip_blocks(adev);
2153                 if (r)
2154                         return r;
2155                 break;
2156         }
2157
2158         if (amdgpu_has_atpx() &&
2159             (amdgpu_is_atpx_hybrid() ||
2160              amdgpu_has_atpx_dgpu_power_cntl()) &&
2161             ((adev->flags & AMD_IS_APU) == 0) &&
2162             !pci_is_thunderbolt_attached(to_pci_dev(dev->dev)))
2163                 adev->flags |= AMD_IS_PX;
2164
2165         if (!(adev->flags & AMD_IS_APU)) {
2166                 parent = pci_upstream_bridge(adev->pdev);
2167                 adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2168         }
2169
2170         amdgpu_amdkfd_device_probe(adev);
2171
2172         adev->pm.pp_feature = amdgpu_pp_feature_mask;
2173         if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2174                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2175         if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2176                 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2177
2178         total = true;
2179         for (i = 0; i < adev->num_ip_blocks; i++) {
2180                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2181                         DRM_ERROR("disabled ip block: %d <%s>\n",
2182                                   i, adev->ip_blocks[i].version->funcs->name);
2183                         adev->ip_blocks[i].status.valid = false;
2184                 } else {
2185                         if (adev->ip_blocks[i].version->funcs->early_init) {
2186                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2187                                 if (r == -ENOENT) {
2188                                         adev->ip_blocks[i].status.valid = false;
2189                                 } else if (r) {
2190                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
2191                                                   adev->ip_blocks[i].version->funcs->name, r);
2192                                         total = false;
2193                                 } else {
2194                                         adev->ip_blocks[i].status.valid = true;
2195                                 }
2196                         } else {
2197                                 adev->ip_blocks[i].status.valid = true;
2198                         }
2199                 }
2200                 /* get the vbios after the asic_funcs are set up */
2201                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2202                         r = amdgpu_device_parse_gpu_info_fw(adev);
2203                         if (r)
2204                                 return r;
2205
2206                         /* Read BIOS */
2207                         if (!amdgpu_get_bios(adev))
2208                                 return -EINVAL;
2209
2210                         r = amdgpu_atombios_init(adev);
2211                         if (r) {
2212                                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2213                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2214                                 return r;
2215                         }
2216
2217                         /*get pf2vf msg info at it's earliest time*/
2218                         if (amdgpu_sriov_vf(adev))
2219                                 amdgpu_virt_init_data_exchange(adev);
2220
2221                 }
2222         }
2223         if (!total)
2224                 return -ENODEV;
2225
2226         adev->cg_flags &= amdgpu_cg_mask;
2227         adev->pg_flags &= amdgpu_pg_mask;
2228
2229         return 0;
2230 }
2231
2232 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2233 {
2234         int i, r;
2235
2236         for (i = 0; i < adev->num_ip_blocks; i++) {
2237                 if (!adev->ip_blocks[i].status.sw)
2238                         continue;
2239                 if (adev->ip_blocks[i].status.hw)
2240                         continue;
2241                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2242                     (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2243                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2244                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2245                         if (r) {
2246                                 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2247                                           adev->ip_blocks[i].version->funcs->name, r);
2248                                 return r;
2249                         }
2250                         adev->ip_blocks[i].status.hw = true;
2251                 }
2252         }
2253
2254         return 0;
2255 }
2256
2257 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2258 {
2259         int i, r;
2260
2261         for (i = 0; i < adev->num_ip_blocks; i++) {
2262                 if (!adev->ip_blocks[i].status.sw)
2263                         continue;
2264                 if (adev->ip_blocks[i].status.hw)
2265                         continue;
2266                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2267                 if (r) {
2268                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2269                                   adev->ip_blocks[i].version->funcs->name, r);
2270                         return r;
2271                 }
2272                 adev->ip_blocks[i].status.hw = true;
2273         }
2274
2275         return 0;
2276 }
2277
2278 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2279 {
2280         int r = 0;
2281         int i;
2282         uint32_t smu_version;
2283
2284         if (adev->asic_type >= CHIP_VEGA10) {
2285                 for (i = 0; i < adev->num_ip_blocks; i++) {
2286                         if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2287                                 continue;
2288
2289                         if (!adev->ip_blocks[i].status.sw)
2290                                 continue;
2291
2292                         /* no need to do the fw loading again if already done*/
2293                         if (adev->ip_blocks[i].status.hw == true)
2294                                 break;
2295
2296                         if (amdgpu_in_reset(adev) || adev->in_suspend) {
2297                                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2298                                 if (r) {
2299                                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2300                                                           adev->ip_blocks[i].version->funcs->name, r);
2301                                         return r;
2302                                 }
2303                         } else {
2304                                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2305                                 if (r) {
2306                                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2307                                                           adev->ip_blocks[i].version->funcs->name, r);
2308                                         return r;
2309                                 }
2310                         }
2311
2312                         adev->ip_blocks[i].status.hw = true;
2313                         break;
2314                 }
2315         }
2316
2317         if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2318                 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2319
2320         return r;
2321 }
2322
2323 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2324 {
2325         long timeout;
2326         int r, i;
2327
2328         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2329                 struct amdgpu_ring *ring = adev->rings[i];
2330
2331                 /* No need to setup the GPU scheduler for rings that don't need it */
2332                 if (!ring || ring->no_scheduler)
2333                         continue;
2334
2335                 switch (ring->funcs->type) {
2336                 case AMDGPU_RING_TYPE_GFX:
2337                         timeout = adev->gfx_timeout;
2338                         break;
2339                 case AMDGPU_RING_TYPE_COMPUTE:
2340                         timeout = adev->compute_timeout;
2341                         break;
2342                 case AMDGPU_RING_TYPE_SDMA:
2343                         timeout = adev->sdma_timeout;
2344                         break;
2345                 default:
2346                         timeout = adev->video_timeout;
2347                         break;
2348                 }
2349
2350                 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
2351                                    ring->num_hw_submission, amdgpu_job_hang_limit,
2352                                    timeout, adev->reset_domain->wq,
2353                                    ring->sched_score, ring->name,
2354                                    adev->dev);
2355                 if (r) {
2356                         DRM_ERROR("Failed to create scheduler on ring %s.\n",
2357                                   ring->name);
2358                         return r;
2359                 }
2360         }
2361
2362         return 0;
2363 }
2364
2365
2366 /**
2367  * amdgpu_device_ip_init - run init for hardware IPs
2368  *
2369  * @adev: amdgpu_device pointer
2370  *
2371  * Main initialization pass for hardware IPs.  The list of all the hardware
2372  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2373  * are run.  sw_init initializes the software state associated with each IP
2374  * and hw_init initializes the hardware associated with each IP.
2375  * Returns 0 on success, negative error code on failure.
2376  */
2377 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2378 {
2379         int i, r;
2380
2381         r = amdgpu_ras_init(adev);
2382         if (r)
2383                 return r;
2384
2385         for (i = 0; i < adev->num_ip_blocks; i++) {
2386                 if (!adev->ip_blocks[i].status.valid)
2387                         continue;
2388                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2389                 if (r) {
2390                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2391                                   adev->ip_blocks[i].version->funcs->name, r);
2392                         goto init_failed;
2393                 }
2394                 adev->ip_blocks[i].status.sw = true;
2395
2396                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2397                         /* need to do common hw init early so everything is set up for gmc */
2398                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2399                         if (r) {
2400                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2401                                 goto init_failed;
2402                         }
2403                         adev->ip_blocks[i].status.hw = true;
2404                 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2405                         /* need to do gmc hw init early so we can allocate gpu mem */
2406                         /* Try to reserve bad pages early */
2407                         if (amdgpu_sriov_vf(adev))
2408                                 amdgpu_virt_exchange_data(adev);
2409
2410                         r = amdgpu_device_mem_scratch_init(adev);
2411                         if (r) {
2412                                 DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r);
2413                                 goto init_failed;
2414                         }
2415                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2416                         if (r) {
2417                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2418                                 goto init_failed;
2419                         }
2420                         r = amdgpu_device_wb_init(adev);
2421                         if (r) {
2422                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2423                                 goto init_failed;
2424                         }
2425                         adev->ip_blocks[i].status.hw = true;
2426
2427                         /* right after GMC hw init, we create CSA */
2428                         if (amdgpu_mcbp) {
2429                                 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2430                                                                AMDGPU_GEM_DOMAIN_VRAM |
2431                                                                AMDGPU_GEM_DOMAIN_GTT,
2432                                                                AMDGPU_CSA_SIZE);
2433                                 if (r) {
2434                                         DRM_ERROR("allocate CSA failed %d\n", r);
2435                                         goto init_failed;
2436                                 }
2437                         }
2438                 }
2439         }
2440
2441         if (amdgpu_sriov_vf(adev))
2442                 amdgpu_virt_init_data_exchange(adev);
2443
2444         r = amdgpu_ib_pool_init(adev);
2445         if (r) {
2446                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2447                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2448                 goto init_failed;
2449         }
2450
2451         r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2452         if (r)
2453                 goto init_failed;
2454
2455         r = amdgpu_device_ip_hw_init_phase1(adev);
2456         if (r)
2457                 goto init_failed;
2458
2459         r = amdgpu_device_fw_loading(adev);
2460         if (r)
2461                 goto init_failed;
2462
2463         r = amdgpu_device_ip_hw_init_phase2(adev);
2464         if (r)
2465                 goto init_failed;
2466
2467         /*
2468          * retired pages will be loaded from eeprom and reserved here,
2469          * it should be called after amdgpu_device_ip_hw_init_phase2  since
2470          * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2471          * for I2C communication which only true at this point.
2472          *
2473          * amdgpu_ras_recovery_init may fail, but the upper only cares the
2474          * failure from bad gpu situation and stop amdgpu init process
2475          * accordingly. For other failed cases, it will still release all
2476          * the resource and print error message, rather than returning one
2477          * negative value to upper level.
2478          *
2479          * Note: theoretically, this should be called before all vram allocations
2480          * to protect retired page from abusing
2481          */
2482         r = amdgpu_ras_recovery_init(adev);
2483         if (r)
2484                 goto init_failed;
2485
2486         /**
2487          * In case of XGMI grab extra reference for reset domain for this device
2488          */
2489         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2490                 if (amdgpu_xgmi_add_device(adev) == 0) {
2491                         if (!amdgpu_sriov_vf(adev)) {
2492                                 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2493
2494                                 if (WARN_ON(!hive)) {
2495                                         r = -ENOENT;
2496                                         goto init_failed;
2497                                 }
2498
2499                                 if (!hive->reset_domain ||
2500                                     !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
2501                                         r = -ENOENT;
2502                                         amdgpu_put_xgmi_hive(hive);
2503                                         goto init_failed;
2504                                 }
2505
2506                                 /* Drop the early temporary reset domain we created for device */
2507                                 amdgpu_reset_put_reset_domain(adev->reset_domain);
2508                                 adev->reset_domain = hive->reset_domain;
2509                                 amdgpu_put_xgmi_hive(hive);
2510                         }
2511                 }
2512         }
2513
2514         r = amdgpu_device_init_schedulers(adev);
2515         if (r)
2516                 goto init_failed;
2517
2518         /* Don't init kfd if whole hive need to be reset during init */
2519         if (!adev->gmc.xgmi.pending_reset)
2520                 amdgpu_amdkfd_device_init(adev);
2521
2522         amdgpu_fru_get_product_info(adev);
2523
2524 init_failed:
2525         if (amdgpu_sriov_vf(adev))
2526                 amdgpu_virt_release_full_gpu(adev, true);
2527
2528         return r;
2529 }
2530
2531 /**
2532  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2533  *
2534  * @adev: amdgpu_device pointer
2535  *
2536  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2537  * this function before a GPU reset.  If the value is retained after a
2538  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2539  */
2540 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2541 {
2542         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2543 }
2544
2545 /**
2546  * amdgpu_device_check_vram_lost - check if vram is valid
2547  *
2548  * @adev: amdgpu_device pointer
2549  *
2550  * Checks the reset magic value written to the gart pointer in VRAM.
2551  * The driver calls this after a GPU reset to see if the contents of
2552  * VRAM is lost or now.
2553  * returns true if vram is lost, false if not.
2554  */
2555 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2556 {
2557         if (memcmp(adev->gart.ptr, adev->reset_magic,
2558                         AMDGPU_RESET_MAGIC_NUM))
2559                 return true;
2560
2561         if (!amdgpu_in_reset(adev))
2562                 return false;
2563
2564         /*
2565          * For all ASICs with baco/mode1 reset, the VRAM is
2566          * always assumed to be lost.
2567          */
2568         switch (amdgpu_asic_reset_method(adev)) {
2569         case AMD_RESET_METHOD_BACO:
2570         case AMD_RESET_METHOD_MODE1:
2571                 return true;
2572         default:
2573                 return false;
2574         }
2575 }
2576
2577 /**
2578  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2579  *
2580  * @adev: amdgpu_device pointer
2581  * @state: clockgating state (gate or ungate)
2582  *
2583  * The list of all the hardware IPs that make up the asic is walked and the
2584  * set_clockgating_state callbacks are run.
2585  * Late initialization pass enabling clockgating for hardware IPs.
2586  * Fini or suspend, pass disabling clockgating for hardware IPs.
2587  * Returns 0 on success, negative error code on failure.
2588  */
2589
2590 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2591                                enum amd_clockgating_state state)
2592 {
2593         int i, j, r;
2594
2595         if (amdgpu_emu_mode == 1)
2596                 return 0;
2597
2598         for (j = 0; j < adev->num_ip_blocks; j++) {
2599                 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2600                 if (!adev->ip_blocks[i].status.late_initialized)
2601                         continue;
2602                 /* skip CG for GFX, SDMA on S0ix */
2603                 if (adev->in_s0ix &&
2604                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2605                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2606                         continue;
2607                 /* skip CG for VCE/UVD, it's handled specially */
2608                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2609                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2610                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2611                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2612                     adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2613                         /* enable clockgating to save power */
2614                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2615                                                                                      state);
2616                         if (r) {
2617                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2618                                           adev->ip_blocks[i].version->funcs->name, r);
2619                                 return r;
2620                         }
2621                 }
2622         }
2623
2624         return 0;
2625 }
2626
2627 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2628                                enum amd_powergating_state state)
2629 {
2630         int i, j, r;
2631
2632         if (amdgpu_emu_mode == 1)
2633                 return 0;
2634
2635         for (j = 0; j < adev->num_ip_blocks; j++) {
2636                 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2637                 if (!adev->ip_blocks[i].status.late_initialized)
2638                         continue;
2639                 /* skip PG for GFX, SDMA on S0ix */
2640                 if (adev->in_s0ix &&
2641                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2642                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2643                         continue;
2644                 /* skip CG for VCE/UVD, it's handled specially */
2645                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2646                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2647                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2648                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2649                     adev->ip_blocks[i].version->funcs->set_powergating_state) {
2650                         /* enable powergating to save power */
2651                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2652                                                                                         state);
2653                         if (r) {
2654                                 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2655                                           adev->ip_blocks[i].version->funcs->name, r);
2656                                 return r;
2657                         }
2658                 }
2659         }
2660         return 0;
2661 }
2662
2663 static int amdgpu_device_enable_mgpu_fan_boost(void)
2664 {
2665         struct amdgpu_gpu_instance *gpu_ins;
2666         struct amdgpu_device *adev;
2667         int i, ret = 0;
2668
2669         mutex_lock(&mgpu_info.mutex);
2670
2671         /*
2672          * MGPU fan boost feature should be enabled
2673          * only when there are two or more dGPUs in
2674          * the system
2675          */
2676         if (mgpu_info.num_dgpu < 2)
2677                 goto out;
2678
2679         for (i = 0; i < mgpu_info.num_dgpu; i++) {
2680                 gpu_ins = &(mgpu_info.gpu_ins[i]);
2681                 adev = gpu_ins->adev;
2682                 if (!(adev->flags & AMD_IS_APU) &&
2683                     !gpu_ins->mgpu_fan_enabled) {
2684                         ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2685                         if (ret)
2686                                 break;
2687
2688                         gpu_ins->mgpu_fan_enabled = 1;
2689                 }
2690         }
2691
2692 out:
2693         mutex_unlock(&mgpu_info.mutex);
2694
2695         return ret;
2696 }
2697
2698 /**
2699  * amdgpu_device_ip_late_init - run late init for hardware IPs
2700  *
2701  * @adev: amdgpu_device pointer
2702  *
2703  * Late initialization pass for hardware IPs.  The list of all the hardware
2704  * IPs that make up the asic is walked and the late_init callbacks are run.
2705  * late_init covers any special initialization that an IP requires
2706  * after all of the have been initialized or something that needs to happen
2707  * late in the init process.
2708  * Returns 0 on success, negative error code on failure.
2709  */
2710 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2711 {
2712         struct amdgpu_gpu_instance *gpu_instance;
2713         int i = 0, r;
2714
2715         for (i = 0; i < adev->num_ip_blocks; i++) {
2716                 if (!adev->ip_blocks[i].status.hw)
2717                         continue;
2718                 if (adev->ip_blocks[i].version->funcs->late_init) {
2719                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2720                         if (r) {
2721                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2722                                           adev->ip_blocks[i].version->funcs->name, r);
2723                                 return r;
2724                         }
2725                 }
2726                 adev->ip_blocks[i].status.late_initialized = true;
2727         }
2728
2729         r = amdgpu_ras_late_init(adev);
2730         if (r) {
2731                 DRM_ERROR("amdgpu_ras_late_init failed %d", r);
2732                 return r;
2733         }
2734
2735         amdgpu_ras_set_error_query_ready(adev, true);
2736
2737         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2738         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2739
2740         amdgpu_device_fill_reset_magic(adev);
2741
2742         r = amdgpu_device_enable_mgpu_fan_boost();
2743         if (r)
2744                 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2745
2746         /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
2747         if (amdgpu_passthrough(adev) && ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1)||
2748                                adev->asic_type == CHIP_ALDEBARAN ))
2749                 amdgpu_dpm_handle_passthrough_sbr(adev, true);
2750
2751         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2752                 mutex_lock(&mgpu_info.mutex);
2753
2754                 /*
2755                  * Reset device p-state to low as this was booted with high.
2756                  *
2757                  * This should be performed only after all devices from the same
2758                  * hive get initialized.
2759                  *
2760                  * However, it's unknown how many device in the hive in advance.
2761                  * As this is counted one by one during devices initializations.
2762                  *
2763                  * So, we wait for all XGMI interlinked devices initialized.
2764                  * This may bring some delays as those devices may come from
2765                  * different hives. But that should be OK.
2766                  */
2767                 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2768                         for (i = 0; i < mgpu_info.num_gpu; i++) {
2769                                 gpu_instance = &(mgpu_info.gpu_ins[i]);
2770                                 if (gpu_instance->adev->flags & AMD_IS_APU)
2771                                         continue;
2772
2773                                 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2774                                                 AMDGPU_XGMI_PSTATE_MIN);
2775                                 if (r) {
2776                                         DRM_ERROR("pstate setting failed (%d).\n", r);
2777                                         break;
2778                                 }
2779                         }
2780                 }
2781
2782                 mutex_unlock(&mgpu_info.mutex);
2783         }
2784
2785         return 0;
2786 }
2787
2788 /**
2789  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
2790  *
2791  * @adev: amdgpu_device pointer
2792  *
2793  * For ASICs need to disable SMC first
2794  */
2795 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
2796 {
2797         int i, r;
2798
2799         if (adev->ip_versions[GC_HWIP][0] > IP_VERSION(9, 0, 0))
2800                 return;
2801
2802         for (i = 0; i < adev->num_ip_blocks; i++) {
2803                 if (!adev->ip_blocks[i].status.hw)
2804                         continue;
2805                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2806                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2807                         /* XXX handle errors */
2808                         if (r) {
2809                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2810                                           adev->ip_blocks[i].version->funcs->name, r);
2811                         }
2812                         adev->ip_blocks[i].status.hw = false;
2813                         break;
2814                 }
2815         }
2816 }
2817
2818 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2819 {
2820         int i, r;
2821
2822         for (i = 0; i < adev->num_ip_blocks; i++) {
2823                 if (!adev->ip_blocks[i].version->funcs->early_fini)
2824                         continue;
2825
2826                 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
2827                 if (r) {
2828                         DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
2829                                   adev->ip_blocks[i].version->funcs->name, r);
2830                 }
2831         }
2832
2833         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2834         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2835
2836         amdgpu_amdkfd_suspend(adev, false);
2837
2838         /* Workaroud for ASICs need to disable SMC first */
2839         amdgpu_device_smu_fini_early(adev);
2840
2841         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2842                 if (!adev->ip_blocks[i].status.hw)
2843                         continue;
2844
2845                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2846                 /* XXX handle errors */
2847                 if (r) {
2848                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2849                                   adev->ip_blocks[i].version->funcs->name, r);
2850                 }
2851
2852                 adev->ip_blocks[i].status.hw = false;
2853         }
2854
2855         if (amdgpu_sriov_vf(adev)) {
2856                 if (amdgpu_virt_release_full_gpu(adev, false))
2857                         DRM_ERROR("failed to release exclusive mode on fini\n");
2858         }
2859
2860         return 0;
2861 }
2862
2863 /**
2864  * amdgpu_device_ip_fini - run fini for hardware IPs
2865  *
2866  * @adev: amdgpu_device pointer
2867  *
2868  * Main teardown pass for hardware IPs.  The list of all the hardware
2869  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2870  * are run.  hw_fini tears down the hardware associated with each IP
2871  * and sw_fini tears down any software state associated with each IP.
2872  * Returns 0 on success, negative error code on failure.
2873  */
2874 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2875 {
2876         int i, r;
2877
2878         if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2879                 amdgpu_virt_release_ras_err_handler_data(adev);
2880
2881         if (adev->gmc.xgmi.num_physical_nodes > 1)
2882                 amdgpu_xgmi_remove_device(adev);
2883
2884         amdgpu_amdkfd_device_fini_sw(adev);
2885
2886         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2887                 if (!adev->ip_blocks[i].status.sw)
2888                         continue;
2889
2890                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2891                         amdgpu_ucode_free_bo(adev);
2892                         amdgpu_free_static_csa(&adev->virt.csa_obj);
2893                         amdgpu_device_wb_fini(adev);
2894                         amdgpu_device_mem_scratch_fini(adev);
2895                         amdgpu_ib_pool_fini(adev);
2896                 }
2897
2898                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2899                 /* XXX handle errors */
2900                 if (r) {
2901                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2902                                   adev->ip_blocks[i].version->funcs->name, r);
2903                 }
2904                 adev->ip_blocks[i].status.sw = false;
2905                 adev->ip_blocks[i].status.valid = false;
2906         }
2907
2908         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2909                 if (!adev->ip_blocks[i].status.late_initialized)
2910                         continue;
2911                 if (adev->ip_blocks[i].version->funcs->late_fini)
2912                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2913                 adev->ip_blocks[i].status.late_initialized = false;
2914         }
2915
2916         amdgpu_ras_fini(adev);
2917
2918         return 0;
2919 }
2920
2921 /**
2922  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2923  *
2924  * @work: work_struct.
2925  */
2926 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2927 {
2928         struct amdgpu_device *adev =
2929                 container_of(work, struct amdgpu_device, delayed_init_work.work);
2930         int r;
2931
2932         r = amdgpu_ib_ring_tests(adev);
2933         if (r)
2934                 DRM_ERROR("ib ring test failed (%d).\n", r);
2935 }
2936
2937 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2938 {
2939         struct amdgpu_device *adev =
2940                 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2941
2942         WARN_ON_ONCE(adev->gfx.gfx_off_state);
2943         WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
2944
2945         if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2946                 adev->gfx.gfx_off_state = true;
2947 }
2948
2949 /**
2950  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2951  *
2952  * @adev: amdgpu_device pointer
2953  *
2954  * Main suspend function for hardware IPs.  The list of all the hardware
2955  * IPs that make up the asic is walked, clockgating is disabled and the
2956  * suspend callbacks are run.  suspend puts the hardware and software state
2957  * in each IP into a state suitable for suspend.
2958  * Returns 0 on success, negative error code on failure.
2959  */
2960 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2961 {
2962         int i, r;
2963
2964         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2965         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2966
2967         /*
2968          * Per PMFW team's suggestion, driver needs to handle gfxoff
2969          * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
2970          * scenario. Add the missing df cstate disablement here.
2971          */
2972         if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
2973                 dev_warn(adev->dev, "Failed to disallow df cstate");
2974
2975         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2976                 if (!adev->ip_blocks[i].status.valid)
2977                         continue;
2978
2979                 /* displays are handled separately */
2980                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
2981                         continue;
2982
2983                 /* XXX handle errors */
2984                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2985                 /* XXX handle errors */
2986                 if (r) {
2987                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
2988                                   adev->ip_blocks[i].version->funcs->name, r);
2989                         return r;
2990                 }
2991
2992                 adev->ip_blocks[i].status.hw = false;
2993         }
2994
2995         return 0;
2996 }
2997
2998 /**
2999  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3000  *
3001  * @adev: amdgpu_device pointer
3002  *
3003  * Main suspend function for hardware IPs.  The list of all the hardware
3004  * IPs that make up the asic is walked, clockgating is disabled and the
3005  * suspend callbacks are run.  suspend puts the hardware and software state
3006  * in each IP into a state suitable for suspend.
3007  * Returns 0 on success, negative error code on failure.
3008  */
3009 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3010 {
3011         int i, r;
3012
3013         if (adev->in_s0ix)
3014                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
3015
3016         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3017                 if (!adev->ip_blocks[i].status.valid)
3018                         continue;
3019                 /* displays are handled in phase1 */
3020                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3021                         continue;
3022                 /* PSP lost connection when err_event_athub occurs */
3023                 if (amdgpu_ras_intr_triggered() &&
3024                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3025                         adev->ip_blocks[i].status.hw = false;
3026                         continue;
3027                 }
3028
3029                 /* skip unnecessary suspend if we do not initialize them yet */
3030                 if (adev->gmc.xgmi.pending_reset &&
3031                     !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3032                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
3033                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3034                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
3035                         adev->ip_blocks[i].status.hw = false;
3036                         continue;
3037                 }
3038
3039                 /* skip suspend of gfx/mes and psp for S0ix
3040                  * gfx is in gfxoff state, so on resume it will exit gfxoff just
3041                  * like at runtime. PSP is also part of the always on hardware
3042                  * so no need to suspend it.
3043                  */
3044                 if (adev->in_s0ix &&
3045                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3046                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3047                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3048                         continue;
3049
3050                 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3051                 if (adev->in_s0ix &&
3052                     (adev->ip_versions[SDMA0_HWIP][0] >= IP_VERSION(5, 0, 0)) &&
3053                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
3054                         continue;
3055
3056                 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3057                  * These are in TMR, hence are expected to be reused by PSP-TOS to reload
3058                  * from this location and RLC Autoload automatically also gets loaded
3059                  * from here based on PMFW -> PSP message during re-init sequence.
3060                  * Therefore, the psp suspend & resume should be skipped to avoid destroy
3061                  * the TMR and reload FWs again for IMU enabled APU ASICs.
3062                  */
3063                 if (amdgpu_in_reset(adev) &&
3064                     (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3065                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3066                         continue;
3067
3068                 /* XXX handle errors */
3069                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3070                 /* XXX handle errors */
3071                 if (r) {
3072                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
3073                                   adev->ip_blocks[i].version->funcs->name, r);
3074                 }
3075                 adev->ip_blocks[i].status.hw = false;
3076                 /* handle putting the SMC in the appropriate state */
3077                 if(!amdgpu_sriov_vf(adev)){
3078                         if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3079                                 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3080                                 if (r) {
3081                                         DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3082                                                         adev->mp1_state, r);
3083                                         return r;
3084                                 }
3085                         }
3086                 }
3087         }
3088
3089         return 0;
3090 }
3091
3092 /**
3093  * amdgpu_device_ip_suspend - run suspend for hardware IPs
3094  *
3095  * @adev: amdgpu_device pointer
3096  *
3097  * Main suspend function for hardware IPs.  The list of all the hardware
3098  * IPs that make up the asic is walked, clockgating is disabled and the
3099  * suspend callbacks are run.  suspend puts the hardware and software state
3100  * in each IP into a state suitable for suspend.
3101  * Returns 0 on success, negative error code on failure.
3102  */
3103 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3104 {
3105         int r;
3106
3107         if (amdgpu_sriov_vf(adev)) {
3108                 amdgpu_virt_fini_data_exchange(adev);
3109                 amdgpu_virt_request_full_gpu(adev, false);
3110         }
3111
3112         r = amdgpu_device_ip_suspend_phase1(adev);
3113         if (r)
3114                 return r;
3115         r = amdgpu_device_ip_suspend_phase2(adev);
3116
3117         if (amdgpu_sriov_vf(adev))
3118                 amdgpu_virt_release_full_gpu(adev, false);
3119
3120         return r;
3121 }
3122
3123 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3124 {
3125         int i, r;
3126
3127         static enum amd_ip_block_type ip_order[] = {
3128                 AMD_IP_BLOCK_TYPE_COMMON,
3129                 AMD_IP_BLOCK_TYPE_GMC,
3130                 AMD_IP_BLOCK_TYPE_PSP,
3131                 AMD_IP_BLOCK_TYPE_IH,
3132         };
3133
3134         for (i = 0; i < adev->num_ip_blocks; i++) {
3135                 int j;
3136                 struct amdgpu_ip_block *block;
3137
3138                 block = &adev->ip_blocks[i];
3139                 block->status.hw = false;
3140
3141                 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3142
3143                         if (block->version->type != ip_order[j] ||
3144                                 !block->status.valid)
3145                                 continue;
3146
3147                         r = block->version->funcs->hw_init(adev);
3148                         DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3149                         if (r)
3150                                 return r;
3151                         block->status.hw = true;
3152                 }
3153         }
3154
3155         return 0;
3156 }
3157
3158 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3159 {
3160         int i, r;
3161
3162         static enum amd_ip_block_type ip_order[] = {
3163                 AMD_IP_BLOCK_TYPE_SMC,
3164                 AMD_IP_BLOCK_TYPE_DCE,
3165                 AMD_IP_BLOCK_TYPE_GFX,
3166                 AMD_IP_BLOCK_TYPE_SDMA,
3167                 AMD_IP_BLOCK_TYPE_UVD,
3168                 AMD_IP_BLOCK_TYPE_VCE,
3169                 AMD_IP_BLOCK_TYPE_VCN
3170         };
3171
3172         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3173                 int j;
3174                 struct amdgpu_ip_block *block;
3175
3176                 for (j = 0; j < adev->num_ip_blocks; j++) {
3177                         block = &adev->ip_blocks[j];
3178
3179                         if (block->version->type != ip_order[i] ||
3180                                 !block->status.valid ||
3181                                 block->status.hw)
3182                                 continue;
3183
3184                         if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3185                                 r = block->version->funcs->resume(adev);
3186                         else
3187                                 r = block->version->funcs->hw_init(adev);
3188
3189                         DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3190                         if (r)
3191                                 return r;
3192                         block->status.hw = true;
3193                 }
3194         }
3195
3196         return 0;
3197 }
3198
3199 /**
3200  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3201  *
3202  * @adev: amdgpu_device pointer
3203  *
3204  * First resume function for hardware IPs.  The list of all the hardware
3205  * IPs that make up the asic is walked and the resume callbacks are run for
3206  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3207  * after a suspend and updates the software state as necessary.  This
3208  * function is also used for restoring the GPU after a GPU reset.
3209  * Returns 0 on success, negative error code on failure.
3210  */
3211 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3212 {
3213         int i, r;
3214
3215         for (i = 0; i < adev->num_ip_blocks; i++) {
3216                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3217                         continue;
3218                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3219                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3220                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3221                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3222
3223                         r = adev->ip_blocks[i].version->funcs->resume(adev);
3224                         if (r) {
3225                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
3226                                           adev->ip_blocks[i].version->funcs->name, r);
3227                                 return r;
3228                         }
3229                         adev->ip_blocks[i].status.hw = true;
3230                 }
3231         }
3232
3233         return 0;
3234 }
3235
3236 /**
3237  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3238  *
3239  * @adev: amdgpu_device pointer
3240  *
3241  * First resume function for hardware IPs.  The list of all the hardware
3242  * IPs that make up the asic is walked and the resume callbacks are run for
3243  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3244  * functional state after a suspend and updates the software state as
3245  * necessary.  This function is also used for restoring the GPU after a GPU
3246  * reset.
3247  * Returns 0 on success, negative error code on failure.
3248  */
3249 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3250 {
3251         int i, r;
3252
3253         for (i = 0; i < adev->num_ip_blocks; i++) {
3254                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3255                         continue;
3256                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3257                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3258                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3259                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3260                         continue;
3261                 r = adev->ip_blocks[i].version->funcs->resume(adev);
3262                 if (r) {
3263                         DRM_ERROR("resume of IP block <%s> failed %d\n",
3264                                   adev->ip_blocks[i].version->funcs->name, r);
3265                         return r;
3266                 }
3267                 adev->ip_blocks[i].status.hw = true;
3268         }
3269
3270         return 0;
3271 }
3272
3273 /**
3274  * amdgpu_device_ip_resume - run resume for hardware IPs
3275  *
3276  * @adev: amdgpu_device pointer
3277  *
3278  * Main resume function for hardware IPs.  The hardware IPs
3279  * are split into two resume functions because they are
3280  * are also used in in recovering from a GPU reset and some additional
3281  * steps need to be take between them.  In this case (S3/S4) they are
3282  * run sequentially.
3283  * Returns 0 on success, negative error code on failure.
3284  */
3285 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3286 {
3287         int r;
3288
3289         r = amdgpu_amdkfd_resume_iommu(adev);
3290         if (r)
3291                 return r;
3292
3293         r = amdgpu_device_ip_resume_phase1(adev);
3294         if (r)
3295                 return r;
3296
3297         r = amdgpu_device_fw_loading(adev);
3298         if (r)
3299                 return r;
3300
3301         r = amdgpu_device_ip_resume_phase2(adev);
3302
3303         return r;
3304 }
3305
3306 /**
3307  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3308  *
3309  * @adev: amdgpu_device pointer
3310  *
3311  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3312  */
3313 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3314 {
3315         if (amdgpu_sriov_vf(adev)) {
3316                 if (adev->is_atom_fw) {
3317                         if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3318                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3319                 } else {
3320                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3321                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3322                 }
3323
3324                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3325                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3326         }
3327 }
3328
3329 /**
3330  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3331  *
3332  * @asic_type: AMD asic type
3333  *
3334  * Check if there is DC (new modesetting infrastructre) support for an asic.
3335  * returns true if DC has support, false if not.
3336  */
3337 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3338 {
3339         switch (asic_type) {
3340 #ifdef CONFIG_DRM_AMDGPU_SI
3341         case CHIP_HAINAN:
3342 #endif
3343         case CHIP_TOPAZ:
3344                 /* chips with no display hardware */
3345                 return false;
3346 #if defined(CONFIG_DRM_AMD_DC)
3347         case CHIP_TAHITI:
3348         case CHIP_PITCAIRN:
3349         case CHIP_VERDE:
3350         case CHIP_OLAND:
3351                 /*
3352                  * We have systems in the wild with these ASICs that require
3353                  * LVDS and VGA support which is not supported with DC.
3354                  *
3355                  * Fallback to the non-DC driver here by default so as not to
3356                  * cause regressions.
3357                  */
3358 #if defined(CONFIG_DRM_AMD_DC_SI)
3359                 return amdgpu_dc > 0;
3360 #else
3361                 return false;
3362 #endif
3363         case CHIP_BONAIRE:
3364         case CHIP_KAVERI:
3365         case CHIP_KABINI:
3366         case CHIP_MULLINS:
3367                 /*
3368                  * We have systems in the wild with these ASICs that require
3369                  * VGA support which is not supported with DC.
3370                  *
3371                  * Fallback to the non-DC driver here by default so as not to
3372                  * cause regressions.
3373                  */
3374                 return amdgpu_dc > 0;
3375         default:
3376                 return amdgpu_dc != 0;
3377 #else
3378         default:
3379                 if (amdgpu_dc > 0)
3380                         DRM_INFO_ONCE("Display Core has been requested via kernel parameter "
3381                                          "but isn't supported by ASIC, ignoring\n");
3382                 return false;
3383 #endif
3384         }
3385 }
3386
3387 /**
3388  * amdgpu_device_has_dc_support - check if dc is supported
3389  *
3390  * @adev: amdgpu_device pointer
3391  *
3392  * Returns true for supported, false for not supported
3393  */
3394 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3395 {
3396         if (adev->enable_virtual_display ||
3397             (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3398                 return false;
3399
3400         return amdgpu_device_asic_has_dc_support(adev->asic_type);
3401 }
3402
3403 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3404 {
3405         struct amdgpu_device *adev =
3406                 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3407         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3408
3409         /* It's a bug to not have a hive within this function */
3410         if (WARN_ON(!hive))
3411                 return;
3412
3413         /*
3414          * Use task barrier to synchronize all xgmi reset works across the
3415          * hive. task_barrier_enter and task_barrier_exit will block
3416          * until all the threads running the xgmi reset works reach
3417          * those points. task_barrier_full will do both blocks.
3418          */
3419         if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3420
3421                 task_barrier_enter(&hive->tb);
3422                 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3423
3424                 if (adev->asic_reset_res)
3425                         goto fail;
3426
3427                 task_barrier_exit(&hive->tb);
3428                 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3429
3430                 if (adev->asic_reset_res)
3431                         goto fail;
3432
3433                 if (adev->mmhub.ras && adev->mmhub.ras->ras_block.hw_ops &&
3434                     adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
3435                         adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(adev);
3436         } else {
3437
3438                 task_barrier_full(&hive->tb);
3439                 adev->asic_reset_res =  amdgpu_asic_reset(adev);
3440         }
3441
3442 fail:
3443         if (adev->asic_reset_res)
3444                 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3445                          adev->asic_reset_res, adev_to_drm(adev)->unique);
3446         amdgpu_put_xgmi_hive(hive);
3447 }
3448
3449 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3450 {
3451         char *input = amdgpu_lockup_timeout;
3452         char *timeout_setting = NULL;
3453         int index = 0;
3454         long timeout;
3455         int ret = 0;
3456
3457         /*
3458          * By default timeout for non compute jobs is 10000
3459          * and 60000 for compute jobs.
3460          * In SR-IOV or passthrough mode, timeout for compute
3461          * jobs are 60000 by default.
3462          */
3463         adev->gfx_timeout = msecs_to_jiffies(10000);
3464         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3465         if (amdgpu_sriov_vf(adev))
3466                 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3467                                         msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3468         else
3469                 adev->compute_timeout =  msecs_to_jiffies(60000);
3470
3471         if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3472                 while ((timeout_setting = strsep(&input, ",")) &&
3473                                 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3474                         ret = kstrtol(timeout_setting, 0, &timeout);
3475                         if (ret)
3476                                 return ret;
3477
3478                         if (timeout == 0) {
3479                                 index++;
3480                                 continue;
3481                         } else if (timeout < 0) {
3482                                 timeout = MAX_SCHEDULE_TIMEOUT;
3483                                 dev_warn(adev->dev, "lockup timeout disabled");
3484                                 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3485                         } else {
3486                                 timeout = msecs_to_jiffies(timeout);
3487                         }
3488
3489                         switch (index++) {
3490                         case 0:
3491                                 adev->gfx_timeout = timeout;
3492                                 break;
3493                         case 1:
3494                                 adev->compute_timeout = timeout;
3495                                 break;
3496                         case 2:
3497                                 adev->sdma_timeout = timeout;
3498                                 break;
3499                         case 3:
3500                                 adev->video_timeout = timeout;
3501                                 break;
3502                         default:
3503                                 break;
3504                         }
3505                 }
3506                 /*
3507                  * There is only one value specified and
3508                  * it should apply to all non-compute jobs.
3509                  */
3510                 if (index == 1) {
3511                         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3512                         if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3513                                 adev->compute_timeout = adev->gfx_timeout;
3514                 }
3515         }
3516
3517         return ret;
3518 }
3519
3520 /**
3521  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3522  *
3523  * @adev: amdgpu_device pointer
3524  *
3525  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3526  */
3527 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3528 {
3529         struct iommu_domain *domain;
3530
3531         domain = iommu_get_domain_for_dev(adev->dev);
3532         if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3533                 adev->ram_is_direct_mapped = true;
3534 }
3535
3536 static const struct attribute *amdgpu_dev_attributes[] = {
3537         &dev_attr_product_name.attr,
3538         &dev_attr_product_number.attr,
3539         &dev_attr_serial_number.attr,
3540         &dev_attr_pcie_replay_count.attr,
3541         NULL
3542 };
3543
3544 /**
3545  * amdgpu_device_init - initialize the driver
3546  *
3547  * @adev: amdgpu_device pointer
3548  * @flags: driver flags
3549  *
3550  * Initializes the driver info and hw (all asics).
3551  * Returns 0 for success or an error on failure.
3552  * Called at driver startup.
3553  */
3554 int amdgpu_device_init(struct amdgpu_device *adev,
3555                        uint32_t flags)
3556 {
3557         struct drm_device *ddev = adev_to_drm(adev);
3558         struct pci_dev *pdev = adev->pdev;
3559         int r, i;
3560         bool px = false;
3561         u32 max_MBps;
3562
3563         adev->shutdown = false;
3564         adev->flags = flags;
3565
3566         if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3567                 adev->asic_type = amdgpu_force_asic_type;
3568         else
3569                 adev->asic_type = flags & AMD_ASIC_MASK;
3570
3571         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3572         if (amdgpu_emu_mode == 1)
3573                 adev->usec_timeout *= 10;
3574         adev->gmc.gart_size = 512 * 1024 * 1024;
3575         adev->accel_working = false;
3576         adev->num_rings = 0;
3577         RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3578         adev->mman.buffer_funcs = NULL;
3579         adev->mman.buffer_funcs_ring = NULL;
3580         adev->vm_manager.vm_pte_funcs = NULL;
3581         adev->vm_manager.vm_pte_num_scheds = 0;
3582         adev->gmc.gmc_funcs = NULL;
3583         adev->harvest_ip_mask = 0x0;
3584         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3585         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3586
3587         adev->smc_rreg = &amdgpu_invalid_rreg;
3588         adev->smc_wreg = &amdgpu_invalid_wreg;
3589         adev->pcie_rreg = &amdgpu_invalid_rreg;
3590         adev->pcie_wreg = &amdgpu_invalid_wreg;
3591         adev->pciep_rreg = &amdgpu_invalid_rreg;
3592         adev->pciep_wreg = &amdgpu_invalid_wreg;
3593         adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3594         adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3595         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3596         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3597         adev->didt_rreg = &amdgpu_invalid_rreg;
3598         adev->didt_wreg = &amdgpu_invalid_wreg;
3599         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3600         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3601         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3602         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3603
3604         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3605                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3606                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3607
3608         /* mutex initialization are all done here so we
3609          * can recall function without having locking issues */
3610         mutex_init(&adev->firmware.mutex);
3611         mutex_init(&adev->pm.mutex);
3612         mutex_init(&adev->gfx.gpu_clock_mutex);
3613         mutex_init(&adev->srbm_mutex);
3614         mutex_init(&adev->gfx.pipe_reserve_mutex);
3615         mutex_init(&adev->gfx.gfx_off_mutex);
3616         mutex_init(&adev->grbm_idx_mutex);
3617         mutex_init(&adev->mn_lock);
3618         mutex_init(&adev->virt.vf_errors.lock);
3619         hash_init(adev->mn_hash);
3620         mutex_init(&adev->psp.mutex);
3621         mutex_init(&adev->notifier_lock);
3622         mutex_init(&adev->pm.stable_pstate_ctx_lock);
3623         mutex_init(&adev->benchmark_mutex);
3624
3625         amdgpu_device_init_apu_flags(adev);
3626
3627         r = amdgpu_device_check_arguments(adev);
3628         if (r)
3629                 return r;
3630
3631         spin_lock_init(&adev->mmio_idx_lock);
3632         spin_lock_init(&adev->smc_idx_lock);
3633         spin_lock_init(&adev->pcie_idx_lock);
3634         spin_lock_init(&adev->uvd_ctx_idx_lock);
3635         spin_lock_init(&adev->didt_idx_lock);
3636         spin_lock_init(&adev->gc_cac_idx_lock);
3637         spin_lock_init(&adev->se_cac_idx_lock);
3638         spin_lock_init(&adev->audio_endpt_idx_lock);
3639         spin_lock_init(&adev->mm_stats.lock);
3640
3641         INIT_LIST_HEAD(&adev->shadow_list);
3642         mutex_init(&adev->shadow_list_lock);
3643
3644         INIT_LIST_HEAD(&adev->reset_list);
3645
3646         INIT_LIST_HEAD(&adev->ras_list);
3647
3648         INIT_DELAYED_WORK(&adev->delayed_init_work,
3649                           amdgpu_device_delayed_init_work_handler);
3650         INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3651                           amdgpu_device_delay_enable_gfx_off);
3652
3653         INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3654
3655         adev->gfx.gfx_off_req_count = 1;
3656         adev->gfx.gfx_off_residency = 0;
3657         adev->gfx.gfx_off_entrycount = 0;
3658         adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3659
3660         atomic_set(&adev->throttling_logging_enabled, 1);
3661         /*
3662          * If throttling continues, logging will be performed every minute
3663          * to avoid log flooding. "-1" is subtracted since the thermal
3664          * throttling interrupt comes every second. Thus, the total logging
3665          * interval is 59 seconds(retelimited printk interval) + 1(waiting
3666          * for throttling interrupt) = 60 seconds.
3667          */
3668         ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3669         ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3670
3671         /* Registers mapping */
3672         /* TODO: block userspace mapping of io register */
3673         if (adev->asic_type >= CHIP_BONAIRE) {
3674                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3675                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3676         } else {
3677                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3678                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3679         }
3680
3681         for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
3682                 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
3683
3684         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3685         if (adev->rmmio == NULL) {
3686                 return -ENOMEM;
3687         }
3688         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3689         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
3690
3691         amdgpu_device_get_pcie_info(adev);
3692
3693         if (amdgpu_mcbp)
3694                 DRM_INFO("MCBP is enabled\n");
3695
3696         /*
3697          * Reset domain needs to be present early, before XGMI hive discovered
3698          * (if any) and intitialized to use reset sem and in_gpu reset flag
3699          * early on during init and before calling to RREG32.
3700          */
3701         adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
3702         if (!adev->reset_domain)
3703                 return -ENOMEM;
3704
3705         /* detect hw virtualization here */
3706         amdgpu_detect_virtualization(adev);
3707
3708         r = amdgpu_device_get_job_timeout_settings(adev);
3709         if (r) {
3710                 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3711                 return r;
3712         }
3713
3714         /* early init functions */
3715         r = amdgpu_device_ip_early_init(adev);
3716         if (r)
3717                 return r;
3718
3719         /* Get rid of things like offb */
3720         r = drm_aperture_remove_conflicting_pci_framebuffers(adev->pdev, &amdgpu_kms_driver);
3721         if (r)
3722                 return r;
3723
3724         /* Enable TMZ based on IP_VERSION */
3725         amdgpu_gmc_tmz_set(adev);
3726
3727         amdgpu_gmc_noretry_set(adev);
3728         /* Need to get xgmi info early to decide the reset behavior*/
3729         if (adev->gmc.xgmi.supported) {
3730                 r = adev->gfxhub.funcs->get_xgmi_info(adev);
3731                 if (r)
3732                         return r;
3733         }
3734
3735         /* enable PCIE atomic ops */
3736         if (amdgpu_sriov_vf(adev))
3737                 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
3738                         adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
3739                         (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3740         else
3741                 adev->have_atomics_support =
3742                         !pci_enable_atomic_ops_to_root(adev->pdev,
3743                                           PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3744                                           PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3745         if (!adev->have_atomics_support)
3746                 dev_info(adev->dev, "PCIE atomic ops is not supported\n");
3747
3748         /* doorbell bar mapping and doorbell index init*/
3749         amdgpu_device_doorbell_init(adev);
3750
3751         if (amdgpu_emu_mode == 1) {
3752                 /* post the asic on emulation mode */
3753                 emu_soc_asic_init(adev);
3754                 goto fence_driver_init;
3755         }
3756
3757         amdgpu_reset_init(adev);
3758
3759         /* detect if we are with an SRIOV vbios */
3760         amdgpu_device_detect_sriov_bios(adev);
3761
3762         /* check if we need to reset the asic
3763          *  E.g., driver was not cleanly unloaded previously, etc.
3764          */
3765         if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3766                 if (adev->gmc.xgmi.num_physical_nodes) {
3767                         dev_info(adev->dev, "Pending hive reset.\n");
3768                         adev->gmc.xgmi.pending_reset = true;
3769                         /* Only need to init necessary block for SMU to handle the reset */
3770                         for (i = 0; i < adev->num_ip_blocks; i++) {
3771                                 if (!adev->ip_blocks[i].status.valid)
3772                                         continue;
3773                                 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3774                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3775                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3776                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3777                                         DRM_DEBUG("IP %s disabled for hw_init.\n",
3778                                                 adev->ip_blocks[i].version->funcs->name);
3779                                         adev->ip_blocks[i].status.hw = true;
3780                                 }
3781                         }
3782                 } else {
3783                         r = amdgpu_asic_reset(adev);
3784                         if (r) {
3785                                 dev_err(adev->dev, "asic reset on init failed\n");
3786                                 goto failed;
3787                         }
3788                 }
3789         }
3790
3791         pci_enable_pcie_error_reporting(adev->pdev);
3792
3793         /* Post card if necessary */
3794         if (amdgpu_device_need_post(adev)) {
3795                 if (!adev->bios) {
3796                         dev_err(adev->dev, "no vBIOS found\n");
3797                         r = -EINVAL;
3798                         goto failed;
3799                 }
3800                 DRM_INFO("GPU posting now...\n");
3801                 r = amdgpu_device_asic_init(adev);
3802                 if (r) {
3803                         dev_err(adev->dev, "gpu post error!\n");
3804                         goto failed;
3805                 }
3806         }
3807
3808         if (adev->is_atom_fw) {
3809                 /* Initialize clocks */
3810                 r = amdgpu_atomfirmware_get_clock_info(adev);
3811                 if (r) {
3812                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
3813                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3814                         goto failed;
3815                 }
3816         } else {
3817                 /* Initialize clocks */
3818                 r = amdgpu_atombios_get_clock_info(adev);
3819                 if (r) {
3820                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
3821                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3822                         goto failed;
3823                 }
3824                 /* init i2c buses */
3825                 if (!amdgpu_device_has_dc_support(adev))
3826                         amdgpu_atombios_i2c_init(adev);
3827         }
3828
3829 fence_driver_init:
3830         /* Fence driver */
3831         r = amdgpu_fence_driver_sw_init(adev);
3832         if (r) {
3833                 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
3834                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
3835                 goto failed;
3836         }
3837
3838         /* init the mode config */
3839         drm_mode_config_init(adev_to_drm(adev));
3840
3841         r = amdgpu_device_ip_init(adev);
3842         if (r) {
3843                 /* failed in exclusive mode due to timeout */
3844                 if (amdgpu_sriov_vf(adev) &&
3845                     !amdgpu_sriov_runtime(adev) &&
3846                     amdgpu_virt_mmio_blocked(adev) &&
3847                     !amdgpu_virt_wait_reset(adev)) {
3848                         dev_err(adev->dev, "VF exclusive mode timeout\n");
3849                         /* Don't send request since VF is inactive. */
3850                         adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
3851                         adev->virt.ops = NULL;
3852                         r = -EAGAIN;
3853                         goto release_ras_con;
3854                 }
3855                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
3856                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
3857                 goto release_ras_con;
3858         }
3859
3860         amdgpu_fence_driver_hw_init(adev);
3861
3862         dev_info(adev->dev,
3863                 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
3864                         adev->gfx.config.max_shader_engines,
3865                         adev->gfx.config.max_sh_per_se,
3866                         adev->gfx.config.max_cu_per_sh,
3867                         adev->gfx.cu_info.number);
3868
3869         adev->accel_working = true;
3870
3871         amdgpu_vm_check_compute_bug(adev);
3872
3873         /* Initialize the buffer migration limit. */
3874         if (amdgpu_moverate >= 0)
3875                 max_MBps = amdgpu_moverate;
3876         else
3877                 max_MBps = 8; /* Allow 8 MB/s. */
3878         /* Get a log2 for easy divisions. */
3879         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
3880
3881         r = amdgpu_pm_sysfs_init(adev);
3882         if (r) {
3883                 adev->pm_sysfs_en = false;
3884                 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
3885         } else
3886                 adev->pm_sysfs_en = true;
3887
3888         r = amdgpu_ucode_sysfs_init(adev);
3889         if (r) {
3890                 adev->ucode_sysfs_en = false;
3891                 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
3892         } else
3893                 adev->ucode_sysfs_en = true;
3894
3895         r = amdgpu_psp_sysfs_init(adev);
3896         if (r) {
3897                 adev->psp_sysfs_en = false;
3898                 if (!amdgpu_sriov_vf(adev))
3899                         DRM_ERROR("Creating psp sysfs failed\n");
3900         } else
3901                 adev->psp_sysfs_en = true;
3902
3903         /*
3904          * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
3905          * Otherwise the mgpu fan boost feature will be skipped due to the
3906          * gpu instance is counted less.
3907          */
3908         amdgpu_register_gpu_instance(adev);
3909
3910         /* enable clockgating, etc. after ib tests, etc. since some blocks require
3911          * explicit gating rather than handling it automatically.
3912          */
3913         if (!adev->gmc.xgmi.pending_reset) {
3914                 r = amdgpu_device_ip_late_init(adev);
3915                 if (r) {
3916                         dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
3917                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
3918                         goto release_ras_con;
3919                 }
3920                 /* must succeed. */
3921                 amdgpu_ras_resume(adev);
3922                 queue_delayed_work(system_wq, &adev->delayed_init_work,
3923                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3924         }
3925
3926         if (amdgpu_sriov_vf(adev))
3927                 flush_delayed_work(&adev->delayed_init_work);
3928
3929         r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
3930         if (r)
3931                 dev_err(adev->dev, "Could not create amdgpu device attr\n");
3932
3933         if (IS_ENABLED(CONFIG_PERF_EVENTS))
3934                 r = amdgpu_pmu_init(adev);
3935         if (r)
3936                 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
3937
3938         /* Have stored pci confspace at hand for restore in sudden PCI error */
3939         if (amdgpu_device_cache_pci_state(adev->pdev))
3940                 pci_restore_state(pdev);
3941
3942         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
3943         /* this will fail for cards that aren't VGA class devices, just
3944          * ignore it */
3945         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3946                 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
3947
3948         if (amdgpu_device_supports_px(ddev)) {
3949                 px = true;
3950                 vga_switcheroo_register_client(adev->pdev,
3951                                                &amdgpu_switcheroo_ops, px);
3952                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
3953         }
3954
3955         if (adev->gmc.xgmi.pending_reset)
3956                 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
3957                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3958
3959         amdgpu_device_check_iommu_direct_map(adev);
3960
3961         return 0;
3962
3963 release_ras_con:
3964         amdgpu_release_ras_context(adev);
3965
3966 failed:
3967         amdgpu_vf_error_trans_all(adev);
3968
3969         return r;
3970 }
3971
3972 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
3973 {
3974
3975         /* Clear all CPU mappings pointing to this device */
3976         unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
3977
3978         /* Unmap all mapped bars - Doorbell, registers and VRAM */
3979         amdgpu_device_doorbell_fini(adev);
3980
3981         iounmap(adev->rmmio);
3982         adev->rmmio = NULL;
3983         if (adev->mman.aper_base_kaddr)
3984                 iounmap(adev->mman.aper_base_kaddr);
3985         adev->mman.aper_base_kaddr = NULL;
3986
3987         /* Memory manager related */
3988         if (!adev->gmc.xgmi.connected_to_cpu) {
3989                 arch_phys_wc_del(adev->gmc.vram_mtrr);
3990                 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
3991         }
3992 }
3993
3994 /**
3995  * amdgpu_device_fini_hw - tear down the driver
3996  *
3997  * @adev: amdgpu_device pointer
3998  *
3999  * Tear down the driver info (all asics).
4000  * Called at driver shutdown.
4001  */
4002 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4003 {
4004         dev_info(adev->dev, "amdgpu: finishing device.\n");
4005         flush_delayed_work(&adev->delayed_init_work);
4006         adev->shutdown = true;
4007
4008         /* make sure IB test finished before entering exclusive mode
4009          * to avoid preemption on IB test
4010          * */
4011         if (amdgpu_sriov_vf(adev)) {
4012                 amdgpu_virt_request_full_gpu(adev, false);
4013                 amdgpu_virt_fini_data_exchange(adev);
4014         }
4015
4016         /* disable all interrupts */
4017         amdgpu_irq_disable_all(adev);
4018         if (adev->mode_info.mode_config_initialized){
4019                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
4020                         drm_helper_force_disable_all(adev_to_drm(adev));
4021                 else
4022                         drm_atomic_helper_shutdown(adev_to_drm(adev));
4023         }
4024         amdgpu_fence_driver_hw_fini(adev);
4025
4026         if (adev->mman.initialized)
4027                 drain_workqueue(adev->mman.bdev.wq);
4028
4029         if (adev->pm_sysfs_en)
4030                 amdgpu_pm_sysfs_fini(adev);
4031         if (adev->ucode_sysfs_en)
4032                 amdgpu_ucode_sysfs_fini(adev);
4033         if (adev->psp_sysfs_en)
4034                 amdgpu_psp_sysfs_fini(adev);
4035         sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4036
4037         /* disable ras feature must before hw fini */
4038         amdgpu_ras_pre_fini(adev);
4039
4040         amdgpu_device_ip_fini_early(adev);
4041
4042         amdgpu_irq_fini_hw(adev);
4043
4044         if (adev->mman.initialized)
4045                 ttm_device_clear_dma_mappings(&adev->mman.bdev);
4046
4047         amdgpu_gart_dummy_page_fini(adev);
4048
4049         if (drm_dev_is_unplugged(adev_to_drm(adev)))
4050                 amdgpu_device_unmap_mmio(adev);
4051
4052 }
4053
4054 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4055 {
4056         int idx;
4057
4058         amdgpu_fence_driver_sw_fini(adev);
4059         amdgpu_device_ip_fini(adev);
4060         amdgpu_ucode_release(&adev->firmware.gpu_info_fw);
4061         adev->accel_working = false;
4062         dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4063
4064         amdgpu_reset_fini(adev);
4065
4066         /* free i2c buses */
4067         if (!amdgpu_device_has_dc_support(adev))
4068                 amdgpu_i2c_fini(adev);
4069
4070         if (amdgpu_emu_mode != 1)
4071                 amdgpu_atombios_fini(adev);
4072
4073         kfree(adev->bios);
4074         adev->bios = NULL;
4075         if (amdgpu_device_supports_px(adev_to_drm(adev))) {
4076                 vga_switcheroo_unregister_client(adev->pdev);
4077                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
4078         }
4079         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4080                 vga_client_unregister(adev->pdev);
4081
4082         if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4083
4084                 iounmap(adev->rmmio);
4085                 adev->rmmio = NULL;
4086                 amdgpu_device_doorbell_fini(adev);
4087                 drm_dev_exit(idx);
4088         }
4089
4090         if (IS_ENABLED(CONFIG_PERF_EVENTS))
4091                 amdgpu_pmu_fini(adev);
4092         if (adev->mman.discovery_bin)
4093                 amdgpu_discovery_fini(adev);
4094
4095         amdgpu_reset_put_reset_domain(adev->reset_domain);
4096         adev->reset_domain = NULL;
4097
4098         kfree(adev->pci_state);
4099
4100 }
4101
4102 /**
4103  * amdgpu_device_evict_resources - evict device resources
4104  * @adev: amdgpu device object
4105  *
4106  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4107  * of the vram memory type. Mainly used for evicting device resources
4108  * at suspend time.
4109  *
4110  */
4111 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4112 {
4113         int ret;
4114
4115         /* No need to evict vram on APUs for suspend to ram or s2idle */
4116         if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4117                 return 0;
4118
4119         ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4120         if (ret)
4121                 DRM_WARN("evicting device resources failed\n");
4122         return ret;
4123 }
4124
4125 /*
4126  * Suspend & resume.
4127  */
4128 /**
4129  * amdgpu_device_suspend - initiate device suspend
4130  *
4131  * @dev: drm dev pointer
4132  * @fbcon : notify the fbdev of suspend
4133  *
4134  * Puts the hw in the suspend state (all asics).
4135  * Returns 0 for success or an error on failure.
4136  * Called at driver suspend.
4137  */
4138 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
4139 {
4140         struct amdgpu_device *adev = drm_to_adev(dev);
4141         int r = 0;
4142
4143         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4144                 return 0;
4145
4146         adev->in_suspend = true;
4147
4148         /* Evict the majority of BOs before grabbing the full access */
4149         r = amdgpu_device_evict_resources(adev);
4150         if (r)
4151                 return r;
4152
4153         if (amdgpu_sriov_vf(adev)) {
4154                 amdgpu_virt_fini_data_exchange(adev);
4155                 r = amdgpu_virt_request_full_gpu(adev, false);
4156                 if (r)
4157                         return r;
4158         }
4159
4160         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4161                 DRM_WARN("smart shift update failed\n");
4162
4163         if (fbcon)
4164                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
4165
4166         cancel_delayed_work_sync(&adev->delayed_init_work);
4167
4168         amdgpu_ras_suspend(adev);
4169
4170         amdgpu_device_ip_suspend_phase1(adev);
4171
4172         if (!adev->in_s0ix)
4173                 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4174
4175         r = amdgpu_device_evict_resources(adev);
4176         if (r)
4177                 return r;
4178
4179         amdgpu_fence_driver_hw_fini(adev);
4180
4181         amdgpu_device_ip_suspend_phase2(adev);
4182
4183         if (amdgpu_sriov_vf(adev))
4184                 amdgpu_virt_release_full_gpu(adev, false);
4185
4186         return 0;
4187 }
4188
4189 /**
4190  * amdgpu_device_resume - initiate device resume
4191  *
4192  * @dev: drm dev pointer
4193  * @fbcon : notify the fbdev of resume
4194  *
4195  * Bring the hw back to operating state (all asics).
4196  * Returns 0 for success or an error on failure.
4197  * Called at driver resume.
4198  */
4199 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4200 {
4201         struct amdgpu_device *adev = drm_to_adev(dev);
4202         int r = 0;
4203
4204         if (amdgpu_sriov_vf(adev)) {
4205                 r = amdgpu_virt_request_full_gpu(adev, true);
4206                 if (r)
4207                         return r;
4208         }
4209
4210         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4211                 return 0;
4212
4213         if (adev->in_s0ix)
4214                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4215
4216         /* post card */
4217         if (amdgpu_device_need_post(adev)) {
4218                 r = amdgpu_device_asic_init(adev);
4219                 if (r)
4220                         dev_err(adev->dev, "amdgpu asic init failed\n");
4221         }
4222
4223         r = amdgpu_device_ip_resume(adev);
4224
4225         if (r) {
4226                 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4227                 goto exit;
4228         }
4229         amdgpu_fence_driver_hw_init(adev);
4230
4231         r = amdgpu_device_ip_late_init(adev);
4232         if (r)
4233                 goto exit;
4234
4235         queue_delayed_work(system_wq, &adev->delayed_init_work,
4236                            msecs_to_jiffies(AMDGPU_RESUME_MS));
4237
4238         if (!adev->in_s0ix) {
4239                 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4240                 if (r)
4241                         goto exit;
4242         }
4243
4244 exit:
4245         if (amdgpu_sriov_vf(adev)) {
4246                 amdgpu_virt_init_data_exchange(adev);
4247                 amdgpu_virt_release_full_gpu(adev, true);
4248         }
4249
4250         if (r)
4251                 return r;
4252
4253         /* Make sure IB tests flushed */
4254         flush_delayed_work(&adev->delayed_init_work);
4255
4256         if (fbcon)
4257                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
4258
4259         amdgpu_ras_resume(adev);
4260
4261         if (adev->mode_info.num_crtc) {
4262                 /*
4263                  * Most of the connector probing functions try to acquire runtime pm
4264                  * refs to ensure that the GPU is powered on when connector polling is
4265                  * performed. Since we're calling this from a runtime PM callback,
4266                  * trying to acquire rpm refs will cause us to deadlock.
4267                  *
4268                  * Since we're guaranteed to be holding the rpm lock, it's safe to
4269                  * temporarily disable the rpm helpers so this doesn't deadlock us.
4270                  */
4271 #ifdef CONFIG_PM
4272                 dev->dev->power.disable_depth++;
4273 #endif
4274                 if (!adev->dc_enabled)
4275                         drm_helper_hpd_irq_event(dev);
4276                 else
4277                         drm_kms_helper_hotplug_event(dev);
4278 #ifdef CONFIG_PM
4279                 dev->dev->power.disable_depth--;
4280 #endif
4281         }
4282         adev->in_suspend = false;
4283
4284         if (adev->enable_mes)
4285                 amdgpu_mes_self_test(adev);
4286
4287         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
4288                 DRM_WARN("smart shift update failed\n");
4289
4290         return 0;
4291 }
4292
4293 /**
4294  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4295  *
4296  * @adev: amdgpu_device pointer
4297  *
4298  * The list of all the hardware IPs that make up the asic is walked and
4299  * the check_soft_reset callbacks are run.  check_soft_reset determines
4300  * if the asic is still hung or not.
4301  * Returns true if any of the IPs are still in a hung state, false if not.
4302  */
4303 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4304 {
4305         int i;
4306         bool asic_hang = false;
4307
4308         if (amdgpu_sriov_vf(adev))
4309                 return true;
4310
4311         if (amdgpu_asic_need_full_reset(adev))
4312                 return true;
4313
4314         for (i = 0; i < adev->num_ip_blocks; i++) {
4315                 if (!adev->ip_blocks[i].status.valid)
4316                         continue;
4317                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4318                         adev->ip_blocks[i].status.hang =
4319                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4320                 if (adev->ip_blocks[i].status.hang) {
4321                         dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4322                         asic_hang = true;
4323                 }
4324         }
4325         return asic_hang;
4326 }
4327
4328 /**
4329  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4330  *
4331  * @adev: amdgpu_device pointer
4332  *
4333  * The list of all the hardware IPs that make up the asic is walked and the
4334  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
4335  * handles any IP specific hardware or software state changes that are
4336  * necessary for a soft reset to succeed.
4337  * Returns 0 on success, negative error code on failure.
4338  */
4339 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4340 {
4341         int i, r = 0;
4342
4343         for (i = 0; i < adev->num_ip_blocks; i++) {
4344                 if (!adev->ip_blocks[i].status.valid)
4345                         continue;
4346                 if (adev->ip_blocks[i].status.hang &&
4347                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4348                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4349                         if (r)
4350                                 return r;
4351                 }
4352         }
4353
4354         return 0;
4355 }
4356
4357 /**
4358  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4359  *
4360  * @adev: amdgpu_device pointer
4361  *
4362  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
4363  * reset is necessary to recover.
4364  * Returns true if a full asic reset is required, false if not.
4365  */
4366 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4367 {
4368         int i;
4369
4370         if (amdgpu_asic_need_full_reset(adev))
4371                 return true;
4372
4373         for (i = 0; i < adev->num_ip_blocks; i++) {
4374                 if (!adev->ip_blocks[i].status.valid)
4375                         continue;
4376                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4377                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4378                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4379                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4380                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4381                         if (adev->ip_blocks[i].status.hang) {
4382                                 dev_info(adev->dev, "Some block need full reset!\n");
4383                                 return true;
4384                         }
4385                 }
4386         }
4387         return false;
4388 }
4389
4390 /**
4391  * amdgpu_device_ip_soft_reset - do a soft reset
4392  *
4393  * @adev: amdgpu_device pointer
4394  *
4395  * The list of all the hardware IPs that make up the asic is walked and the
4396  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
4397  * IP specific hardware or software state changes that are necessary to soft
4398  * reset the IP.
4399  * Returns 0 on success, negative error code on failure.
4400  */
4401 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4402 {
4403         int i, r = 0;
4404
4405         for (i = 0; i < adev->num_ip_blocks; i++) {
4406                 if (!adev->ip_blocks[i].status.valid)
4407                         continue;
4408                 if (adev->ip_blocks[i].status.hang &&
4409                     adev->ip_blocks[i].version->funcs->soft_reset) {
4410                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4411                         if (r)
4412                                 return r;
4413                 }
4414         }
4415
4416         return 0;
4417 }
4418
4419 /**
4420  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4421  *
4422  * @adev: amdgpu_device pointer
4423  *
4424  * The list of all the hardware IPs that make up the asic is walked and the
4425  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4426  * handles any IP specific hardware or software state changes that are
4427  * necessary after the IP has been soft reset.
4428  * Returns 0 on success, negative error code on failure.
4429  */
4430 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4431 {
4432         int i, r = 0;
4433
4434         for (i = 0; i < adev->num_ip_blocks; i++) {
4435                 if (!adev->ip_blocks[i].status.valid)
4436                         continue;
4437                 if (adev->ip_blocks[i].status.hang &&
4438                     adev->ip_blocks[i].version->funcs->post_soft_reset)
4439                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4440                 if (r)
4441                         return r;
4442         }
4443
4444         return 0;
4445 }
4446
4447 /**
4448  * amdgpu_device_recover_vram - Recover some VRAM contents
4449  *
4450  * @adev: amdgpu_device pointer
4451  *
4452  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4453  * restore things like GPUVM page tables after a GPU reset where
4454  * the contents of VRAM might be lost.
4455  *
4456  * Returns:
4457  * 0 on success, negative error code on failure.
4458  */
4459 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4460 {
4461         struct dma_fence *fence = NULL, *next = NULL;
4462         struct amdgpu_bo *shadow;
4463         struct amdgpu_bo_vm *vmbo;
4464         long r = 1, tmo;
4465
4466         if (amdgpu_sriov_runtime(adev))
4467                 tmo = msecs_to_jiffies(8000);
4468         else
4469                 tmo = msecs_to_jiffies(100);
4470
4471         dev_info(adev->dev, "recover vram bo from shadow start\n");
4472         mutex_lock(&adev->shadow_list_lock);
4473         list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4474                 shadow = &vmbo->bo;
4475                 /* No need to recover an evicted BO */
4476                 if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4477                     shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4478                     shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4479                         continue;
4480
4481                 r = amdgpu_bo_restore_shadow(shadow, &next);
4482                 if (r)
4483                         break;
4484
4485                 if (fence) {
4486                         tmo = dma_fence_wait_timeout(fence, false, tmo);
4487                         dma_fence_put(fence);
4488                         fence = next;
4489                         if (tmo == 0) {
4490                                 r = -ETIMEDOUT;
4491                                 break;
4492                         } else if (tmo < 0) {
4493                                 r = tmo;
4494                                 break;
4495                         }
4496                 } else {
4497                         fence = next;
4498                 }
4499         }
4500         mutex_unlock(&adev->shadow_list_lock);
4501
4502         if (fence)
4503                 tmo = dma_fence_wait_timeout(fence, false, tmo);
4504         dma_fence_put(fence);
4505
4506         if (r < 0 || tmo <= 0) {
4507                 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4508                 return -EIO;
4509         }
4510
4511         dev_info(adev->dev, "recover vram bo from shadow done\n");
4512         return 0;
4513 }
4514
4515
4516 /**
4517  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4518  *
4519  * @adev: amdgpu_device pointer
4520  * @from_hypervisor: request from hypervisor
4521  *
4522  * do VF FLR and reinitialize Asic
4523  * return 0 means succeeded otherwise failed
4524  */
4525 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4526                                      bool from_hypervisor)
4527 {
4528         int r;
4529         struct amdgpu_hive_info *hive = NULL;
4530         int retry_limit = 0;
4531
4532 retry:
4533         amdgpu_amdkfd_pre_reset(adev);
4534
4535         if (from_hypervisor)
4536                 r = amdgpu_virt_request_full_gpu(adev, true);
4537         else
4538                 r = amdgpu_virt_reset_gpu(adev);
4539         if (r)
4540                 return r;
4541
4542         /* Resume IP prior to SMC */
4543         r = amdgpu_device_ip_reinit_early_sriov(adev);
4544         if (r)
4545                 goto error;
4546
4547         amdgpu_virt_init_data_exchange(adev);
4548
4549         r = amdgpu_device_fw_loading(adev);
4550         if (r)
4551                 return r;
4552
4553         /* now we are okay to resume SMC/CP/SDMA */
4554         r = amdgpu_device_ip_reinit_late_sriov(adev);
4555         if (r)
4556                 goto error;
4557
4558         hive = amdgpu_get_xgmi_hive(adev);
4559         /* Update PSP FW topology after reset */
4560         if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
4561                 r = amdgpu_xgmi_update_topology(hive, adev);
4562
4563         if (hive)
4564                 amdgpu_put_xgmi_hive(hive);
4565
4566         if (!r) {
4567                 amdgpu_irq_gpu_reset_resume_helper(adev);
4568                 r = amdgpu_ib_ring_tests(adev);
4569
4570                 amdgpu_amdkfd_post_reset(adev);
4571         }
4572
4573 error:
4574         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4575                 amdgpu_inc_vram_lost(adev);
4576                 r = amdgpu_device_recover_vram(adev);
4577         }
4578         amdgpu_virt_release_full_gpu(adev, true);
4579
4580         if (AMDGPU_RETRY_SRIOV_RESET(r)) {
4581                 if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) {
4582                         retry_limit++;
4583                         goto retry;
4584                 } else
4585                         DRM_ERROR("GPU reset retry is beyond the retry limit\n");
4586         }
4587
4588         return r;
4589 }
4590
4591 /**
4592  * amdgpu_device_has_job_running - check if there is any job in mirror list
4593  *
4594  * @adev: amdgpu_device pointer
4595  *
4596  * check if there is any job in mirror list
4597  */
4598 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4599 {
4600         int i;
4601         struct drm_sched_job *job;
4602
4603         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4604                 struct amdgpu_ring *ring = adev->rings[i];
4605
4606                 if (!ring || !ring->sched.thread)
4607                         continue;
4608
4609                 spin_lock(&ring->sched.job_list_lock);
4610                 job = list_first_entry_or_null(&ring->sched.pending_list,
4611                                                struct drm_sched_job, list);
4612                 spin_unlock(&ring->sched.job_list_lock);
4613                 if (job)
4614                         return true;
4615         }
4616         return false;
4617 }
4618
4619 /**
4620  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4621  *
4622  * @adev: amdgpu_device pointer
4623  *
4624  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4625  * a hung GPU.
4626  */
4627 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4628 {
4629
4630         if (amdgpu_gpu_recovery == 0)
4631                 goto disabled;
4632
4633         /* Skip soft reset check in fatal error mode */
4634         if (!amdgpu_ras_is_poison_mode_supported(adev))
4635                 return true;
4636
4637         if (amdgpu_sriov_vf(adev))
4638                 return true;
4639
4640         if (amdgpu_gpu_recovery == -1) {
4641                 switch (adev->asic_type) {
4642 #ifdef CONFIG_DRM_AMDGPU_SI
4643                 case CHIP_VERDE:
4644                 case CHIP_TAHITI:
4645                 case CHIP_PITCAIRN:
4646                 case CHIP_OLAND:
4647                 case CHIP_HAINAN:
4648 #endif
4649 #ifdef CONFIG_DRM_AMDGPU_CIK
4650                 case CHIP_KAVERI:
4651                 case CHIP_KABINI:
4652                 case CHIP_MULLINS:
4653 #endif
4654                 case CHIP_CARRIZO:
4655                 case CHIP_STONEY:
4656                 case CHIP_CYAN_SKILLFISH:
4657                         goto disabled;
4658                 default:
4659                         break;
4660                 }
4661         }
4662
4663         return true;
4664
4665 disabled:
4666                 dev_info(adev->dev, "GPU recovery disabled.\n");
4667                 return false;
4668 }
4669
4670 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4671 {
4672         u32 i;
4673         int ret = 0;
4674
4675         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4676
4677         dev_info(adev->dev, "GPU mode1 reset\n");
4678
4679         /* disable BM */
4680         pci_clear_master(adev->pdev);
4681
4682         amdgpu_device_cache_pci_state(adev->pdev);
4683
4684         if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4685                 dev_info(adev->dev, "GPU smu mode1 reset\n");
4686                 ret = amdgpu_dpm_mode1_reset(adev);
4687         } else {
4688                 dev_info(adev->dev, "GPU psp mode1 reset\n");
4689                 ret = psp_gpu_reset(adev);
4690         }
4691
4692         if (ret)
4693                 dev_err(adev->dev, "GPU mode1 reset failed\n");
4694
4695         amdgpu_device_load_pci_state(adev->pdev);
4696
4697         /* wait for asic to come out of reset */
4698         for (i = 0; i < adev->usec_timeout; i++) {
4699                 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4700
4701                 if (memsize != 0xffffffff)
4702                         break;
4703                 udelay(1);
4704         }
4705
4706         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4707         return ret;
4708 }
4709
4710 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4711                                  struct amdgpu_reset_context *reset_context)
4712 {
4713         int i, r = 0;
4714         struct amdgpu_job *job = NULL;
4715         bool need_full_reset =
4716                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4717
4718         if (reset_context->reset_req_dev == adev)
4719                 job = reset_context->job;
4720
4721         if (amdgpu_sriov_vf(adev)) {
4722                 /* stop the data exchange thread */
4723                 amdgpu_virt_fini_data_exchange(adev);
4724         }
4725
4726         amdgpu_fence_driver_isr_toggle(adev, true);
4727
4728         /* block all schedulers and reset given job's ring */
4729         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4730                 struct amdgpu_ring *ring = adev->rings[i];
4731
4732                 if (!ring || !ring->sched.thread)
4733                         continue;
4734
4735                 /*clear job fence from fence drv to avoid force_completion
4736                  *leave NULL and vm flush fence in fence drv */
4737                 amdgpu_fence_driver_clear_job_fences(ring);
4738
4739                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
4740                 amdgpu_fence_driver_force_completion(ring);
4741         }
4742
4743         amdgpu_fence_driver_isr_toggle(adev, false);
4744
4745         if (job && job->vm)
4746                 drm_sched_increase_karma(&job->base);
4747
4748         r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
4749         /* If reset handler not implemented, continue; otherwise return */
4750         if (r == -ENOSYS)
4751                 r = 0;
4752         else
4753                 return r;
4754
4755         /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
4756         if (!amdgpu_sriov_vf(adev)) {
4757
4758                 if (!need_full_reset)
4759                         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
4760
4761                 if (!need_full_reset && amdgpu_gpu_recovery &&
4762                     amdgpu_device_ip_check_soft_reset(adev)) {
4763                         amdgpu_device_ip_pre_soft_reset(adev);
4764                         r = amdgpu_device_ip_soft_reset(adev);
4765                         amdgpu_device_ip_post_soft_reset(adev);
4766                         if (r || amdgpu_device_ip_check_soft_reset(adev)) {
4767                                 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
4768                                 need_full_reset = true;
4769                         }
4770                 }
4771
4772                 if (need_full_reset)
4773                         r = amdgpu_device_ip_suspend(adev);
4774                 if (need_full_reset)
4775                         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4776                 else
4777                         clear_bit(AMDGPU_NEED_FULL_RESET,
4778                                   &reset_context->flags);
4779         }
4780
4781         return r;
4782 }
4783
4784 static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev)
4785 {
4786         int i;
4787
4788         lockdep_assert_held(&adev->reset_domain->sem);
4789
4790         for (i = 0; i < adev->num_regs; i++) {
4791                 adev->reset_dump_reg_value[i] = RREG32(adev->reset_dump_reg_list[i]);
4792                 trace_amdgpu_reset_reg_dumps(adev->reset_dump_reg_list[i],
4793                                              adev->reset_dump_reg_value[i]);
4794         }
4795
4796         return 0;
4797 }
4798
4799 #ifdef CONFIG_DEV_COREDUMP
4800 static ssize_t amdgpu_devcoredump_read(char *buffer, loff_t offset,
4801                 size_t count, void *data, size_t datalen)
4802 {
4803         struct drm_printer p;
4804         struct amdgpu_device *adev = data;
4805         struct drm_print_iterator iter;
4806         int i;
4807
4808         iter.data = buffer;
4809         iter.offset = 0;
4810         iter.start = offset;
4811         iter.remain = count;
4812
4813         p = drm_coredump_printer(&iter);
4814
4815         drm_printf(&p, "**** AMDGPU Device Coredump ****\n");
4816         drm_printf(&p, "kernel: " UTS_RELEASE "\n");
4817         drm_printf(&p, "module: " KBUILD_MODNAME "\n");
4818         drm_printf(&p, "time: %lld.%09ld\n", adev->reset_time.tv_sec, adev->reset_time.tv_nsec);
4819         if (adev->reset_task_info.pid)
4820                 drm_printf(&p, "process_name: %s PID: %d\n",
4821                            adev->reset_task_info.process_name,
4822                            adev->reset_task_info.pid);
4823
4824         if (adev->reset_vram_lost)
4825                 drm_printf(&p, "VRAM is lost due to GPU reset!\n");
4826         if (adev->num_regs) {
4827                 drm_printf(&p, "AMDGPU register dumps:\nOffset:     Value:\n");
4828
4829                 for (i = 0; i < adev->num_regs; i++)
4830                         drm_printf(&p, "0x%08x: 0x%08x\n",
4831                                    adev->reset_dump_reg_list[i],
4832                                    adev->reset_dump_reg_value[i]);
4833         }
4834
4835         return count - iter.remain;
4836 }
4837
4838 static void amdgpu_devcoredump_free(void *data)
4839 {
4840 }
4841
4842 static void amdgpu_reset_capture_coredumpm(struct amdgpu_device *adev)
4843 {
4844         struct drm_device *dev = adev_to_drm(adev);
4845
4846         ktime_get_ts64(&adev->reset_time);
4847         dev_coredumpm(dev->dev, THIS_MODULE, adev, 0, GFP_KERNEL,
4848                       amdgpu_devcoredump_read, amdgpu_devcoredump_free);
4849 }
4850 #endif
4851
4852 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
4853                          struct amdgpu_reset_context *reset_context)
4854 {
4855         struct amdgpu_device *tmp_adev = NULL;
4856         bool need_full_reset, skip_hw_reset, vram_lost = false;
4857         int r = 0;
4858         bool gpu_reset_for_dev_remove = 0;
4859
4860         /* Try reset handler method first */
4861         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
4862                                     reset_list);
4863         amdgpu_reset_reg_dumps(tmp_adev);
4864
4865         reset_context->reset_device_list = device_list_handle;
4866         r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
4867         /* If reset handler not implemented, continue; otherwise return */
4868         if (r == -ENOSYS)
4869                 r = 0;
4870         else
4871                 return r;
4872
4873         /* Reset handler not implemented, use the default method */
4874         need_full_reset =
4875                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4876         skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
4877
4878         gpu_reset_for_dev_remove =
4879                 test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
4880                         test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4881
4882         /*
4883          * ASIC reset has to be done on all XGMI hive nodes ASAP
4884          * to allow proper links negotiation in FW (within 1 sec)
4885          */
4886         if (!skip_hw_reset && need_full_reset) {
4887                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4888                         /* For XGMI run all resets in parallel to speed up the process */
4889                         if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4890                                 tmp_adev->gmc.xgmi.pending_reset = false;
4891                                 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
4892                                         r = -EALREADY;
4893                         } else
4894                                 r = amdgpu_asic_reset(tmp_adev);
4895
4896                         if (r) {
4897                                 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
4898                                          r, adev_to_drm(tmp_adev)->unique);
4899                                 break;
4900                         }
4901                 }
4902
4903                 /* For XGMI wait for all resets to complete before proceed */
4904                 if (!r) {
4905                         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4906                                 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4907                                         flush_work(&tmp_adev->xgmi_reset_work);
4908                                         r = tmp_adev->asic_reset_res;
4909                                         if (r)
4910                                                 break;
4911                                 }
4912                         }
4913                 }
4914         }
4915
4916         if (!r && amdgpu_ras_intr_triggered()) {
4917                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4918                         if (tmp_adev->mmhub.ras && tmp_adev->mmhub.ras->ras_block.hw_ops &&
4919                             tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
4920                                 tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(tmp_adev);
4921                 }
4922
4923                 amdgpu_ras_intr_cleared();
4924         }
4925
4926         /* Since the mode1 reset affects base ip blocks, the
4927          * phase1 ip blocks need to be resumed. Otherwise there
4928          * will be a BIOS signature error and the psp bootloader
4929          * can't load kdb on the next amdgpu install.
4930          */
4931         if (gpu_reset_for_dev_remove) {
4932                 list_for_each_entry(tmp_adev, device_list_handle, reset_list)
4933                         amdgpu_device_ip_resume_phase1(tmp_adev);
4934
4935                 goto end;
4936         }
4937
4938         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4939                 if (need_full_reset) {
4940                         /* post card */
4941                         r = amdgpu_device_asic_init(tmp_adev);
4942                         if (r) {
4943                                 dev_warn(tmp_adev->dev, "asic atom init failed!");
4944                         } else {
4945                                 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
4946                                 r = amdgpu_amdkfd_resume_iommu(tmp_adev);
4947                                 if (r)
4948                                         goto out;
4949
4950                                 r = amdgpu_device_ip_resume_phase1(tmp_adev);
4951                                 if (r)
4952                                         goto out;
4953
4954                                 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
4955 #ifdef CONFIG_DEV_COREDUMP
4956                                 tmp_adev->reset_vram_lost = vram_lost;
4957                                 memset(&tmp_adev->reset_task_info, 0,
4958                                                 sizeof(tmp_adev->reset_task_info));
4959                                 if (reset_context->job && reset_context->job->vm)
4960                                         tmp_adev->reset_task_info =
4961                                                 reset_context->job->vm->task_info;
4962                                 amdgpu_reset_capture_coredumpm(tmp_adev);
4963 #endif
4964                                 if (vram_lost) {
4965                                         DRM_INFO("VRAM is lost due to GPU reset!\n");
4966                                         amdgpu_inc_vram_lost(tmp_adev);
4967                                 }
4968
4969                                 r = amdgpu_device_fw_loading(tmp_adev);
4970                                 if (r)
4971                                         return r;
4972
4973                                 r = amdgpu_device_ip_resume_phase2(tmp_adev);
4974                                 if (r)
4975                                         goto out;
4976
4977                                 if (vram_lost)
4978                                         amdgpu_device_fill_reset_magic(tmp_adev);
4979
4980                                 /*
4981                                  * Add this ASIC as tracked as reset was already
4982                                  * complete successfully.
4983                                  */
4984                                 amdgpu_register_gpu_instance(tmp_adev);
4985
4986                                 if (!reset_context->hive &&
4987                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4988                                         amdgpu_xgmi_add_device(tmp_adev);
4989
4990                                 r = amdgpu_device_ip_late_init(tmp_adev);
4991                                 if (r)
4992                                         goto out;
4993
4994                                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
4995
4996                                 /*
4997                                  * The GPU enters bad state once faulty pages
4998                                  * by ECC has reached the threshold, and ras
4999                                  * recovery is scheduled next. So add one check
5000                                  * here to break recovery if it indeed exceeds
5001                                  * bad page threshold, and remind user to
5002                                  * retire this GPU or setting one bigger
5003                                  * bad_page_threshold value to fix this once
5004                                  * probing driver again.
5005                                  */
5006                                 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
5007                                         /* must succeed. */
5008                                         amdgpu_ras_resume(tmp_adev);
5009                                 } else {
5010                                         r = -EINVAL;
5011                                         goto out;
5012                                 }
5013
5014                                 /* Update PSP FW topology after reset */
5015                                 if (reset_context->hive &&
5016                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5017                                         r = amdgpu_xgmi_update_topology(
5018                                                 reset_context->hive, tmp_adev);
5019                         }
5020                 }
5021
5022 out:
5023                 if (!r) {
5024                         amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5025                         r = amdgpu_ib_ring_tests(tmp_adev);
5026                         if (r) {
5027                                 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5028                                 need_full_reset = true;
5029                                 r = -EAGAIN;
5030                                 goto end;
5031                         }
5032                 }
5033
5034                 if (!r)
5035                         r = amdgpu_device_recover_vram(tmp_adev);
5036                 else
5037                         tmp_adev->asic_reset_res = r;
5038         }
5039
5040 end:
5041         if (need_full_reset)
5042                 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5043         else
5044                 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5045         return r;
5046 }
5047
5048 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5049 {
5050
5051         switch (amdgpu_asic_reset_method(adev)) {
5052         case AMD_RESET_METHOD_MODE1:
5053                 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5054                 break;
5055         case AMD_RESET_METHOD_MODE2:
5056                 adev->mp1_state = PP_MP1_STATE_RESET;
5057                 break;
5058         default:
5059                 adev->mp1_state = PP_MP1_STATE_NONE;
5060                 break;
5061         }
5062 }
5063
5064 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5065 {
5066         amdgpu_vf_error_trans_all(adev);
5067         adev->mp1_state = PP_MP1_STATE_NONE;
5068 }
5069
5070 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5071 {
5072         struct pci_dev *p = NULL;
5073
5074         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5075                         adev->pdev->bus->number, 1);
5076         if (p) {
5077                 pm_runtime_enable(&(p->dev));
5078                 pm_runtime_resume(&(p->dev));
5079         }
5080
5081         pci_dev_put(p);
5082 }
5083
5084 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5085 {
5086         enum amd_reset_method reset_method;
5087         struct pci_dev *p = NULL;
5088         u64 expires;
5089
5090         /*
5091          * For now, only BACO and mode1 reset are confirmed
5092          * to suffer the audio issue without proper suspended.
5093          */
5094         reset_method = amdgpu_asic_reset_method(adev);
5095         if ((reset_method != AMD_RESET_METHOD_BACO) &&
5096              (reset_method != AMD_RESET_METHOD_MODE1))
5097                 return -EINVAL;
5098
5099         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5100                         adev->pdev->bus->number, 1);
5101         if (!p)
5102                 return -ENODEV;
5103
5104         expires = pm_runtime_autosuspend_expiration(&(p->dev));
5105         if (!expires)
5106                 /*
5107                  * If we cannot get the audio device autosuspend delay,
5108                  * a fixed 4S interval will be used. Considering 3S is
5109                  * the audio controller default autosuspend delay setting.
5110                  * 4S used here is guaranteed to cover that.
5111                  */
5112                 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5113
5114         while (!pm_runtime_status_suspended(&(p->dev))) {
5115                 if (!pm_runtime_suspend(&(p->dev)))
5116                         break;
5117
5118                 if (expires < ktime_get_mono_fast_ns()) {
5119                         dev_warn(adev->dev, "failed to suspend display audio\n");
5120                         pci_dev_put(p);
5121                         /* TODO: abort the succeeding gpu reset? */
5122                         return -ETIMEDOUT;
5123                 }
5124         }
5125
5126         pm_runtime_disable(&(p->dev));
5127
5128         pci_dev_put(p);
5129         return 0;
5130 }
5131
5132 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5133 {
5134         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5135
5136 #if defined(CONFIG_DEBUG_FS)
5137         if (!amdgpu_sriov_vf(adev))
5138                 cancel_work(&adev->reset_work);
5139 #endif
5140
5141         if (adev->kfd.dev)
5142                 cancel_work(&adev->kfd.reset_work);
5143
5144         if (amdgpu_sriov_vf(adev))
5145                 cancel_work(&adev->virt.flr_work);
5146
5147         if (con && adev->ras_enabled)
5148                 cancel_work(&con->recovery_work);
5149
5150 }
5151
5152 /**
5153  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5154  *
5155  * @adev: amdgpu_device pointer
5156  * @job: which job trigger hang
5157  *
5158  * Attempt to reset the GPU if it has hung (all asics).
5159  * Attempt to do soft-reset or full-reset and reinitialize Asic
5160  * Returns 0 for success or an error on failure.
5161  */
5162
5163 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5164                               struct amdgpu_job *job,
5165                               struct amdgpu_reset_context *reset_context)
5166 {
5167         struct list_head device_list, *device_list_handle =  NULL;
5168         bool job_signaled = false;
5169         struct amdgpu_hive_info *hive = NULL;
5170         struct amdgpu_device *tmp_adev = NULL;
5171         int i, r = 0;
5172         bool need_emergency_restart = false;
5173         bool audio_suspended = false;
5174         bool gpu_reset_for_dev_remove = false;
5175
5176         gpu_reset_for_dev_remove =
5177                         test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5178                                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5179
5180         /*
5181          * Special case: RAS triggered and full reset isn't supported
5182          */
5183         need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5184
5185         /*
5186          * Flush RAM to disk so that after reboot
5187          * the user can read log and see why the system rebooted.
5188          */
5189         if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {
5190                 DRM_WARN("Emergency reboot.");
5191
5192                 ksys_sync_helper();
5193                 emergency_restart();
5194         }
5195
5196         dev_info(adev->dev, "GPU %s begin!\n",
5197                 need_emergency_restart ? "jobs stop":"reset");
5198
5199         if (!amdgpu_sriov_vf(adev))
5200                 hive = amdgpu_get_xgmi_hive(adev);
5201         if (hive)
5202                 mutex_lock(&hive->hive_lock);
5203
5204         reset_context->job = job;
5205         reset_context->hive = hive;
5206         /*
5207          * Build list of devices to reset.
5208          * In case we are in XGMI hive mode, resort the device list
5209          * to put adev in the 1st position.
5210          */
5211         INIT_LIST_HEAD(&device_list);
5212         if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5213                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5214                         list_add_tail(&tmp_adev->reset_list, &device_list);
5215                         if (gpu_reset_for_dev_remove && adev->shutdown)
5216                                 tmp_adev->shutdown = true;
5217                 }
5218                 if (!list_is_first(&adev->reset_list, &device_list))
5219                         list_rotate_to_front(&adev->reset_list, &device_list);
5220                 device_list_handle = &device_list;
5221         } else {
5222                 list_add_tail(&adev->reset_list, &device_list);
5223                 device_list_handle = &device_list;
5224         }
5225
5226         /* We need to lock reset domain only once both for XGMI and single device */
5227         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5228                                     reset_list);
5229         amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5230
5231         /* block all schedulers and reset given job's ring */
5232         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5233
5234                 amdgpu_device_set_mp1_state(tmp_adev);
5235
5236                 /*
5237                  * Try to put the audio codec into suspend state
5238                  * before gpu reset started.
5239                  *
5240                  * Due to the power domain of the graphics device
5241                  * is shared with AZ power domain. Without this,
5242                  * we may change the audio hardware from behind
5243                  * the audio driver's back. That will trigger
5244                  * some audio codec errors.
5245                  */
5246                 if (!amdgpu_device_suspend_display_audio(tmp_adev))
5247                         audio_suspended = true;
5248
5249                 amdgpu_ras_set_error_query_ready(tmp_adev, false);
5250
5251                 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5252
5253                 if (!amdgpu_sriov_vf(tmp_adev))
5254                         amdgpu_amdkfd_pre_reset(tmp_adev);
5255
5256                 /*
5257                  * Mark these ASICs to be reseted as untracked first
5258                  * And add them back after reset completed
5259                  */
5260                 amdgpu_unregister_gpu_instance(tmp_adev);
5261
5262                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true);
5263
5264                 /* disable ras on ALL IPs */
5265                 if (!need_emergency_restart &&
5266                       amdgpu_device_ip_need_full_reset(tmp_adev))
5267                         amdgpu_ras_suspend(tmp_adev);
5268
5269                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5270                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5271
5272                         if (!ring || !ring->sched.thread)
5273                                 continue;
5274
5275                         drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5276
5277                         if (need_emergency_restart)
5278                                 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5279                 }
5280                 atomic_inc(&tmp_adev->gpu_reset_counter);
5281         }
5282
5283         if (need_emergency_restart)
5284                 goto skip_sched_resume;
5285
5286         /*
5287          * Must check guilty signal here since after this point all old
5288          * HW fences are force signaled.
5289          *
5290          * job->base holds a reference to parent fence
5291          */
5292         if (job && dma_fence_is_signaled(&job->hw_fence)) {
5293                 job_signaled = true;
5294                 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5295                 goto skip_hw_reset;
5296         }
5297
5298 retry:  /* Rest of adevs pre asic reset from XGMI hive. */
5299         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5300                 if (gpu_reset_for_dev_remove) {
5301                         /* Workaroud for ASICs need to disable SMC first */
5302                         amdgpu_device_smu_fini_early(tmp_adev);
5303                 }
5304                 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5305                 /*TODO Should we stop ?*/
5306                 if (r) {
5307                         dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5308                                   r, adev_to_drm(tmp_adev)->unique);
5309                         tmp_adev->asic_reset_res = r;
5310                 }
5311
5312                 /*
5313                  * Drop all pending non scheduler resets. Scheduler resets
5314                  * were already dropped during drm_sched_stop
5315                  */
5316                 amdgpu_device_stop_pending_resets(tmp_adev);
5317         }
5318
5319         /* Actual ASIC resets if needed.*/
5320         /* Host driver will handle XGMI hive reset for SRIOV */
5321         if (amdgpu_sriov_vf(adev)) {
5322                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
5323                 if (r)
5324                         adev->asic_reset_res = r;
5325
5326                 /* Aldebaran supports ras in SRIOV, so need resume ras during reset */
5327                 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
5328                         amdgpu_ras_resume(adev);
5329         } else {
5330                 r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5331                 if (r && r == -EAGAIN)
5332                         goto retry;
5333
5334                 if (!r && gpu_reset_for_dev_remove)
5335                         goto recover_end;
5336         }
5337
5338 skip_hw_reset:
5339
5340         /* Post ASIC reset for all devs .*/
5341         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5342
5343                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5344                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5345
5346                         if (!ring || !ring->sched.thread)
5347                                 continue;
5348
5349                         drm_sched_start(&ring->sched, true);
5350                 }
5351
5352                 if (adev->enable_mes && adev->ip_versions[GC_HWIP][0] != IP_VERSION(11, 0, 3))
5353                         amdgpu_mes_self_test(tmp_adev);
5354
5355                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) {
5356                         drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5357                 }
5358
5359                 if (tmp_adev->asic_reset_res)
5360                         r = tmp_adev->asic_reset_res;
5361
5362                 tmp_adev->asic_reset_res = 0;
5363
5364                 if (r) {
5365                         /* bad news, how to tell it to userspace ? */
5366                         dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5367                         amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5368                 } else {
5369                         dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5370                         if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5371                                 DRM_WARN("smart shift update failed\n");
5372                 }
5373         }
5374
5375 skip_sched_resume:
5376         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5377                 /* unlock kfd: SRIOV would do it separately */
5378                 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5379                         amdgpu_amdkfd_post_reset(tmp_adev);
5380
5381                 /* kfd_post_reset will do nothing if kfd device is not initialized,
5382                  * need to bring up kfd here if it's not be initialized before
5383                  */
5384                 if (!adev->kfd.init_complete)
5385                         amdgpu_amdkfd_device_init(adev);
5386
5387                 if (audio_suspended)
5388                         amdgpu_device_resume_display_audio(tmp_adev);
5389
5390                 amdgpu_device_unset_mp1_state(tmp_adev);
5391
5392                 amdgpu_ras_set_error_query_ready(tmp_adev, true);
5393         }
5394
5395 recover_end:
5396         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5397                                             reset_list);
5398         amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
5399
5400         if (hive) {
5401                 mutex_unlock(&hive->hive_lock);
5402                 amdgpu_put_xgmi_hive(hive);
5403         }
5404
5405         if (r)
5406                 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5407
5408         atomic_set(&adev->reset_domain->reset_res, r);
5409         return r;
5410 }
5411
5412 /**
5413  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5414  *
5415  * @adev: amdgpu_device pointer
5416  *
5417  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5418  * and lanes) of the slot the device is in. Handles APUs and
5419  * virtualized environments where PCIE config space may not be available.
5420  */
5421 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5422 {
5423         struct pci_dev *pdev;
5424         enum pci_bus_speed speed_cap, platform_speed_cap;
5425         enum pcie_link_width platform_link_width;
5426
5427         if (amdgpu_pcie_gen_cap)
5428                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5429
5430         if (amdgpu_pcie_lane_cap)
5431                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5432
5433         /* covers APUs as well */
5434         if (pci_is_root_bus(adev->pdev->bus)) {
5435                 if (adev->pm.pcie_gen_mask == 0)
5436                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5437                 if (adev->pm.pcie_mlw_mask == 0)
5438                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5439                 return;
5440         }
5441
5442         if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5443                 return;
5444
5445         pcie_bandwidth_available(adev->pdev, NULL,
5446                                  &platform_speed_cap, &platform_link_width);
5447
5448         if (adev->pm.pcie_gen_mask == 0) {
5449                 /* asic caps */
5450                 pdev = adev->pdev;
5451                 speed_cap = pcie_get_speed_cap(pdev);
5452                 if (speed_cap == PCI_SPEED_UNKNOWN) {
5453                         adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5454                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5455                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5456                 } else {
5457                         if (speed_cap == PCIE_SPEED_32_0GT)
5458                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5459                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5460                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5461                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5462                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5463                         else if (speed_cap == PCIE_SPEED_16_0GT)
5464                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5465                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5466                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5467                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5468                         else if (speed_cap == PCIE_SPEED_8_0GT)
5469                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5470                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5471                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5472                         else if (speed_cap == PCIE_SPEED_5_0GT)
5473                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5474                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5475                         else
5476                                 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5477                 }
5478                 /* platform caps */
5479                 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5480                         adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5481                                                    CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5482                 } else {
5483                         if (platform_speed_cap == PCIE_SPEED_32_0GT)
5484                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5485                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5486                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5487                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5488                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5489                         else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5490                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5491                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5492                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5493                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5494                         else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5495                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5496                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5497                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5498                         else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5499                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5500                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5501                         else
5502                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5503
5504                 }
5505         }
5506         if (adev->pm.pcie_mlw_mask == 0) {
5507                 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5508                         adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5509                 } else {
5510                         switch (platform_link_width) {
5511                         case PCIE_LNK_X32:
5512                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5513                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5514                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5515                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5516                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5517                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5518                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5519                                 break;
5520                         case PCIE_LNK_X16:
5521                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5522                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5523                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5524                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5525                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5526                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5527                                 break;
5528                         case PCIE_LNK_X12:
5529                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5530                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5531                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5532                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5533                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5534                                 break;
5535                         case PCIE_LNK_X8:
5536                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5537                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5538                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5539                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5540                                 break;
5541                         case PCIE_LNK_X4:
5542                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5543                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5544                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5545                                 break;
5546                         case PCIE_LNK_X2:
5547                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5548                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5549                                 break;
5550                         case PCIE_LNK_X1:
5551                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5552                                 break;
5553                         default:
5554                                 break;
5555                         }
5556                 }
5557         }
5558 }
5559
5560 /**
5561  * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
5562  *
5563  * @adev: amdgpu_device pointer
5564  * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
5565  *
5566  * Return true if @peer_adev can access (DMA) @adev through the PCIe
5567  * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
5568  * @peer_adev.
5569  */
5570 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
5571                                       struct amdgpu_device *peer_adev)
5572 {
5573 #ifdef CONFIG_HSA_AMD_P2P
5574         uint64_t address_mask = peer_adev->dev->dma_mask ?
5575                 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
5576         resource_size_t aper_limit =
5577                 adev->gmc.aper_base + adev->gmc.aper_size - 1;
5578         bool p2p_access =
5579                 !adev->gmc.xgmi.connected_to_cpu &&
5580                 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
5581
5582         return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
5583                 adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
5584                 !(adev->gmc.aper_base & address_mask ||
5585                   aper_limit & address_mask));
5586 #else
5587         return false;
5588 #endif
5589 }
5590
5591 int amdgpu_device_baco_enter(struct drm_device *dev)
5592 {
5593         struct amdgpu_device *adev = drm_to_adev(dev);
5594         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5595
5596         if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5597                 return -ENOTSUPP;
5598
5599         if (ras && adev->ras_enabled &&
5600             adev->nbio.funcs->enable_doorbell_interrupt)
5601                 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5602
5603         return amdgpu_dpm_baco_enter(adev);
5604 }
5605
5606 int amdgpu_device_baco_exit(struct drm_device *dev)
5607 {
5608         struct amdgpu_device *adev = drm_to_adev(dev);
5609         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5610         int ret = 0;
5611
5612         if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5613                 return -ENOTSUPP;
5614
5615         ret = amdgpu_dpm_baco_exit(adev);
5616         if (ret)
5617                 return ret;
5618
5619         if (ras && adev->ras_enabled &&
5620             adev->nbio.funcs->enable_doorbell_interrupt)
5621                 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5622
5623         if (amdgpu_passthrough(adev) &&
5624             adev->nbio.funcs->clear_doorbell_interrupt)
5625                 adev->nbio.funcs->clear_doorbell_interrupt(adev);
5626
5627         return 0;
5628 }
5629
5630 /**
5631  * amdgpu_pci_error_detected - Called when a PCI error is detected.
5632  * @pdev: PCI device struct
5633  * @state: PCI channel state
5634  *
5635  * Description: Called when a PCI error is detected.
5636  *
5637  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5638  */
5639 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5640 {
5641         struct drm_device *dev = pci_get_drvdata(pdev);
5642         struct amdgpu_device *adev = drm_to_adev(dev);
5643         int i;
5644
5645         DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5646
5647         if (adev->gmc.xgmi.num_physical_nodes > 1) {
5648                 DRM_WARN("No support for XGMI hive yet...");
5649                 return PCI_ERS_RESULT_DISCONNECT;
5650         }
5651
5652         adev->pci_channel_state = state;
5653
5654         switch (state) {
5655         case pci_channel_io_normal:
5656                 return PCI_ERS_RESULT_CAN_RECOVER;
5657         /* Fatal error, prepare for slot reset */
5658         case pci_channel_io_frozen:
5659                 /*
5660                  * Locking adev->reset_domain->sem will prevent any external access
5661                  * to GPU during PCI error recovery
5662                  */
5663                 amdgpu_device_lock_reset_domain(adev->reset_domain);
5664                 amdgpu_device_set_mp1_state(adev);
5665
5666                 /*
5667                  * Block any work scheduling as we do for regular GPU reset
5668                  * for the duration of the recovery
5669                  */
5670                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5671                         struct amdgpu_ring *ring = adev->rings[i];
5672
5673                         if (!ring || !ring->sched.thread)
5674                                 continue;
5675
5676                         drm_sched_stop(&ring->sched, NULL);
5677                 }
5678                 atomic_inc(&adev->gpu_reset_counter);
5679                 return PCI_ERS_RESULT_NEED_RESET;
5680         case pci_channel_io_perm_failure:
5681                 /* Permanent error, prepare for device removal */
5682                 return PCI_ERS_RESULT_DISCONNECT;
5683         }
5684
5685         return PCI_ERS_RESULT_NEED_RESET;
5686 }
5687
5688 /**
5689  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5690  * @pdev: pointer to PCI device
5691  */
5692 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5693 {
5694
5695         DRM_INFO("PCI error: mmio enabled callback!!\n");
5696
5697         /* TODO - dump whatever for debugging purposes */
5698
5699         /* This called only if amdgpu_pci_error_detected returns
5700          * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5701          * works, no need to reset slot.
5702          */
5703
5704         return PCI_ERS_RESULT_RECOVERED;
5705 }
5706
5707 /**
5708  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5709  * @pdev: PCI device struct
5710  *
5711  * Description: This routine is called by the pci error recovery
5712  * code after the PCI slot has been reset, just before we
5713  * should resume normal operations.
5714  */
5715 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5716 {
5717         struct drm_device *dev = pci_get_drvdata(pdev);
5718         struct amdgpu_device *adev = drm_to_adev(dev);
5719         int r, i;
5720         struct amdgpu_reset_context reset_context;
5721         u32 memsize;
5722         struct list_head device_list;
5723
5724         DRM_INFO("PCI error: slot reset callback!!\n");
5725
5726         memset(&reset_context, 0, sizeof(reset_context));
5727
5728         INIT_LIST_HEAD(&device_list);
5729         list_add_tail(&adev->reset_list, &device_list);
5730
5731         /* wait for asic to come out of reset */
5732         msleep(500);
5733
5734         /* Restore PCI confspace */
5735         amdgpu_device_load_pci_state(pdev);
5736
5737         /* confirm  ASIC came out of reset */
5738         for (i = 0; i < adev->usec_timeout; i++) {
5739                 memsize = amdgpu_asic_get_config_memsize(adev);
5740
5741                 if (memsize != 0xffffffff)
5742                         break;
5743                 udelay(1);
5744         }
5745         if (memsize == 0xffffffff) {
5746                 r = -ETIME;
5747                 goto out;
5748         }
5749
5750         reset_context.method = AMD_RESET_METHOD_NONE;
5751         reset_context.reset_req_dev = adev;
5752         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5753         set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
5754
5755         adev->no_hw_access = true;
5756         r = amdgpu_device_pre_asic_reset(adev, &reset_context);
5757         adev->no_hw_access = false;
5758         if (r)
5759                 goto out;
5760
5761         r = amdgpu_do_asic_reset(&device_list, &reset_context);
5762
5763 out:
5764         if (!r) {
5765                 if (amdgpu_device_cache_pci_state(adev->pdev))
5766                         pci_restore_state(adev->pdev);
5767
5768                 DRM_INFO("PCIe error recovery succeeded\n");
5769         } else {
5770                 DRM_ERROR("PCIe error recovery failed, err:%d", r);
5771                 amdgpu_device_unset_mp1_state(adev);
5772                 amdgpu_device_unlock_reset_domain(adev->reset_domain);
5773         }
5774
5775         return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
5776 }
5777
5778 /**
5779  * amdgpu_pci_resume() - resume normal ops after PCI reset
5780  * @pdev: pointer to PCI device
5781  *
5782  * Called when the error recovery driver tells us that its
5783  * OK to resume normal operation.
5784  */
5785 void amdgpu_pci_resume(struct pci_dev *pdev)
5786 {
5787         struct drm_device *dev = pci_get_drvdata(pdev);
5788         struct amdgpu_device *adev = drm_to_adev(dev);
5789         int i;
5790
5791
5792         DRM_INFO("PCI error: resume callback!!\n");
5793
5794         /* Only continue execution for the case of pci_channel_io_frozen */
5795         if (adev->pci_channel_state != pci_channel_io_frozen)
5796                 return;
5797
5798         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5799                 struct amdgpu_ring *ring = adev->rings[i];
5800
5801                 if (!ring || !ring->sched.thread)
5802                         continue;
5803
5804                 drm_sched_start(&ring->sched, true);
5805         }
5806
5807         amdgpu_device_unset_mp1_state(adev);
5808         amdgpu_device_unlock_reset_domain(adev->reset_domain);
5809 }
5810
5811 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
5812 {
5813         struct drm_device *dev = pci_get_drvdata(pdev);
5814         struct amdgpu_device *adev = drm_to_adev(dev);
5815         int r;
5816
5817         r = pci_save_state(pdev);
5818         if (!r) {
5819                 kfree(adev->pci_state);
5820
5821                 adev->pci_state = pci_store_saved_state(pdev);
5822
5823                 if (!adev->pci_state) {
5824                         DRM_ERROR("Failed to store PCI saved state");
5825                         return false;
5826                 }
5827         } else {
5828                 DRM_WARN("Failed to save PCI state, err:%d\n", r);
5829                 return false;
5830         }
5831
5832         return true;
5833 }
5834
5835 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
5836 {
5837         struct drm_device *dev = pci_get_drvdata(pdev);
5838         struct amdgpu_device *adev = drm_to_adev(dev);
5839         int r;
5840
5841         if (!adev->pci_state)
5842                 return false;
5843
5844         r = pci_load_saved_state(pdev, adev->pci_state);
5845
5846         if (!r) {
5847                 pci_restore_state(pdev);
5848         } else {
5849                 DRM_WARN("Failed to load PCI state, err:%d\n", r);
5850                 return false;
5851         }
5852
5853         return true;
5854 }
5855
5856 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
5857                 struct amdgpu_ring *ring)
5858 {
5859 #ifdef CONFIG_X86_64
5860         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
5861                 return;
5862 #endif
5863         if (adev->gmc.xgmi.connected_to_cpu)
5864                 return;
5865
5866         if (ring && ring->funcs->emit_hdp_flush)
5867                 amdgpu_ring_emit_hdp_flush(ring);
5868         else
5869                 amdgpu_asic_flush_hdp(adev, ring);
5870 }
5871
5872 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
5873                 struct amdgpu_ring *ring)
5874 {
5875 #ifdef CONFIG_X86_64
5876         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
5877                 return;
5878 #endif
5879         if (adev->gmc.xgmi.connected_to_cpu)
5880                 return;
5881
5882         amdgpu_asic_invalidate_hdp(adev, ring);
5883 }
5884
5885 int amdgpu_in_reset(struct amdgpu_device *adev)
5886 {
5887         return atomic_read(&adev->reset_domain->in_gpu_reset);
5888 }
5889
5890 /**
5891  * amdgpu_device_halt() - bring hardware to some kind of halt state
5892  *
5893  * @adev: amdgpu_device pointer
5894  *
5895  * Bring hardware to some kind of halt state so that no one can touch it
5896  * any more. It will help to maintain error context when error occurred.
5897  * Compare to a simple hang, the system will keep stable at least for SSH
5898  * access. Then it should be trivial to inspect the hardware state and
5899  * see what's going on. Implemented as following:
5900  *
5901  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
5902  *    clears all CPU mappings to device, disallows remappings through page faults
5903  * 2. amdgpu_irq_disable_all() disables all interrupts
5904  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
5905  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
5906  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
5907  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
5908  *    flush any in flight DMA operations
5909  */
5910 void amdgpu_device_halt(struct amdgpu_device *adev)
5911 {
5912         struct pci_dev *pdev = adev->pdev;
5913         struct drm_device *ddev = adev_to_drm(adev);
5914
5915         drm_dev_unplug(ddev);
5916
5917         amdgpu_irq_disable_all(adev);
5918
5919         amdgpu_fence_driver_hw_fini(adev);
5920
5921         adev->no_hw_access = true;
5922
5923         amdgpu_device_unmap_mmio(adev);
5924
5925         pci_disable_device(pdev);
5926         pci_wait_for_pending_transaction(pdev);
5927 }
5928
5929 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
5930                                 u32 reg)
5931 {
5932         unsigned long flags, address, data;
5933         u32 r;
5934
5935         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
5936         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
5937
5938         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
5939         WREG32(address, reg * 4);
5940         (void)RREG32(address);
5941         r = RREG32(data);
5942         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
5943         return r;
5944 }
5945
5946 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
5947                                 u32 reg, u32 v)
5948 {
5949         unsigned long flags, address, data;
5950
5951         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
5952         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
5953
5954         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
5955         WREG32(address, reg * 4);
5956         (void)RREG32(address);
5957         WREG32(data, v);
5958         (void)RREG32(data);
5959         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
5960 }
5961
5962 /**
5963  * amdgpu_device_switch_gang - switch to a new gang
5964  * @adev: amdgpu_device pointer
5965  * @gang: the gang to switch to
5966  *
5967  * Try to switch to a new gang.
5968  * Returns: NULL if we switched to the new gang or a reference to the current
5969  * gang leader.
5970  */
5971 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
5972                                             struct dma_fence *gang)
5973 {
5974         struct dma_fence *old = NULL;
5975
5976         do {
5977                 dma_fence_put(old);
5978                 rcu_read_lock();
5979                 old = dma_fence_get_rcu_safe(&adev->gang_submit);
5980                 rcu_read_unlock();
5981
5982                 if (old == gang)
5983                         break;
5984
5985                 if (!dma_fence_is_signaled(old))
5986                         return old;
5987
5988         } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
5989                          old, gang) != old);
5990
5991         dma_fence_put(old);
5992         return NULL;
5993 }
5994
5995 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
5996 {
5997         switch (adev->asic_type) {
5998 #ifdef CONFIG_DRM_AMDGPU_SI
5999         case CHIP_HAINAN:
6000 #endif
6001         case CHIP_TOPAZ:
6002                 /* chips with no display hardware */
6003                 return false;
6004 #ifdef CONFIG_DRM_AMDGPU_SI
6005         case CHIP_TAHITI:
6006         case CHIP_PITCAIRN:
6007         case CHIP_VERDE:
6008         case CHIP_OLAND:
6009 #endif
6010 #ifdef CONFIG_DRM_AMDGPU_CIK
6011         case CHIP_BONAIRE:
6012         case CHIP_HAWAII:
6013         case CHIP_KAVERI:
6014         case CHIP_KABINI:
6015         case CHIP_MULLINS:
6016 #endif
6017         case CHIP_TONGA:
6018         case CHIP_FIJI:
6019         case CHIP_POLARIS10:
6020         case CHIP_POLARIS11:
6021         case CHIP_POLARIS12:
6022         case CHIP_VEGAM:
6023         case CHIP_CARRIZO:
6024         case CHIP_STONEY:
6025                 /* chips with display hardware */
6026                 return true;
6027         default:
6028                 /* IP discovery */
6029                 if (!adev->ip_versions[DCE_HWIP][0] ||
6030                     (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6031                         return false;
6032                 return true;
6033         }
6034 }