drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/power_supply.h>
  29 #include <linux/kthread.h>
  30 #include <linux/module.h>
  31 #include <linux/console.h>
  32 #include <linux/slab.h>
  33 #include <linux/iommu.h>
  34
  35 #include <drm/drm_atomic_helper.h>
  36 #include <drm/drm_probe_helper.h>
  37 #include <drm/amdgpu_drm.h>
  38 #include <linux/vgaarb.h>
  39 #include <linux/vga_switcheroo.h>
  40 #include <linux/efi.h>
  41 #include "amdgpu.h"
  42 #include "amdgpu_trace.h"
  43 #include "amdgpu_i2c.h"
  44 #include "atom.h"
  45 #include "amdgpu_atombios.h"
  46 #include "amdgpu_atomfirmware.h"
  47 #include "amd_pcie.h"
  48 #ifdef CONFIG_DRM_AMDGPU_SI
  49 #include "si.h"
  50 #endif
  51 #ifdef CONFIG_DRM_AMDGPU_CIK
  52 #include "cik.h"
  53 #endif
  54 #include "vi.h"
  55 #include "soc15.h"
  56 #include "nv.h"
  57 #include "bif/bif_4_1_d.h"
  58 #include <linux/pci.h>
  59 #include <linux/firmware.h>
  60 #include "amdgpu_vf_error.h"
  61
  62 #include "amdgpu_amdkfd.h"
  63 #include "amdgpu_pm.h"
  64
  65 #include "amdgpu_xgmi.h"
  66 #include "amdgpu_ras.h"
  67 #include "amdgpu_pmu.h"
  68 #include "amdgpu_fru_eeprom.h"
  69 #include "amdgpu_reset.h"
  70
  71 #include <linux/suspend.h>
  72 #include <drm/task_barrier.h>
  73 #include <linux/pm_runtime.h>
  74
  75 #include <drm/drm_drv.h>
  76
  77 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  78 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
  79 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  80 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
  81 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
  82 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
  83 MODULE_FIRMWARE("amdgpu/renoir_gpu_info.bin");
  84 MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin");
  85 MODULE_FIRMWARE("amdgpu/navi14_gpu_info.bin");
  86 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
  87 MODULE_FIRMWARE("amdgpu/vangogh_gpu_info.bin");
  88 MODULE_FIRMWARE("amdgpu/yellow_carp_gpu_info.bin");
  89
  90 #define AMDGPU_RESUME_MS                2000
  91
  92 const char *amdgpu_asic_name[] = {
  93         "TAHITI",
  94         "PITCAIRN",
  95         "VERDE",
  96         "OLAND",
  97         "HAINAN",
  98         "BONAIRE",
  99         "KAVERI",
 100         "KABINI",
 101         "HAWAII",
 102         "MULLINS",
 103         "TOPAZ",
 104         "TONGA",
 105         "FIJI",
 106         "CARRIZO",
 107         "STONEY",
 108         "POLARIS10",
 109         "POLARIS11",
 110         "POLARIS12",
 111         "VEGAM",
 112         "VEGA10",
 113         "VEGA12",
 114         "VEGA20",
 115         "RAVEN",
 116         "ARCTURUS",
 117         "RENOIR",
 118         "ALDEBARAN",
 119         "NAVI10",
 120         "CYAN_SKILLFISH",
 121         "NAVI14",
 122         "NAVI12",
 123         "SIENNA_CICHLID",
 124         "NAVY_FLOUNDER",
 125         "VANGOGH",
 126         "DIMGREY_CAVEFISH",
 127         "BEIGE_GOBY",
 128         "YELLOW_CARP",
 129         "IP DISCOVERY",
 130         "LAST",
 131 };
 132
 133 /**
 134  * DOC: pcie_replay_count
 135  *
 136  * The amdgpu driver provides a sysfs API for reporting the total number
 137  * of PCIe replays (NAKs)
 138  * The file pcie_replay_count is used for this and returns the total
 139  * number of replays as a sum of the NAKs generated and NAKs received
 140  */
 141
 142 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
 143                 struct device_attribute *attr, char *buf)
 144 {
 145         struct drm_device *ddev = dev_get_drvdata(dev);
 146         struct amdgpu_device *adev = drm_to_adev(ddev);
 147         uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
 148
 149         return sysfs_emit(buf, "%llu\n", cnt);
 150 }
 151
 152 static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
 153                 amdgpu_device_get_pcie_replay_count, NULL);
 154
 155 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 156
 157 /**
 158  * DOC: product_name
 159  *
 160  * The amdgpu driver provides a sysfs API for reporting the product name
 161  * for the device
 162  * The file serial_number is used for this and returns the product name
 163  * as returned from the FRU.
 164  * NOTE: This is only available for certain server cards
 165  */
 166
 167 static ssize_t amdgpu_device_get_product_name(struct device *dev,
 168                 struct device_attribute *attr, char *buf)
 169 {
 170         struct drm_device *ddev = dev_get_drvdata(dev);
 171         struct amdgpu_device *adev = drm_to_adev(ddev);
 172
 173         return sysfs_emit(buf, "%s\n", adev->product_name);
 174 }
 175
 176 static DEVICE_ATTR(product_name, S_IRUGO,
 177                 amdgpu_device_get_product_name, NULL);
 178
 179 /**
 180  * DOC: product_number
 181  *
 182  * The amdgpu driver provides a sysfs API for reporting the part number
 183  * for the device
 184  * The file serial_number is used for this and returns the part number
 185  * as returned from the FRU.
 186  * NOTE: This is only available for certain server cards
 187  */
 188
 189 static ssize_t amdgpu_device_get_product_number(struct device *dev,
 190                 struct device_attribute *attr, char *buf)
 191 {
 192         struct drm_device *ddev = dev_get_drvdata(dev);
 193         struct amdgpu_device *adev = drm_to_adev(ddev);
 194
 195         return sysfs_emit(buf, "%s\n", adev->product_number);
 196 }
 197
 198 static DEVICE_ATTR(product_number, S_IRUGO,
 199                 amdgpu_device_get_product_number, NULL);
 200
 201 /**
 202  * DOC: serial_number
 203  *
 204  * The amdgpu driver provides a sysfs API for reporting the serial number
 205  * for the device
 206  * The file serial_number is used for this and returns the serial number
 207  * as returned from the FRU.
 208  * NOTE: This is only available for certain server cards
 209  */
 210
 211 static ssize_t amdgpu_device_get_serial_number(struct device *dev,
 212                 struct device_attribute *attr, char *buf)
 213 {
 214         struct drm_device *ddev = dev_get_drvdata(dev);
 215         struct amdgpu_device *adev = drm_to_adev(ddev);
 216
 217         return sysfs_emit(buf, "%s\n", adev->serial);
 218 }
 219
 220 static DEVICE_ATTR(serial_number, S_IRUGO,
 221                 amdgpu_device_get_serial_number, NULL);
 222
 223 /**
 224  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
 225  *
 226  * @dev: drm_device pointer
 227  *
 228  * Returns true if the device is a dGPU with ATPX power control,
 229  * otherwise return false.
 230  */
 231 bool amdgpu_device_supports_px(struct drm_device *dev)
 232 {
 233         struct amdgpu_device *adev = drm_to_adev(dev);
 234
 235         if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
 236                 return true;
 237         return false;
 238 }
 239
 240 /**
 241  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
 242  *
 243  * @dev: drm_device pointer
 244  *
 245  * Returns true if the device is a dGPU with ACPI power control,
 246  * otherwise return false.
 247  */
 248 bool amdgpu_device_supports_boco(struct drm_device *dev)
 249 {
 250         struct amdgpu_device *adev = drm_to_adev(dev);
 251
 252         if (adev->has_pr3 ||
 253             ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
 254                 return true;
 255         return false;
 256 }
 257
 258 /**
 259  * amdgpu_device_supports_baco - Does the device support BACO
 260  *
 261  * @dev: drm_device pointer
 262  *
 263  * Returns true if the device supporte BACO,
 264  * otherwise return false.
 265  */
 266 bool amdgpu_device_supports_baco(struct drm_device *dev)
 267 {
 268         struct amdgpu_device *adev = drm_to_adev(dev);
 269
 270         return amdgpu_asic_supports_baco(adev);
 271 }
 272
 273 /**
 274  * amdgpu_device_supports_smart_shift - Is the device dGPU with
 275  * smart shift support
 276  *
 277  * @dev: drm_device pointer
 278  *
 279  * Returns true if the device is a dGPU with Smart Shift support,
 280  * otherwise returns false.
 281  */
 282 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
 283 {
 284         return (amdgpu_device_supports_boco(dev) &&
 285                 amdgpu_acpi_is_power_shift_control_supported());
 286 }
 287
 288 /*
 289  * VRAM access helper functions
 290  */
 291
 292 /**
 293  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
 294  *
 295  * @adev: amdgpu_device pointer
 296  * @pos: offset of the buffer in vram
 297  * @buf: virtual address of the buffer in system memory
 298  * @size: read/write size, sizeof(@buf) must > @size
 299  * @write: true - write to vram, otherwise - read from vram
 300  */
 301 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
 302                              void *buf, size_t size, bool write)
 303 {
 304         unsigned long flags;
 305         uint32_t hi = ~0, tmp = 0;
 306         uint32_t *data = buf;
 307         uint64_t last;
 308         int idx;
 309
 310         if (!drm_dev_enter(adev_to_drm(adev), &idx))
 311                 return;
 312
 313         BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
 314
 315         spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 316         for (last = pos + size; pos < last; pos += 4) {
 317                 tmp = pos >> 31;
 318
 319                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
 320                 if (tmp != hi) {
 321                         WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
 322                         hi = tmp;
 323                 }
 324                 if (write)
 325                         WREG32_NO_KIQ(mmMM_DATA, *data++);
 326                 else
 327                         *data++ = RREG32_NO_KIQ(mmMM_DATA);
 328         }
 329
 330         spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 331         drm_dev_exit(idx);
 332 }
 333
 334 /**
 335  * amdgpu_device_aper_access - access vram by vram aperature
 336  *
 337  * @adev: amdgpu_device pointer
 338  * @pos: offset of the buffer in vram
 339  * @buf: virtual address of the buffer in system memory
 340  * @size: read/write size, sizeof(@buf) must > @size
 341  * @write: true - write to vram, otherwise - read from vram
 342  *
 343  * The return value means how many bytes have been transferred.
 344  */
 345 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
 346                                  void *buf, size_t size, bool write)
 347 {
 348 #ifdef CONFIG_64BIT
 349         void __iomem *addr;
 350         size_t count = 0;
 351         uint64_t last;
 352
 353         if (!adev->mman.aper_base_kaddr)
 354                 return 0;
 355
 356         last = min(pos + size, adev->gmc.visible_vram_size);
 357         if (last > pos) {
 358                 addr = adev->mman.aper_base_kaddr + pos;
 359                 count = last - pos;
 360
 361                 if (write) {
 362                         memcpy_toio(addr, buf, count);
 363                         mb();
 364                         amdgpu_device_flush_hdp(adev, NULL);
 365                 } else {
 366                         amdgpu_device_invalidate_hdp(adev, NULL);
 367                         mb();
 368                         memcpy_fromio(buf, addr, count);
 369                 }
 370
 371         }
 372
 373         return count;
 374 #else
 375         return 0;
 376 #endif
 377 }
 378
 379 /**
 380  * amdgpu_device_vram_access - read/write a buffer in vram
 381  *
 382  * @adev: amdgpu_device pointer
 383  * @pos: offset of the buffer in vram
 384  * @buf: virtual address of the buffer in system memory
 385  * @size: read/write size, sizeof(@buf) must > @size
 386  * @write: true - write to vram, otherwise - read from vram
 387  */
 388 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
 389                                void *buf, size_t size, bool write)
 390 {
 391         size_t count;
 392
 393         /* try to using vram apreature to access vram first */
 394         count = amdgpu_device_aper_access(adev, pos, buf, size, write);
 395         size -= count;
 396         if (size) {
 397                 /* using MM to access rest vram */
 398                 pos += count;
 399                 buf += count;
 400                 amdgpu_device_mm_access(adev, pos, buf, size, write);
 401         }
 402 }
 403
 404 /*
 405  * register access helper functions.
 406  */
 407
 408 /* Check if hw access should be skipped because of hotplug or device error */
 409 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
 410 {
 411         if (adev->no_hw_access)
 412                 return true;
 413
 414 #ifdef CONFIG_LOCKDEP
 415         /*
 416          * This is a bit complicated to understand, so worth a comment. What we assert
 417          * here is that the GPU reset is not running on another thread in parallel.
 418          *
 419          * For this we trylock the read side of the reset semaphore, if that succeeds
 420          * we know that the reset is not running in paralell.
 421          *
 422          * If the trylock fails we assert that we are either already holding the read
 423          * side of the lock or are the reset thread itself and hold the write side of
 424          * the lock.
 425          */
 426         if (in_task()) {
 427                 if (down_read_trylock(&adev->reset_sem))
 428                         up_read(&adev->reset_sem);
 429                 else
 430                         lockdep_assert_held(&adev->reset_sem);
 431         }
 432 #endif
 433         return false;
 434 }
 435
 436 /**
 437  * amdgpu_device_rreg - read a memory mapped IO or indirect register
 438  *
 439  * @adev: amdgpu_device pointer
 440  * @reg: dword aligned register offset
 441  * @acc_flags: access flags which require special behavior
 442  *
 443  * Returns the 32 bit value from the offset specified.
 444  */
 445 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 446                             uint32_t reg, uint32_t acc_flags)
 447 {
 448         uint32_t ret;
 449
 450         if (amdgpu_device_skip_hw_access(adev))
 451                 return 0;
 452
 453         if ((reg * 4) < adev->rmmio_size) {
 454                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 455                     amdgpu_sriov_runtime(adev) &&
 456                     down_read_trylock(&adev->reset_sem)) {
 457                         ret = amdgpu_kiq_rreg(adev, reg);
 458                         up_read(&adev->reset_sem);
 459                 } else {
 460                         ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 461                 }
 462         } else {
 463                 ret = adev->pcie_rreg(adev, reg * 4);
 464         }
 465
 466         trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
 467
 468         return ret;
 469 }
 470
 471 /*
 472  * MMIO register read with bytes helper functions
 473  * @offset:bytes offset from MMIO start
 474  *
 475 */
 476
 477 /**
 478  * amdgpu_mm_rreg8 - read a memory mapped IO register
 479  *
 480  * @adev: amdgpu_device pointer
 481  * @offset: byte aligned register offset
 482  *
 483  * Returns the 8 bit value from the offset specified.
 484  */
 485 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
 486 {
 487         if (amdgpu_device_skip_hw_access(adev))
 488                 return 0;
 489
 490         if (offset < adev->rmmio_size)
 491                 return (readb(adev->rmmio + offset));
 492         BUG();
 493 }
 494
 495 /*
 496  * MMIO register write with bytes helper functions
 497  * @offset:bytes offset from MMIO start
 498  * @value: the value want to be written to the register
 499  *
 500 */
 501 /**
 502  * amdgpu_mm_wreg8 - read a memory mapped IO register
 503  *
 504  * @adev: amdgpu_device pointer
 505  * @offset: byte aligned register offset
 506  * @value: 8 bit value to write
 507  *
 508  * Writes the value specified to the offset specified.
 509  */
 510 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
 511 {
 512         if (amdgpu_device_skip_hw_access(adev))
 513                 return;
 514
 515         if (offset < adev->rmmio_size)
 516                 writeb(value, adev->rmmio + offset);
 517         else
 518                 BUG();
 519 }
 520
 521 /**
 522  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
 523  *
 524  * @adev: amdgpu_device pointer
 525  * @reg: dword aligned register offset
 526  * @v: 32 bit value to write to the register
 527  * @acc_flags: access flags which require special behavior
 528  *
 529  * Writes the value specified to the offset specified.
 530  */
 531 void amdgpu_device_wreg(struct amdgpu_device *adev,
 532                         uint32_t reg, uint32_t v,
 533                         uint32_t acc_flags)
 534 {
 535         if (amdgpu_device_skip_hw_access(adev))
 536                 return;
 537
 538         if ((reg * 4) < adev->rmmio_size) {
 539                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 540                     amdgpu_sriov_runtime(adev) &&
 541                     down_read_trylock(&adev->reset_sem)) {
 542                         amdgpu_kiq_wreg(adev, reg, v);
 543                         up_read(&adev->reset_sem);
 544                 } else {
 545                         writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 546                 }
 547         } else {
 548                 adev->pcie_wreg(adev, reg * 4, v);
 549         }
 550
 551         trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
 552 }
 553
 554 /**
 555  * amdgpu_mm_wreg_mmio_rlc -  write register either with mmio or with RLC path if in range
 556  *
 557  * this function is invoked only the debugfs register access
 558  */
 559 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
 560                              uint32_t reg, uint32_t v)
 561 {
 562         if (amdgpu_device_skip_hw_access(adev))
 563                 return;
 564
 565         if (amdgpu_sriov_fullaccess(adev) &&
 566             adev->gfx.rlc.funcs &&
 567             adev->gfx.rlc.funcs->is_rlcg_access_range) {
 568                 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
 569                         return adev->gfx.rlc.funcs->sriov_wreg(adev, reg, v, 0, 0);
 570         } else {
 571                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 572         }
 573 }
 574
 575 /**
 576  * amdgpu_mm_rdoorbell - read a doorbell dword
 577  *
 578  * @adev: amdgpu_device pointer
 579  * @index: doorbell index
 580  *
 581  * Returns the value in the doorbell aperture at the
 582  * requested doorbell index (CIK).
 583  */
 584 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 585 {
 586         if (amdgpu_device_skip_hw_access(adev))
 587                 return 0;
 588
 589         if (index < adev->doorbell.num_doorbells) {
 590                 return readl(adev->doorbell.ptr + index);
 591         } else {
 592                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 593                 return 0;
 594         }
 595 }
 596
 597 /**
 598  * amdgpu_mm_wdoorbell - write a doorbell dword
 599  *
 600  * @adev: amdgpu_device pointer
 601  * @index: doorbell index
 602  * @v: value to write
 603  *
 604  * Writes @v to the doorbell aperture at the
 605  * requested doorbell index (CIK).
 606  */
 607 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 608 {
 609         if (amdgpu_device_skip_hw_access(adev))
 610                 return;
 611
 612         if (index < adev->doorbell.num_doorbells) {
 613                 writel(v, adev->doorbell.ptr + index);
 614         } else {
 615                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 616         }
 617 }
 618
 619 /**
 620  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 621  *
 622  * @adev: amdgpu_device pointer
 623  * @index: doorbell index
 624  *
 625  * Returns the value in the doorbell aperture at the
 626  * requested doorbell index (VEGA10+).
 627  */
 628 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 629 {
 630         if (amdgpu_device_skip_hw_access(adev))
 631                 return 0;
 632
 633         if (index < adev->doorbell.num_doorbells) {
 634                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 635         } else {
 636                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 637                 return 0;
 638         }
 639 }
 640
 641 /**
 642  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 643  *
 644  * @adev: amdgpu_device pointer
 645  * @index: doorbell index
 646  * @v: value to write
 647  *
 648  * Writes @v to the doorbell aperture at the
 649  * requested doorbell index (VEGA10+).
 650  */
 651 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 652 {
 653         if (amdgpu_device_skip_hw_access(adev))
 654                 return;
 655
 656         if (index < adev->doorbell.num_doorbells) {
 657                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 658         } else {
 659                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 660         }
 661 }
 662
 663 /**
 664  * amdgpu_device_indirect_rreg - read an indirect register
 665  *
 666  * @adev: amdgpu_device pointer
 667  * @pcie_index: mmio register offset
 668  * @pcie_data: mmio register offset
 669  * @reg_addr: indirect register address to read from
 670  *
 671  * Returns the value of indirect register @reg_addr
 672  */
 673 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
 674                                 u32 pcie_index, u32 pcie_data,
 675                                 u32 reg_addr)
 676 {
 677         unsigned long flags;
 678         u32 r;
 679         void __iomem *pcie_index_offset;
 680         void __iomem *pcie_data_offset;
 681
 682         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 683         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 684         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 685
 686         writel(reg_addr, pcie_index_offset);
 687         readl(pcie_index_offset);
 688         r = readl(pcie_data_offset);
 689         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 690
 691         return r;
 692 }
 693
 694 /**
 695  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
 696  *
 697  * @adev: amdgpu_device pointer
 698  * @pcie_index: mmio register offset
 699  * @pcie_data: mmio register offset
 700  * @reg_addr: indirect register address to read from
 701  *
 702  * Returns the value of indirect register @reg_addr
 703  */
 704 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
 705                                   u32 pcie_index, u32 pcie_data,
 706                                   u32 reg_addr)
 707 {
 708         unsigned long flags;
 709         u64 r;
 710         void __iomem *pcie_index_offset;
 711         void __iomem *pcie_data_offset;
 712
 713         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 714         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 715         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 716
 717         /* read low 32 bits */
 718         writel(reg_addr, pcie_index_offset);
 719         readl(pcie_index_offset);
 720         r = readl(pcie_data_offset);
 721         /* read high 32 bits */
 722         writel(reg_addr + 4, pcie_index_offset);
 723         readl(pcie_index_offset);
 724         r |= ((u64)readl(pcie_data_offset) << 32);
 725         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 726
 727         return r;
 728 }
 729
 730 /**
 731  * amdgpu_device_indirect_wreg - write an indirect register address
 732  *
 733  * @adev: amdgpu_device pointer
 734  * @pcie_index: mmio register offset
 735  * @pcie_data: mmio register offset
 736  * @reg_addr: indirect register offset
 737  * @reg_data: indirect register data
 738  *
 739  */
 740 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
 741                                  u32 pcie_index, u32 pcie_data,
 742                                  u32 reg_addr, u32 reg_data)
 743 {
 744         unsigned long flags;
 745         void __iomem *pcie_index_offset;
 746         void __iomem *pcie_data_offset;
 747
 748         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 749         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 750         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 751
 752         writel(reg_addr, pcie_index_offset);
 753         readl(pcie_index_offset);
 754         writel(reg_data, pcie_data_offset);
 755         readl(pcie_data_offset);
 756         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 757 }
 758
 759 /**
 760  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
 761  *
 762  * @adev: amdgpu_device pointer
 763  * @pcie_index: mmio register offset
 764  * @pcie_data: mmio register offset
 765  * @reg_addr: indirect register offset
 766  * @reg_data: indirect register data
 767  *
 768  */
 769 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
 770                                    u32 pcie_index, u32 pcie_data,
 771                                    u32 reg_addr, u64 reg_data)
 772 {
 773         unsigned long flags;
 774         void __iomem *pcie_index_offset;
 775         void __iomem *pcie_data_offset;
 776
 777         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 778         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 779         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 780
 781         /* write low 32 bits */
 782         writel(reg_addr, pcie_index_offset);
 783         readl(pcie_index_offset);
 784         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
 785         readl(pcie_data_offset);
 786         /* write high 32 bits */
 787         writel(reg_addr + 4, pcie_index_offset);
 788         readl(pcie_index_offset);
 789         writel((u32)(reg_data >> 32), pcie_data_offset);
 790         readl(pcie_data_offset);
 791         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 792 }
 793
 794 /**
 795  * amdgpu_invalid_rreg - dummy reg read function
 796  *
 797  * @adev: amdgpu_device pointer
 798  * @reg: offset of register
 799  *
 800  * Dummy register read function.  Used for register blocks
 801  * that certain asics don't have (all asics).
 802  * Returns the value in the register.
 803  */
 804 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 805 {
 806         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 807         BUG();
 808         return 0;
 809 }
 810
 811 /**
 812  * amdgpu_invalid_wreg - dummy reg write function
 813  *
 814  * @adev: amdgpu_device pointer
 815  * @reg: offset of register
 816  * @v: value to write to the register
 817  *
 818  * Dummy register read function.  Used for register blocks
 819  * that certain asics don't have (all asics).
 820  */
 821 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 822 {
 823         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 824                   reg, v);
 825         BUG();
 826 }
 827
 828 /**
 829  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
 830  *
 831  * @adev: amdgpu_device pointer
 832  * @reg: offset of register
 833  *
 834  * Dummy register read function.  Used for register blocks
 835  * that certain asics don't have (all asics).
 836  * Returns the value in the register.
 837  */
 838 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
 839 {
 840         DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
 841         BUG();
 842         return 0;
 843 }
 844
 845 /**
 846  * amdgpu_invalid_wreg64 - dummy reg write function
 847  *
 848  * @adev: amdgpu_device pointer
 849  * @reg: offset of register
 850  * @v: value to write to the register
 851  *
 852  * Dummy register read function.  Used for register blocks
 853  * that certain asics don't have (all asics).
 854  */
 855 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
 856 {
 857         DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
 858                   reg, v);
 859         BUG();
 860 }
 861
 862 /**
 863  * amdgpu_block_invalid_rreg - dummy reg read function
 864  *
 865  * @adev: amdgpu_device pointer
 866  * @block: offset of instance
 867  * @reg: offset of register
 868  *
 869  * Dummy register read function.  Used for register blocks
 870  * that certain asics don't have (all asics).
 871  * Returns the value in the register.
 872  */
 873 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 874                                           uint32_t block, uint32_t reg)
 875 {
 876         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 877                   reg, block);
 878         BUG();
 879         return 0;
 880 }
 881
 882 /**
 883  * amdgpu_block_invalid_wreg - dummy reg write function
 884  *
 885  * @adev: amdgpu_device pointer
 886  * @block: offset of instance
 887  * @reg: offset of register
 888  * @v: value to write to the register
 889  *
 890  * Dummy register read function.  Used for register blocks
 891  * that certain asics don't have (all asics).
 892  */
 893 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 894                                       uint32_t block,
 895                                       uint32_t reg, uint32_t v)
 896 {
 897         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 898                   reg, block, v);
 899         BUG();
 900 }
 901
 902 /**
 903  * amdgpu_device_asic_init - Wrapper for atom asic_init
 904  *
 905  * @adev: amdgpu_device pointer
 906  *
 907  * Does any asic specific work and then calls atom asic init.
 908  */
 909 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
 910 {
 911         amdgpu_asic_pre_asic_init(adev);
 912
 913         return amdgpu_atom_asic_init(adev->mode_info.atom_context);
 914 }
 915
 916 /**
 917  * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
 918  *
 919  * @adev: amdgpu_device pointer
 920  *
 921  * Allocates a scratch page of VRAM for use by various things in the
 922  * driver.
 923  */
 924 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
 925 {
 926         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
 927                                        PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 928                                        &adev->vram_scratch.robj,
 929                                        &adev->vram_scratch.gpu_addr,
 930                                        (void **)&adev->vram_scratch.ptr);
 931 }
 932
 933 /**
 934  * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
 935  *
 936  * @adev: amdgpu_device pointer
 937  *
 938  * Frees the VRAM scratch page.
 939  */
 940 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
 941 {
 942         amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
 943 }
 944
 945 /**
 946  * amdgpu_device_program_register_sequence - program an array of registers.
 947  *
 948  * @adev: amdgpu_device pointer
 949  * @registers: pointer to the register array
 950  * @array_size: size of the register array
 951  *
 952  * Programs an array or registers with and and or masks.
 953  * This is a helper for setting golden registers.
 954  */
 955 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
 956                                              const u32 *registers,
 957                                              const u32 array_size)
 958 {
 959         u32 tmp, reg, and_mask, or_mask;
 960         int i;
 961
 962         if (array_size % 3)
 963                 return;
 964
 965         for (i = 0; i < array_size; i +=3) {
 966                 reg = registers[i + 0];
 967                 and_mask = registers[i + 1];
 968                 or_mask = registers[i + 2];
 969
 970                 if (and_mask == 0xffffffff) {
 971                         tmp = or_mask;
 972                 } else {
 973                         tmp = RREG32(reg);
 974                         tmp &= ~and_mask;
 975                         if (adev->family >= AMDGPU_FAMILY_AI)
 976                                 tmp |= (or_mask & and_mask);
 977                         else
 978                                 tmp |= or_mask;
 979                 }
 980                 WREG32(reg, tmp);
 981         }
 982 }
 983
 984 /**
 985  * amdgpu_device_pci_config_reset - reset the GPU
 986  *
 987  * @adev: amdgpu_device pointer
 988  *
 989  * Resets the GPU using the pci config reset sequence.
 990  * Only applicable to asics prior to vega10.
 991  */
 992 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
 993 {
 994         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
 995 }
 996
 997 /**
 998  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
 999  *
1000  * @adev: amdgpu_device pointer
1001  *
1002  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1003  */
1004 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1005 {
1006         return pci_reset_function(adev->pdev);
1007 }
1008
1009 /*
1010  * GPU doorbell aperture helpers function.
1011  */
1012 /**
1013  * amdgpu_device_doorbell_init - Init doorbell driver information.
1014  *
1015  * @adev: amdgpu_device pointer
1016  *
1017  * Init doorbell driver information (CIK)
1018  * Returns 0 on success, error on failure.
1019  */
1020 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
1021 {
1022
1023         /* No doorbell on SI hardware generation */
1024         if (adev->asic_type < CHIP_BONAIRE) {
1025                 adev->doorbell.base = 0;
1026                 adev->doorbell.size = 0;
1027                 adev->doorbell.num_doorbells = 0;
1028                 adev->doorbell.ptr = NULL;
1029                 return 0;
1030         }
1031
1032         if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
1033                 return -EINVAL;
1034
1035         amdgpu_asic_init_doorbell_index(adev);
1036
1037         /* doorbell bar mapping */
1038         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
1039         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
1040
1041         adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
1042                                              adev->doorbell_index.max_assignment+1);
1043         if (adev->doorbell.num_doorbells == 0)
1044                 return -EINVAL;
1045
1046         /* For Vega, reserve and map two pages on doorbell BAR since SDMA
1047          * paging queue doorbell use the second page. The
1048          * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
1049          * doorbells are in the first page. So with paging queue enabled,
1050          * the max num_doorbells should + 1 page (0x400 in dword)
1051          */
1052         if (adev->asic_type >= CHIP_VEGA10)
1053                 adev->doorbell.num_doorbells += 0x400;
1054
1055         adev->doorbell.ptr = ioremap(adev->doorbell.base,
1056                                      adev->doorbell.num_doorbells *
1057                                      sizeof(u32));
1058         if (adev->doorbell.ptr == NULL)
1059                 return -ENOMEM;
1060
1061         return 0;
1062 }
1063
1064 /**
1065  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
1066  *
1067  * @adev: amdgpu_device pointer
1068  *
1069  * Tear down doorbell driver information (CIK)
1070  */
1071 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
1072 {
1073         iounmap(adev->doorbell.ptr);
1074         adev->doorbell.ptr = NULL;
1075 }
1076
1077
1078
1079 /*
1080  * amdgpu_device_wb_*()
1081  * Writeback is the method by which the GPU updates special pages in memory
1082  * with the status of certain GPU events (fences, ring pointers,etc.).
1083  */
1084
1085 /**
1086  * amdgpu_device_wb_fini - Disable Writeback and free memory
1087  *
1088  * @adev: amdgpu_device pointer
1089  *
1090  * Disables Writeback and frees the Writeback memory (all asics).
1091  * Used at driver shutdown.
1092  */
1093 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1094 {
1095         if (adev->wb.wb_obj) {
1096                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1097                                       &adev->wb.gpu_addr,
1098                                       (void **)&adev->wb.wb);
1099                 adev->wb.wb_obj = NULL;
1100         }
1101 }
1102
1103 /**
1104  * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1105  *
1106  * @adev: amdgpu_device pointer
1107  *
1108  * Initializes writeback and allocates writeback memory (all asics).
1109  * Used at driver startup.
1110  * Returns 0 on success or an -error on failure.
1111  */
1112 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1113 {
1114         int r;
1115
1116         if (adev->wb.wb_obj == NULL) {
1117                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1118                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1119                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1120                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
1121                                             (void **)&adev->wb.wb);
1122                 if (r) {
1123                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1124                         return r;
1125                 }
1126
1127                 adev->wb.num_wb = AMDGPU_MAX_WB;
1128                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1129
1130                 /* clear wb memory */
1131                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1132         }
1133
1134         return 0;
1135 }
1136
1137 /**
1138  * amdgpu_device_wb_get - Allocate a wb entry
1139  *
1140  * @adev: amdgpu_device pointer
1141  * @wb: wb index
1142  *
1143  * Allocate a wb slot for use by the driver (all asics).
1144  * Returns 0 on success or -EINVAL on failure.
1145  */
1146 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1147 {
1148         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1149
1150         if (offset < adev->wb.num_wb) {
1151                 __set_bit(offset, adev->wb.used);
1152                 *wb = offset << 3; /* convert to dw offset */
1153                 return 0;
1154         } else {
1155                 return -EINVAL;
1156         }
1157 }
1158
1159 /**
1160  * amdgpu_device_wb_free - Free a wb entry
1161  *
1162  * @adev: amdgpu_device pointer
1163  * @wb: wb index
1164  *
1165  * Free a wb slot allocated for use by the driver (all asics)
1166  */
1167 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1168 {
1169         wb >>= 3;
1170         if (wb < adev->wb.num_wb)
1171                 __clear_bit(wb, adev->wb.used);
1172 }
1173
1174 /**
1175  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1176  *
1177  * @adev: amdgpu_device pointer
1178  *
1179  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1180  * to fail, but if any of the BARs is not accessible after the size we abort
1181  * driver loading by returning -ENODEV.
1182  */
1183 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1184 {
1185         int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1186         struct pci_bus *root;
1187         struct resource *res;
1188         unsigned i;
1189         u16 cmd;
1190         int r;
1191
1192         /* Bypass for VF */
1193         if (amdgpu_sriov_vf(adev))
1194                 return 0;
1195
1196         /* skip if the bios has already enabled large BAR */
1197         if (adev->gmc.real_vram_size &&
1198             (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1199                 return 0;
1200
1201         /* Check if the root BUS has 64bit memory resources */
1202         root = adev->pdev->bus;
1203         while (root->parent)
1204                 root = root->parent;
1205
1206         pci_bus_for_each_resource(root, res, i) {
1207                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1208                     res->start > 0x100000000ull)
1209                         break;
1210         }
1211
1212         /* Trying to resize is pointless without a root hub window above 4GB */
1213         if (!res)
1214                 return 0;
1215
1216         /* Limit the BAR size to what is available */
1217         rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1218                         rbar_size);
1219
1220         /* Disable memory decoding while we change the BAR addresses and size */
1221         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1222         pci_write_config_word(adev->pdev, PCI_COMMAND,
1223                               cmd & ~PCI_COMMAND_MEMORY);
1224
1225         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1226         amdgpu_device_doorbell_fini(adev);
1227         if (adev->asic_type >= CHIP_BONAIRE)
1228                 pci_release_resource(adev->pdev, 2);
1229
1230         pci_release_resource(adev->pdev, 0);
1231
1232         r = pci_resize_resource(adev->pdev, 0, rbar_size);
1233         if (r == -ENOSPC)
1234                 DRM_INFO("Not enough PCI address space for a large BAR.");
1235         else if (r && r != -ENOTSUPP)
1236                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1237
1238         pci_assign_unassigned_bus_resources(adev->pdev->bus);
1239
1240         /* When the doorbell or fb BAR isn't available we have no chance of
1241          * using the device.
1242          */
1243         r = amdgpu_device_doorbell_init(adev);
1244         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1245                 return -ENODEV;
1246
1247         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1248
1249         return 0;
1250 }
1251
1252 /*
1253  * GPU helpers function.
1254  */
1255 /**
1256  * amdgpu_device_need_post - check if the hw need post or not
1257  *
1258  * @adev: amdgpu_device pointer
1259  *
1260  * Check if the asic has been initialized (all asics) at driver startup
1261  * or post is needed if  hw reset is performed.
1262  * Returns true if need or false if not.
1263  */
1264 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1265 {
1266         uint32_t reg;
1267
1268         if (amdgpu_sriov_vf(adev))
1269                 return false;
1270
1271         if (amdgpu_passthrough(adev)) {
1272                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1273                  * some old smc fw still need driver do vPost otherwise gpu hang, while
1274                  * those smc fw version above 22.15 doesn't have this flaw, so we force
1275                  * vpost executed for smc version below 22.15
1276                  */
1277                 if (adev->asic_type == CHIP_FIJI) {
1278                         int err;
1279                         uint32_t fw_ver;
1280                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1281                         /* force vPost if error occured */
1282                         if (err)
1283                                 return true;
1284
1285                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1286                         if (fw_ver < 0x00160e00)
1287                                 return true;
1288                 }
1289         }
1290
1291         /* Don't post if we need to reset whole hive on init */
1292         if (adev->gmc.xgmi.pending_reset)
1293                 return false;
1294
1295         if (adev->has_hw_reset) {
1296                 adev->has_hw_reset = false;
1297                 return true;
1298         }
1299
1300         /* bios scratch used on CIK+ */
1301         if (adev->asic_type >= CHIP_BONAIRE)
1302                 return amdgpu_atombios_scratch_need_asic_init(adev);
1303
1304         /* check MEM_SIZE for older asics */
1305         reg = amdgpu_asic_get_config_memsize(adev);
1306
1307         if ((reg != 0) && (reg != 0xffffffff))
1308                 return false;
1309
1310         return true;
1311 }
1312
1313 /* if we get transitioned to only one device, take VGA back */
1314 /**
1315  * amdgpu_device_vga_set_decode - enable/disable vga decode
1316  *
1317  * @pdev: PCI device pointer
1318  * @state: enable/disable vga decode
1319  *
1320  * Enable/disable vga decode (all asics).
1321  * Returns VGA resource flags.
1322  */
1323 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1324                 bool state)
1325 {
1326         struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1327         amdgpu_asic_set_vga_state(adev, state);
1328         if (state)
1329                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1330                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1331         else
1332                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1333 }
1334
1335 /**
1336  * amdgpu_device_check_block_size - validate the vm block size
1337  *
1338  * @adev: amdgpu_device pointer
1339  *
1340  * Validates the vm block size specified via module parameter.
1341  * The vm block size defines number of bits in page table versus page directory,
1342  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1343  * page table and the remaining bits are in the page directory.
1344  */
1345 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1346 {
1347         /* defines number of bits in page table versus page directory,
1348          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1349          * page table and the remaining bits are in the page directory */
1350         if (amdgpu_vm_block_size == -1)
1351                 return;
1352
1353         if (amdgpu_vm_block_size < 9) {
1354                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1355                          amdgpu_vm_block_size);
1356                 amdgpu_vm_block_size = -1;
1357         }
1358 }
1359
1360 /**
1361  * amdgpu_device_check_vm_size - validate the vm size
1362  *
1363  * @adev: amdgpu_device pointer
1364  *
1365  * Validates the vm size in GB specified via module parameter.
1366  * The VM size is the size of the GPU virtual memory space in GB.
1367  */
1368 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1369 {
1370         /* no need to check the default value */
1371         if (amdgpu_vm_size == -1)
1372                 return;
1373
1374         if (amdgpu_vm_size < 1) {
1375                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1376                          amdgpu_vm_size);
1377                 amdgpu_vm_size = -1;
1378         }
1379 }
1380
1381 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1382 {
1383         struct sysinfo si;
1384         bool is_os_64 = (sizeof(void *) == 8);
1385         uint64_t total_memory;
1386         uint64_t dram_size_seven_GB = 0x1B8000000;
1387         uint64_t dram_size_three_GB = 0xB8000000;
1388
1389         if (amdgpu_smu_memory_pool_size == 0)
1390                 return;
1391
1392         if (!is_os_64) {
1393                 DRM_WARN("Not 64-bit OS, feature not supported\n");
1394                 goto def_value;
1395         }
1396         si_meminfo(&si);
1397         total_memory = (uint64_t)si.totalram * si.mem_unit;
1398
1399         if ((amdgpu_smu_memory_pool_size == 1) ||
1400                 (amdgpu_smu_memory_pool_size == 2)) {
1401                 if (total_memory < dram_size_three_GB)
1402                         goto def_value1;
1403         } else if ((amdgpu_smu_memory_pool_size == 4) ||
1404                 (amdgpu_smu_memory_pool_size == 8)) {
1405                 if (total_memory < dram_size_seven_GB)
1406                         goto def_value1;
1407         } else {
1408                 DRM_WARN("Smu memory pool size not supported\n");
1409                 goto def_value;
1410         }
1411         adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1412
1413         return;
1414
1415 def_value1:
1416         DRM_WARN("No enough system memory\n");
1417 def_value:
1418         adev->pm.smu_prv_buffer_size = 0;
1419 }
1420
1421 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1422 {
1423         if (!(adev->flags & AMD_IS_APU) ||
1424             adev->asic_type < CHIP_RAVEN)
1425                 return 0;
1426
1427         switch (adev->asic_type) {
1428         case CHIP_RAVEN:
1429                 if (adev->pdev->device == 0x15dd)
1430                         adev->apu_flags |= AMD_APU_IS_RAVEN;
1431                 if (adev->pdev->device == 0x15d8)
1432                         adev->apu_flags |= AMD_APU_IS_PICASSO;
1433                 break;
1434         case CHIP_RENOIR:
1435                 if ((adev->pdev->device == 0x1636) ||
1436                     (adev->pdev->device == 0x164c))
1437                         adev->apu_flags |= AMD_APU_IS_RENOIR;
1438                 else
1439                         adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1440                 break;
1441         case CHIP_VANGOGH:
1442                 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1443                 break;
1444         case CHIP_YELLOW_CARP:
1445                 break;
1446         case CHIP_CYAN_SKILLFISH:
1447                 if (adev->pdev->device == 0x13FE)
1448                         adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1449                 break;
1450         default:
1451                 return -EINVAL;
1452         }
1453
1454         return 0;
1455 }
1456
1457 /**
1458  * amdgpu_device_check_arguments - validate module params
1459  *
1460  * @adev: amdgpu_device pointer
1461  *
1462  * Validates certain module parameters and updates
1463  * the associated values used by the driver (all asics).
1464  */
1465 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1466 {
1467         if (amdgpu_sched_jobs < 4) {
1468                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1469                          amdgpu_sched_jobs);
1470                 amdgpu_sched_jobs = 4;
1471         } else if (!is_power_of_2(amdgpu_sched_jobs)){
1472                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1473                          amdgpu_sched_jobs);
1474                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1475         }
1476
1477         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1478                 /* gart size must be greater or equal to 32M */
1479                 dev_warn(adev->dev, "gart size (%d) too small\n",
1480                          amdgpu_gart_size);
1481                 amdgpu_gart_size = -1;
1482         }
1483
1484         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1485                 /* gtt size must be greater or equal to 32M */
1486                 dev_warn(adev->dev, "gtt size (%d) too small\n",
1487                                  amdgpu_gtt_size);
1488                 amdgpu_gtt_size = -1;
1489         }
1490
1491         /* valid range is between 4 and 9 inclusive */
1492         if (amdgpu_vm_fragment_size != -1 &&
1493             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1494                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1495                 amdgpu_vm_fragment_size = -1;
1496         }
1497
1498         if (amdgpu_sched_hw_submission < 2) {
1499                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1500                          amdgpu_sched_hw_submission);
1501                 amdgpu_sched_hw_submission = 2;
1502         } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1503                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1504                          amdgpu_sched_hw_submission);
1505                 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1506         }
1507
1508         amdgpu_device_check_smu_prv_buffer_size(adev);
1509
1510         amdgpu_device_check_vm_size(adev);
1511
1512         amdgpu_device_check_block_size(adev);
1513
1514         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1515
1516         amdgpu_gmc_tmz_set(adev);
1517
1518         amdgpu_gmc_noretry_set(adev);
1519
1520         return 0;
1521 }
1522
1523 /**
1524  * amdgpu_switcheroo_set_state - set switcheroo state
1525  *
1526  * @pdev: pci dev pointer
1527  * @state: vga_switcheroo state
1528  *
1529  * Callback for the switcheroo driver.  Suspends or resumes the
1530  * the asics before or after it is powered up using ACPI methods.
1531  */
1532 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1533                                         enum vga_switcheroo_state state)
1534 {
1535         struct drm_device *dev = pci_get_drvdata(pdev);
1536         int r;
1537
1538         if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1539                 return;
1540
1541         if (state == VGA_SWITCHEROO_ON) {
1542                 pr_info("switched on\n");
1543                 /* don't suspend or resume card normally */
1544                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1545
1546                 pci_set_power_state(pdev, PCI_D0);
1547                 amdgpu_device_load_pci_state(pdev);
1548                 r = pci_enable_device(pdev);
1549                 if (r)
1550                         DRM_WARN("pci_enable_device failed (%d)\n", r);
1551                 amdgpu_device_resume(dev, true);
1552
1553                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1554         } else {
1555                 pr_info("switched off\n");
1556                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1557                 amdgpu_device_suspend(dev, true);
1558                 amdgpu_device_cache_pci_state(pdev);
1559                 /* Shut down the device */
1560                 pci_disable_device(pdev);
1561                 pci_set_power_state(pdev, PCI_D3cold);
1562                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1563         }
1564 }
1565
1566 /**
1567  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1568  *
1569  * @pdev: pci dev pointer
1570  *
1571  * Callback for the switcheroo driver.  Check of the switcheroo
1572  * state can be changed.
1573  * Returns true if the state can be changed, false if not.
1574  */
1575 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1576 {
1577         struct drm_device *dev = pci_get_drvdata(pdev);
1578
1579         /*
1580         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1581         * locking inversion with the driver load path. And the access here is
1582         * completely racy anyway. So don't bother with locking for now.
1583         */
1584         return atomic_read(&dev->open_count) == 0;
1585 }
1586
1587 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1588         .set_gpu_state = amdgpu_switcheroo_set_state,
1589         .reprobe = NULL,
1590         .can_switch = amdgpu_switcheroo_can_switch,
1591 };
1592
1593 /**
1594  * amdgpu_device_ip_set_clockgating_state - set the CG state
1595  *
1596  * @dev: amdgpu_device pointer
1597  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1598  * @state: clockgating state (gate or ungate)
1599  *
1600  * Sets the requested clockgating state for all instances of
1601  * the hardware IP specified.
1602  * Returns the error code from the last instance.
1603  */
1604 int amdgpu_device_ip_set_clockgating_state(void *dev,
1605                                            enum amd_ip_block_type block_type,
1606                                            enum amd_clockgating_state state)
1607 {
1608         struct amdgpu_device *adev = dev;
1609         int i, r = 0;
1610
1611         for (i = 0; i < adev->num_ip_blocks; i++) {
1612                 if (!adev->ip_blocks[i].status.valid)
1613                         continue;
1614                 if (adev->ip_blocks[i].version->type != block_type)
1615                         continue;
1616                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1617                         continue;
1618                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1619                         (void *)adev, state);
1620                 if (r)
1621                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1622                                   adev->ip_blocks[i].version->funcs->name, r);
1623         }
1624         return r;
1625 }
1626
1627 /**
1628  * amdgpu_device_ip_set_powergating_state - set the PG state
1629  *
1630  * @dev: amdgpu_device pointer
1631  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1632  * @state: powergating state (gate or ungate)
1633  *
1634  * Sets the requested powergating state for all instances of
1635  * the hardware IP specified.
1636  * Returns the error code from the last instance.
1637  */
1638 int amdgpu_device_ip_set_powergating_state(void *dev,
1639                                            enum amd_ip_block_type block_type,
1640                                            enum amd_powergating_state state)
1641 {
1642         struct amdgpu_device *adev = dev;
1643         int i, r = 0;
1644
1645         for (i = 0; i < adev->num_ip_blocks; i++) {
1646                 if (!adev->ip_blocks[i].status.valid)
1647                         continue;
1648                 if (adev->ip_blocks[i].version->type != block_type)
1649                         continue;
1650                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1651                         continue;
1652                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1653                         (void *)adev, state);
1654                 if (r)
1655                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1656                                   adev->ip_blocks[i].version->funcs->name, r);
1657         }
1658         return r;
1659 }
1660
1661 /**
1662  * amdgpu_device_ip_get_clockgating_state - get the CG state
1663  *
1664  * @adev: amdgpu_device pointer
1665  * @flags: clockgating feature flags
1666  *
1667  * Walks the list of IPs on the device and updates the clockgating
1668  * flags for each IP.
1669  * Updates @flags with the feature flags for each hardware IP where
1670  * clockgating is enabled.
1671  */
1672 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1673                                             u32 *flags)
1674 {
1675         int i;
1676
1677         for (i = 0; i < adev->num_ip_blocks; i++) {
1678                 if (!adev->ip_blocks[i].status.valid)
1679                         continue;
1680                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1681                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1682         }
1683 }
1684
1685 /**
1686  * amdgpu_device_ip_wait_for_idle - wait for idle
1687  *
1688  * @adev: amdgpu_device pointer
1689  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1690  *
1691  * Waits for the request hardware IP to be idle.
1692  * Returns 0 for success or a negative error code on failure.
1693  */
1694 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1695                                    enum amd_ip_block_type block_type)
1696 {
1697         int i, r;
1698
1699         for (i = 0; i < adev->num_ip_blocks; i++) {
1700                 if (!adev->ip_blocks[i].status.valid)
1701                         continue;
1702                 if (adev->ip_blocks[i].version->type == block_type) {
1703                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1704                         if (r)
1705                                 return r;
1706                         break;
1707                 }
1708         }
1709         return 0;
1710
1711 }
1712
1713 /**
1714  * amdgpu_device_ip_is_idle - is the hardware IP idle
1715  *
1716  * @adev: amdgpu_device pointer
1717  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1718  *
1719  * Check if the hardware IP is idle or not.
1720  * Returns true if it the IP is idle, false if not.
1721  */
1722 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1723                               enum amd_ip_block_type block_type)
1724 {
1725         int i;
1726
1727         for (i = 0; i < adev->num_ip_blocks; i++) {
1728                 if (!adev->ip_blocks[i].status.valid)
1729                         continue;
1730                 if (adev->ip_blocks[i].version->type == block_type)
1731                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1732         }
1733         return true;
1734
1735 }
1736
1737 /**
1738  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1739  *
1740  * @adev: amdgpu_device pointer
1741  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1742  *
1743  * Returns a pointer to the hardware IP block structure
1744  * if it exists for the asic, otherwise NULL.
1745  */
1746 struct amdgpu_ip_block *
1747 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1748                               enum amd_ip_block_type type)
1749 {
1750         int i;
1751
1752         for (i = 0; i < adev->num_ip_blocks; i++)
1753                 if (adev->ip_blocks[i].version->type == type)
1754                         return &adev->ip_blocks[i];
1755
1756         return NULL;
1757 }
1758
1759 /**
1760  * amdgpu_device_ip_block_version_cmp
1761  *
1762  * @adev: amdgpu_device pointer
1763  * @type: enum amd_ip_block_type
1764  * @major: major version
1765  * @minor: minor version
1766  *
1767  * return 0 if equal or greater
1768  * return 1 if smaller or the ip_block doesn't exist
1769  */
1770 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1771                                        enum amd_ip_block_type type,
1772                                        u32 major, u32 minor)
1773 {
1774         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1775
1776         if (ip_block && ((ip_block->version->major > major) ||
1777                         ((ip_block->version->major == major) &&
1778                         (ip_block->version->minor >= minor))))
1779                 return 0;
1780
1781         return 1;
1782 }
1783
1784 /**
1785  * amdgpu_device_ip_block_add
1786  *
1787  * @adev: amdgpu_device pointer
1788  * @ip_block_version: pointer to the IP to add
1789  *
1790  * Adds the IP block driver information to the collection of IPs
1791  * on the asic.
1792  */
1793 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1794                                const struct amdgpu_ip_block_version *ip_block_version)
1795 {
1796         if (!ip_block_version)
1797                 return -EINVAL;
1798
1799         switch (ip_block_version->type) {
1800         case AMD_IP_BLOCK_TYPE_VCN:
1801                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
1802                         return 0;
1803                 break;
1804         case AMD_IP_BLOCK_TYPE_JPEG:
1805                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
1806                         return 0;
1807                 break;
1808         default:
1809                 break;
1810         }
1811
1812         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1813                   ip_block_version->funcs->name);
1814
1815         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1816
1817         return 0;
1818 }
1819
1820 /**
1821  * amdgpu_device_enable_virtual_display - enable virtual display feature
1822  *
1823  * @adev: amdgpu_device pointer
1824  *
1825  * Enabled the virtual display feature if the user has enabled it via
1826  * the module parameter virtual_display.  This feature provides a virtual
1827  * display hardware on headless boards or in virtualized environments.
1828  * This function parses and validates the configuration string specified by
1829  * the user and configues the virtual display configuration (number of
1830  * virtual connectors, crtcs, etc.) specified.
1831  */
1832 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1833 {
1834         adev->enable_virtual_display = false;
1835
1836         if (amdgpu_virtual_display) {
1837                 const char *pci_address_name = pci_name(adev->pdev);
1838                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1839
1840                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1841                 pciaddstr_tmp = pciaddstr;
1842                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1843                         pciaddname = strsep(&pciaddname_tmp, ",");
1844                         if (!strcmp("all", pciaddname)
1845                             || !strcmp(pci_address_name, pciaddname)) {
1846                                 long num_crtc;
1847                                 int res = -1;
1848
1849                                 adev->enable_virtual_display = true;
1850
1851                                 if (pciaddname_tmp)
1852                                         res = kstrtol(pciaddname_tmp, 10,
1853                                                       &num_crtc);
1854
1855                                 if (!res) {
1856                                         if (num_crtc < 1)
1857                                                 num_crtc = 1;
1858                                         if (num_crtc > 6)
1859                                                 num_crtc = 6;
1860                                         adev->mode_info.num_crtc = num_crtc;
1861                                 } else {
1862                                         adev->mode_info.num_crtc = 1;
1863                                 }
1864                                 break;
1865                         }
1866                 }
1867
1868                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1869                          amdgpu_virtual_display, pci_address_name,
1870                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1871
1872                 kfree(pciaddstr);
1873         }
1874 }
1875
1876 /**
1877  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1878  *
1879  * @adev: amdgpu_device pointer
1880  *
1881  * Parses the asic configuration parameters specified in the gpu info
1882  * firmware and makes them availale to the driver for use in configuring
1883  * the asic.
1884  * Returns 0 on success, -EINVAL on failure.
1885  */
1886 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1887 {
1888         const char *chip_name;
1889         char fw_name[40];
1890         int err;
1891         const struct gpu_info_firmware_header_v1_0 *hdr;
1892
1893         adev->firmware.gpu_info_fw = NULL;
1894
1895         if (adev->mman.discovery_bin) {
1896                 amdgpu_discovery_get_gfx_info(adev);
1897
1898                 /*
1899                  * FIXME: The bounding box is still needed by Navi12, so
1900                  * temporarily read it from gpu_info firmware. Should be droped
1901                  * when DAL no longer needs it.
1902                  */
1903                 if (adev->asic_type != CHIP_NAVI12)
1904                         return 0;
1905         }
1906
1907         switch (adev->asic_type) {
1908 #ifdef CONFIG_DRM_AMDGPU_SI
1909         case CHIP_VERDE:
1910         case CHIP_TAHITI:
1911         case CHIP_PITCAIRN:
1912         case CHIP_OLAND:
1913         case CHIP_HAINAN:
1914 #endif
1915 #ifdef CONFIG_DRM_AMDGPU_CIK
1916         case CHIP_BONAIRE:
1917         case CHIP_HAWAII:
1918         case CHIP_KAVERI:
1919         case CHIP_KABINI:
1920         case CHIP_MULLINS:
1921 #endif
1922         case CHIP_TOPAZ:
1923         case CHIP_TONGA:
1924         case CHIP_FIJI:
1925         case CHIP_POLARIS10:
1926         case CHIP_POLARIS11:
1927         case CHIP_POLARIS12:
1928         case CHIP_VEGAM:
1929         case CHIP_CARRIZO:
1930         case CHIP_STONEY:
1931         case CHIP_VEGA20:
1932         case CHIP_ALDEBARAN:
1933         case CHIP_SIENNA_CICHLID:
1934         case CHIP_NAVY_FLOUNDER:
1935         case CHIP_DIMGREY_CAVEFISH:
1936         case CHIP_BEIGE_GOBY:
1937         default:
1938                 return 0;
1939         case CHIP_VEGA10:
1940                 chip_name = "vega10";
1941                 break;
1942         case CHIP_VEGA12:
1943                 chip_name = "vega12";
1944                 break;
1945         case CHIP_RAVEN:
1946                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1947                         chip_name = "raven2";
1948                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1949                         chip_name = "picasso";
1950                 else
1951                         chip_name = "raven";
1952                 break;
1953         case CHIP_ARCTURUS:
1954                 chip_name = "arcturus";
1955                 break;
1956         case CHIP_RENOIR:
1957                 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1958                         chip_name = "renoir";
1959                 else
1960                         chip_name = "green_sardine";
1961                 break;
1962         case CHIP_NAVI10:
1963                 chip_name = "navi10";
1964                 break;
1965         case CHIP_NAVI14:
1966                 chip_name = "navi14";
1967                 break;
1968         case CHIP_NAVI12:
1969                 chip_name = "navi12";
1970                 break;
1971         case CHIP_VANGOGH:
1972                 chip_name = "vangogh";
1973                 break;
1974         case CHIP_YELLOW_CARP:
1975                 chip_name = "yellow_carp";
1976                 break;
1977         }
1978
1979         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1980         err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1981         if (err) {
1982                 dev_err(adev->dev,
1983                         "Failed to load gpu_info firmware \"%s\"\n",
1984                         fw_name);
1985                 goto out;
1986         }
1987         err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1988         if (err) {
1989                 dev_err(adev->dev,
1990                         "Failed to validate gpu_info firmware \"%s\"\n",
1991                         fw_name);
1992                 goto out;
1993         }
1994
1995         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1996         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1997
1998         switch (hdr->version_major) {
1999         case 1:
2000         {
2001                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2002                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2003                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2004
2005                 /*
2006                  * Should be droped when DAL no longer needs it.
2007                  */
2008                 if (adev->asic_type == CHIP_NAVI12)
2009                         goto parse_soc_bounding_box;
2010
2011                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2012                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2013                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2014                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2015                 adev->gfx.config.max_texture_channel_caches =
2016                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
2017                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2018                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2019                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2020                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2021                 adev->gfx.config.double_offchip_lds_buf =
2022                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2023                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2024                 adev->gfx.cu_info.max_waves_per_simd =
2025                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2026                 adev->gfx.cu_info.max_scratch_slots_per_cu =
2027                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2028                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2029                 if (hdr->version_minor >= 1) {
2030                         const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2031                                 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2032                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2033                         adev->gfx.config.num_sc_per_sh =
2034                                 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2035                         adev->gfx.config.num_packer_per_sc =
2036                                 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2037                 }
2038
2039 parse_soc_bounding_box:
2040                 /*
2041                  * soc bounding box info is not integrated in disocovery table,
2042                  * we always need to parse it from gpu info firmware if needed.
2043                  */
2044                 if (hdr->version_minor == 2) {
2045                         const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2046                                 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2047                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2048                         adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2049                 }
2050                 break;
2051         }
2052         default:
2053                 dev_err(adev->dev,
2054                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2055                 err = -EINVAL;
2056                 goto out;
2057         }
2058 out:
2059         return err;
2060 }
2061
2062 /**
2063  * amdgpu_device_ip_early_init - run early init for hardware IPs
2064  *
2065  * @adev: amdgpu_device pointer
2066  *
2067  * Early initialization pass for hardware IPs.  The hardware IPs that make
2068  * up each asic are discovered each IP's early_init callback is run.  This
2069  * is the first stage in initializing the asic.
2070  * Returns 0 on success, negative error code on failure.
2071  */
2072 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2073 {
2074         int i, r;
2075
2076         amdgpu_device_enable_virtual_display(adev);
2077
2078         if (amdgpu_sriov_vf(adev)) {
2079                 r = amdgpu_virt_request_full_gpu(adev, true);
2080                 if (r)
2081                         return r;
2082         }
2083
2084         switch (adev->asic_type) {
2085 #ifdef CONFIG_DRM_AMDGPU_SI
2086         case CHIP_VERDE:
2087         case CHIP_TAHITI:
2088         case CHIP_PITCAIRN:
2089         case CHIP_OLAND:
2090         case CHIP_HAINAN:
2091                 adev->family = AMDGPU_FAMILY_SI;
2092                 r = si_set_ip_blocks(adev);
2093                 if (r)
2094                         return r;
2095                 break;
2096 #endif
2097 #ifdef CONFIG_DRM_AMDGPU_CIK
2098         case CHIP_BONAIRE:
2099         case CHIP_HAWAII:
2100         case CHIP_KAVERI:
2101         case CHIP_KABINI:
2102         case CHIP_MULLINS:
2103                 if (adev->flags & AMD_IS_APU)
2104                         adev->family = AMDGPU_FAMILY_KV;
2105                 else
2106                         adev->family = AMDGPU_FAMILY_CI;
2107
2108                 r = cik_set_ip_blocks(adev);
2109                 if (r)
2110                         return r;
2111                 break;
2112 #endif
2113         case CHIP_TOPAZ:
2114         case CHIP_TONGA:
2115         case CHIP_FIJI:
2116         case CHIP_POLARIS10:
2117         case CHIP_POLARIS11:
2118         case CHIP_POLARIS12:
2119         case CHIP_VEGAM:
2120         case CHIP_CARRIZO:
2121         case CHIP_STONEY:
2122                 if (adev->flags & AMD_IS_APU)
2123                         adev->family = AMDGPU_FAMILY_CZ;
2124                 else
2125                         adev->family = AMDGPU_FAMILY_VI;
2126
2127                 r = vi_set_ip_blocks(adev);
2128                 if (r)
2129                         return r;
2130                 break;
2131         default:
2132                 r = amdgpu_discovery_set_ip_blocks(adev);
2133                 if (r)
2134                         return r;
2135                 break;
2136         }
2137
2138         amdgpu_amdkfd_device_probe(adev);
2139
2140         adev->pm.pp_feature = amdgpu_pp_feature_mask;
2141         if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2142                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2143         if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2144                 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2145
2146         for (i = 0; i < adev->num_ip_blocks; i++) {
2147                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2148                         DRM_ERROR("disabled ip block: %d <%s>\n",
2149                                   i, adev->ip_blocks[i].version->funcs->name);
2150                         adev->ip_blocks[i].status.valid = false;
2151                 } else {
2152                         if (adev->ip_blocks[i].version->funcs->early_init) {
2153                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2154                                 if (r == -ENOENT) {
2155                                         adev->ip_blocks[i].status.valid = false;
2156                                 } else if (r) {
2157                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
2158                                                   adev->ip_blocks[i].version->funcs->name, r);
2159                                         return r;
2160                                 } else {
2161                                         adev->ip_blocks[i].status.valid = true;
2162                                 }
2163                         } else {
2164                                 adev->ip_blocks[i].status.valid = true;
2165                         }
2166                 }
2167                 /* get the vbios after the asic_funcs are set up */
2168                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2169                         r = amdgpu_device_parse_gpu_info_fw(adev);
2170                         if (r)
2171                                 return r;
2172
2173                         /* Read BIOS */
2174                         if (!amdgpu_get_bios(adev))
2175                                 return -EINVAL;
2176
2177                         r = amdgpu_atombios_init(adev);
2178                         if (r) {
2179                                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2180                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2181                                 return r;
2182                         }
2183
2184                         /*get pf2vf msg info at it's earliest time*/
2185                         if (amdgpu_sriov_vf(adev))
2186                                 amdgpu_virt_init_data_exchange(adev);
2187
2188                 }
2189         }
2190
2191         adev->cg_flags &= amdgpu_cg_mask;
2192         adev->pg_flags &= amdgpu_pg_mask;
2193
2194         return 0;
2195 }
2196
2197 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2198 {
2199         int i, r;
2200
2201         for (i = 0; i < adev->num_ip_blocks; i++) {
2202                 if (!adev->ip_blocks[i].status.sw)
2203                         continue;
2204                 if (adev->ip_blocks[i].status.hw)
2205                         continue;
2206                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2207                     (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2208                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2209                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2210                         if (r) {
2211                                 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2212                                           adev->ip_blocks[i].version->funcs->name, r);
2213                                 return r;
2214                         }
2215                         adev->ip_blocks[i].status.hw = true;
2216                 }
2217         }
2218
2219         return 0;
2220 }
2221
2222 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2223 {
2224         int i, r;
2225
2226         for (i = 0; i < adev->num_ip_blocks; i++) {
2227                 if (!adev->ip_blocks[i].status.sw)
2228                         continue;
2229                 if (adev->ip_blocks[i].status.hw)
2230                         continue;
2231                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2232                 if (r) {
2233                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2234                                   adev->ip_blocks[i].version->funcs->name, r);
2235                         return r;
2236                 }
2237                 adev->ip_blocks[i].status.hw = true;
2238         }
2239
2240         return 0;
2241 }
2242
2243 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2244 {
2245         int r = 0;
2246         int i;
2247         uint32_t smu_version;
2248
2249         if (adev->asic_type >= CHIP_VEGA10) {
2250                 for (i = 0; i < adev->num_ip_blocks; i++) {
2251                         if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2252                                 continue;
2253
2254                         if (!adev->ip_blocks[i].status.sw)
2255                                 continue;
2256
2257                         /* no need to do the fw loading again if already done*/
2258                         if (adev->ip_blocks[i].status.hw == true)
2259                                 break;
2260
2261                         if (amdgpu_in_reset(adev) || adev->in_suspend) {
2262                                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2263                                 if (r) {
2264                                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2265                                                           adev->ip_blocks[i].version->funcs->name, r);
2266                                         return r;
2267                                 }
2268                         } else {
2269                                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2270                                 if (r) {
2271                                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2272                                                           adev->ip_blocks[i].version->funcs->name, r);
2273                                         return r;
2274                                 }
2275                         }
2276
2277                         adev->ip_blocks[i].status.hw = true;
2278                         break;
2279                 }
2280         }
2281
2282         if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2283                 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2284
2285         return r;
2286 }
2287
2288 /**
2289  * amdgpu_device_ip_init - run init for hardware IPs
2290  *
2291  * @adev: amdgpu_device pointer
2292  *
2293  * Main initialization pass for hardware IPs.  The list of all the hardware
2294  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2295  * are run.  sw_init initializes the software state associated with each IP
2296  * and hw_init initializes the hardware associated with each IP.
2297  * Returns 0 on success, negative error code on failure.
2298  */
2299 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2300 {
2301         int i, r;
2302
2303         r = amdgpu_ras_init(adev);
2304         if (r)
2305                 return r;
2306
2307         for (i = 0; i < adev->num_ip_blocks; i++) {
2308                 if (!adev->ip_blocks[i].status.valid)
2309                         continue;
2310                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2311                 if (r) {
2312                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2313                                   adev->ip_blocks[i].version->funcs->name, r);
2314                         goto init_failed;
2315                 }
2316                 adev->ip_blocks[i].status.sw = true;
2317
2318                 /* need to do gmc hw init early so we can allocate gpu mem */
2319                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2320                         /* Try to reserve bad pages early */
2321                         if (amdgpu_sriov_vf(adev))
2322                                 amdgpu_virt_exchange_data(adev);
2323
2324                         r = amdgpu_device_vram_scratch_init(adev);
2325                         if (r) {
2326                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
2327                                 goto init_failed;
2328                         }
2329                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2330                         if (r) {
2331                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2332                                 goto init_failed;
2333                         }
2334                         r = amdgpu_device_wb_init(adev);
2335                         if (r) {
2336                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2337                                 goto init_failed;
2338                         }
2339                         adev->ip_blocks[i].status.hw = true;
2340
2341                         /* right after GMC hw init, we create CSA */
2342                         if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
2343                                 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2344                                                                 AMDGPU_GEM_DOMAIN_VRAM,
2345                                                                 AMDGPU_CSA_SIZE);
2346                                 if (r) {
2347                                         DRM_ERROR("allocate CSA failed %d\n", r);
2348                                         goto init_failed;
2349                                 }
2350                         }
2351                 }
2352         }
2353
2354         if (amdgpu_sriov_vf(adev))
2355                 amdgpu_virt_exchange_data(adev);
2356
2357         r = amdgpu_ib_pool_init(adev);
2358         if (r) {
2359                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2360                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2361                 goto init_failed;
2362         }
2363
2364         r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2365         if (r)
2366                 goto init_failed;
2367
2368         r = amdgpu_device_ip_hw_init_phase1(adev);
2369         if (r)
2370                 goto init_failed;
2371
2372         r = amdgpu_device_fw_loading(adev);
2373         if (r)
2374                 goto init_failed;
2375
2376         r = amdgpu_device_ip_hw_init_phase2(adev);
2377         if (r)
2378                 goto init_failed;
2379
2380         /*
2381          * retired pages will be loaded from eeprom and reserved here,
2382          * it should be called after amdgpu_device_ip_hw_init_phase2  since
2383          * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2384          * for I2C communication which only true at this point.
2385          *
2386          * amdgpu_ras_recovery_init may fail, but the upper only cares the
2387          * failure from bad gpu situation and stop amdgpu init process
2388          * accordingly. For other failed cases, it will still release all
2389          * the resource and print error message, rather than returning one
2390          * negative value to upper level.
2391          *
2392          * Note: theoretically, this should be called before all vram allocations
2393          * to protect retired page from abusing
2394          */
2395         r = amdgpu_ras_recovery_init(adev);
2396         if (r)
2397                 goto init_failed;
2398
2399         if (adev->gmc.xgmi.num_physical_nodes > 1)
2400                 amdgpu_xgmi_add_device(adev);
2401
2402         /* Don't init kfd if whole hive need to be reset during init */
2403         if (!adev->gmc.xgmi.pending_reset)
2404                 amdgpu_amdkfd_device_init(adev);
2405
2406         amdgpu_fru_get_product_info(adev);
2407
2408 init_failed:
2409         if (amdgpu_sriov_vf(adev))
2410                 amdgpu_virt_release_full_gpu(adev, true);
2411
2412         return r;
2413 }
2414
2415 /**
2416  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2417  *
2418  * @adev: amdgpu_device pointer
2419  *
2420  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2421  * this function before a GPU reset.  If the value is retained after a
2422  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2423  */
2424 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2425 {
2426         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2427 }
2428
2429 /**
2430  * amdgpu_device_check_vram_lost - check if vram is valid
2431  *
2432  * @adev: amdgpu_device pointer
2433  *
2434  * Checks the reset magic value written to the gart pointer in VRAM.
2435  * The driver calls this after a GPU reset to see if the contents of
2436  * VRAM is lost or now.
2437  * returns true if vram is lost, false if not.
2438  */
2439 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2440 {
2441         if (memcmp(adev->gart.ptr, adev->reset_magic,
2442                         AMDGPU_RESET_MAGIC_NUM))
2443                 return true;
2444
2445         if (!amdgpu_in_reset(adev))
2446                 return false;
2447
2448         /*
2449          * For all ASICs with baco/mode1 reset, the VRAM is
2450          * always assumed to be lost.
2451          */
2452         switch (amdgpu_asic_reset_method(adev)) {
2453         case AMD_RESET_METHOD_BACO:
2454         case AMD_RESET_METHOD_MODE1:
2455                 return true;
2456         default:
2457                 return false;
2458         }
2459 }
2460
2461 /**
2462  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2463  *
2464  * @adev: amdgpu_device pointer
2465  * @state: clockgating state (gate or ungate)
2466  *
2467  * The list of all the hardware IPs that make up the asic is walked and the
2468  * set_clockgating_state callbacks are run.
2469  * Late initialization pass enabling clockgating for hardware IPs.
2470  * Fini or suspend, pass disabling clockgating for hardware IPs.
2471  * Returns 0 on success, negative error code on failure.
2472  */
2473
2474 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2475                                enum amd_clockgating_state state)
2476 {
2477         int i, j, r;
2478
2479         if (amdgpu_emu_mode == 1)
2480                 return 0;
2481
2482         for (j = 0; j < adev->num_ip_blocks; j++) {
2483                 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2484                 if (!adev->ip_blocks[i].status.late_initialized)
2485                         continue;
2486                 /* skip CG for GFX on S0ix */
2487                 if (adev->in_s0ix &&
2488                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2489                         continue;
2490                 /* skip CG for VCE/UVD, it's handled specially */
2491                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2492                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2493                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2494                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2495                     adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2496                         /* enable clockgating to save power */
2497                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2498                                                                                      state);
2499                         if (r) {
2500                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2501                                           adev->ip_blocks[i].version->funcs->name, r);
2502                                 return r;
2503                         }
2504                 }
2505         }
2506
2507         return 0;
2508 }
2509
2510 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2511                                enum amd_powergating_state state)
2512 {
2513         int i, j, r;
2514
2515         if (amdgpu_emu_mode == 1)
2516                 return 0;
2517
2518         for (j = 0; j < adev->num_ip_blocks; j++) {
2519                 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2520                 if (!adev->ip_blocks[i].status.late_initialized)
2521                         continue;
2522                 /* skip PG for GFX on S0ix */
2523                 if (adev->in_s0ix &&
2524                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2525                         continue;
2526                 /* skip CG for VCE/UVD, it's handled specially */
2527                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2528                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2529                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2530                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2531                     adev->ip_blocks[i].version->funcs->set_powergating_state) {
2532                         /* enable powergating to save power */
2533                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2534                                                                                         state);
2535                         if (r) {
2536                                 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2537                                           adev->ip_blocks[i].version->funcs->name, r);
2538                                 return r;
2539                         }
2540                 }
2541         }
2542         return 0;
2543 }
2544
2545 static int amdgpu_device_enable_mgpu_fan_boost(void)
2546 {
2547         struct amdgpu_gpu_instance *gpu_ins;
2548         struct amdgpu_device *adev;
2549         int i, ret = 0;
2550
2551         mutex_lock(&mgpu_info.mutex);
2552
2553         /*
2554          * MGPU fan boost feature should be enabled
2555          * only when there are two or more dGPUs in
2556          * the system
2557          */
2558         if (mgpu_info.num_dgpu < 2)
2559                 goto out;
2560
2561         for (i = 0; i < mgpu_info.num_dgpu; i++) {
2562                 gpu_ins = &(mgpu_info.gpu_ins[i]);
2563                 adev = gpu_ins->adev;
2564                 if (!(adev->flags & AMD_IS_APU) &&
2565                     !gpu_ins->mgpu_fan_enabled) {
2566                         ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2567                         if (ret)
2568                                 break;
2569
2570                         gpu_ins->mgpu_fan_enabled = 1;
2571                 }
2572         }
2573
2574 out:
2575         mutex_unlock(&mgpu_info.mutex);
2576
2577         return ret;
2578 }
2579
2580 /**
2581  * amdgpu_device_ip_late_init - run late init for hardware IPs
2582  *
2583  * @adev: amdgpu_device pointer
2584  *
2585  * Late initialization pass for hardware IPs.  The list of all the hardware
2586  * IPs that make up the asic is walked and the late_init callbacks are run.
2587  * late_init covers any special initialization that an IP requires
2588  * after all of the have been initialized or something that needs to happen
2589  * late in the init process.
2590  * Returns 0 on success, negative error code on failure.
2591  */
2592 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2593 {
2594         struct amdgpu_gpu_instance *gpu_instance;
2595         int i = 0, r;
2596
2597         for (i = 0; i < adev->num_ip_blocks; i++) {
2598                 if (!adev->ip_blocks[i].status.hw)
2599                         continue;
2600                 if (adev->ip_blocks[i].version->funcs->late_init) {
2601                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2602                         if (r) {
2603                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2604                                           adev->ip_blocks[i].version->funcs->name, r);
2605                                 return r;
2606                         }
2607                 }
2608                 adev->ip_blocks[i].status.late_initialized = true;
2609         }
2610
2611         amdgpu_ras_set_error_query_ready(adev, true);
2612
2613         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2614         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2615
2616         amdgpu_device_fill_reset_magic(adev);
2617
2618         r = amdgpu_device_enable_mgpu_fan_boost();
2619         if (r)
2620                 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2621
2622         /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
2623         if (amdgpu_passthrough(adev) && ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1)||
2624                                adev->asic_type == CHIP_ALDEBARAN ))
2625                 smu_handle_passthrough_sbr(&adev->smu, true);
2626
2627         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2628                 mutex_lock(&mgpu_info.mutex);
2629
2630                 /*
2631                  * Reset device p-state to low as this was booted with high.
2632                  *
2633                  * This should be performed only after all devices from the same
2634                  * hive get initialized.
2635                  *
2636                  * However, it's unknown how many device in the hive in advance.
2637                  * As this is counted one by one during devices initializations.
2638                  *
2639                  * So, we wait for all XGMI interlinked devices initialized.
2640                  * This may bring some delays as those devices may come from
2641                  * different hives. But that should be OK.
2642                  */
2643                 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2644                         for (i = 0; i < mgpu_info.num_gpu; i++) {
2645                                 gpu_instance = &(mgpu_info.gpu_ins[i]);
2646                                 if (gpu_instance->adev->flags & AMD_IS_APU)
2647                                         continue;
2648
2649                                 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2650                                                 AMDGPU_XGMI_PSTATE_MIN);
2651                                 if (r) {
2652                                         DRM_ERROR("pstate setting failed (%d).\n", r);
2653                                         break;
2654                                 }
2655                         }
2656                 }
2657
2658                 mutex_unlock(&mgpu_info.mutex);
2659         }
2660
2661         return 0;
2662 }
2663
2664 /**
2665  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
2666  *
2667  * @adev: amdgpu_device pointer
2668  *
2669  * For ASICs need to disable SMC first
2670  */
2671 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
2672 {
2673         int i, r;
2674
2675         if (adev->ip_versions[GC_HWIP][0] > IP_VERSION(9, 0, 0))
2676                 return;
2677
2678         for (i = 0; i < adev->num_ip_blocks; i++) {
2679                 if (!adev->ip_blocks[i].status.hw)
2680                         continue;
2681                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2682                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2683                         /* XXX handle errors */
2684                         if (r) {
2685                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2686                                           adev->ip_blocks[i].version->funcs->name, r);
2687                         }
2688                         adev->ip_blocks[i].status.hw = false;
2689                         break;
2690                 }
2691         }
2692 }
2693
2694 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2695 {
2696         int i, r;
2697
2698         for (i = 0; i < adev->num_ip_blocks; i++) {
2699                 if (!adev->ip_blocks[i].version->funcs->early_fini)
2700                         continue;
2701
2702                 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
2703                 if (r) {
2704                         DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
2705                                   adev->ip_blocks[i].version->funcs->name, r);
2706                 }
2707         }
2708
2709         amdgpu_amdkfd_suspend(adev, false);
2710
2711         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2712         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2713
2714         /* Workaroud for ASICs need to disable SMC first */
2715         amdgpu_device_smu_fini_early(adev);
2716
2717         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2718                 if (!adev->ip_blocks[i].status.hw)
2719                         continue;
2720
2721                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2722                 /* XXX handle errors */
2723                 if (r) {
2724                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2725                                   adev->ip_blocks[i].version->funcs->name, r);
2726                 }
2727
2728                 adev->ip_blocks[i].status.hw = false;
2729         }
2730
2731         if (amdgpu_sriov_vf(adev)) {
2732                 if (amdgpu_virt_release_full_gpu(adev, false))
2733                         DRM_ERROR("failed to release exclusive mode on fini\n");
2734         }
2735
2736         return 0;
2737 }
2738
2739 /**
2740  * amdgpu_device_ip_fini - run fini for hardware IPs
2741  *
2742  * @adev: amdgpu_device pointer
2743  *
2744  * Main teardown pass for hardware IPs.  The list of all the hardware
2745  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2746  * are run.  hw_fini tears down the hardware associated with each IP
2747  * and sw_fini tears down any software state associated with each IP.
2748  * Returns 0 on success, negative error code on failure.
2749  */
2750 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2751 {
2752         int i, r;
2753
2754         if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2755                 amdgpu_virt_release_ras_err_handler_data(adev);
2756
2757         if (adev->gmc.xgmi.num_physical_nodes > 1)
2758                 amdgpu_xgmi_remove_device(adev);
2759
2760         amdgpu_amdkfd_device_fini_sw(adev);
2761
2762         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2763                 if (!adev->ip_blocks[i].status.sw)
2764                         continue;
2765
2766                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2767                         amdgpu_ucode_free_bo(adev);
2768                         amdgpu_free_static_csa(&adev->virt.csa_obj);
2769                         amdgpu_device_wb_fini(adev);
2770                         amdgpu_device_vram_scratch_fini(adev);
2771                         amdgpu_ib_pool_fini(adev);
2772                 }
2773
2774                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2775                 /* XXX handle errors */
2776                 if (r) {
2777                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2778                                   adev->ip_blocks[i].version->funcs->name, r);
2779                 }
2780                 adev->ip_blocks[i].status.sw = false;
2781                 adev->ip_blocks[i].status.valid = false;
2782         }
2783
2784         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2785                 if (!adev->ip_blocks[i].status.late_initialized)
2786                         continue;
2787                 if (adev->ip_blocks[i].version->funcs->late_fini)
2788                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2789                 adev->ip_blocks[i].status.late_initialized = false;
2790         }
2791
2792         amdgpu_ras_fini(adev);
2793
2794         return 0;
2795 }
2796
2797 /**
2798  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2799  *
2800  * @work: work_struct.
2801  */
2802 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2803 {
2804         struct amdgpu_device *adev =
2805                 container_of(work, struct amdgpu_device, delayed_init_work.work);
2806         int r;
2807
2808         r = amdgpu_ib_ring_tests(adev);
2809         if (r)
2810                 DRM_ERROR("ib ring test failed (%d).\n", r);
2811 }
2812
2813 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2814 {
2815         struct amdgpu_device *adev =
2816                 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2817
2818         WARN_ON_ONCE(adev->gfx.gfx_off_state);
2819         WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
2820
2821         if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2822                 adev->gfx.gfx_off_state = true;
2823 }
2824
2825 /**
2826  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2827  *
2828  * @adev: amdgpu_device pointer
2829  *
2830  * Main suspend function for hardware IPs.  The list of all the hardware
2831  * IPs that make up the asic is walked, clockgating is disabled and the
2832  * suspend callbacks are run.  suspend puts the hardware and software state
2833  * in each IP into a state suitable for suspend.
2834  * Returns 0 on success, negative error code on failure.
2835  */
2836 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2837 {
2838         int i, r;
2839
2840         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2841         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2842
2843         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2844                 if (!adev->ip_blocks[i].status.valid)
2845                         continue;
2846
2847                 /* displays are handled separately */
2848                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
2849                         continue;
2850
2851                 /* XXX handle errors */
2852                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2853                 /* XXX handle errors */
2854                 if (r) {
2855                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
2856                                   adev->ip_blocks[i].version->funcs->name, r);
2857                         return r;
2858                 }
2859
2860                 adev->ip_blocks[i].status.hw = false;
2861         }
2862
2863         return 0;
2864 }
2865
2866 /**
2867  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2868  *
2869  * @adev: amdgpu_device pointer
2870  *
2871  * Main suspend function for hardware IPs.  The list of all the hardware
2872  * IPs that make up the asic is walked, clockgating is disabled and the
2873  * suspend callbacks are run.  suspend puts the hardware and software state
2874  * in each IP into a state suitable for suspend.
2875  * Returns 0 on success, negative error code on failure.
2876  */
2877 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2878 {
2879         int i, r;
2880
2881         if (adev->in_s0ix)
2882                 amdgpu_gfx_state_change_set(adev, sGpuChangeState_D3Entry);
2883
2884         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2885                 if (!adev->ip_blocks[i].status.valid)
2886                         continue;
2887                 /* displays are handled in phase1 */
2888                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
2889                         continue;
2890                 /* PSP lost connection when err_event_athub occurs */
2891                 if (amdgpu_ras_intr_triggered() &&
2892                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2893                         adev->ip_blocks[i].status.hw = false;
2894                         continue;
2895                 }
2896
2897                 /* skip unnecessary suspend if we do not initialize them yet */
2898                 if (adev->gmc.xgmi.pending_reset &&
2899                     !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2900                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
2901                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2902                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
2903                         adev->ip_blocks[i].status.hw = false;
2904                         continue;
2905                 }
2906
2907                 /* skip suspend of gfx and psp for S0ix
2908                  * gfx is in gfxoff state, so on resume it will exit gfxoff just
2909                  * like at runtime. PSP is also part of the always on hardware
2910                  * so no need to suspend it.
2911                  */
2912                 if (adev->in_s0ix &&
2913                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
2914                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX))
2915                         continue;
2916
2917                 /* XXX handle errors */
2918                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2919                 /* XXX handle errors */
2920                 if (r) {
2921                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
2922                                   adev->ip_blocks[i].version->funcs->name, r);
2923                 }
2924                 adev->ip_blocks[i].status.hw = false;
2925                 /* handle putting the SMC in the appropriate state */
2926                 if(!amdgpu_sriov_vf(adev)){
2927                         if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2928                                 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
2929                                 if (r) {
2930                                         DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
2931                                                         adev->mp1_state, r);
2932                                         return r;
2933                                 }
2934                         }
2935                 }
2936         }
2937
2938         return 0;
2939 }
2940
2941 /**
2942  * amdgpu_device_ip_suspend - run suspend for hardware IPs
2943  *
2944  * @adev: amdgpu_device pointer
2945  *
2946  * Main suspend function for hardware IPs.  The list of all the hardware
2947  * IPs that make up the asic is walked, clockgating is disabled and the
2948  * suspend callbacks are run.  suspend puts the hardware and software state
2949  * in each IP into a state suitable for suspend.
2950  * Returns 0 on success, negative error code on failure.
2951  */
2952 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
2953 {
2954         int r;
2955
2956         if (amdgpu_sriov_vf(adev)) {
2957                 amdgpu_virt_fini_data_exchange(adev);
2958                 amdgpu_virt_request_full_gpu(adev, false);
2959         }
2960
2961         r = amdgpu_device_ip_suspend_phase1(adev);
2962         if (r)
2963                 return r;
2964         r = amdgpu_device_ip_suspend_phase2(adev);
2965
2966         if (amdgpu_sriov_vf(adev))
2967                 amdgpu_virt_release_full_gpu(adev, false);
2968
2969         return r;
2970 }
2971
2972 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
2973 {
2974         int i, r;
2975
2976         static enum amd_ip_block_type ip_order[] = {
2977                 AMD_IP_BLOCK_TYPE_GMC,
2978                 AMD_IP_BLOCK_TYPE_COMMON,
2979                 AMD_IP_BLOCK_TYPE_PSP,
2980                 AMD_IP_BLOCK_TYPE_IH,
2981         };
2982
2983         for (i = 0; i < adev->num_ip_blocks; i++) {
2984                 int j;
2985                 struct amdgpu_ip_block *block;
2986
2987                 block = &adev->ip_blocks[i];
2988                 block->status.hw = false;
2989
2990                 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
2991
2992                         if (block->version->type != ip_order[j] ||
2993                                 !block->status.valid)
2994                                 continue;
2995
2996                         r = block->version->funcs->hw_init(adev);
2997                         DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2998                         if (r)
2999                                 return r;
3000                         block->status.hw = true;
3001                 }
3002         }
3003
3004         return 0;
3005 }
3006
3007 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3008 {
3009         int i, r;
3010
3011         static enum amd_ip_block_type ip_order[] = {
3012                 AMD_IP_BLOCK_TYPE_SMC,
3013                 AMD_IP_BLOCK_TYPE_DCE,
3014                 AMD_IP_BLOCK_TYPE_GFX,
3015                 AMD_IP_BLOCK_TYPE_SDMA,
3016                 AMD_IP_BLOCK_TYPE_UVD,
3017                 AMD_IP_BLOCK_TYPE_VCE,
3018                 AMD_IP_BLOCK_TYPE_VCN
3019         };
3020
3021         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3022                 int j;
3023                 struct amdgpu_ip_block *block;
3024
3025                 for (j = 0; j < adev->num_ip_blocks; j++) {
3026                         block = &adev->ip_blocks[j];
3027
3028                         if (block->version->type != ip_order[i] ||
3029                                 !block->status.valid ||
3030                                 block->status.hw)
3031                                 continue;
3032
3033                         if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3034                                 r = block->version->funcs->resume(adev);
3035                         else
3036                                 r = block->version->funcs->hw_init(adev);
3037
3038                         DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3039                         if (r)
3040                                 return r;
3041                         block->status.hw = true;
3042                 }
3043         }
3044
3045         return 0;
3046 }
3047
3048 /**
3049  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3050  *
3051  * @adev: amdgpu_device pointer
3052  *
3053  * First resume function for hardware IPs.  The list of all the hardware
3054  * IPs that make up the asic is walked and the resume callbacks are run for
3055  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3056  * after a suspend and updates the software state as necessary.  This
3057  * function is also used for restoring the GPU after a GPU reset.
3058  * Returns 0 on success, negative error code on failure.
3059  */
3060 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3061 {
3062         int i, r;
3063
3064         for (i = 0; i < adev->num_ip_blocks; i++) {
3065                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3066                         continue;
3067                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3068                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3069                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
3070
3071                         r = adev->ip_blocks[i].version->funcs->resume(adev);
3072                         if (r) {
3073                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
3074                                           adev->ip_blocks[i].version->funcs->name, r);
3075                                 return r;
3076                         }
3077                         adev->ip_blocks[i].status.hw = true;
3078                 }
3079         }
3080
3081         return 0;
3082 }
3083
3084 /**
3085  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3086  *
3087  * @adev: amdgpu_device pointer
3088  *
3089  * First resume function for hardware IPs.  The list of all the hardware
3090  * IPs that make up the asic is walked and the resume callbacks are run for
3091  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3092  * functional state after a suspend and updates the software state as
3093  * necessary.  This function is also used for restoring the GPU after a GPU
3094  * reset.
3095  * Returns 0 on success, negative error code on failure.
3096  */
3097 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3098 {
3099         int i, r;
3100
3101         for (i = 0; i < adev->num_ip_blocks; i++) {
3102                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3103                         continue;
3104                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3105                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3106                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3107                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3108                         continue;
3109                 r = adev->ip_blocks[i].version->funcs->resume(adev);
3110                 if (r) {
3111                         DRM_ERROR("resume of IP block <%s> failed %d\n",
3112                                   adev->ip_blocks[i].version->funcs->name, r);
3113                         return r;
3114                 }
3115                 adev->ip_blocks[i].status.hw = true;
3116         }
3117
3118         return 0;
3119 }
3120
3121 /**
3122  * amdgpu_device_ip_resume - run resume for hardware IPs
3123  *
3124  * @adev: amdgpu_device pointer
3125  *
3126  * Main resume function for hardware IPs.  The hardware IPs
3127  * are split into two resume functions because they are
3128  * are also used in in recovering from a GPU reset and some additional
3129  * steps need to be take between them.  In this case (S3/S4) they are
3130  * run sequentially.
3131  * Returns 0 on success, negative error code on failure.
3132  */
3133 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3134 {
3135         int r;
3136
3137         r = amdgpu_amdkfd_resume_iommu(adev);
3138         if (r)
3139                 return r;
3140
3141         r = amdgpu_device_ip_resume_phase1(adev);
3142         if (r)
3143                 return r;
3144
3145         r = amdgpu_device_fw_loading(adev);
3146         if (r)
3147                 return r;
3148
3149         r = amdgpu_device_ip_resume_phase2(adev);
3150
3151         return r;
3152 }
3153
3154 /**
3155  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3156  *
3157  * @adev: amdgpu_device pointer
3158  *
3159  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3160  */
3161 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3162 {
3163         if (amdgpu_sriov_vf(adev)) {
3164                 if (adev->is_atom_fw) {
3165                         if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3166                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3167                 } else {
3168                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3169                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3170                 }
3171
3172                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3173                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3174         }
3175 }
3176
3177 /**
3178  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3179  *
3180  * @asic_type: AMD asic type
3181  *
3182  * Check if there is DC (new modesetting infrastructre) support for an asic.
3183  * returns true if DC has support, false if not.
3184  */
3185 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3186 {
3187         switch (asic_type) {
3188 #ifdef CONFIG_DRM_AMDGPU_SI
3189         case CHIP_HAINAN:
3190 #endif
3191         case CHIP_TOPAZ:
3192                 /* chips with no display hardware */
3193                 return false;
3194 #if defined(CONFIG_DRM_AMD_DC)
3195         case CHIP_TAHITI:
3196         case CHIP_PITCAIRN:
3197         case CHIP_VERDE:
3198         case CHIP_OLAND:
3199                 /*
3200                  * We have systems in the wild with these ASICs that require
3201                  * LVDS and VGA support which is not supported with DC.
3202                  *
3203                  * Fallback to the non-DC driver here by default so as not to
3204                  * cause regressions.
3205                  */
3206 #if defined(CONFIG_DRM_AMD_DC_SI)
3207                 return amdgpu_dc > 0;
3208 #else
3209                 return false;
3210 #endif
3211         case CHIP_BONAIRE:
3212         case CHIP_KAVERI:
3213         case CHIP_KABINI:
3214         case CHIP_MULLINS:
3215                 /*
3216                  * We have systems in the wild with these ASICs that require
3217                  * LVDS and VGA support which is not supported with DC.
3218                  *
3219                  * Fallback to the non-DC driver here by default so as not to
3220                  * cause regressions.
3221                  */
3222                 return amdgpu_dc > 0;
3223         case CHIP_HAWAII:
3224         case CHIP_CARRIZO:
3225         case CHIP_STONEY:
3226         case CHIP_POLARIS10:
3227         case CHIP_POLARIS11:
3228         case CHIP_POLARIS12:
3229         case CHIP_VEGAM:
3230         case CHIP_TONGA:
3231         case CHIP_FIJI:
3232         case CHIP_VEGA10:
3233         case CHIP_VEGA12:
3234         case CHIP_VEGA20:
3235 #if defined(CONFIG_DRM_AMD_DC_DCN)
3236         case CHIP_RAVEN:
3237         case CHIP_NAVI10:
3238         case CHIP_NAVI14:
3239         case CHIP_NAVI12:
3240         case CHIP_RENOIR:
3241         case CHIP_CYAN_SKILLFISH:
3242         case CHIP_SIENNA_CICHLID:
3243         case CHIP_NAVY_FLOUNDER:
3244         case CHIP_DIMGREY_CAVEFISH:
3245         case CHIP_BEIGE_GOBY:
3246         case CHIP_VANGOGH:
3247         case CHIP_YELLOW_CARP:
3248 #endif
3249         default:
3250                 return amdgpu_dc != 0;
3251 #else
3252         default:
3253                 if (amdgpu_dc > 0)
3254                         DRM_INFO_ONCE("Display Core has been requested via kernel parameter "
3255                                          "but isn't supported by ASIC, ignoring\n");
3256                 return false;
3257 #endif
3258         }
3259 }
3260
3261 /**
3262  * amdgpu_device_has_dc_support - check if dc is supported
3263  *
3264  * @adev: amdgpu_device pointer
3265  *
3266  * Returns true for supported, false for not supported
3267  */
3268 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3269 {
3270         if (amdgpu_sriov_vf(adev) ||
3271             adev->enable_virtual_display ||
3272             (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3273                 return false;
3274
3275         return amdgpu_device_asic_has_dc_support(adev->asic_type);
3276 }
3277
3278 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3279 {
3280         struct amdgpu_device *adev =
3281                 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3282         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3283
3284         /* It's a bug to not have a hive within this function */
3285         if (WARN_ON(!hive))
3286                 return;
3287
3288         /*
3289          * Use task barrier to synchronize all xgmi reset works across the
3290          * hive. task_barrier_enter and task_barrier_exit will block
3291          * until all the threads running the xgmi reset works reach
3292          * those points. task_barrier_full will do both blocks.
3293          */
3294         if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3295
3296                 task_barrier_enter(&hive->tb);
3297                 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3298
3299                 if (adev->asic_reset_res)
3300                         goto fail;
3301
3302                 task_barrier_exit(&hive->tb);
3303                 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3304
3305                 if (adev->asic_reset_res)
3306                         goto fail;
3307
3308                 if (adev->mmhub.ras_funcs &&
3309                     adev->mmhub.ras_funcs->reset_ras_error_count)
3310                         adev->mmhub.ras_funcs->reset_ras_error_count(adev);
3311         } else {
3312
3313                 task_barrier_full(&hive->tb);
3314                 adev->asic_reset_res =  amdgpu_asic_reset(adev);
3315         }
3316
3317 fail:
3318         if (adev->asic_reset_res)
3319                 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3320                          adev->asic_reset_res, adev_to_drm(adev)->unique);
3321         amdgpu_put_xgmi_hive(hive);
3322 }
3323
3324 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3325 {
3326         char *input = amdgpu_lockup_timeout;
3327         char *timeout_setting = NULL;
3328         int index = 0;
3329         long timeout;
3330         int ret = 0;
3331
3332         /*
3333          * By default timeout for non compute jobs is 10000
3334          * and 60000 for compute jobs.
3335          * In SR-IOV or passthrough mode, timeout for compute
3336          * jobs are 60000 by default.
3337          */
3338         adev->gfx_timeout = msecs_to_jiffies(10000);
3339         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3340         if (amdgpu_sriov_vf(adev))
3341                 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3342                                         msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3343         else
3344                 adev->compute_timeout =  msecs_to_jiffies(60000);
3345
3346         if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3347                 while ((timeout_setting = strsep(&input, ",")) &&
3348                                 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3349                         ret = kstrtol(timeout_setting, 0, &timeout);
3350                         if (ret)
3351                                 return ret;
3352
3353                         if (timeout == 0) {
3354                                 index++;
3355                                 continue;
3356                         } else if (timeout < 0) {
3357                                 timeout = MAX_SCHEDULE_TIMEOUT;
3358                                 dev_warn(adev->dev, "lockup timeout disabled");
3359                                 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3360                         } else {
3361                                 timeout = msecs_to_jiffies(timeout);
3362                         }
3363
3364                         switch (index++) {
3365                         case 0:
3366                                 adev->gfx_timeout = timeout;
3367                                 break;
3368                         case 1:
3369                                 adev->compute_timeout = timeout;
3370                                 break;
3371                         case 2:
3372                                 adev->sdma_timeout = timeout;
3373                                 break;
3374                         case 3:
3375                                 adev->video_timeout = timeout;
3376                                 break;
3377                         default:
3378                                 break;
3379                         }
3380                 }
3381                 /*
3382                  * There is only one value specified and
3383                  * it should apply to all non-compute jobs.
3384                  */
3385                 if (index == 1) {
3386                         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3387                         if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3388                                 adev->compute_timeout = adev->gfx_timeout;
3389                 }
3390         }
3391
3392         return ret;
3393 }
3394
3395 /**
3396  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3397  *
3398  * @adev: amdgpu_device pointer
3399  *
3400  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3401  */
3402 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3403 {
3404         struct iommu_domain *domain;
3405
3406         domain = iommu_get_domain_for_dev(adev->dev);
3407         if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3408                 adev->ram_is_direct_mapped = true;
3409 }
3410
3411 static const struct attribute *amdgpu_dev_attributes[] = {
3412         &dev_attr_product_name.attr,
3413         &dev_attr_product_number.attr,
3414         &dev_attr_serial_number.attr,
3415         &dev_attr_pcie_replay_count.attr,
3416         NULL
3417 };
3418
3419 /**
3420  * amdgpu_device_init - initialize the driver
3421  *
3422  * @adev: amdgpu_device pointer
3423  * @flags: driver flags
3424  *
3425  * Initializes the driver info and hw (all asics).
3426  * Returns 0 for success or an error on failure.
3427  * Called at driver startup.
3428  */
3429 int amdgpu_device_init(struct amdgpu_device *adev,
3430                        uint32_t flags)
3431 {
3432         struct drm_device *ddev = adev_to_drm(adev);
3433         struct pci_dev *pdev = adev->pdev;
3434         int r, i;
3435         bool px = false;
3436         u32 max_MBps;
3437
3438         adev->shutdown = false;
3439         adev->flags = flags;
3440
3441         if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3442                 adev->asic_type = amdgpu_force_asic_type;
3443         else
3444                 adev->asic_type = flags & AMD_ASIC_MASK;
3445
3446         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3447         if (amdgpu_emu_mode == 1)
3448                 adev->usec_timeout *= 10;
3449         adev->gmc.gart_size = 512 * 1024 * 1024;
3450         adev->accel_working = false;
3451         adev->num_rings = 0;
3452         adev->mman.buffer_funcs = NULL;
3453         adev->mman.buffer_funcs_ring = NULL;
3454         adev->vm_manager.vm_pte_funcs = NULL;
3455         adev->vm_manager.vm_pte_num_scheds = 0;
3456         adev->gmc.gmc_funcs = NULL;
3457         adev->harvest_ip_mask = 0x0;
3458         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3459         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3460
3461         adev->smc_rreg = &amdgpu_invalid_rreg;
3462         adev->smc_wreg = &amdgpu_invalid_wreg;
3463         adev->pcie_rreg = &amdgpu_invalid_rreg;
3464         adev->pcie_wreg = &amdgpu_invalid_wreg;
3465         adev->pciep_rreg = &amdgpu_invalid_rreg;
3466         adev->pciep_wreg = &amdgpu_invalid_wreg;
3467         adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3468         adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3469         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3470         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3471         adev->didt_rreg = &amdgpu_invalid_rreg;
3472         adev->didt_wreg = &amdgpu_invalid_wreg;
3473         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3474         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3475         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3476         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3477
3478         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3479                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3480                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3481
3482         /* mutex initialization are all done here so we
3483          * can recall function without having locking issues */
3484         mutex_init(&adev->firmware.mutex);
3485         mutex_init(&adev->pm.mutex);
3486         mutex_init(&adev->gfx.gpu_clock_mutex);
3487         mutex_init(&adev->srbm_mutex);
3488         mutex_init(&adev->gfx.pipe_reserve_mutex);
3489         mutex_init(&adev->gfx.gfx_off_mutex);
3490         mutex_init(&adev->grbm_idx_mutex);
3491         mutex_init(&adev->mn_lock);
3492         mutex_init(&adev->virt.vf_errors.lock);
3493         hash_init(adev->mn_hash);
3494         atomic_set(&adev->in_gpu_reset, 0);
3495         init_rwsem(&adev->reset_sem);
3496         mutex_init(&adev->psp.mutex);
3497         mutex_init(&adev->notifier_lock);
3498
3499         r = amdgpu_device_init_apu_flags(adev);
3500         if (r)
3501                 return r;
3502
3503         r = amdgpu_device_check_arguments(adev);
3504         if (r)
3505                 return r;
3506
3507         spin_lock_init(&adev->mmio_idx_lock);
3508         spin_lock_init(&adev->smc_idx_lock);
3509         spin_lock_init(&adev->pcie_idx_lock);
3510         spin_lock_init(&adev->uvd_ctx_idx_lock);
3511         spin_lock_init(&adev->didt_idx_lock);
3512         spin_lock_init(&adev->gc_cac_idx_lock);
3513         spin_lock_init(&adev->se_cac_idx_lock);
3514         spin_lock_init(&adev->audio_endpt_idx_lock);
3515         spin_lock_init(&adev->mm_stats.lock);
3516
3517         INIT_LIST_HEAD(&adev->shadow_list);
3518         mutex_init(&adev->shadow_list_lock);
3519
3520         INIT_LIST_HEAD(&adev->reset_list);
3521
3522         INIT_DELAYED_WORK(&adev->delayed_init_work,
3523                           amdgpu_device_delayed_init_work_handler);
3524         INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3525                           amdgpu_device_delay_enable_gfx_off);
3526
3527         INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3528
3529         adev->gfx.gfx_off_req_count = 1;
3530         adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3531
3532         atomic_set(&adev->throttling_logging_enabled, 1);
3533         /*
3534          * If throttling continues, logging will be performed every minute
3535          * to avoid log flooding. "-1" is subtracted since the thermal
3536          * throttling interrupt comes every second. Thus, the total logging
3537          * interval is 59 seconds(retelimited printk interval) + 1(waiting
3538          * for throttling interrupt) = 60 seconds.
3539          */
3540         ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3541         ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3542
3543         /* Registers mapping */
3544         /* TODO: block userspace mapping of io register */
3545         if (adev->asic_type >= CHIP_BONAIRE) {
3546                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3547                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3548         } else {
3549                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3550                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3551         }
3552
3553         for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
3554                 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
3555
3556         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3557         if (adev->rmmio == NULL) {
3558                 return -ENOMEM;
3559         }
3560         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3561         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
3562
3563         amdgpu_device_get_pcie_info(adev);
3564
3565         if (amdgpu_mcbp)
3566                 DRM_INFO("MCBP is enabled\n");
3567
3568         if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10)
3569                 adev->enable_mes = true;
3570
3571         /* detect hw virtualization here */
3572         amdgpu_detect_virtualization(adev);
3573
3574         r = amdgpu_device_get_job_timeout_settings(adev);
3575         if (r) {
3576                 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3577                 return r;
3578         }
3579
3580         /* early init functions */
3581         r = amdgpu_device_ip_early_init(adev);
3582         if (r)
3583                 return r;
3584
3585         /* Need to get xgmi info early to decide the reset behavior*/
3586         if (adev->gmc.xgmi.supported) {
3587                 r = adev->gfxhub.funcs->get_xgmi_info(adev);
3588                 if (r)
3589                         return r;
3590         }
3591
3592         /* enable PCIE atomic ops */
3593         if (amdgpu_sriov_vf(adev))
3594                 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
3595                         adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_enabled_flags ==
3596                         (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3597         else
3598                 adev->have_atomics_support =
3599                         !pci_enable_atomic_ops_to_root(adev->pdev,
3600                                           PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3601                                           PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3602         if (!adev->have_atomics_support)
3603                 dev_info(adev->dev, "PCIE atomic ops is not supported\n");
3604
3605         /* doorbell bar mapping and doorbell index init*/
3606         amdgpu_device_doorbell_init(adev);
3607
3608         if (amdgpu_emu_mode == 1) {
3609                 /* post the asic on emulation mode */
3610                 emu_soc_asic_init(adev);
3611                 goto fence_driver_init;
3612         }
3613
3614         amdgpu_reset_init(adev);
3615
3616         /* detect if we are with an SRIOV vbios */
3617         amdgpu_device_detect_sriov_bios(adev);
3618
3619         /* check if we need to reset the asic
3620          *  E.g., driver was not cleanly unloaded previously, etc.
3621          */
3622         if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3623                 if (adev->gmc.xgmi.num_physical_nodes) {
3624                         dev_info(adev->dev, "Pending hive reset.\n");
3625                         adev->gmc.xgmi.pending_reset = true;
3626                         /* Only need to init necessary block for SMU to handle the reset */
3627                         for (i = 0; i < adev->num_ip_blocks; i++) {
3628                                 if (!adev->ip_blocks[i].status.valid)
3629                                         continue;
3630                                 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3631                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3632                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3633                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3634                                         DRM_DEBUG("IP %s disabled for hw_init.\n",
3635                                                 adev->ip_blocks[i].version->funcs->name);
3636                                         adev->ip_blocks[i].status.hw = true;
3637                                 }
3638                         }
3639                 } else {
3640                         r = amdgpu_asic_reset(adev);
3641                         if (r) {
3642                                 dev_err(adev->dev, "asic reset on init failed\n");
3643                                 goto failed;
3644                         }
3645                 }
3646         }
3647
3648         pci_enable_pcie_error_reporting(adev->pdev);
3649
3650         /* Post card if necessary */
3651         if (amdgpu_device_need_post(adev)) {
3652                 if (!adev->bios) {
3653                         dev_err(adev->dev, "no vBIOS found\n");
3654                         r = -EINVAL;
3655                         goto failed;
3656                 }
3657                 DRM_INFO("GPU posting now...\n");
3658                 r = amdgpu_device_asic_init(adev);
3659                 if (r) {
3660                         dev_err(adev->dev, "gpu post error!\n");
3661                         goto failed;
3662                 }
3663         }
3664
3665         if (adev->is_atom_fw) {
3666                 /* Initialize clocks */
3667                 r = amdgpu_atomfirmware_get_clock_info(adev);
3668                 if (r) {
3669                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
3670                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3671                         goto failed;
3672                 }
3673         } else {
3674                 /* Initialize clocks */
3675                 r = amdgpu_atombios_get_clock_info(adev);
3676                 if (r) {
3677                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
3678                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3679                         goto failed;
3680                 }
3681                 /* init i2c buses */
3682                 if (!amdgpu_device_has_dc_support(adev))
3683                         amdgpu_atombios_i2c_init(adev);
3684         }
3685
3686 fence_driver_init:
3687         /* Fence driver */
3688         r = amdgpu_fence_driver_sw_init(adev);
3689         if (r) {
3690                 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
3691                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
3692                 goto failed;
3693         }
3694
3695         /* init the mode config */
3696         drm_mode_config_init(adev_to_drm(adev));
3697
3698         r = amdgpu_device_ip_init(adev);
3699         if (r) {
3700                 /* failed in exclusive mode due to timeout */
3701                 if (amdgpu_sriov_vf(adev) &&
3702                     !amdgpu_sriov_runtime(adev) &&
3703                     amdgpu_virt_mmio_blocked(adev) &&
3704                     !amdgpu_virt_wait_reset(adev)) {
3705                         dev_err(adev->dev, "VF exclusive mode timeout\n");
3706                         /* Don't send request since VF is inactive. */
3707                         adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
3708                         adev->virt.ops = NULL;
3709                         r = -EAGAIN;
3710                         goto release_ras_con;
3711                 }
3712                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
3713                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
3714                 goto release_ras_con;
3715         }
3716
3717         amdgpu_fence_driver_hw_init(adev);
3718
3719         dev_info(adev->dev,
3720                 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
3721                         adev->gfx.config.max_shader_engines,
3722                         adev->gfx.config.max_sh_per_se,
3723                         adev->gfx.config.max_cu_per_sh,
3724                         adev->gfx.cu_info.number);
3725
3726         adev->accel_working = true;
3727
3728         amdgpu_vm_check_compute_bug(adev);
3729
3730         /* Initialize the buffer migration limit. */
3731         if (amdgpu_moverate >= 0)
3732                 max_MBps = amdgpu_moverate;
3733         else
3734                 max_MBps = 8; /* Allow 8 MB/s. */
3735         /* Get a log2 for easy divisions. */
3736         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
3737
3738         r = amdgpu_pm_sysfs_init(adev);
3739         if (r) {
3740                 adev->pm_sysfs_en = false;
3741                 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
3742         } else
3743                 adev->pm_sysfs_en = true;
3744
3745         r = amdgpu_ucode_sysfs_init(adev);
3746         if (r) {
3747                 adev->ucode_sysfs_en = false;
3748                 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
3749         } else
3750                 adev->ucode_sysfs_en = true;
3751
3752         if ((amdgpu_testing & 1)) {
3753                 if (adev->accel_working)
3754                         amdgpu_test_moves(adev);
3755                 else
3756                         DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
3757         }
3758         if (amdgpu_benchmarking) {
3759                 if (adev->accel_working)
3760                         amdgpu_benchmark(adev, amdgpu_benchmarking);
3761                 else
3762                         DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
3763         }
3764
3765         /*
3766          * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
3767          * Otherwise the mgpu fan boost feature will be skipped due to the
3768          * gpu instance is counted less.
3769          */
3770         amdgpu_register_gpu_instance(adev);
3771
3772         /* enable clockgating, etc. after ib tests, etc. since some blocks require
3773          * explicit gating rather than handling it automatically.
3774          */
3775         if (!adev->gmc.xgmi.pending_reset) {
3776                 r = amdgpu_device_ip_late_init(adev);
3777                 if (r) {
3778                         dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
3779                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
3780                         goto release_ras_con;
3781                 }
3782                 /* must succeed. */
3783                 amdgpu_ras_resume(adev);
3784                 queue_delayed_work(system_wq, &adev->delayed_init_work,
3785                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3786         }
3787
3788         if (amdgpu_sriov_vf(adev))
3789                 flush_delayed_work(&adev->delayed_init_work);
3790
3791         r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
3792         if (r)
3793                 dev_err(adev->dev, "Could not create amdgpu device attr\n");
3794
3795         if (IS_ENABLED(CONFIG_PERF_EVENTS))
3796                 r = amdgpu_pmu_init(adev);
3797         if (r)
3798                 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
3799
3800         /* Have stored pci confspace at hand for restore in sudden PCI error */
3801         if (amdgpu_device_cache_pci_state(adev->pdev))
3802                 pci_restore_state(pdev);
3803
3804         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
3805         /* this will fail for cards that aren't VGA class devices, just
3806          * ignore it */
3807         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3808                 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
3809
3810         if (amdgpu_device_supports_px(ddev)) {
3811                 px = true;
3812                 vga_switcheroo_register_client(adev->pdev,
3813                                                &amdgpu_switcheroo_ops, px);
3814                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
3815         }
3816
3817         if (adev->gmc.xgmi.pending_reset)
3818                 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
3819                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3820
3821         amdgpu_device_check_iommu_direct_map(adev);
3822
3823         return 0;
3824
3825 release_ras_con:
3826         amdgpu_release_ras_context(adev);
3827
3828 failed:
3829         amdgpu_vf_error_trans_all(adev);
3830
3831         return r;
3832 }
3833
3834 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
3835 {
3836
3837         /* Clear all CPU mappings pointing to this device */
3838         unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
3839
3840         /* Unmap all mapped bars - Doorbell, registers and VRAM */
3841         amdgpu_device_doorbell_fini(adev);
3842
3843         iounmap(adev->rmmio);
3844         adev->rmmio = NULL;
3845         if (adev->mman.aper_base_kaddr)
3846                 iounmap(adev->mman.aper_base_kaddr);
3847         adev->mman.aper_base_kaddr = NULL;
3848
3849         /* Memory manager related */
3850         if (!adev->gmc.xgmi.connected_to_cpu) {
3851                 arch_phys_wc_del(adev->gmc.vram_mtrr);
3852                 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
3853         }
3854 }
3855
3856 /**
3857  * amdgpu_device_fini_hw - tear down the driver
3858  *
3859  * @adev: amdgpu_device pointer
3860  *
3861  * Tear down the driver info (all asics).
3862  * Called at driver shutdown.
3863  */
3864 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
3865 {
3866         dev_info(adev->dev, "amdgpu: finishing device.\n");
3867         flush_delayed_work(&adev->delayed_init_work);
3868         if (adev->mman.initialized) {
3869                 flush_delayed_work(&adev->mman.bdev.wq);
3870                 ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
3871         }
3872         adev->shutdown = true;
3873
3874         /* make sure IB test finished before entering exclusive mode
3875          * to avoid preemption on IB test
3876          * */
3877         if (amdgpu_sriov_vf(adev)) {
3878                 amdgpu_virt_request_full_gpu(adev, false);
3879                 amdgpu_virt_fini_data_exchange(adev);
3880         }
3881
3882         /* disable all interrupts */
3883         amdgpu_irq_disable_all(adev);
3884         if (adev->mode_info.mode_config_initialized){
3885                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
3886                         drm_helper_force_disable_all(adev_to_drm(adev));
3887                 else
3888                         drm_atomic_helper_shutdown(adev_to_drm(adev));
3889         }
3890         amdgpu_fence_driver_hw_fini(adev);
3891
3892         if (adev->pm_sysfs_en)
3893                 amdgpu_pm_sysfs_fini(adev);
3894         if (adev->ucode_sysfs_en)
3895                 amdgpu_ucode_sysfs_fini(adev);
3896         sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
3897
3898         /* disable ras feature must before hw fini */
3899         amdgpu_ras_pre_fini(adev);
3900
3901         amdgpu_device_ip_fini_early(adev);
3902
3903         amdgpu_irq_fini_hw(adev);
3904
3905         if (adev->mman.initialized)
3906                 ttm_device_clear_dma_mappings(&adev->mman.bdev);
3907
3908         amdgpu_gart_dummy_page_fini(adev);
3909
3910         if (drm_dev_is_unplugged(adev_to_drm(adev)))
3911                 amdgpu_device_unmap_mmio(adev);
3912
3913 }
3914
3915 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
3916 {
3917         int idx;
3918
3919         amdgpu_fence_driver_sw_fini(adev);
3920         amdgpu_device_ip_fini(adev);
3921         release_firmware(adev->firmware.gpu_info_fw);
3922         adev->firmware.gpu_info_fw = NULL;
3923         adev->accel_working = false;
3924
3925         amdgpu_reset_fini(adev);
3926
3927         /* free i2c buses */
3928         if (!amdgpu_device_has_dc_support(adev))
3929                 amdgpu_i2c_fini(adev);
3930
3931         if (amdgpu_emu_mode != 1)
3932                 amdgpu_atombios_fini(adev);
3933
3934         kfree(adev->bios);
3935         adev->bios = NULL;
3936         if (amdgpu_device_supports_px(adev_to_drm(adev))) {
3937                 vga_switcheroo_unregister_client(adev->pdev);
3938                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
3939         }
3940         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3941                 vga_client_unregister(adev->pdev);
3942
3943         if (drm_dev_enter(adev_to_drm(adev), &idx)) {
3944
3945                 iounmap(adev->rmmio);
3946                 adev->rmmio = NULL;
3947                 amdgpu_device_doorbell_fini(adev);
3948                 drm_dev_exit(idx);
3949         }
3950
3951         if (IS_ENABLED(CONFIG_PERF_EVENTS))
3952                 amdgpu_pmu_fini(adev);
3953         if (adev->mman.discovery_bin)
3954                 amdgpu_discovery_fini(adev);
3955
3956         kfree(adev->pci_state);
3957
3958 }
3959
3960 /**
3961  * amdgpu_device_evict_resources - evict device resources
3962  * @adev: amdgpu device object
3963  *
3964  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
3965  * of the vram memory type. Mainly used for evicting device resources
3966  * at suspend time.
3967  *
3968  */
3969 static void amdgpu_device_evict_resources(struct amdgpu_device *adev)
3970 {
3971         /* No need to evict vram on APUs for suspend to ram or s2idle */
3972         if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
3973                 return;
3974
3975         if (amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM))
3976                 DRM_WARN("evicting device resources failed\n");
3977
3978 }
3979
3980 /*
3981  * Suspend & resume.
3982  */
3983 /**
3984  * amdgpu_device_suspend - initiate device suspend
3985  *
3986  * @dev: drm dev pointer
3987  * @fbcon : notify the fbdev of suspend
3988  *
3989  * Puts the hw in the suspend state (all asics).
3990  * Returns 0 for success or an error on failure.
3991  * Called at driver suspend.
3992  */
3993 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
3994 {
3995         struct amdgpu_device *adev = drm_to_adev(dev);
3996
3997         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3998                 return 0;
3999
4000         adev->in_suspend = true;
4001
4002         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4003                 DRM_WARN("smart shift update failed\n");
4004
4005         drm_kms_helper_poll_disable(dev);
4006
4007         if (fbcon)
4008                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
4009
4010         cancel_delayed_work_sync(&adev->delayed_init_work);
4011
4012         amdgpu_ras_suspend(adev);
4013
4014         amdgpu_device_ip_suspend_phase1(adev);
4015
4016         if (!adev->in_s0ix)
4017                 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4018
4019         amdgpu_device_evict_resources(adev);
4020
4021         amdgpu_fence_driver_hw_fini(adev);
4022
4023         amdgpu_device_ip_suspend_phase2(adev);
4024
4025         return 0;
4026 }
4027
4028 /**
4029  * amdgpu_device_resume - initiate device resume
4030  *
4031  * @dev: drm dev pointer
4032  * @fbcon : notify the fbdev of resume
4033  *
4034  * Bring the hw back to operating state (all asics).
4035  * Returns 0 for success or an error on failure.
4036  * Called at driver resume.
4037  */
4038 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4039 {
4040         struct amdgpu_device *adev = drm_to_adev(dev);
4041         int r = 0;
4042
4043         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4044                 return 0;
4045
4046         if (adev->in_s0ix)
4047                 amdgpu_gfx_state_change_set(adev, sGpuChangeState_D0Entry);
4048
4049         /* post card */
4050         if (amdgpu_device_need_post(adev)) {
4051                 r = amdgpu_device_asic_init(adev);
4052                 if (r)
4053                         dev_err(adev->dev, "amdgpu asic init failed\n");
4054         }
4055
4056         r = amdgpu_device_ip_resume(adev);
4057         if (r) {
4058                 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4059                 return r;
4060         }
4061         amdgpu_fence_driver_hw_init(adev);
4062
4063         r = amdgpu_device_ip_late_init(adev);
4064         if (r)
4065                 return r;
4066
4067         queue_delayed_work(system_wq, &adev->delayed_init_work,
4068                            msecs_to_jiffies(AMDGPU_RESUME_MS));
4069
4070         if (!adev->in_s0ix) {
4071                 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4072                 if (r)
4073                         return r;
4074         }
4075
4076         /* Make sure IB tests flushed */
4077         flush_delayed_work(&adev->delayed_init_work);
4078
4079         if (fbcon)
4080                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
4081
4082         drm_kms_helper_poll_enable(dev);
4083
4084         amdgpu_ras_resume(adev);
4085
4086         /*
4087          * Most of the connector probing functions try to acquire runtime pm
4088          * refs to ensure that the GPU is powered on when connector polling is
4089          * performed. Since we're calling this from a runtime PM callback,
4090          * trying to acquire rpm refs will cause us to deadlock.
4091          *
4092          * Since we're guaranteed to be holding the rpm lock, it's safe to
4093          * temporarily disable the rpm helpers so this doesn't deadlock us.
4094          */
4095 #ifdef CONFIG_PM
4096         dev->dev->power.disable_depth++;
4097 #endif
4098         if (!amdgpu_device_has_dc_support(adev))
4099                 drm_helper_hpd_irq_event(dev);
4100         else
4101                 drm_kms_helper_hotplug_event(dev);
4102 #ifdef CONFIG_PM
4103         dev->dev->power.disable_depth--;
4104 #endif
4105         adev->in_suspend = false;
4106
4107         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
4108                 DRM_WARN("smart shift update failed\n");
4109
4110         return 0;
4111 }
4112
4113 /**
4114  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4115  *
4116  * @adev: amdgpu_device pointer
4117  *
4118  * The list of all the hardware IPs that make up the asic is walked and
4119  * the check_soft_reset callbacks are run.  check_soft_reset determines
4120  * if the asic is still hung or not.
4121  * Returns true if any of the IPs are still in a hung state, false if not.
4122  */
4123 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4124 {
4125         int i;
4126         bool asic_hang = false;
4127
4128         if (amdgpu_sriov_vf(adev))
4129                 return true;
4130
4131         if (amdgpu_asic_need_full_reset(adev))
4132                 return true;
4133
4134         for (i = 0; i < adev->num_ip_blocks; i++) {
4135                 if (!adev->ip_blocks[i].status.valid)
4136                         continue;
4137                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4138                         adev->ip_blocks[i].status.hang =
4139                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4140                 if (adev->ip_blocks[i].status.hang) {
4141                         dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4142                         asic_hang = true;
4143                 }
4144         }
4145         return asic_hang;
4146 }
4147
4148 /**
4149  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4150  *
4151  * @adev: amdgpu_device pointer
4152  *
4153  * The list of all the hardware IPs that make up the asic is walked and the
4154  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
4155  * handles any IP specific hardware or software state changes that are
4156  * necessary for a soft reset to succeed.
4157  * Returns 0 on success, negative error code on failure.
4158  */
4159 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4160 {
4161         int i, r = 0;
4162
4163         for (i = 0; i < adev->num_ip_blocks; i++) {
4164                 if (!adev->ip_blocks[i].status.valid)
4165                         continue;
4166                 if (adev->ip_blocks[i].status.hang &&
4167                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4168                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4169                         if (r)
4170                                 return r;
4171                 }
4172         }
4173
4174         return 0;
4175 }
4176
4177 /**
4178  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4179  *
4180  * @adev: amdgpu_device pointer
4181  *
4182  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
4183  * reset is necessary to recover.
4184  * Returns true if a full asic reset is required, false if not.
4185  */
4186 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4187 {
4188         int i;
4189
4190         if (amdgpu_asic_need_full_reset(adev))
4191                 return true;
4192
4193         for (i = 0; i < adev->num_ip_blocks; i++) {
4194                 if (!adev->ip_blocks[i].status.valid)
4195                         continue;
4196                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4197                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4198                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4199                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4200                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4201                         if (adev->ip_blocks[i].status.hang) {
4202                                 dev_info(adev->dev, "Some block need full reset!\n");
4203                                 return true;
4204                         }
4205                 }
4206         }
4207         return false;
4208 }
4209
4210 /**
4211  * amdgpu_device_ip_soft_reset - do a soft reset
4212  *
4213  * @adev: amdgpu_device pointer
4214  *
4215  * The list of all the hardware IPs that make up the asic is walked and the
4216  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
4217  * IP specific hardware or software state changes that are necessary to soft
4218  * reset the IP.
4219  * Returns 0 on success, negative error code on failure.
4220  */
4221 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4222 {
4223         int i, r = 0;
4224
4225         for (i = 0; i < adev->num_ip_blocks; i++) {
4226                 if (!adev->ip_blocks[i].status.valid)
4227                         continue;
4228                 if (adev->ip_blocks[i].status.hang &&
4229                     adev->ip_blocks[i].version->funcs->soft_reset) {
4230                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4231                         if (r)
4232                                 return r;
4233                 }
4234         }
4235
4236         return 0;
4237 }
4238
4239 /**
4240  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4241  *
4242  * @adev: amdgpu_device pointer
4243  *
4244  * The list of all the hardware IPs that make up the asic is walked and the
4245  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4246  * handles any IP specific hardware or software state changes that are
4247  * necessary after the IP has been soft reset.
4248  * Returns 0 on success, negative error code on failure.
4249  */
4250 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4251 {
4252         int i, r = 0;
4253
4254         for (i = 0; i < adev->num_ip_blocks; i++) {
4255                 if (!adev->ip_blocks[i].status.valid)
4256                         continue;
4257                 if (adev->ip_blocks[i].status.hang &&
4258                     adev->ip_blocks[i].version->funcs->post_soft_reset)
4259                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4260                 if (r)
4261                         return r;
4262         }
4263
4264         return 0;
4265 }
4266
4267 /**
4268  * amdgpu_device_recover_vram - Recover some VRAM contents
4269  *
4270  * @adev: amdgpu_device pointer
4271  *
4272  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4273  * restore things like GPUVM page tables after a GPU reset where
4274  * the contents of VRAM might be lost.
4275  *
4276  * Returns:
4277  * 0 on success, negative error code on failure.
4278  */
4279 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4280 {
4281         struct dma_fence *fence = NULL, *next = NULL;
4282         struct amdgpu_bo *shadow;
4283         struct amdgpu_bo_vm *vmbo;
4284         long r = 1, tmo;
4285
4286         if (amdgpu_sriov_runtime(adev))
4287                 tmo = msecs_to_jiffies(8000);
4288         else
4289                 tmo = msecs_to_jiffies(100);
4290
4291         dev_info(adev->dev, "recover vram bo from shadow start\n");
4292         mutex_lock(&adev->shadow_list_lock);
4293         list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4294                 shadow = &vmbo->bo;
4295                 /* No need to recover an evicted BO */
4296                 if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4297                     shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4298                     shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4299                         continue;
4300
4301                 r = amdgpu_bo_restore_shadow(shadow, &next);
4302                 if (r)
4303                         break;
4304
4305                 if (fence) {
4306                         tmo = dma_fence_wait_timeout(fence, false, tmo);
4307                         dma_fence_put(fence);
4308                         fence = next;
4309                         if (tmo == 0) {
4310                                 r = -ETIMEDOUT;
4311                                 break;
4312                         } else if (tmo < 0) {
4313                                 r = tmo;
4314                                 break;
4315                         }
4316                 } else {
4317                         fence = next;
4318                 }
4319         }
4320         mutex_unlock(&adev->shadow_list_lock);
4321
4322         if (fence)
4323                 tmo = dma_fence_wait_timeout(fence, false, tmo);
4324         dma_fence_put(fence);
4325
4326         if (r < 0 || tmo <= 0) {
4327                 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4328                 return -EIO;
4329         }
4330
4331         dev_info(adev->dev, "recover vram bo from shadow done\n");
4332         return 0;
4333 }
4334
4335
4336 /**
4337  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4338  *
4339  * @adev: amdgpu_device pointer
4340  * @from_hypervisor: request from hypervisor
4341  *
4342  * do VF FLR and reinitialize Asic
4343  * return 0 means succeeded otherwise failed
4344  */
4345 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4346                                      bool from_hypervisor)
4347 {
4348         int r;
4349         struct amdgpu_hive_info *hive = NULL;
4350
4351         amdgpu_amdkfd_pre_reset(adev);
4352
4353         amdgpu_amdkfd_pre_reset(adev);
4354
4355         if (from_hypervisor)
4356                 r = amdgpu_virt_request_full_gpu(adev, true);
4357         else
4358                 r = amdgpu_virt_reset_gpu(adev);
4359         if (r)
4360                 return r;
4361
4362         /* Resume IP prior to SMC */
4363         r = amdgpu_device_ip_reinit_early_sriov(adev);
4364         if (r)
4365                 goto error;
4366
4367         amdgpu_virt_init_data_exchange(adev);
4368
4369         r = amdgpu_device_fw_loading(adev);
4370         if (r)
4371                 return r;
4372
4373         /* now we are okay to resume SMC/CP/SDMA */
4374         r = amdgpu_device_ip_reinit_late_sriov(adev);
4375         if (r)
4376                 goto error;
4377
4378         hive = amdgpu_get_xgmi_hive(adev);
4379         /* Update PSP FW topology after reset */
4380         if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
4381                 r = amdgpu_xgmi_update_topology(hive, adev);
4382
4383         if (hive)
4384                 amdgpu_put_xgmi_hive(hive);
4385
4386         if (!r) {
4387                 amdgpu_irq_gpu_reset_resume_helper(adev);
4388                 r = amdgpu_ib_ring_tests(adev);
4389                 amdgpu_amdkfd_post_reset(adev);
4390         }
4391
4392 error:
4393         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4394                 amdgpu_inc_vram_lost(adev);
4395                 r = amdgpu_device_recover_vram(adev);
4396         }
4397         amdgpu_virt_release_full_gpu(adev, true);
4398
4399         return r;
4400 }
4401
4402 /**
4403  * amdgpu_device_has_job_running - check if there is any job in mirror list
4404  *
4405  * @adev: amdgpu_device pointer
4406  *
4407  * check if there is any job in mirror list
4408  */
4409 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4410 {
4411         int i;
4412         struct drm_sched_job *job;
4413
4414         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4415                 struct amdgpu_ring *ring = adev->rings[i];
4416
4417                 if (!ring || !ring->sched.thread)
4418                         continue;
4419
4420                 spin_lock(&ring->sched.job_list_lock);
4421                 job = list_first_entry_or_null(&ring->sched.pending_list,
4422                                                struct drm_sched_job, list);
4423                 spin_unlock(&ring->sched.job_list_lock);
4424                 if (job)
4425                         return true;
4426         }
4427         return false;
4428 }
4429
4430 /**
4431  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4432  *
4433  * @adev: amdgpu_device pointer
4434  *
4435  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4436  * a hung GPU.
4437  */
4438 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4439 {
4440         if (!amdgpu_device_ip_check_soft_reset(adev)) {
4441                 dev_info(adev->dev, "Timeout, but no hardware hang detected.\n");
4442                 return false;
4443         }
4444
4445         if (amdgpu_gpu_recovery == 0)
4446                 goto disabled;
4447
4448         if (amdgpu_sriov_vf(adev))
4449                 return true;
4450
4451         if (amdgpu_gpu_recovery == -1) {
4452                 switch (adev->asic_type) {
4453                 case CHIP_BONAIRE:
4454                 case CHIP_HAWAII:
4455                 case CHIP_TOPAZ:
4456                 case CHIP_TONGA:
4457                 case CHIP_FIJI:
4458                 case CHIP_POLARIS10:
4459                 case CHIP_POLARIS11:
4460                 case CHIP_POLARIS12:
4461                 case CHIP_VEGAM:
4462                 case CHIP_VEGA20:
4463                 case CHIP_VEGA10:
4464                 case CHIP_VEGA12:
4465                 case CHIP_RAVEN:
4466                 case CHIP_ARCTURUS:
4467                 case CHIP_RENOIR:
4468                 case CHIP_NAVI10:
4469                 case CHIP_NAVI14:
4470                 case CHIP_NAVI12:
4471                 case CHIP_SIENNA_CICHLID:
4472                 case CHIP_NAVY_FLOUNDER:
4473                 case CHIP_DIMGREY_CAVEFISH:
4474                 case CHIP_BEIGE_GOBY:
4475                 case CHIP_VANGOGH:
4476                 case CHIP_ALDEBARAN:
4477                         break;
4478                 default:
4479                         goto disabled;
4480                 }
4481         }
4482
4483         return true;
4484
4485 disabled:
4486                 dev_info(adev->dev, "GPU recovery disabled.\n");
4487                 return false;
4488 }
4489
4490 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4491 {
4492         u32 i;
4493         int ret = 0;
4494
4495         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4496
4497         dev_info(adev->dev, "GPU mode1 reset\n");
4498
4499         /* disable BM */
4500         pci_clear_master(adev->pdev);
4501
4502         amdgpu_device_cache_pci_state(adev->pdev);
4503
4504         if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4505                 dev_info(adev->dev, "GPU smu mode1 reset\n");
4506                 ret = amdgpu_dpm_mode1_reset(adev);
4507         } else {
4508                 dev_info(adev->dev, "GPU psp mode1 reset\n");
4509                 ret = psp_gpu_reset(adev);
4510         }
4511
4512         if (ret)
4513                 dev_err(adev->dev, "GPU mode1 reset failed\n");
4514
4515         amdgpu_device_load_pci_state(adev->pdev);
4516
4517         /* wait for asic to come out of reset */
4518         for (i = 0; i < adev->usec_timeout; i++) {
4519                 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4520
4521                 if (memsize != 0xffffffff)
4522                         break;
4523                 udelay(1);
4524         }
4525
4526         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4527         return ret;
4528 }
4529
4530 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4531                                  struct amdgpu_reset_context *reset_context)
4532 {
4533         int i, r = 0;
4534         struct amdgpu_job *job = NULL;
4535         bool need_full_reset =
4536                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4537
4538         if (reset_context->reset_req_dev == adev)
4539                 job = reset_context->job;
4540
4541         if (amdgpu_sriov_vf(adev)) {
4542                 /* stop the data exchange thread */
4543                 amdgpu_virt_fini_data_exchange(adev);
4544         }
4545
4546         /* block all schedulers and reset given job's ring */
4547         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4548                 struct amdgpu_ring *ring = adev->rings[i];
4549
4550                 if (!ring || !ring->sched.thread)
4551                         continue;
4552
4553                 /*clear job fence from fence drv to avoid force_completion
4554                  *leave NULL and vm flush fence in fence drv */
4555                 amdgpu_fence_driver_clear_job_fences(ring);
4556
4557                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
4558                 amdgpu_fence_driver_force_completion(ring);
4559         }
4560
4561         if (job && job->vm)
4562                 drm_sched_increase_karma(&job->base);
4563
4564         r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
4565         /* If reset handler not implemented, continue; otherwise return */
4566         if (r == -ENOSYS)
4567                 r = 0;
4568         else
4569                 return r;
4570
4571         /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
4572         if (!amdgpu_sriov_vf(adev)) {
4573
4574                 if (!need_full_reset)
4575                         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
4576
4577                 if (!need_full_reset) {
4578                         amdgpu_device_ip_pre_soft_reset(adev);
4579                         r = amdgpu_device_ip_soft_reset(adev);
4580                         amdgpu_device_ip_post_soft_reset(adev);
4581                         if (r || amdgpu_device_ip_check_soft_reset(adev)) {
4582                                 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
4583                                 need_full_reset = true;
4584                         }
4585                 }
4586
4587                 if (need_full_reset)
4588                         r = amdgpu_device_ip_suspend(adev);
4589                 if (need_full_reset)
4590                         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4591                 else
4592                         clear_bit(AMDGPU_NEED_FULL_RESET,
4593                                   &reset_context->flags);
4594         }
4595
4596         return r;
4597 }
4598
4599 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
4600                          struct amdgpu_reset_context *reset_context)
4601 {
4602         struct amdgpu_device *tmp_adev = NULL;
4603         bool need_full_reset, skip_hw_reset, vram_lost = false;
4604         int r = 0;
4605
4606         /* Try reset handler method first */
4607         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
4608                                     reset_list);
4609         r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
4610         /* If reset handler not implemented, continue; otherwise return */
4611         if (r == -ENOSYS)
4612                 r = 0;
4613         else
4614                 return r;
4615
4616         /* Reset handler not implemented, use the default method */
4617         need_full_reset =
4618                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4619         skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
4620
4621         /*
4622          * ASIC reset has to be done on all XGMI hive nodes ASAP
4623          * to allow proper links negotiation in FW (within 1 sec)
4624          */
4625         if (!skip_hw_reset && need_full_reset) {
4626                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4627                         /* For XGMI run all resets in parallel to speed up the process */
4628                         if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4629                                 tmp_adev->gmc.xgmi.pending_reset = false;
4630                                 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
4631                                         r = -EALREADY;
4632                         } else
4633                                 r = amdgpu_asic_reset(tmp_adev);
4634
4635                         if (r) {
4636                                 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
4637                                          r, adev_to_drm(tmp_adev)->unique);
4638                                 break;
4639                         }
4640                 }
4641
4642                 /* For XGMI wait for all resets to complete before proceed */
4643                 if (!r) {
4644                         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4645                                 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4646                                         flush_work(&tmp_adev->xgmi_reset_work);
4647                                         r = tmp_adev->asic_reset_res;
4648                                         if (r)
4649                                                 break;
4650                                 }
4651                         }
4652                 }
4653         }
4654
4655         if (!r && amdgpu_ras_intr_triggered()) {
4656                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4657                         if (tmp_adev->mmhub.ras_funcs &&
4658                             tmp_adev->mmhub.ras_funcs->reset_ras_error_count)
4659                                 tmp_adev->mmhub.ras_funcs->reset_ras_error_count(tmp_adev);
4660                 }
4661
4662                 amdgpu_ras_intr_cleared();
4663         }
4664
4665         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4666                 if (need_full_reset) {
4667                         /* post card */
4668                         r = amdgpu_device_asic_init(tmp_adev);
4669                         if (r) {
4670                                 dev_warn(tmp_adev->dev, "asic atom init failed!");
4671                         } else {
4672                                 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
4673                                 r = amdgpu_amdkfd_resume_iommu(tmp_adev);
4674                                 if (r)
4675                                         goto out;
4676
4677                                 r = amdgpu_device_ip_resume_phase1(tmp_adev);
4678                                 if (r)
4679                                         goto out;
4680
4681                                 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
4682                                 if (vram_lost) {
4683                                         DRM_INFO("VRAM is lost due to GPU reset!\n");
4684                                         amdgpu_inc_vram_lost(tmp_adev);
4685                                 }
4686
4687                                 r = amdgpu_device_fw_loading(tmp_adev);
4688                                 if (r)
4689                                         return r;
4690
4691                                 r = amdgpu_device_ip_resume_phase2(tmp_adev);
4692                                 if (r)
4693                                         goto out;
4694
4695                                 if (vram_lost)
4696                                         amdgpu_device_fill_reset_magic(tmp_adev);
4697
4698                                 /*
4699                                  * Add this ASIC as tracked as reset was already
4700                                  * complete successfully.
4701                                  */
4702                                 amdgpu_register_gpu_instance(tmp_adev);
4703
4704                                 if (!reset_context->hive &&
4705                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4706                                         amdgpu_xgmi_add_device(tmp_adev);
4707
4708                                 r = amdgpu_device_ip_late_init(tmp_adev);
4709                                 if (r)
4710                                         goto out;
4711
4712                                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
4713
4714                                 /*
4715                                  * The GPU enters bad state once faulty pages
4716                                  * by ECC has reached the threshold, and ras
4717                                  * recovery is scheduled next. So add one check
4718                                  * here to break recovery if it indeed exceeds
4719                                  * bad page threshold, and remind user to
4720                                  * retire this GPU or setting one bigger
4721                                  * bad_page_threshold value to fix this once
4722                                  * probing driver again.
4723                                  */
4724                                 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
4725                                         /* must succeed. */
4726                                         amdgpu_ras_resume(tmp_adev);
4727                                 } else {
4728                                         r = -EINVAL;
4729                                         goto out;
4730                                 }
4731
4732                                 /* Update PSP FW topology after reset */
4733                                 if (reset_context->hive &&
4734                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4735                                         r = amdgpu_xgmi_update_topology(
4736                                                 reset_context->hive, tmp_adev);
4737                         }
4738                 }
4739
4740 out:
4741                 if (!r) {
4742                         amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
4743                         r = amdgpu_ib_ring_tests(tmp_adev);
4744                         if (r) {
4745                                 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
4746                                 need_full_reset = true;
4747                                 r = -EAGAIN;
4748                                 goto end;
4749                         }
4750                 }
4751
4752                 if (!r)
4753                         r = amdgpu_device_recover_vram(tmp_adev);
4754                 else
4755                         tmp_adev->asic_reset_res = r;
4756         }
4757
4758 end:
4759         if (need_full_reset)
4760                 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4761         else
4762                 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4763         return r;
4764 }
4765
4766 static bool amdgpu_device_lock_adev(struct amdgpu_device *adev,
4767                                 struct amdgpu_hive_info *hive)
4768 {
4769         if (atomic_cmpxchg(&adev->in_gpu_reset, 0, 1) != 0)
4770                 return false;
4771
4772         if (hive) {
4773                 down_write_nest_lock(&adev->reset_sem, &hive->hive_lock);
4774         } else {
4775                 down_write(&adev->reset_sem);
4776         }
4777
4778         switch (amdgpu_asic_reset_method(adev)) {
4779         case AMD_RESET_METHOD_MODE1:
4780                 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
4781                 break;
4782         case AMD_RESET_METHOD_MODE2:
4783                 adev->mp1_state = PP_MP1_STATE_RESET;
4784                 break;
4785         default:
4786                 adev->mp1_state = PP_MP1_STATE_NONE;
4787                 break;
4788         }
4789
4790         return true;
4791 }
4792
4793 static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
4794 {
4795         amdgpu_vf_error_trans_all(adev);
4796         adev->mp1_state = PP_MP1_STATE_NONE;
4797         atomic_set(&adev->in_gpu_reset, 0);
4798         up_write(&adev->reset_sem);
4799 }
4800
4801 /*
4802  * to lockup a list of amdgpu devices in a hive safely, if not a hive
4803  * with multiple nodes, it will be similar as amdgpu_device_lock_adev.
4804  *
4805  * unlock won't require roll back.
4806  */
4807 static int amdgpu_device_lock_hive_adev(struct amdgpu_device *adev, struct amdgpu_hive_info *hive)
4808 {
4809         struct amdgpu_device *tmp_adev = NULL;
4810
4811         if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
4812                 if (!hive) {
4813                         dev_err(adev->dev, "Hive is NULL while device has multiple xgmi nodes");
4814                         return -ENODEV;
4815                 }
4816                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
4817                         if (!amdgpu_device_lock_adev(tmp_adev, hive))
4818                                 goto roll_back;
4819                 }
4820         } else if (!amdgpu_device_lock_adev(adev, hive))
4821                 return -EAGAIN;
4822
4823         return 0;
4824 roll_back:
4825         if (!list_is_first(&tmp_adev->gmc.xgmi.head, &hive->device_list)) {
4826                 /*
4827                  * if the lockup iteration break in the middle of a hive,
4828                  * it may means there may has a race issue,
4829                  * or a hive device locked up independently.
4830                  * we may be in trouble and may not, so will try to roll back
4831                  * the lock and give out a warnning.
4832                  */
4833                 dev_warn(tmp_adev->dev, "Hive lock iteration broke in the middle. Rolling back to unlock");
4834                 list_for_each_entry_continue_reverse(tmp_adev, &hive->device_list, gmc.xgmi.head) {
4835                         amdgpu_device_unlock_adev(tmp_adev);
4836                 }
4837         }
4838         return -EAGAIN;
4839 }
4840
4841 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
4842 {
4843         struct pci_dev *p = NULL;
4844
4845         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4846                         adev->pdev->bus->number, 1);
4847         if (p) {
4848                 pm_runtime_enable(&(p->dev));
4849                 pm_runtime_resume(&(p->dev));
4850         }
4851 }
4852
4853 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
4854 {
4855         enum amd_reset_method reset_method;
4856         struct pci_dev *p = NULL;
4857         u64 expires;
4858
4859         /*
4860          * For now, only BACO and mode1 reset are confirmed
4861          * to suffer the audio issue without proper suspended.
4862          */
4863         reset_method = amdgpu_asic_reset_method(adev);
4864         if ((reset_method != AMD_RESET_METHOD_BACO) &&
4865              (reset_method != AMD_RESET_METHOD_MODE1))
4866                 return -EINVAL;
4867
4868         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4869                         adev->pdev->bus->number, 1);
4870         if (!p)
4871                 return -ENODEV;
4872
4873         expires = pm_runtime_autosuspend_expiration(&(p->dev));
4874         if (!expires)
4875                 /*
4876                  * If we cannot get the audio device autosuspend delay,
4877                  * a fixed 4S interval will be used. Considering 3S is
4878                  * the audio controller default autosuspend delay setting.
4879                  * 4S used here is guaranteed to cover that.
4880                  */
4881                 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
4882
4883         while (!pm_runtime_status_suspended(&(p->dev))) {
4884                 if (!pm_runtime_suspend(&(p->dev)))
4885                         break;
4886
4887                 if (expires < ktime_get_mono_fast_ns()) {
4888                         dev_warn(adev->dev, "failed to suspend display audio\n");
4889                         /* TODO: abort the succeeding gpu reset? */
4890                         return -ETIMEDOUT;
4891                 }
4892         }
4893
4894         pm_runtime_disable(&(p->dev));
4895
4896         return 0;
4897 }
4898
4899 static void amdgpu_device_recheck_guilty_jobs(
4900         struct amdgpu_device *adev, struct list_head *device_list_handle,
4901         struct amdgpu_reset_context *reset_context)
4902 {
4903         int i, r = 0;
4904
4905         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4906                 struct amdgpu_ring *ring = adev->rings[i];
4907                 int ret = 0;
4908                 struct drm_sched_job *s_job;
4909
4910                 if (!ring || !ring->sched.thread)
4911                         continue;
4912
4913                 s_job = list_first_entry_or_null(&ring->sched.pending_list,
4914                                 struct drm_sched_job, list);
4915                 if (s_job == NULL)
4916                         continue;
4917
4918                 /* clear job's guilty and depend the folowing step to decide the real one */
4919                 drm_sched_reset_karma(s_job);
4920                 /* for the real bad job, it will be resubmitted twice, adding a dma_fence_get
4921                  * to make sure fence is balanced */
4922                 dma_fence_get(s_job->s_fence->parent);
4923                 drm_sched_resubmit_jobs_ext(&ring->sched, 1);
4924
4925                 ret = dma_fence_wait_timeout(s_job->s_fence->parent, false, ring->sched.timeout);
4926                 if (ret == 0) { /* timeout */
4927                         DRM_ERROR("Found the real bad job! ring:%s, job_id:%llx\n",
4928                                                 ring->sched.name, s_job->id);
4929
4930                         /* set guilty */
4931                         drm_sched_increase_karma(s_job);
4932 retry:
4933                         /* do hw reset */
4934                         if (amdgpu_sriov_vf(adev)) {
4935                                 amdgpu_virt_fini_data_exchange(adev);
4936                                 r = amdgpu_device_reset_sriov(adev, false);
4937                                 if (r)
4938                                         adev->asic_reset_res = r;
4939                         } else {
4940                                 clear_bit(AMDGPU_SKIP_HW_RESET,
4941                                           &reset_context->flags);
4942                                 r = amdgpu_do_asic_reset(device_list_handle,
4943                                                          reset_context);
4944                                 if (r && r == -EAGAIN)
4945                                         goto retry;
4946                         }
4947
4948                         /*
4949                          * add reset counter so that the following
4950                          * resubmitted job could flush vmid
4951                          */
4952                         atomic_inc(&adev->gpu_reset_counter);
4953                         continue;
4954                 }
4955
4956                 /* got the hw fence, signal finished fence */
4957                 atomic_dec(ring->sched.score);
4958                 dma_fence_put(s_job->s_fence->parent);
4959                 dma_fence_get(&s_job->s_fence->finished);
4960                 dma_fence_signal(&s_job->s_fence->finished);
4961                 dma_fence_put(&s_job->s_fence->finished);
4962
4963                 /* remove node from list and free the job */
4964                 spin_lock(&ring->sched.job_list_lock);
4965                 list_del_init(&s_job->list);
4966                 spin_unlock(&ring->sched.job_list_lock);
4967                 ring->sched.ops->free_job(s_job);
4968         }
4969 }
4970
4971 /**
4972  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
4973  *
4974  * @adev: amdgpu_device pointer
4975  * @job: which job trigger hang
4976  *
4977  * Attempt to reset the GPU if it has hung (all asics).
4978  * Attempt to do soft-reset or full-reset and reinitialize Asic
4979  * Returns 0 for success or an error on failure.
4980  */
4981
4982 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
4983                               struct amdgpu_job *job)
4984 {
4985         struct list_head device_list, *device_list_handle =  NULL;
4986         bool job_signaled = false;
4987         struct amdgpu_hive_info *hive = NULL;
4988         struct amdgpu_device *tmp_adev = NULL;
4989         int i, r = 0;
4990         bool need_emergency_restart = false;
4991         bool audio_suspended = false;
4992         int tmp_vram_lost_counter;
4993         struct amdgpu_reset_context reset_context;
4994
4995         memset(&reset_context, 0, sizeof(reset_context));
4996
4997         /*
4998          * Special case: RAS triggered and full reset isn't supported
4999          */
5000         need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5001
5002         /*
5003          * Flush RAM to disk so that after reboot
5004          * the user can read log and see why the system rebooted.
5005          */
5006         if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {
5007                 DRM_WARN("Emergency reboot.");
5008
5009                 ksys_sync_helper();
5010                 emergency_restart();
5011         }
5012
5013         dev_info(adev->dev, "GPU %s begin!\n",
5014                 need_emergency_restart ? "jobs stop":"reset");
5015
5016         /*
5017          * Here we trylock to avoid chain of resets executing from
5018          * either trigger by jobs on different adevs in XGMI hive or jobs on
5019          * different schedulers for same device while this TO handler is running.
5020          * We always reset all schedulers for device and all devices for XGMI
5021          * hive so that should take care of them too.
5022          */
5023         if (!amdgpu_sriov_vf(adev))
5024                 hive = amdgpu_get_xgmi_hive(adev);
5025         if (hive) {
5026                 if (atomic_cmpxchg(&hive->in_reset, 0, 1) != 0) {
5027                         DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress",
5028                                 job ? job->base.id : -1, hive->hive_id);
5029                         amdgpu_put_xgmi_hive(hive);
5030                         if (job && job->vm)
5031                                 drm_sched_increase_karma(&job->base);
5032                         return 0;
5033                 }
5034                 mutex_lock(&hive->hive_lock);
5035         }
5036
5037         reset_context.method = AMD_RESET_METHOD_NONE;
5038         reset_context.reset_req_dev = adev;
5039         reset_context.job = job;
5040         reset_context.hive = hive;
5041         clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5042
5043         /*
5044          * lock the device before we try to operate the linked list
5045          * if didn't get the device lock, don't touch the linked list since
5046          * others may iterating it.
5047          */
5048         r = amdgpu_device_lock_hive_adev(adev, hive);
5049         if (r) {
5050                 dev_info(adev->dev, "Bailing on TDR for s_job:%llx, as another already in progress",
5051                                         job ? job->base.id : -1);
5052
5053                 /* even we skipped this reset, still need to set the job to guilty */
5054                 if (job && job->vm)
5055                         drm_sched_increase_karma(&job->base);
5056                 goto skip_recovery;
5057         }
5058
5059         /*
5060          * Build list of devices to reset.
5061          * In case we are in XGMI hive mode, resort the device list
5062          * to put adev in the 1st position.
5063          */
5064         INIT_LIST_HEAD(&device_list);
5065         if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5066                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
5067                         list_add_tail(&tmp_adev->reset_list, &device_list);
5068                 if (!list_is_first(&adev->reset_list, &device_list))
5069                         list_rotate_to_front(&adev->reset_list, &device_list);
5070                 device_list_handle = &device_list;
5071         } else {
5072                 list_add_tail(&adev->reset_list, &device_list);
5073                 device_list_handle = &device_list;
5074         }
5075
5076         /* block all schedulers and reset given job's ring */
5077         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5078                 /*
5079                  * Try to put the audio codec into suspend state
5080                  * before gpu reset started.
5081                  *
5082                  * Due to the power domain of the graphics device
5083                  * is shared with AZ power domain. Without this,
5084                  * we may change the audio hardware from behind
5085                  * the audio driver's back. That will trigger
5086                  * some audio codec errors.
5087                  */
5088                 if (!amdgpu_device_suspend_display_audio(tmp_adev))
5089                         audio_suspended = true;
5090
5091                 amdgpu_ras_set_error_query_ready(tmp_adev, false);
5092
5093                 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5094
5095                 if (!amdgpu_sriov_vf(tmp_adev))
5096                         amdgpu_amdkfd_pre_reset(tmp_adev);
5097
5098                 /*
5099                  * Mark these ASICs to be reseted as untracked first
5100                  * And add them back after reset completed
5101                  */
5102                 amdgpu_unregister_gpu_instance(tmp_adev);
5103
5104                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
5105
5106                 /* disable ras on ALL IPs */
5107                 if (!need_emergency_restart &&
5108                       amdgpu_device_ip_need_full_reset(tmp_adev))
5109                         amdgpu_ras_suspend(tmp_adev);
5110
5111                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5112                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5113
5114                         if (!ring || !ring->sched.thread)
5115                                 continue;
5116
5117                         drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5118
5119                         if (need_emergency_restart)
5120                                 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5121                 }
5122                 atomic_inc(&tmp_adev->gpu_reset_counter);
5123         }
5124
5125         if (need_emergency_restart)
5126                 goto skip_sched_resume;
5127
5128         /*
5129          * Must check guilty signal here since after this point all old
5130          * HW fences are force signaled.
5131          *
5132          * job->base holds a reference to parent fence
5133          */
5134         if (job && job->base.s_fence->parent &&
5135             dma_fence_is_signaled(job->base.s_fence->parent)) {
5136                 job_signaled = true;
5137                 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5138                 goto skip_hw_reset;
5139         }
5140
5141 retry:  /* Rest of adevs pre asic reset from XGMI hive. */
5142         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5143                 r = amdgpu_device_pre_asic_reset(tmp_adev, &reset_context);
5144                 /*TODO Should we stop ?*/
5145                 if (r) {
5146                         dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5147                                   r, adev_to_drm(tmp_adev)->unique);
5148                         tmp_adev->asic_reset_res = r;
5149                 }
5150         }
5151
5152         tmp_vram_lost_counter = atomic_read(&((adev)->vram_lost_counter));
5153         /* Actual ASIC resets if needed.*/
5154         /* Host driver will handle XGMI hive reset for SRIOV */
5155         if (amdgpu_sriov_vf(adev)) {
5156                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
5157                 if (r)
5158                         adev->asic_reset_res = r;
5159         } else {
5160                 r = amdgpu_do_asic_reset(device_list_handle, &reset_context);
5161                 if (r && r == -EAGAIN)
5162                         goto retry;
5163         }
5164
5165 skip_hw_reset:
5166
5167         /* Post ASIC reset for all devs .*/
5168         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5169
5170                 /*
5171                  * Sometimes a later bad compute job can block a good gfx job as gfx
5172                  * and compute ring share internal GC HW mutually. We add an additional
5173                  * guilty jobs recheck step to find the real guilty job, it synchronously
5174                  * submits and pends for the first job being signaled. If it gets timeout,
5175                  * we identify it as a real guilty job.
5176                  */
5177                 if (amdgpu_gpu_recovery == 2 &&
5178                         !(tmp_vram_lost_counter < atomic_read(&adev->vram_lost_counter)))
5179                         amdgpu_device_recheck_guilty_jobs(
5180                                 tmp_adev, device_list_handle, &reset_context);
5181
5182                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5183                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5184
5185                         if (!ring || !ring->sched.thread)
5186                                 continue;
5187
5188                         /* No point to resubmit jobs if we didn't HW reset*/
5189                         if (!tmp_adev->asic_reset_res && !job_signaled)
5190                                 drm_sched_resubmit_jobs(&ring->sched);
5191
5192                         drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);
5193                 }
5194
5195                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) {
5196                         drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5197                 }
5198
5199                 tmp_adev->asic_reset_res = 0;
5200
5201                 if (r) {
5202                         /* bad news, how to tell it to userspace ? */
5203                         dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5204                         amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5205                 } else {
5206                         dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5207                         if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5208                                 DRM_WARN("smart shift update failed\n");
5209                 }
5210         }
5211
5212 skip_sched_resume:
5213         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5214                 /* unlock kfd: SRIOV would do it separately */
5215                 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5216                         amdgpu_amdkfd_post_reset(tmp_adev);
5217
5218                 /* kfd_post_reset will do nothing if kfd device is not initialized,
5219                  * need to bring up kfd here if it's not be initialized before
5220                  */
5221                 if (!adev->kfd.init_complete)
5222                         amdgpu_amdkfd_device_init(adev);
5223
5224                 if (audio_suspended)
5225                         amdgpu_device_resume_display_audio(tmp_adev);
5226                 amdgpu_device_unlock_adev(tmp_adev);
5227         }
5228
5229 skip_recovery:
5230         if (hive) {
5231                 atomic_set(&hive->in_reset, 0);
5232                 mutex_unlock(&hive->hive_lock);
5233                 amdgpu_put_xgmi_hive(hive);
5234         }
5235
5236         if (r && r != -EAGAIN)
5237                 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5238         return r;
5239 }
5240
5241 /**
5242  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5243  *
5244  * @adev: amdgpu_device pointer
5245  *
5246  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5247  * and lanes) of the slot the device is in. Handles APUs and
5248  * virtualized environments where PCIE config space may not be available.
5249  */
5250 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5251 {
5252         struct pci_dev *pdev;
5253         enum pci_bus_speed speed_cap, platform_speed_cap;
5254         enum pcie_link_width platform_link_width;
5255
5256         if (amdgpu_pcie_gen_cap)
5257                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5258
5259         if (amdgpu_pcie_lane_cap)
5260                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5261
5262         /* covers APUs as well */
5263         if (pci_is_root_bus(adev->pdev->bus)) {
5264                 if (adev->pm.pcie_gen_mask == 0)
5265                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5266                 if (adev->pm.pcie_mlw_mask == 0)
5267                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5268                 return;
5269         }
5270
5271         if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5272                 return;
5273
5274         pcie_bandwidth_available(adev->pdev, NULL,
5275                                  &platform_speed_cap, &platform_link_width);
5276
5277         if (adev->pm.pcie_gen_mask == 0) {
5278                 /* asic caps */
5279                 pdev = adev->pdev;
5280                 speed_cap = pcie_get_speed_cap(pdev);
5281                 if (speed_cap == PCI_SPEED_UNKNOWN) {
5282                         adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5283                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5284                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5285                 } else {
5286                         if (speed_cap == PCIE_SPEED_32_0GT)
5287                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5288                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5289                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5290                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5291                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5292                         else if (speed_cap == PCIE_SPEED_16_0GT)
5293                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5294                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5295                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5296                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5297                         else if (speed_cap == PCIE_SPEED_8_0GT)
5298                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5299                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5300                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5301                         else if (speed_cap == PCIE_SPEED_5_0GT)
5302                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5303                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5304                         else
5305                                 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5306                 }
5307                 /* platform caps */
5308                 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5309                         adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5310                                                    CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5311                 } else {
5312                         if (platform_speed_cap == PCIE_SPEED_32_0GT)
5313                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5314                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5315                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5316                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5317                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5318                         else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5319                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5320                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5321                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5322                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5323                         else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5324                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5325                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5326                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5327                         else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5328                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5329                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5330                         else
5331                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5332
5333                 }
5334         }
5335         if (adev->pm.pcie_mlw_mask == 0) {
5336                 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5337                         adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5338                 } else {
5339                         switch (platform_link_width) {
5340                         case PCIE_LNK_X32:
5341                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5342                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5343                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5344                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5345                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5346                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5347                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5348                                 break;
5349                         case PCIE_LNK_X16:
5350                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5351                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5352                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5353                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5354                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5355                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5356                                 break;
5357                         case PCIE_LNK_X12:
5358                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5359                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5360                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5361                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5362                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5363                                 break;
5364                         case PCIE_LNK_X8:
5365                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5366                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5367                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5368                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5369                                 break;
5370                         case PCIE_LNK_X4:
5371                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5372                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5373                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5374                                 break;
5375                         case PCIE_LNK_X2:
5376                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5377                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5378                                 break;
5379                         case PCIE_LNK_X1:
5380                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5381                                 break;
5382                         default:
5383                                 break;
5384                         }
5385                 }
5386         }
5387 }
5388
5389 int amdgpu_device_baco_enter(struct drm_device *dev)
5390 {
5391         struct amdgpu_device *adev = drm_to_adev(dev);
5392         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5393
5394         if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5395                 return -ENOTSUPP;
5396
5397         if (ras && adev->ras_enabled &&
5398             adev->nbio.funcs->enable_doorbell_interrupt)
5399                 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5400
5401         return amdgpu_dpm_baco_enter(adev);
5402 }
5403
5404 int amdgpu_device_baco_exit(struct drm_device *dev)
5405 {
5406         struct amdgpu_device *adev = drm_to_adev(dev);
5407         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5408         int ret = 0;
5409
5410         if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5411                 return -ENOTSUPP;
5412
5413         ret = amdgpu_dpm_baco_exit(adev);
5414         if (ret)
5415                 return ret;
5416
5417         if (ras && adev->ras_enabled &&
5418             adev->nbio.funcs->enable_doorbell_interrupt)
5419                 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5420
5421         if (amdgpu_passthrough(adev) &&
5422             adev->nbio.funcs->clear_doorbell_interrupt)
5423                 adev->nbio.funcs->clear_doorbell_interrupt(adev);
5424
5425         return 0;
5426 }
5427
5428 static void amdgpu_cancel_all_tdr(struct amdgpu_device *adev)
5429 {
5430         int i;
5431
5432         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5433                 struct amdgpu_ring *ring = adev->rings[i];
5434
5435                 if (!ring || !ring->sched.thread)
5436                         continue;
5437
5438                 cancel_delayed_work_sync(&ring->sched.work_tdr);
5439         }
5440 }
5441
5442 /**
5443  * amdgpu_pci_error_detected - Called when a PCI error is detected.
5444  * @pdev: PCI device struct
5445  * @state: PCI channel state
5446  *
5447  * Description: Called when a PCI error is detected.
5448  *
5449  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5450  */
5451 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5452 {
5453         struct drm_device *dev = pci_get_drvdata(pdev);
5454         struct amdgpu_device *adev = drm_to_adev(dev);
5455         int i;
5456
5457         DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5458
5459         if (adev->gmc.xgmi.num_physical_nodes > 1) {
5460                 DRM_WARN("No support for XGMI hive yet...");
5461                 return PCI_ERS_RESULT_DISCONNECT;
5462         }
5463
5464         adev->pci_channel_state = state;
5465
5466         switch (state) {
5467         case pci_channel_io_normal:
5468                 return PCI_ERS_RESULT_CAN_RECOVER;
5469         /* Fatal error, prepare for slot reset */
5470         case pci_channel_io_frozen:
5471                 /*
5472                  * Cancel and wait for all TDRs in progress if failing to
5473                  * set  adev->in_gpu_reset in amdgpu_device_lock_adev
5474                  *
5475                  * Locking adev->reset_sem will prevent any external access
5476                  * to GPU during PCI error recovery
5477                  */
5478                 while (!amdgpu_device_lock_adev(adev, NULL))
5479                         amdgpu_cancel_all_tdr(adev);
5480
5481                 /*
5482                  * Block any work scheduling as we do for regular GPU reset
5483                  * for the duration of the recovery
5484                  */
5485                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5486                         struct amdgpu_ring *ring = adev->rings[i];
5487
5488                         if (!ring || !ring->sched.thread)
5489                                 continue;
5490
5491                         drm_sched_stop(&ring->sched, NULL);
5492                 }
5493                 atomic_inc(&adev->gpu_reset_counter);
5494                 return PCI_ERS_RESULT_NEED_RESET;
5495         case pci_channel_io_perm_failure:
5496                 /* Permanent error, prepare for device removal */
5497                 return PCI_ERS_RESULT_DISCONNECT;
5498         }
5499
5500         return PCI_ERS_RESULT_NEED_RESET;
5501 }
5502
5503 /**
5504  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5505  * @pdev: pointer to PCI device
5506  */
5507 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5508 {
5509
5510         DRM_INFO("PCI error: mmio enabled callback!!\n");
5511
5512         /* TODO - dump whatever for debugging purposes */
5513
5514         /* This called only if amdgpu_pci_error_detected returns
5515          * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5516          * works, no need to reset slot.
5517          */
5518
5519         return PCI_ERS_RESULT_RECOVERED;
5520 }
5521
5522 /**
5523  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5524  * @pdev: PCI device struct
5525  *
5526  * Description: This routine is called by the pci error recovery
5527  * code after the PCI slot has been reset, just before we
5528  * should resume normal operations.
5529  */
5530 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5531 {
5532         struct drm_device *dev = pci_get_drvdata(pdev);
5533         struct amdgpu_device *adev = drm_to_adev(dev);
5534         int r, i;
5535         struct amdgpu_reset_context reset_context;
5536         u32 memsize;
5537         struct list_head device_list;
5538
5539         DRM_INFO("PCI error: slot reset callback!!\n");
5540
5541         memset(&reset_context, 0, sizeof(reset_context));
5542
5543         INIT_LIST_HEAD(&device_list);
5544         list_add_tail(&adev->reset_list, &device_list);
5545
5546         /* wait for asic to come out of reset */
5547         msleep(500);
5548
5549         /* Restore PCI confspace */
5550         amdgpu_device_load_pci_state(pdev);
5551
5552         /* confirm  ASIC came out of reset */
5553         for (i = 0; i < adev->usec_timeout; i++) {
5554                 memsize = amdgpu_asic_get_config_memsize(adev);
5555
5556                 if (memsize != 0xffffffff)
5557                         break;
5558                 udelay(1);
5559         }
5560         if (memsize == 0xffffffff) {
5561                 r = -ETIME;
5562                 goto out;
5563         }
5564
5565         reset_context.method = AMD_RESET_METHOD_NONE;
5566         reset_context.reset_req_dev = adev;
5567         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5568         set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
5569
5570         adev->no_hw_access = true;
5571         r = amdgpu_device_pre_asic_reset(adev, &reset_context);
5572         adev->no_hw_access = false;
5573         if (r)
5574                 goto out;
5575
5576         r = amdgpu_do_asic_reset(&device_list, &reset_context);
5577
5578 out:
5579         if (!r) {
5580                 if (amdgpu_device_cache_pci_state(adev->pdev))
5581                         pci_restore_state(adev->pdev);
5582
5583                 DRM_INFO("PCIe error recovery succeeded\n");
5584         } else {
5585                 DRM_ERROR("PCIe error recovery failed, err:%d", r);
5586                 amdgpu_device_unlock_adev(adev);
5587         }
5588
5589         return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
5590 }
5591
5592 /**
5593  * amdgpu_pci_resume() - resume normal ops after PCI reset
5594  * @pdev: pointer to PCI device
5595  *
5596  * Called when the error recovery driver tells us that its
5597  * OK to resume normal operation.
5598  */
5599 void amdgpu_pci_resume(struct pci_dev *pdev)
5600 {
5601         struct drm_device *dev = pci_get_drvdata(pdev);
5602         struct amdgpu_device *adev = drm_to_adev(dev);
5603         int i;
5604
5605
5606         DRM_INFO("PCI error: resume callback!!\n");
5607
5608         /* Only continue execution for the case of pci_channel_io_frozen */
5609         if (adev->pci_channel_state != pci_channel_io_frozen)
5610                 return;
5611
5612         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5613                 struct amdgpu_ring *ring = adev->rings[i];
5614
5615                 if (!ring || !ring->sched.thread)
5616                         continue;
5617
5618
5619                 drm_sched_resubmit_jobs(&ring->sched);
5620                 drm_sched_start(&ring->sched, true);
5621         }
5622
5623         amdgpu_device_unlock_adev(adev);
5624 }
5625
5626 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
5627 {
5628         struct drm_device *dev = pci_get_drvdata(pdev);
5629         struct amdgpu_device *adev = drm_to_adev(dev);
5630         int r;
5631
5632         r = pci_save_state(pdev);
5633         if (!r) {
5634                 kfree(adev->pci_state);
5635
5636                 adev->pci_state = pci_store_saved_state(pdev);
5637
5638                 if (!adev->pci_state) {
5639                         DRM_ERROR("Failed to store PCI saved state");
5640                         return false;
5641                 }
5642         } else {
5643                 DRM_WARN("Failed to save PCI state, err:%d\n", r);
5644                 return false;
5645         }
5646
5647         return true;
5648 }
5649
5650 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
5651 {
5652         struct drm_device *dev = pci_get_drvdata(pdev);
5653         struct amdgpu_device *adev = drm_to_adev(dev);
5654         int r;
5655
5656         if (!adev->pci_state)
5657                 return false;
5658
5659         r = pci_load_saved_state(pdev, adev->pci_state);
5660
5661         if (!r) {
5662                 pci_restore_state(pdev);
5663         } else {
5664                 DRM_WARN("Failed to load PCI state, err:%d\n", r);
5665                 return false;
5666         }
5667
5668         return true;
5669 }
5670
5671 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
5672                 struct amdgpu_ring *ring)
5673 {
5674 #ifdef CONFIG_X86_64
5675         if (adev->flags & AMD_IS_APU)
5676                 return;
5677 #endif
5678         if (adev->gmc.xgmi.connected_to_cpu)
5679                 return;
5680
5681         if (ring && ring->funcs->emit_hdp_flush)
5682                 amdgpu_ring_emit_hdp_flush(ring);
5683         else
5684                 amdgpu_asic_flush_hdp(adev, ring);
5685 }
5686
5687 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
5688                 struct amdgpu_ring *ring)
5689 {
5690 #ifdef CONFIG_X86_64
5691         if (adev->flags & AMD_IS_APU)
5692                 return;
5693 #endif
5694         if (adev->gmc.xgmi.connected_to_cpu)
5695                 return;
5696
5697         amdgpu_asic_invalidate_hdp(adev, ring);
5698 }
5699
5700 /**
5701  * amdgpu_device_halt() - bring hardware to some kind of halt state
5702  *
5703  * @adev: amdgpu_device pointer
5704  *
5705  * Bring hardware to some kind of halt state so that no one can touch it
5706  * any more. It will help to maintain error context when error occurred.
5707  * Compare to a simple hang, the system will keep stable at least for SSH
5708  * access. Then it should be trivial to inspect the hardware state and
5709  * see what's going on. Implemented as following:
5710  *
5711  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
5712  *    clears all CPU mappings to device, disallows remappings through page faults
5713  * 2. amdgpu_irq_disable_all() disables all interrupts
5714  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
5715  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
5716  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
5717  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
5718  *    flush any in flight DMA operations
5719  */
5720 void amdgpu_device_halt(struct amdgpu_device *adev)
5721 {
5722         struct pci_dev *pdev = adev->pdev;
5723         struct drm_device *ddev = adev_to_drm(adev);
5724
5725         drm_dev_unplug(ddev);
5726
5727         amdgpu_irq_disable_all(adev);
5728
5729         amdgpu_fence_driver_hw_fini(adev);
5730
5731         adev->no_hw_access = true;
5732
5733         amdgpu_device_unmap_mmio(adev);
5734
5735         pci_disable_device(pdev);
5736         pci_wait_for_pending_transaction(pdev);
5737 }