drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/power_supply.h>
  29 #include <linux/kthread.h>
  30 #include <linux/module.h>
  31 #include <linux/console.h>
  32 #include <linux/slab.h>
  33 #include <linux/iommu.h>
  34 #include <linux/pci.h>
  35 #include <linux/devcoredump.h>
  36 #include <generated/utsrelease.h>
  37 #include <linux/pci-p2pdma.h>
  38 #include <linux/apple-gmux.h>
  39
  40 #include <drm/drm_aperture.h>
  41 #include <drm/drm_atomic_helper.h>
  42 #include <drm/drm_crtc_helper.h>
  43 #include <drm/drm_fb_helper.h>
  44 #include <drm/drm_probe_helper.h>
  45 #include <drm/amdgpu_drm.h>
  46 #include <linux/vgaarb.h>
  47 #include <linux/vga_switcheroo.h>
  48 #include <linux/efi.h>
  49 #include "amdgpu.h"
  50 #include "amdgpu_trace.h"
  51 #include "amdgpu_i2c.h"
  52 #include "atom.h"
  53 #include "amdgpu_atombios.h"
  54 #include "amdgpu_atomfirmware.h"
  55 #include "amd_pcie.h"
  56 #ifdef CONFIG_DRM_AMDGPU_SI
  57 #include "si.h"
  58 #endif
  59 #ifdef CONFIG_DRM_AMDGPU_CIK
  60 #include "cik.h"
  61 #endif
  62 #include "vi.h"
  63 #include "soc15.h"
  64 #include "nv.h"
  65 #include "bif/bif_4_1_d.h"
  66 #include <linux/firmware.h>
  67 #include "amdgpu_vf_error.h"
  68
  69 #include "amdgpu_amdkfd.h"
  70 #include "amdgpu_pm.h"
  71
  72 #include "amdgpu_xgmi.h"
  73 #include "amdgpu_ras.h"
  74 #include "amdgpu_pmu.h"
  75 #include "amdgpu_fru_eeprom.h"
  76 #include "amdgpu_reset.h"
  77
  78 #include <linux/suspend.h>
  79 #include <drm/task_barrier.h>
  80 #include <linux/pm_runtime.h>
  81
  82 #include <drm/drm_drv.h>
  83
  84 #if IS_ENABLED(CONFIG_X86)
  85 #include <asm/intel-family.h>
  86 #endif
  87
  88 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  89 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
  90 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  91 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
  92 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
  93 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
  94 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
  95
  96 #define AMDGPU_RESUME_MS                2000
  97 #define AMDGPU_MAX_RETRY_LIMIT          2
  98 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
  99
 100 static const struct drm_driver amdgpu_kms_driver;
 101
 102 const char *amdgpu_asic_name[] = {
 103         "TAHITI",
 104         "PITCAIRN",
 105         "VERDE",
 106         "OLAND",
 107         "HAINAN",
 108         "BONAIRE",
 109         "KAVERI",
 110         "KABINI",
 111         "HAWAII",
 112         "MULLINS",
 113         "TOPAZ",
 114         "TONGA",
 115         "FIJI",
 116         "CARRIZO",
 117         "STONEY",
 118         "POLARIS10",
 119         "POLARIS11",
 120         "POLARIS12",
 121         "VEGAM",
 122         "VEGA10",
 123         "VEGA12",
 124         "VEGA20",
 125         "RAVEN",
 126         "ARCTURUS",
 127         "RENOIR",
 128         "ALDEBARAN",
 129         "NAVI10",
 130         "CYAN_SKILLFISH",
 131         "NAVI14",
 132         "NAVI12",
 133         "SIENNA_CICHLID",
 134         "NAVY_FLOUNDER",
 135         "VANGOGH",
 136         "DIMGREY_CAVEFISH",
 137         "BEIGE_GOBY",
 138         "YELLOW_CARP",
 139         "IP DISCOVERY",
 140         "LAST",
 141 };
 142
 143 /**
 144  * DOC: pcie_replay_count
 145  *
 146  * The amdgpu driver provides a sysfs API for reporting the total number
 147  * of PCIe replays (NAKs)
 148  * The file pcie_replay_count is used for this and returns the total
 149  * number of replays as a sum of the NAKs generated and NAKs received
 150  */
 151
 152 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
 153                 struct device_attribute *attr, char *buf)
 154 {
 155         struct drm_device *ddev = dev_get_drvdata(dev);
 156         struct amdgpu_device *adev = drm_to_adev(ddev);
 157         uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
 158
 159         return sysfs_emit(buf, "%llu\n", cnt);
 160 }
 161
 162 static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
 163                 amdgpu_device_get_pcie_replay_count, NULL);
 164
 165 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 166
 167 /**
 168  * DOC: product_name
 169  *
 170  * The amdgpu driver provides a sysfs API for reporting the product name
 171  * for the device
 172  * The file product_name is used for this and returns the product name
 173  * as returned from the FRU.
 174  * NOTE: This is only available for certain server cards
 175  */
 176
 177 static ssize_t amdgpu_device_get_product_name(struct device *dev,
 178                 struct device_attribute *attr, char *buf)
 179 {
 180         struct drm_device *ddev = dev_get_drvdata(dev);
 181         struct amdgpu_device *adev = drm_to_adev(ddev);
 182
 183         return sysfs_emit(buf, "%s\n", adev->product_name);
 184 }
 185
 186 static DEVICE_ATTR(product_name, S_IRUGO,
 187                 amdgpu_device_get_product_name, NULL);
 188
 189 /**
 190  * DOC: product_number
 191  *
 192  * The amdgpu driver provides a sysfs API for reporting the part number
 193  * for the device
 194  * The file product_number is used for this and returns the part number
 195  * as returned from the FRU.
 196  * NOTE: This is only available for certain server cards
 197  */
 198
 199 static ssize_t amdgpu_device_get_product_number(struct device *dev,
 200                 struct device_attribute *attr, char *buf)
 201 {
 202         struct drm_device *ddev = dev_get_drvdata(dev);
 203         struct amdgpu_device *adev = drm_to_adev(ddev);
 204
 205         return sysfs_emit(buf, "%s\n", adev->product_number);
 206 }
 207
 208 static DEVICE_ATTR(product_number, S_IRUGO,
 209                 amdgpu_device_get_product_number, NULL);
 210
 211 /**
 212  * DOC: serial_number
 213  *
 214  * The amdgpu driver provides a sysfs API for reporting the serial number
 215  * for the device
 216  * The file serial_number is used for this and returns the serial number
 217  * as returned from the FRU.
 218  * NOTE: This is only available for certain server cards
 219  */
 220
 221 static ssize_t amdgpu_device_get_serial_number(struct device *dev,
 222                 struct device_attribute *attr, char *buf)
 223 {
 224         struct drm_device *ddev = dev_get_drvdata(dev);
 225         struct amdgpu_device *adev = drm_to_adev(ddev);
 226
 227         return sysfs_emit(buf, "%s\n", adev->serial);
 228 }
 229
 230 static DEVICE_ATTR(serial_number, S_IRUGO,
 231                 amdgpu_device_get_serial_number, NULL);
 232
 233 /**
 234  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
 235  *
 236  * @dev: drm_device pointer
 237  *
 238  * Returns true if the device is a dGPU with ATPX power control,
 239  * otherwise return false.
 240  */
 241 bool amdgpu_device_supports_px(struct drm_device *dev)
 242 {
 243         struct amdgpu_device *adev = drm_to_adev(dev);
 244
 245         if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
 246                 return true;
 247         return false;
 248 }
 249
 250 /**
 251  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
 252  *
 253  * @dev: drm_device pointer
 254  *
 255  * Returns true if the device is a dGPU with ACPI power control,
 256  * otherwise return false.
 257  */
 258 bool amdgpu_device_supports_boco(struct drm_device *dev)
 259 {
 260         struct amdgpu_device *adev = drm_to_adev(dev);
 261
 262         if (adev->has_pr3 ||
 263             ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
 264                 return true;
 265         return false;
 266 }
 267
 268 /**
 269  * amdgpu_device_supports_baco - Does the device support BACO
 270  *
 271  * @dev: drm_device pointer
 272  *
 273  * Returns true if the device supporte BACO,
 274  * otherwise return false.
 275  */
 276 bool amdgpu_device_supports_baco(struct drm_device *dev)
 277 {
 278         struct amdgpu_device *adev = drm_to_adev(dev);
 279
 280         return amdgpu_asic_supports_baco(adev);
 281 }
 282
 283 /**
 284  * amdgpu_device_supports_smart_shift - Is the device dGPU with
 285  * smart shift support
 286  *
 287  * @dev: drm_device pointer
 288  *
 289  * Returns true if the device is a dGPU with Smart Shift support,
 290  * otherwise returns false.
 291  */
 292 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
 293 {
 294         return (amdgpu_device_supports_boco(dev) &&
 295                 amdgpu_acpi_is_power_shift_control_supported());
 296 }
 297
 298 /*
 299  * VRAM access helper functions
 300  */
 301
 302 /**
 303  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
 304  *
 305  * @adev: amdgpu_device pointer
 306  * @pos: offset of the buffer in vram
 307  * @buf: virtual address of the buffer in system memory
 308  * @size: read/write size, sizeof(@buf) must > @size
 309  * @write: true - write to vram, otherwise - read from vram
 310  */
 311 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
 312                              void *buf, size_t size, bool write)
 313 {
 314         unsigned long flags;
 315         uint32_t hi = ~0, tmp = 0;
 316         uint32_t *data = buf;
 317         uint64_t last;
 318         int idx;
 319
 320         if (!drm_dev_enter(adev_to_drm(adev), &idx))
 321                 return;
 322
 323         BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
 324
 325         spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 326         for (last = pos + size; pos < last; pos += 4) {
 327                 tmp = pos >> 31;
 328
 329                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
 330                 if (tmp != hi) {
 331                         WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
 332                         hi = tmp;
 333                 }
 334                 if (write)
 335                         WREG32_NO_KIQ(mmMM_DATA, *data++);
 336                 else
 337                         *data++ = RREG32_NO_KIQ(mmMM_DATA);
 338         }
 339
 340         spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 341         drm_dev_exit(idx);
 342 }
 343
 344 /**
 345  * amdgpu_device_aper_access - access vram by vram aperature
 346  *
 347  * @adev: amdgpu_device pointer
 348  * @pos: offset of the buffer in vram
 349  * @buf: virtual address of the buffer in system memory
 350  * @size: read/write size, sizeof(@buf) must > @size
 351  * @write: true - write to vram, otherwise - read from vram
 352  *
 353  * The return value means how many bytes have been transferred.
 354  */
 355 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
 356                                  void *buf, size_t size, bool write)
 357 {
 358 #ifdef CONFIG_64BIT
 359         void __iomem *addr;
 360         size_t count = 0;
 361         uint64_t last;
 362
 363         if (!adev->mman.aper_base_kaddr)
 364                 return 0;
 365
 366         last = min(pos + size, adev->gmc.visible_vram_size);
 367         if (last > pos) {
 368                 addr = adev->mman.aper_base_kaddr + pos;
 369                 count = last - pos;
 370
 371                 if (write) {
 372                         memcpy_toio(addr, buf, count);
 373                         mb();
 374                         amdgpu_device_flush_hdp(adev, NULL);
 375                 } else {
 376                         amdgpu_device_invalidate_hdp(adev, NULL);
 377                         mb();
 378                         memcpy_fromio(buf, addr, count);
 379                 }
 380
 381         }
 382
 383         return count;
 384 #else
 385         return 0;
 386 #endif
 387 }
 388
 389 /**
 390  * amdgpu_device_vram_access - read/write a buffer in vram
 391  *
 392  * @adev: amdgpu_device pointer
 393  * @pos: offset of the buffer in vram
 394  * @buf: virtual address of the buffer in system memory
 395  * @size: read/write size, sizeof(@buf) must > @size
 396  * @write: true - write to vram, otherwise - read from vram
 397  */
 398 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
 399                                void *buf, size_t size, bool write)
 400 {
 401         size_t count;
 402
 403         /* try to using vram apreature to access vram first */
 404         count = amdgpu_device_aper_access(adev, pos, buf, size, write);
 405         size -= count;
 406         if (size) {
 407                 /* using MM to access rest vram */
 408                 pos += count;
 409                 buf += count;
 410                 amdgpu_device_mm_access(adev, pos, buf, size, write);
 411         }
 412 }
 413
 414 /*
 415  * register access helper functions.
 416  */
 417
 418 /* Check if hw access should be skipped because of hotplug or device error */
 419 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
 420 {
 421         if (adev->no_hw_access)
 422                 return true;
 423
 424 #ifdef CONFIG_LOCKDEP
 425         /*
 426          * This is a bit complicated to understand, so worth a comment. What we assert
 427          * here is that the GPU reset is not running on another thread in parallel.
 428          *
 429          * For this we trylock the read side of the reset semaphore, if that succeeds
 430          * we know that the reset is not running in paralell.
 431          *
 432          * If the trylock fails we assert that we are either already holding the read
 433          * side of the lock or are the reset thread itself and hold the write side of
 434          * the lock.
 435          */
 436         if (in_task()) {
 437                 if (down_read_trylock(&adev->reset_domain->sem))
 438                         up_read(&adev->reset_domain->sem);
 439                 else
 440                         lockdep_assert_held(&adev->reset_domain->sem);
 441         }
 442 #endif
 443         return false;
 444 }
 445
 446 /**
 447  * amdgpu_device_rreg - read a memory mapped IO or indirect register
 448  *
 449  * @adev: amdgpu_device pointer
 450  * @reg: dword aligned register offset
 451  * @acc_flags: access flags which require special behavior
 452  *
 453  * Returns the 32 bit value from the offset specified.
 454  */
 455 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 456                             uint32_t reg, uint32_t acc_flags)
 457 {
 458         uint32_t ret;
 459
 460         if (amdgpu_device_skip_hw_access(adev))
 461                 return 0;
 462
 463         if ((reg * 4) < adev->rmmio_size) {
 464                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 465                     amdgpu_sriov_runtime(adev) &&
 466                     down_read_trylock(&adev->reset_domain->sem)) {
 467                         ret = amdgpu_kiq_rreg(adev, reg);
 468                         up_read(&adev->reset_domain->sem);
 469                 } else {
 470                         ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 471                 }
 472         } else {
 473                 ret = adev->pcie_rreg(adev, reg * 4);
 474         }
 475
 476         trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
 477
 478         return ret;
 479 }
 480
 481 /*
 482  * MMIO register read with bytes helper functions
 483  * @offset:bytes offset from MMIO start
 484  *
 485 */
 486
 487 /**
 488  * amdgpu_mm_rreg8 - read a memory mapped IO register
 489  *
 490  * @adev: amdgpu_device pointer
 491  * @offset: byte aligned register offset
 492  *
 493  * Returns the 8 bit value from the offset specified.
 494  */
 495 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
 496 {
 497         if (amdgpu_device_skip_hw_access(adev))
 498                 return 0;
 499
 500         if (offset < adev->rmmio_size)
 501                 return (readb(adev->rmmio + offset));
 502         BUG();
 503 }
 504
 505 /*
 506  * MMIO register write with bytes helper functions
 507  * @offset:bytes offset from MMIO start
 508  * @value: the value want to be written to the register
 509  *
 510 */
 511 /**
 512  * amdgpu_mm_wreg8 - read a memory mapped IO register
 513  *
 514  * @adev: amdgpu_device pointer
 515  * @offset: byte aligned register offset
 516  * @value: 8 bit value to write
 517  *
 518  * Writes the value specified to the offset specified.
 519  */
 520 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
 521 {
 522         if (amdgpu_device_skip_hw_access(adev))
 523                 return;
 524
 525         if (offset < adev->rmmio_size)
 526                 writeb(value, adev->rmmio + offset);
 527         else
 528                 BUG();
 529 }
 530
 531 /**
 532  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
 533  *
 534  * @adev: amdgpu_device pointer
 535  * @reg: dword aligned register offset
 536  * @v: 32 bit value to write to the register
 537  * @acc_flags: access flags which require special behavior
 538  *
 539  * Writes the value specified to the offset specified.
 540  */
 541 void amdgpu_device_wreg(struct amdgpu_device *adev,
 542                         uint32_t reg, uint32_t v,
 543                         uint32_t acc_flags)
 544 {
 545         if (amdgpu_device_skip_hw_access(adev))
 546                 return;
 547
 548         if ((reg * 4) < adev->rmmio_size) {
 549                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 550                     amdgpu_sriov_runtime(adev) &&
 551                     down_read_trylock(&adev->reset_domain->sem)) {
 552                         amdgpu_kiq_wreg(adev, reg, v);
 553                         up_read(&adev->reset_domain->sem);
 554                 } else {
 555                         writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 556                 }
 557         } else {
 558                 adev->pcie_wreg(adev, reg * 4, v);
 559         }
 560
 561         trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
 562 }
 563
 564 /**
 565  * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
 566  *
 567  * @adev: amdgpu_device pointer
 568  * @reg: mmio/rlc register
 569  * @v: value to write
 570  *
 571  * this function is invoked only for the debugfs register access
 572  */
 573 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
 574                              uint32_t reg, uint32_t v)
 575 {
 576         if (amdgpu_device_skip_hw_access(adev))
 577                 return;
 578
 579         if (amdgpu_sriov_fullaccess(adev) &&
 580             adev->gfx.rlc.funcs &&
 581             adev->gfx.rlc.funcs->is_rlcg_access_range) {
 582                 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
 583                         return amdgpu_sriov_wreg(adev, reg, v, 0, 0);
 584         } else if ((reg * 4) >= adev->rmmio_size) {
 585                 adev->pcie_wreg(adev, reg * 4, v);
 586         } else {
 587                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 588         }
 589 }
 590
 591 /**
 592  * amdgpu_mm_rdoorbell - read a doorbell dword
 593  *
 594  * @adev: amdgpu_device pointer
 595  * @index: doorbell index
 596  *
 597  * Returns the value in the doorbell aperture at the
 598  * requested doorbell index (CIK).
 599  */
 600 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 601 {
 602         if (amdgpu_device_skip_hw_access(adev))
 603                 return 0;
 604
 605         if (index < adev->doorbell.num_kernel_doorbells) {
 606                 return readl(adev->doorbell.ptr + index);
 607         } else {
 608                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 609                 return 0;
 610         }
 611 }
 612
 613 /**
 614  * amdgpu_mm_wdoorbell - write a doorbell dword
 615  *
 616  * @adev: amdgpu_device pointer
 617  * @index: doorbell index
 618  * @v: value to write
 619  *
 620  * Writes @v to the doorbell aperture at the
 621  * requested doorbell index (CIK).
 622  */
 623 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 624 {
 625         if (amdgpu_device_skip_hw_access(adev))
 626                 return;
 627
 628         if (index < adev->doorbell.num_kernel_doorbells) {
 629                 writel(v, adev->doorbell.ptr + index);
 630         } else {
 631                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 632         }
 633 }
 634
 635 /**
 636  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 637  *
 638  * @adev: amdgpu_device pointer
 639  * @index: doorbell index
 640  *
 641  * Returns the value in the doorbell aperture at the
 642  * requested doorbell index (VEGA10+).
 643  */
 644 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 645 {
 646         if (amdgpu_device_skip_hw_access(adev))
 647                 return 0;
 648
 649         if (index < adev->doorbell.num_kernel_doorbells) {
 650                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 651         } else {
 652                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 653                 return 0;
 654         }
 655 }
 656
 657 /**
 658  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 659  *
 660  * @adev: amdgpu_device pointer
 661  * @index: doorbell index
 662  * @v: value to write
 663  *
 664  * Writes @v to the doorbell aperture at the
 665  * requested doorbell index (VEGA10+).
 666  */
 667 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 668 {
 669         if (amdgpu_device_skip_hw_access(adev))
 670                 return;
 671
 672         if (index < adev->doorbell.num_kernel_doorbells) {
 673                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 674         } else {
 675                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 676         }
 677 }
 678
 679 /**
 680  * amdgpu_device_indirect_rreg - read an indirect register
 681  *
 682  * @adev: amdgpu_device pointer
 683  * @reg_addr: indirect register address to read from
 684  *
 685  * Returns the value of indirect register @reg_addr
 686  */
 687 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
 688                                 u32 reg_addr)
 689 {
 690         unsigned long flags, pcie_index, pcie_data;
 691         void __iomem *pcie_index_offset;
 692         void __iomem *pcie_data_offset;
 693         u32 r;
 694
 695         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 696         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 697
 698         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 699         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 700         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 701
 702         writel(reg_addr, pcie_index_offset);
 703         readl(pcie_index_offset);
 704         r = readl(pcie_data_offset);
 705         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 706
 707         return r;
 708 }
 709
 710 u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,
 711                                     u64 reg_addr)
 712 {
 713         unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
 714         u32 r;
 715         void __iomem *pcie_index_offset;
 716         void __iomem *pcie_index_hi_offset;
 717         void __iomem *pcie_data_offset;
 718
 719         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 720         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 721         if (adev->nbio.funcs->get_pcie_index_hi_offset)
 722                 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
 723         else
 724                 pcie_index_hi = 0;
 725
 726         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 727         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 728         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 729         if (pcie_index_hi != 0)
 730                 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
 731                                 pcie_index_hi * 4;
 732
 733         writel(reg_addr, pcie_index_offset);
 734         readl(pcie_index_offset);
 735         if (pcie_index_hi != 0) {
 736                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 737                 readl(pcie_index_hi_offset);
 738         }
 739         r = readl(pcie_data_offset);
 740
 741         /* clear the high bits */
 742         if (pcie_index_hi != 0) {
 743                 writel(0, pcie_index_hi_offset);
 744                 readl(pcie_index_hi_offset);
 745         }
 746
 747         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 748
 749         return r;
 750 }
 751
 752 /**
 753  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
 754  *
 755  * @adev: amdgpu_device pointer
 756  * @reg_addr: indirect register address to read from
 757  *
 758  * Returns the value of indirect register @reg_addr
 759  */
 760 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
 761                                   u32 reg_addr)
 762 {
 763         unsigned long flags, pcie_index, pcie_data;
 764         void __iomem *pcie_index_offset;
 765         void __iomem *pcie_data_offset;
 766         u64 r;
 767
 768         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 769         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 770
 771         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 772         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 773         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 774
 775         /* read low 32 bits */
 776         writel(reg_addr, pcie_index_offset);
 777         readl(pcie_index_offset);
 778         r = readl(pcie_data_offset);
 779         /* read high 32 bits */
 780         writel(reg_addr + 4, pcie_index_offset);
 781         readl(pcie_index_offset);
 782         r |= ((u64)readl(pcie_data_offset) << 32);
 783         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 784
 785         return r;
 786 }
 787
 788 /**
 789  * amdgpu_device_indirect_wreg - write an indirect register address
 790  *
 791  * @adev: amdgpu_device pointer
 792  * @reg_addr: indirect register offset
 793  * @reg_data: indirect register data
 794  *
 795  */
 796 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
 797                                  u32 reg_addr, u32 reg_data)
 798 {
 799         unsigned long flags, pcie_index, pcie_data;
 800         void __iomem *pcie_index_offset;
 801         void __iomem *pcie_data_offset;
 802
 803         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 804         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 805
 806         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 807         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 808         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 809
 810         writel(reg_addr, pcie_index_offset);
 811         readl(pcie_index_offset);
 812         writel(reg_data, pcie_data_offset);
 813         readl(pcie_data_offset);
 814         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 815 }
 816
 817 void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,
 818                                      u64 reg_addr, u32 reg_data)
 819 {
 820         unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
 821         void __iomem *pcie_index_offset;
 822         void __iomem *pcie_index_hi_offset;
 823         void __iomem *pcie_data_offset;
 824
 825         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 826         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 827         if (adev->nbio.funcs->get_pcie_index_hi_offset)
 828                 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
 829         else
 830                 pcie_index_hi = 0;
 831
 832         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 833         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 834         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 835         if (pcie_index_hi != 0)
 836                 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
 837                                 pcie_index_hi * 4;
 838
 839         writel(reg_addr, pcie_index_offset);
 840         readl(pcie_index_offset);
 841         if (pcie_index_hi != 0) {
 842                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 843                 readl(pcie_index_hi_offset);
 844         }
 845         writel(reg_data, pcie_data_offset);
 846         readl(pcie_data_offset);
 847
 848         /* clear the high bits */
 849         if (pcie_index_hi != 0) {
 850                 writel(0, pcie_index_hi_offset);
 851                 readl(pcie_index_hi_offset);
 852         }
 853
 854         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 855 }
 856
 857 /**
 858  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
 859  *
 860  * @adev: amdgpu_device pointer
 861  * @reg_addr: indirect register offset
 862  * @reg_data: indirect register data
 863  *
 864  */
 865 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
 866                                    u32 reg_addr, u64 reg_data)
 867 {
 868         unsigned long flags, pcie_index, pcie_data;
 869         void __iomem *pcie_index_offset;
 870         void __iomem *pcie_data_offset;
 871
 872         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 873         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 874
 875         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 876         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 877         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 878
 879         /* write low 32 bits */
 880         writel(reg_addr, pcie_index_offset);
 881         readl(pcie_index_offset);
 882         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
 883         readl(pcie_data_offset);
 884         /* write high 32 bits */
 885         writel(reg_addr + 4, pcie_index_offset);
 886         readl(pcie_index_offset);
 887         writel((u32)(reg_data >> 32), pcie_data_offset);
 888         readl(pcie_data_offset);
 889         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 890 }
 891
 892 /**
 893  * amdgpu_device_get_rev_id - query device rev_id
 894  *
 895  * @adev: amdgpu_device pointer
 896  *
 897  * Return device rev_id
 898  */
 899 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
 900 {
 901         return adev->nbio.funcs->get_rev_id(adev);
 902 }
 903
 904 /**
 905  * amdgpu_invalid_rreg - dummy reg read function
 906  *
 907  * @adev: amdgpu_device pointer
 908  * @reg: offset of register
 909  *
 910  * Dummy register read function.  Used for register blocks
 911  * that certain asics don't have (all asics).
 912  * Returns the value in the register.
 913  */
 914 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 915 {
 916         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 917         BUG();
 918         return 0;
 919 }
 920
 921 static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg)
 922 {
 923         DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
 924         BUG();
 925         return 0;
 926 }
 927
 928 /**
 929  * amdgpu_invalid_wreg - dummy reg write function
 930  *
 931  * @adev: amdgpu_device pointer
 932  * @reg: offset of register
 933  * @v: value to write to the register
 934  *
 935  * Dummy register read function.  Used for register blocks
 936  * that certain asics don't have (all asics).
 937  */
 938 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 939 {
 940         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 941                   reg, v);
 942         BUG();
 943 }
 944
 945 static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v)
 946 {
 947         DRM_ERROR("Invalid callback to write register 0x%llX with 0x%08X\n",
 948                   reg, v);
 949         BUG();
 950 }
 951
 952 /**
 953  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
 954  *
 955  * @adev: amdgpu_device pointer
 956  * @reg: offset of register
 957  *
 958  * Dummy register read function.  Used for register blocks
 959  * that certain asics don't have (all asics).
 960  * Returns the value in the register.
 961  */
 962 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
 963 {
 964         DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
 965         BUG();
 966         return 0;
 967 }
 968
 969 /**
 970  * amdgpu_invalid_wreg64 - dummy reg write function
 971  *
 972  * @adev: amdgpu_device pointer
 973  * @reg: offset of register
 974  * @v: value to write to the register
 975  *
 976  * Dummy register read function.  Used for register blocks
 977  * that certain asics don't have (all asics).
 978  */
 979 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
 980 {
 981         DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
 982                   reg, v);
 983         BUG();
 984 }
 985
 986 /**
 987  * amdgpu_block_invalid_rreg - dummy reg read function
 988  *
 989  * @adev: amdgpu_device pointer
 990  * @block: offset of instance
 991  * @reg: offset of register
 992  *
 993  * Dummy register read function.  Used for register blocks
 994  * that certain asics don't have (all asics).
 995  * Returns the value in the register.
 996  */
 997 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 998                                           uint32_t block, uint32_t reg)
 999 {
1000         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
1001                   reg, block);
1002         BUG();
1003         return 0;
1004 }
1005
1006 /**
1007  * amdgpu_block_invalid_wreg - dummy reg write function
1008  *
1009  * @adev: amdgpu_device pointer
1010  * @block: offset of instance
1011  * @reg: offset of register
1012  * @v: value to write to the register
1013  *
1014  * Dummy register read function.  Used for register blocks
1015  * that certain asics don't have (all asics).
1016  */
1017 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
1018                                       uint32_t block,
1019                                       uint32_t reg, uint32_t v)
1020 {
1021         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
1022                   reg, block, v);
1023         BUG();
1024 }
1025
1026 /**
1027  * amdgpu_device_asic_init - Wrapper for atom asic_init
1028  *
1029  * @adev: amdgpu_device pointer
1030  *
1031  * Does any asic specific work and then calls atom asic init.
1032  */
1033 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
1034 {
1035         amdgpu_asic_pre_asic_init(adev);
1036
1037         if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3) ||
1038             adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0))
1039                 return amdgpu_atomfirmware_asic_init(adev, true);
1040         else
1041                 return amdgpu_atom_asic_init(adev->mode_info.atom_context);
1042 }
1043
1044 /**
1045  * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
1046  *
1047  * @adev: amdgpu_device pointer
1048  *
1049  * Allocates a scratch page of VRAM for use by various things in the
1050  * driver.
1051  */
1052 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
1053 {
1054         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
1055                                        AMDGPU_GEM_DOMAIN_VRAM |
1056                                        AMDGPU_GEM_DOMAIN_GTT,
1057                                        &adev->mem_scratch.robj,
1058                                        &adev->mem_scratch.gpu_addr,
1059                                        (void **)&adev->mem_scratch.ptr);
1060 }
1061
1062 /**
1063  * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1064  *
1065  * @adev: amdgpu_device pointer
1066  *
1067  * Frees the VRAM scratch page.
1068  */
1069 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
1070 {
1071         amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
1072 }
1073
1074 /**
1075  * amdgpu_device_program_register_sequence - program an array of registers.
1076  *
1077  * @adev: amdgpu_device pointer
1078  * @registers: pointer to the register array
1079  * @array_size: size of the register array
1080  *
1081  * Programs an array or registers with and and or masks.
1082  * This is a helper for setting golden registers.
1083  */
1084 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1085                                              const u32 *registers,
1086                                              const u32 array_size)
1087 {
1088         u32 tmp, reg, and_mask, or_mask;
1089         int i;
1090
1091         if (array_size % 3)
1092                 return;
1093
1094         for (i = 0; i < array_size; i += 3) {
1095                 reg = registers[i + 0];
1096                 and_mask = registers[i + 1];
1097                 or_mask = registers[i + 2];
1098
1099                 if (and_mask == 0xffffffff) {
1100                         tmp = or_mask;
1101                 } else {
1102                         tmp = RREG32(reg);
1103                         tmp &= ~and_mask;
1104                         if (adev->family >= AMDGPU_FAMILY_AI)
1105                                 tmp |= (or_mask & and_mask);
1106                         else
1107                                 tmp |= or_mask;
1108                 }
1109                 WREG32(reg, tmp);
1110         }
1111 }
1112
1113 /**
1114  * amdgpu_device_pci_config_reset - reset the GPU
1115  *
1116  * @adev: amdgpu_device pointer
1117  *
1118  * Resets the GPU using the pci config reset sequence.
1119  * Only applicable to asics prior to vega10.
1120  */
1121 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1122 {
1123         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1124 }
1125
1126 /**
1127  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1128  *
1129  * @adev: amdgpu_device pointer
1130  *
1131  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1132  */
1133 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1134 {
1135         return pci_reset_function(adev->pdev);
1136 }
1137
1138 /*
1139  * GPU doorbell aperture helpers function.
1140  */
1141 /**
1142  * amdgpu_device_doorbell_init - Init doorbell driver information.
1143  *
1144  * @adev: amdgpu_device pointer
1145  *
1146  * Init doorbell driver information (CIK)
1147  * Returns 0 on success, error on failure.
1148  */
1149 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
1150 {
1151
1152         /* No doorbell on SI hardware generation */
1153         if (adev->asic_type < CHIP_BONAIRE) {
1154                 adev->doorbell.base = 0;
1155                 adev->doorbell.size = 0;
1156                 adev->doorbell.num_kernel_doorbells = 0;
1157                 adev->doorbell.ptr = NULL;
1158                 return 0;
1159         }
1160
1161         if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
1162                 return -EINVAL;
1163
1164         amdgpu_asic_init_doorbell_index(adev);
1165
1166         /* doorbell bar mapping */
1167         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
1168         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
1169
1170         if (adev->enable_mes) {
1171                 adev->doorbell.num_kernel_doorbells =
1172                         adev->doorbell.size / sizeof(u32);
1173         } else {
1174                 adev->doorbell.num_kernel_doorbells =
1175                         min_t(u32, adev->doorbell.size / sizeof(u32),
1176                               adev->doorbell_index.max_assignment+1);
1177                 if (adev->doorbell.num_kernel_doorbells == 0)
1178                         return -EINVAL;
1179
1180                 /* For Vega, reserve and map two pages on doorbell BAR since SDMA
1181                  * paging queue doorbell use the second page. The
1182                  * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
1183                  * doorbells are in the first page. So with paging queue enabled,
1184                  * the max num_kernel_doorbells should + 1 page (0x400 in dword)
1185                  */
1186                 if (adev->ip_versions[SDMA0_HWIP][0] >= IP_VERSION(4, 0, 0) &&
1187                     adev->ip_versions[SDMA0_HWIP][0] < IP_VERSION(4, 2, 0))
1188                         adev->doorbell.num_kernel_doorbells += 0x400;
1189         }
1190
1191         adev->doorbell.ptr = ioremap(adev->doorbell.base,
1192                                      adev->doorbell.num_kernel_doorbells *
1193                                      sizeof(u32));
1194         if (adev->doorbell.ptr == NULL)
1195                 return -ENOMEM;
1196
1197         return 0;
1198 }
1199
1200 /**
1201  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
1202  *
1203  * @adev: amdgpu_device pointer
1204  *
1205  * Tear down doorbell driver information (CIK)
1206  */
1207 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
1208 {
1209         iounmap(adev->doorbell.ptr);
1210         adev->doorbell.ptr = NULL;
1211 }
1212
1213
1214
1215 /*
1216  * amdgpu_device_wb_*()
1217  * Writeback is the method by which the GPU updates special pages in memory
1218  * with the status of certain GPU events (fences, ring pointers,etc.).
1219  */
1220
1221 /**
1222  * amdgpu_device_wb_fini - Disable Writeback and free memory
1223  *
1224  * @adev: amdgpu_device pointer
1225  *
1226  * Disables Writeback and frees the Writeback memory (all asics).
1227  * Used at driver shutdown.
1228  */
1229 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1230 {
1231         if (adev->wb.wb_obj) {
1232                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1233                                       &adev->wb.gpu_addr,
1234                                       (void **)&adev->wb.wb);
1235                 adev->wb.wb_obj = NULL;
1236         }
1237 }
1238
1239 /**
1240  * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1241  *
1242  * @adev: amdgpu_device pointer
1243  *
1244  * Initializes writeback and allocates writeback memory (all asics).
1245  * Used at driver startup.
1246  * Returns 0 on success or an -error on failure.
1247  */
1248 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1249 {
1250         int r;
1251
1252         if (adev->wb.wb_obj == NULL) {
1253                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1254                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1255                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1256                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
1257                                             (void **)&adev->wb.wb);
1258                 if (r) {
1259                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1260                         return r;
1261                 }
1262
1263                 adev->wb.num_wb = AMDGPU_MAX_WB;
1264                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1265
1266                 /* clear wb memory */
1267                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1268         }
1269
1270         return 0;
1271 }
1272
1273 /**
1274  * amdgpu_device_wb_get - Allocate a wb entry
1275  *
1276  * @adev: amdgpu_device pointer
1277  * @wb: wb index
1278  *
1279  * Allocate a wb slot for use by the driver (all asics).
1280  * Returns 0 on success or -EINVAL on failure.
1281  */
1282 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1283 {
1284         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1285
1286         if (offset < adev->wb.num_wb) {
1287                 __set_bit(offset, adev->wb.used);
1288                 *wb = offset << 3; /* convert to dw offset */
1289                 return 0;
1290         } else {
1291                 return -EINVAL;
1292         }
1293 }
1294
1295 /**
1296  * amdgpu_device_wb_free - Free a wb entry
1297  *
1298  * @adev: amdgpu_device pointer
1299  * @wb: wb index
1300  *
1301  * Free a wb slot allocated for use by the driver (all asics)
1302  */
1303 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1304 {
1305         wb >>= 3;
1306         if (wb < adev->wb.num_wb)
1307                 __clear_bit(wb, adev->wb.used);
1308 }
1309
1310 /**
1311  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1312  *
1313  * @adev: amdgpu_device pointer
1314  *
1315  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1316  * to fail, but if any of the BARs is not accessible after the size we abort
1317  * driver loading by returning -ENODEV.
1318  */
1319 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1320 {
1321         int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1322         struct pci_bus *root;
1323         struct resource *res;
1324         unsigned i;
1325         u16 cmd;
1326         int r;
1327
1328         /* Bypass for VF */
1329         if (amdgpu_sriov_vf(adev))
1330                 return 0;
1331
1332         /* skip if the bios has already enabled large BAR */
1333         if (adev->gmc.real_vram_size &&
1334             (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1335                 return 0;
1336
1337         /* Check if the root BUS has 64bit memory resources */
1338         root = adev->pdev->bus;
1339         while (root->parent)
1340                 root = root->parent;
1341
1342         pci_bus_for_each_resource(root, res, i) {
1343                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1344                     res->start > 0x100000000ull)
1345                         break;
1346         }
1347
1348         /* Trying to resize is pointless without a root hub window above 4GB */
1349         if (!res)
1350                 return 0;
1351
1352         /* Limit the BAR size to what is available */
1353         rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1354                         rbar_size);
1355
1356         /* Disable memory decoding while we change the BAR addresses and size */
1357         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1358         pci_write_config_word(adev->pdev, PCI_COMMAND,
1359                               cmd & ~PCI_COMMAND_MEMORY);
1360
1361         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1362         amdgpu_device_doorbell_fini(adev);
1363         if (adev->asic_type >= CHIP_BONAIRE)
1364                 pci_release_resource(adev->pdev, 2);
1365
1366         pci_release_resource(adev->pdev, 0);
1367
1368         r = pci_resize_resource(adev->pdev, 0, rbar_size);
1369         if (r == -ENOSPC)
1370                 DRM_INFO("Not enough PCI address space for a large BAR.");
1371         else if (r && r != -ENOTSUPP)
1372                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1373
1374         pci_assign_unassigned_bus_resources(adev->pdev->bus);
1375
1376         /* When the doorbell or fb BAR isn't available we have no chance of
1377          * using the device.
1378          */
1379         r = amdgpu_device_doorbell_init(adev);
1380         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1381                 return -ENODEV;
1382
1383         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1384
1385         return 0;
1386 }
1387
1388 static bool amdgpu_device_read_bios(struct amdgpu_device *adev)
1389 {
1390         if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU)) {
1391                 return false;
1392         }
1393
1394         return true;
1395 }
1396
1397 /*
1398  * GPU helpers function.
1399  */
1400 /**
1401  * amdgpu_device_need_post - check if the hw need post or not
1402  *
1403  * @adev: amdgpu_device pointer
1404  *
1405  * Check if the asic has been initialized (all asics) at driver startup
1406  * or post is needed if  hw reset is performed.
1407  * Returns true if need or false if not.
1408  */
1409 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1410 {
1411         uint32_t reg;
1412
1413         if (amdgpu_sriov_vf(adev))
1414                 return false;
1415
1416         if (!amdgpu_device_read_bios(adev))
1417                 return false;
1418
1419         if (amdgpu_passthrough(adev)) {
1420                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1421                  * some old smc fw still need driver do vPost otherwise gpu hang, while
1422                  * those smc fw version above 22.15 doesn't have this flaw, so we force
1423                  * vpost executed for smc version below 22.15
1424                  */
1425                 if (adev->asic_type == CHIP_FIJI) {
1426                         int err;
1427                         uint32_t fw_ver;
1428                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1429                         /* force vPost if error occured */
1430                         if (err)
1431                                 return true;
1432
1433                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1434                         if (fw_ver < 0x00160e00)
1435                                 return true;
1436                 }
1437         }
1438
1439         /* Don't post if we need to reset whole hive on init */
1440         if (adev->gmc.xgmi.pending_reset)
1441                 return false;
1442
1443         if (adev->has_hw_reset) {
1444                 adev->has_hw_reset = false;
1445                 return true;
1446         }
1447
1448         /* bios scratch used on CIK+ */
1449         if (adev->asic_type >= CHIP_BONAIRE)
1450                 return amdgpu_atombios_scratch_need_asic_init(adev);
1451
1452         /* check MEM_SIZE for older asics */
1453         reg = amdgpu_asic_get_config_memsize(adev);
1454
1455         if ((reg != 0) && (reg != 0xffffffff))
1456                 return false;
1457
1458         return true;
1459 }
1460
1461 /*
1462  * Intel hosts such as Raptor Lake and Sapphire Rapids don't support dynamic
1463  * speed switching. Until we have confirmation from Intel that a specific host
1464  * supports it, it's safer that we keep it disabled for all.
1465  *
1466  * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
1467  * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1468  */
1469 bool amdgpu_device_pcie_dynamic_switching_supported(void)
1470 {
1471 #if IS_ENABLED(CONFIG_X86)
1472         struct cpuinfo_x86 *c = &cpu_data(0);
1473
1474         if (c->x86_vendor == X86_VENDOR_INTEL)
1475                 return false;
1476 #endif
1477         return true;
1478 }
1479
1480 /**
1481  * amdgpu_device_should_use_aspm - check if the device should program ASPM
1482  *
1483  * @adev: amdgpu_device pointer
1484  *
1485  * Confirm whether the module parameter and pcie bridge agree that ASPM should
1486  * be set for this device.
1487  *
1488  * Returns true if it should be used or false if not.
1489  */
1490 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1491 {
1492         switch (amdgpu_aspm) {
1493         case -1:
1494                 break;
1495         case 0:
1496                 return false;
1497         case 1:
1498                 return true;
1499         default:
1500                 return false;
1501         }
1502         return pcie_aspm_enabled(adev->pdev);
1503 }
1504
1505 bool amdgpu_device_aspm_support_quirk(void)
1506 {
1507 #if IS_ENABLED(CONFIG_X86)
1508         struct cpuinfo_x86 *c = &cpu_data(0);
1509
1510         return !(c->x86 == 6 && c->x86_model == INTEL_FAM6_ALDERLAKE);
1511 #else
1512         return true;
1513 #endif
1514 }
1515
1516 /* if we get transitioned to only one device, take VGA back */
1517 /**
1518  * amdgpu_device_vga_set_decode - enable/disable vga decode
1519  *
1520  * @pdev: PCI device pointer
1521  * @state: enable/disable vga decode
1522  *
1523  * Enable/disable vga decode (all asics).
1524  * Returns VGA resource flags.
1525  */
1526 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1527                 bool state)
1528 {
1529         struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1530         amdgpu_asic_set_vga_state(adev, state);
1531         if (state)
1532                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1533                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1534         else
1535                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1536 }
1537
1538 /**
1539  * amdgpu_device_check_block_size - validate the vm block size
1540  *
1541  * @adev: amdgpu_device pointer
1542  *
1543  * Validates the vm block size specified via module parameter.
1544  * The vm block size defines number of bits in page table versus page directory,
1545  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1546  * page table and the remaining bits are in the page directory.
1547  */
1548 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1549 {
1550         /* defines number of bits in page table versus page directory,
1551          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1552          * page table and the remaining bits are in the page directory */
1553         if (amdgpu_vm_block_size == -1)
1554                 return;
1555
1556         if (amdgpu_vm_block_size < 9) {
1557                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1558                          amdgpu_vm_block_size);
1559                 amdgpu_vm_block_size = -1;
1560         }
1561 }
1562
1563 /**
1564  * amdgpu_device_check_vm_size - validate the vm size
1565  *
1566  * @adev: amdgpu_device pointer
1567  *
1568  * Validates the vm size in GB specified via module parameter.
1569  * The VM size is the size of the GPU virtual memory space in GB.
1570  */
1571 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1572 {
1573         /* no need to check the default value */
1574         if (amdgpu_vm_size == -1)
1575                 return;
1576
1577         if (amdgpu_vm_size < 1) {
1578                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1579                          amdgpu_vm_size);
1580                 amdgpu_vm_size = -1;
1581         }
1582 }
1583
1584 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1585 {
1586         struct sysinfo si;
1587         bool is_os_64 = (sizeof(void *) == 8);
1588         uint64_t total_memory;
1589         uint64_t dram_size_seven_GB = 0x1B8000000;
1590         uint64_t dram_size_three_GB = 0xB8000000;
1591
1592         if (amdgpu_smu_memory_pool_size == 0)
1593                 return;
1594
1595         if (!is_os_64) {
1596                 DRM_WARN("Not 64-bit OS, feature not supported\n");
1597                 goto def_value;
1598         }
1599         si_meminfo(&si);
1600         total_memory = (uint64_t)si.totalram * si.mem_unit;
1601
1602         if ((amdgpu_smu_memory_pool_size == 1) ||
1603                 (amdgpu_smu_memory_pool_size == 2)) {
1604                 if (total_memory < dram_size_three_GB)
1605                         goto def_value1;
1606         } else if ((amdgpu_smu_memory_pool_size == 4) ||
1607                 (amdgpu_smu_memory_pool_size == 8)) {
1608                 if (total_memory < dram_size_seven_GB)
1609                         goto def_value1;
1610         } else {
1611                 DRM_WARN("Smu memory pool size not supported\n");
1612                 goto def_value;
1613         }
1614         adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1615
1616         return;
1617
1618 def_value1:
1619         DRM_WARN("No enough system memory\n");
1620 def_value:
1621         adev->pm.smu_prv_buffer_size = 0;
1622 }
1623
1624 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1625 {
1626         if (!(adev->flags & AMD_IS_APU) ||
1627             adev->asic_type < CHIP_RAVEN)
1628                 return 0;
1629
1630         switch (adev->asic_type) {
1631         case CHIP_RAVEN:
1632                 if (adev->pdev->device == 0x15dd)
1633                         adev->apu_flags |= AMD_APU_IS_RAVEN;
1634                 if (adev->pdev->device == 0x15d8)
1635                         adev->apu_flags |= AMD_APU_IS_PICASSO;
1636                 break;
1637         case CHIP_RENOIR:
1638                 if ((adev->pdev->device == 0x1636) ||
1639                     (adev->pdev->device == 0x164c))
1640                         adev->apu_flags |= AMD_APU_IS_RENOIR;
1641                 else
1642                         adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1643                 break;
1644         case CHIP_VANGOGH:
1645                 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1646                 break;
1647         case CHIP_YELLOW_CARP:
1648                 break;
1649         case CHIP_CYAN_SKILLFISH:
1650                 if ((adev->pdev->device == 0x13FE) ||
1651                     (adev->pdev->device == 0x143F))
1652                         adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1653                 break;
1654         default:
1655                 break;
1656         }
1657
1658         return 0;
1659 }
1660
1661 /**
1662  * amdgpu_device_check_arguments - validate module params
1663  *
1664  * @adev: amdgpu_device pointer
1665  *
1666  * Validates certain module parameters and updates
1667  * the associated values used by the driver (all asics).
1668  */
1669 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1670 {
1671         if (amdgpu_sched_jobs < 4) {
1672                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1673                          amdgpu_sched_jobs);
1674                 amdgpu_sched_jobs = 4;
1675         } else if (!is_power_of_2(amdgpu_sched_jobs)) {
1676                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1677                          amdgpu_sched_jobs);
1678                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1679         }
1680
1681         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1682                 /* gart size must be greater or equal to 32M */
1683                 dev_warn(adev->dev, "gart size (%d) too small\n",
1684                          amdgpu_gart_size);
1685                 amdgpu_gart_size = -1;
1686         }
1687
1688         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1689                 /* gtt size must be greater or equal to 32M */
1690                 dev_warn(adev->dev, "gtt size (%d) too small\n",
1691                                  amdgpu_gtt_size);
1692                 amdgpu_gtt_size = -1;
1693         }
1694
1695         /* valid range is between 4 and 9 inclusive */
1696         if (amdgpu_vm_fragment_size != -1 &&
1697             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1698                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1699                 amdgpu_vm_fragment_size = -1;
1700         }
1701
1702         if (amdgpu_sched_hw_submission < 2) {
1703                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1704                          amdgpu_sched_hw_submission);
1705                 amdgpu_sched_hw_submission = 2;
1706         } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1707                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1708                          amdgpu_sched_hw_submission);
1709                 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1710         }
1711
1712         if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1713                 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1714                 amdgpu_reset_method = -1;
1715         }
1716
1717         amdgpu_device_check_smu_prv_buffer_size(adev);
1718
1719         amdgpu_device_check_vm_size(adev);
1720
1721         amdgpu_device_check_block_size(adev);
1722
1723         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1724
1725         return 0;
1726 }
1727
1728 /**
1729  * amdgpu_switcheroo_set_state - set switcheroo state
1730  *
1731  * @pdev: pci dev pointer
1732  * @state: vga_switcheroo state
1733  *
1734  * Callback for the switcheroo driver.  Suspends or resumes
1735  * the asics before or after it is powered up using ACPI methods.
1736  */
1737 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1738                                         enum vga_switcheroo_state state)
1739 {
1740         struct drm_device *dev = pci_get_drvdata(pdev);
1741         int r;
1742
1743         if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1744                 return;
1745
1746         if (state == VGA_SWITCHEROO_ON) {
1747                 pr_info("switched on\n");
1748                 /* don't suspend or resume card normally */
1749                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1750
1751                 pci_set_power_state(pdev, PCI_D0);
1752                 amdgpu_device_load_pci_state(pdev);
1753                 r = pci_enable_device(pdev);
1754                 if (r)
1755                         DRM_WARN("pci_enable_device failed (%d)\n", r);
1756                 amdgpu_device_resume(dev, true);
1757
1758                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1759         } else {
1760                 pr_info("switched off\n");
1761                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1762                 amdgpu_device_suspend(dev, true);
1763                 amdgpu_device_cache_pci_state(pdev);
1764                 /* Shut down the device */
1765                 pci_disable_device(pdev);
1766                 pci_set_power_state(pdev, PCI_D3cold);
1767                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1768         }
1769 }
1770
1771 /**
1772  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1773  *
1774  * @pdev: pci dev pointer
1775  *
1776  * Callback for the switcheroo driver.  Check of the switcheroo
1777  * state can be changed.
1778  * Returns true if the state can be changed, false if not.
1779  */
1780 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1781 {
1782         struct drm_device *dev = pci_get_drvdata(pdev);
1783
1784         /*
1785         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1786         * locking inversion with the driver load path. And the access here is
1787         * completely racy anyway. So don't bother with locking for now.
1788         */
1789         return atomic_read(&dev->open_count) == 0;
1790 }
1791
1792 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1793         .set_gpu_state = amdgpu_switcheroo_set_state,
1794         .reprobe = NULL,
1795         .can_switch = amdgpu_switcheroo_can_switch,
1796 };
1797
1798 /**
1799  * amdgpu_device_ip_set_clockgating_state - set the CG state
1800  *
1801  * @dev: amdgpu_device pointer
1802  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1803  * @state: clockgating state (gate or ungate)
1804  *
1805  * Sets the requested clockgating state for all instances of
1806  * the hardware IP specified.
1807  * Returns the error code from the last instance.
1808  */
1809 int amdgpu_device_ip_set_clockgating_state(void *dev,
1810                                            enum amd_ip_block_type block_type,
1811                                            enum amd_clockgating_state state)
1812 {
1813         struct amdgpu_device *adev = dev;
1814         int i, r = 0;
1815
1816         for (i = 0; i < adev->num_ip_blocks; i++) {
1817                 if (!adev->ip_blocks[i].status.valid)
1818                         continue;
1819                 if (adev->ip_blocks[i].version->type != block_type)
1820                         continue;
1821                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1822                         continue;
1823                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1824                         (void *)adev, state);
1825                 if (r)
1826                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1827                                   adev->ip_blocks[i].version->funcs->name, r);
1828         }
1829         return r;
1830 }
1831
1832 /**
1833  * amdgpu_device_ip_set_powergating_state - set the PG state
1834  *
1835  * @dev: amdgpu_device pointer
1836  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1837  * @state: powergating state (gate or ungate)
1838  *
1839  * Sets the requested powergating state for all instances of
1840  * the hardware IP specified.
1841  * Returns the error code from the last instance.
1842  */
1843 int amdgpu_device_ip_set_powergating_state(void *dev,
1844                                            enum amd_ip_block_type block_type,
1845                                            enum amd_powergating_state state)
1846 {
1847         struct amdgpu_device *adev = dev;
1848         int i, r = 0;
1849
1850         for (i = 0; i < adev->num_ip_blocks; i++) {
1851                 if (!adev->ip_blocks[i].status.valid)
1852                         continue;
1853                 if (adev->ip_blocks[i].version->type != block_type)
1854                         continue;
1855                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1856                         continue;
1857                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1858                         (void *)adev, state);
1859                 if (r)
1860                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1861                                   adev->ip_blocks[i].version->funcs->name, r);
1862         }
1863         return r;
1864 }
1865
1866 /**
1867  * amdgpu_device_ip_get_clockgating_state - get the CG state
1868  *
1869  * @adev: amdgpu_device pointer
1870  * @flags: clockgating feature flags
1871  *
1872  * Walks the list of IPs on the device and updates the clockgating
1873  * flags for each IP.
1874  * Updates @flags with the feature flags for each hardware IP where
1875  * clockgating is enabled.
1876  */
1877 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1878                                             u64 *flags)
1879 {
1880         int i;
1881
1882         for (i = 0; i < adev->num_ip_blocks; i++) {
1883                 if (!adev->ip_blocks[i].status.valid)
1884                         continue;
1885                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1886                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1887         }
1888 }
1889
1890 /**
1891  * amdgpu_device_ip_wait_for_idle - wait for idle
1892  *
1893  * @adev: amdgpu_device pointer
1894  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1895  *
1896  * Waits for the request hardware IP to be idle.
1897  * Returns 0 for success or a negative error code on failure.
1898  */
1899 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1900                                    enum amd_ip_block_type block_type)
1901 {
1902         int i, r;
1903
1904         for (i = 0; i < adev->num_ip_blocks; i++) {
1905                 if (!adev->ip_blocks[i].status.valid)
1906                         continue;
1907                 if (adev->ip_blocks[i].version->type == block_type) {
1908                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1909                         if (r)
1910                                 return r;
1911                         break;
1912                 }
1913         }
1914         return 0;
1915
1916 }
1917
1918 /**
1919  * amdgpu_device_ip_is_idle - is the hardware IP idle
1920  *
1921  * @adev: amdgpu_device pointer
1922  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1923  *
1924  * Check if the hardware IP is idle or not.
1925  * Returns true if it the IP is idle, false if not.
1926  */
1927 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1928                               enum amd_ip_block_type block_type)
1929 {
1930         int i;
1931
1932         for (i = 0; i < adev->num_ip_blocks; i++) {
1933                 if (!adev->ip_blocks[i].status.valid)
1934                         continue;
1935                 if (adev->ip_blocks[i].version->type == block_type)
1936                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1937         }
1938         return true;
1939
1940 }
1941
1942 /**
1943  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1944  *
1945  * @adev: amdgpu_device pointer
1946  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1947  *
1948  * Returns a pointer to the hardware IP block structure
1949  * if it exists for the asic, otherwise NULL.
1950  */
1951 struct amdgpu_ip_block *
1952 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1953                               enum amd_ip_block_type type)
1954 {
1955         int i;
1956
1957         for (i = 0; i < adev->num_ip_blocks; i++)
1958                 if (adev->ip_blocks[i].version->type == type)
1959                         return &adev->ip_blocks[i];
1960
1961         return NULL;
1962 }
1963
1964 /**
1965  * amdgpu_device_ip_block_version_cmp
1966  *
1967  * @adev: amdgpu_device pointer
1968  * @type: enum amd_ip_block_type
1969  * @major: major version
1970  * @minor: minor version
1971  *
1972  * return 0 if equal or greater
1973  * return 1 if smaller or the ip_block doesn't exist
1974  */
1975 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1976                                        enum amd_ip_block_type type,
1977                                        u32 major, u32 minor)
1978 {
1979         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1980
1981         if (ip_block && ((ip_block->version->major > major) ||
1982                         ((ip_block->version->major == major) &&
1983                         (ip_block->version->minor >= minor))))
1984                 return 0;
1985
1986         return 1;
1987 }
1988
1989 /**
1990  * amdgpu_device_ip_block_add
1991  *
1992  * @adev: amdgpu_device pointer
1993  * @ip_block_version: pointer to the IP to add
1994  *
1995  * Adds the IP block driver information to the collection of IPs
1996  * on the asic.
1997  */
1998 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1999                                const struct amdgpu_ip_block_version *ip_block_version)
2000 {
2001         if (!ip_block_version)
2002                 return -EINVAL;
2003
2004         switch (ip_block_version->type) {
2005         case AMD_IP_BLOCK_TYPE_VCN:
2006                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
2007                         return 0;
2008                 break;
2009         case AMD_IP_BLOCK_TYPE_JPEG:
2010                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
2011                         return 0;
2012                 break;
2013         default:
2014                 break;
2015         }
2016
2017         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
2018                   ip_block_version->funcs->name);
2019
2020         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
2021
2022         return 0;
2023 }
2024
2025 /**
2026  * amdgpu_device_enable_virtual_display - enable virtual display feature
2027  *
2028  * @adev: amdgpu_device pointer
2029  *
2030  * Enabled the virtual display feature if the user has enabled it via
2031  * the module parameter virtual_display.  This feature provides a virtual
2032  * display hardware on headless boards or in virtualized environments.
2033  * This function parses and validates the configuration string specified by
2034  * the user and configues the virtual display configuration (number of
2035  * virtual connectors, crtcs, etc.) specified.
2036  */
2037 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
2038 {
2039         adev->enable_virtual_display = false;
2040
2041         if (amdgpu_virtual_display) {
2042                 const char *pci_address_name = pci_name(adev->pdev);
2043                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
2044
2045                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
2046                 pciaddstr_tmp = pciaddstr;
2047                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
2048                         pciaddname = strsep(&pciaddname_tmp, ",");
2049                         if (!strcmp("all", pciaddname)
2050                             || !strcmp(pci_address_name, pciaddname)) {
2051                                 long num_crtc;
2052                                 int res = -1;
2053
2054                                 adev->enable_virtual_display = true;
2055
2056                                 if (pciaddname_tmp)
2057                                         res = kstrtol(pciaddname_tmp, 10,
2058                                                       &num_crtc);
2059
2060                                 if (!res) {
2061                                         if (num_crtc < 1)
2062                                                 num_crtc = 1;
2063                                         if (num_crtc > 6)
2064                                                 num_crtc = 6;
2065                                         adev->mode_info.num_crtc = num_crtc;
2066                                 } else {
2067                                         adev->mode_info.num_crtc = 1;
2068                                 }
2069                                 break;
2070                         }
2071                 }
2072
2073                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
2074                          amdgpu_virtual_display, pci_address_name,
2075                          adev->enable_virtual_display, adev->mode_info.num_crtc);
2076
2077                 kfree(pciaddstr);
2078         }
2079 }
2080
2081 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
2082 {
2083         if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
2084                 adev->mode_info.num_crtc = 1;
2085                 adev->enable_virtual_display = true;
2086                 DRM_INFO("virtual_display:%d, num_crtc:%d\n",
2087                          adev->enable_virtual_display, adev->mode_info.num_crtc);
2088         }
2089 }
2090
2091 /**
2092  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
2093  *
2094  * @adev: amdgpu_device pointer
2095  *
2096  * Parses the asic configuration parameters specified in the gpu info
2097  * firmware and makes them availale to the driver for use in configuring
2098  * the asic.
2099  * Returns 0 on success, -EINVAL on failure.
2100  */
2101 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
2102 {
2103         const char *chip_name;
2104         char fw_name[40];
2105         int err;
2106         const struct gpu_info_firmware_header_v1_0 *hdr;
2107
2108         adev->firmware.gpu_info_fw = NULL;
2109
2110         if (adev->mman.discovery_bin) {
2111                 /*
2112                  * FIXME: The bounding box is still needed by Navi12, so
2113                  * temporarily read it from gpu_info firmware. Should be dropped
2114                  * when DAL no longer needs it.
2115                  */
2116                 if (adev->asic_type != CHIP_NAVI12)
2117                         return 0;
2118         }
2119
2120         switch (adev->asic_type) {
2121         default:
2122                 return 0;
2123         case CHIP_VEGA10:
2124                 chip_name = "vega10";
2125                 break;
2126         case CHIP_VEGA12:
2127                 chip_name = "vega12";
2128                 break;
2129         case CHIP_RAVEN:
2130                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2131                         chip_name = "raven2";
2132                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
2133                         chip_name = "picasso";
2134                 else
2135                         chip_name = "raven";
2136                 break;
2137         case CHIP_ARCTURUS:
2138                 chip_name = "arcturus";
2139                 break;
2140         case CHIP_NAVI12:
2141                 chip_name = "navi12";
2142                 break;
2143         }
2144
2145         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
2146         err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, fw_name);
2147         if (err) {
2148                 dev_err(adev->dev,
2149                         "Failed to get gpu_info firmware \"%s\"\n",
2150                         fw_name);
2151                 goto out;
2152         }
2153
2154         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2155         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2156
2157         switch (hdr->version_major) {
2158         case 1:
2159         {
2160                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2161                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2162                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2163
2164                 /*
2165                  * Should be droped when DAL no longer needs it.
2166                  */
2167                 if (adev->asic_type == CHIP_NAVI12)
2168                         goto parse_soc_bounding_box;
2169
2170                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2171                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2172                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2173                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2174                 adev->gfx.config.max_texture_channel_caches =
2175                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
2176                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2177                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2178                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2179                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2180                 adev->gfx.config.double_offchip_lds_buf =
2181                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2182                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2183                 adev->gfx.cu_info.max_waves_per_simd =
2184                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2185                 adev->gfx.cu_info.max_scratch_slots_per_cu =
2186                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2187                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2188                 if (hdr->version_minor >= 1) {
2189                         const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2190                                 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2191                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2192                         adev->gfx.config.num_sc_per_sh =
2193                                 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2194                         adev->gfx.config.num_packer_per_sc =
2195                                 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2196                 }
2197
2198 parse_soc_bounding_box:
2199                 /*
2200                  * soc bounding box info is not integrated in disocovery table,
2201                  * we always need to parse it from gpu info firmware if needed.
2202                  */
2203                 if (hdr->version_minor == 2) {
2204                         const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2205                                 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2206                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2207                         adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2208                 }
2209                 break;
2210         }
2211         default:
2212                 dev_err(adev->dev,
2213                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2214                 err = -EINVAL;
2215                 goto out;
2216         }
2217 out:
2218         return err;
2219 }
2220
2221 /**
2222  * amdgpu_device_ip_early_init - run early init for hardware IPs
2223  *
2224  * @adev: amdgpu_device pointer
2225  *
2226  * Early initialization pass for hardware IPs.  The hardware IPs that make
2227  * up each asic are discovered each IP's early_init callback is run.  This
2228  * is the first stage in initializing the asic.
2229  * Returns 0 on success, negative error code on failure.
2230  */
2231 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2232 {
2233         struct drm_device *dev = adev_to_drm(adev);
2234         struct pci_dev *parent;
2235         int i, r;
2236         bool total;
2237
2238         amdgpu_device_enable_virtual_display(adev);
2239
2240         if (amdgpu_sriov_vf(adev)) {
2241                 r = amdgpu_virt_request_full_gpu(adev, true);
2242                 if (r)
2243                         return r;
2244         }
2245
2246         switch (adev->asic_type) {
2247 #ifdef CONFIG_DRM_AMDGPU_SI
2248         case CHIP_VERDE:
2249         case CHIP_TAHITI:
2250         case CHIP_PITCAIRN:
2251         case CHIP_OLAND:
2252         case CHIP_HAINAN:
2253                 adev->family = AMDGPU_FAMILY_SI;
2254                 r = si_set_ip_blocks(adev);
2255                 if (r)
2256                         return r;
2257                 break;
2258 #endif
2259 #ifdef CONFIG_DRM_AMDGPU_CIK
2260         case CHIP_BONAIRE:
2261         case CHIP_HAWAII:
2262         case CHIP_KAVERI:
2263         case CHIP_KABINI:
2264         case CHIP_MULLINS:
2265                 if (adev->flags & AMD_IS_APU)
2266                         adev->family = AMDGPU_FAMILY_KV;
2267                 else
2268                         adev->family = AMDGPU_FAMILY_CI;
2269
2270                 r = cik_set_ip_blocks(adev);
2271                 if (r)
2272                         return r;
2273                 break;
2274 #endif
2275         case CHIP_TOPAZ:
2276         case CHIP_TONGA:
2277         case CHIP_FIJI:
2278         case CHIP_POLARIS10:
2279         case CHIP_POLARIS11:
2280         case CHIP_POLARIS12:
2281         case CHIP_VEGAM:
2282         case CHIP_CARRIZO:
2283         case CHIP_STONEY:
2284                 if (adev->flags & AMD_IS_APU)
2285                         adev->family = AMDGPU_FAMILY_CZ;
2286                 else
2287                         adev->family = AMDGPU_FAMILY_VI;
2288
2289                 r = vi_set_ip_blocks(adev);
2290                 if (r)
2291                         return r;
2292                 break;
2293         default:
2294                 r = amdgpu_discovery_set_ip_blocks(adev);
2295                 if (r)
2296                         return r;
2297                 break;
2298         }
2299
2300         if (amdgpu_has_atpx() &&
2301             (amdgpu_is_atpx_hybrid() ||
2302              amdgpu_has_atpx_dgpu_power_cntl()) &&
2303             ((adev->flags & AMD_IS_APU) == 0) &&
2304             !pci_is_thunderbolt_attached(to_pci_dev(dev->dev)))
2305                 adev->flags |= AMD_IS_PX;
2306
2307         if (!(adev->flags & AMD_IS_APU)) {
2308                 parent = pci_upstream_bridge(adev->pdev);
2309                 adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2310         }
2311
2312
2313         adev->pm.pp_feature = amdgpu_pp_feature_mask;
2314         if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2315                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2316         if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2317                 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2318
2319         total = true;
2320         for (i = 0; i < adev->num_ip_blocks; i++) {
2321                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2322                         DRM_WARN("disabled ip block: %d <%s>\n",
2323                                   i, adev->ip_blocks[i].version->funcs->name);
2324                         adev->ip_blocks[i].status.valid = false;
2325                 } else {
2326                         if (adev->ip_blocks[i].version->funcs->early_init) {
2327                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2328                                 if (r == -ENOENT) {
2329                                         adev->ip_blocks[i].status.valid = false;
2330                                 } else if (r) {
2331                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
2332                                                   adev->ip_blocks[i].version->funcs->name, r);
2333                                         total = false;
2334                                 } else {
2335                                         adev->ip_blocks[i].status.valid = true;
2336                                 }
2337                         } else {
2338                                 adev->ip_blocks[i].status.valid = true;
2339                         }
2340                 }
2341                 /* get the vbios after the asic_funcs are set up */
2342                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2343                         r = amdgpu_device_parse_gpu_info_fw(adev);
2344                         if (r)
2345                                 return r;
2346
2347                         /* Read BIOS */
2348                         if (amdgpu_device_read_bios(adev)) {
2349                                 if (!amdgpu_get_bios(adev))
2350                                         return -EINVAL;
2351
2352                                 r = amdgpu_atombios_init(adev);
2353                                 if (r) {
2354                                         dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2355                                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2356                                         return r;
2357                                 }
2358                         }
2359
2360                         /*get pf2vf msg info at it's earliest time*/
2361                         if (amdgpu_sriov_vf(adev))
2362                                 amdgpu_virt_init_data_exchange(adev);
2363
2364                 }
2365         }
2366         if (!total)
2367                 return -ENODEV;
2368
2369         amdgpu_amdkfd_device_probe(adev);
2370         adev->cg_flags &= amdgpu_cg_mask;
2371         adev->pg_flags &= amdgpu_pg_mask;
2372
2373         return 0;
2374 }
2375
2376 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2377 {
2378         int i, r;
2379
2380         for (i = 0; i < adev->num_ip_blocks; i++) {
2381                 if (!adev->ip_blocks[i].status.sw)
2382                         continue;
2383                 if (adev->ip_blocks[i].status.hw)
2384                         continue;
2385                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2386                     (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2387                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2388                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2389                         if (r) {
2390                                 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2391                                           adev->ip_blocks[i].version->funcs->name, r);
2392                                 return r;
2393                         }
2394                         adev->ip_blocks[i].status.hw = true;
2395                 }
2396         }
2397
2398         return 0;
2399 }
2400
2401 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2402 {
2403         int i, r;
2404
2405         for (i = 0; i < adev->num_ip_blocks; i++) {
2406                 if (!adev->ip_blocks[i].status.sw)
2407                         continue;
2408                 if (adev->ip_blocks[i].status.hw)
2409                         continue;
2410                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2411                 if (r) {
2412                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2413                                   adev->ip_blocks[i].version->funcs->name, r);
2414                         return r;
2415                 }
2416                 adev->ip_blocks[i].status.hw = true;
2417         }
2418
2419         return 0;
2420 }
2421
2422 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2423 {
2424         int r = 0;
2425         int i;
2426         uint32_t smu_version;
2427
2428         if (adev->asic_type >= CHIP_VEGA10) {
2429                 for (i = 0; i < adev->num_ip_blocks; i++) {
2430                         if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2431                                 continue;
2432
2433                         if (!adev->ip_blocks[i].status.sw)
2434                                 continue;
2435
2436                         /* no need to do the fw loading again if already done*/
2437                         if (adev->ip_blocks[i].status.hw == true)
2438                                 break;
2439
2440                         if (amdgpu_in_reset(adev) || adev->in_suspend) {
2441                                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2442                                 if (r) {
2443                                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2444                                                           adev->ip_blocks[i].version->funcs->name, r);
2445                                         return r;
2446                                 }
2447                         } else {
2448                                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2449                                 if (r) {
2450                                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2451                                                           adev->ip_blocks[i].version->funcs->name, r);
2452                                         return r;
2453                                 }
2454                         }
2455
2456                         adev->ip_blocks[i].status.hw = true;
2457                         break;
2458                 }
2459         }
2460
2461         if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2462                 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2463
2464         return r;
2465 }
2466
2467 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2468 {
2469         long timeout;
2470         int r, i;
2471
2472         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2473                 struct amdgpu_ring *ring = adev->rings[i];
2474
2475                 /* No need to setup the GPU scheduler for rings that don't need it */
2476                 if (!ring || ring->no_scheduler)
2477                         continue;
2478
2479                 switch (ring->funcs->type) {
2480                 case AMDGPU_RING_TYPE_GFX:
2481                         timeout = adev->gfx_timeout;
2482                         break;
2483                 case AMDGPU_RING_TYPE_COMPUTE:
2484                         timeout = adev->compute_timeout;
2485                         break;
2486                 case AMDGPU_RING_TYPE_SDMA:
2487                         timeout = adev->sdma_timeout;
2488                         break;
2489                 default:
2490                         timeout = adev->video_timeout;
2491                         break;
2492                 }
2493
2494                 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
2495                                    ring->num_hw_submission, 0,
2496                                    timeout, adev->reset_domain->wq,
2497                                    ring->sched_score, ring->name,
2498                                    adev->dev);
2499                 if (r) {
2500                         DRM_ERROR("Failed to create scheduler on ring %s.\n",
2501                                   ring->name);
2502                         return r;
2503                 }
2504         }
2505
2506         amdgpu_xcp_update_partition_sched_list(adev);
2507
2508         return 0;
2509 }
2510
2511
2512 /**
2513  * amdgpu_device_ip_init - run init for hardware IPs
2514  *
2515  * @adev: amdgpu_device pointer
2516  *
2517  * Main initialization pass for hardware IPs.  The list of all the hardware
2518  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2519  * are run.  sw_init initializes the software state associated with each IP
2520  * and hw_init initializes the hardware associated with each IP.
2521  * Returns 0 on success, negative error code on failure.
2522  */
2523 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2524 {
2525         int i, r;
2526
2527         r = amdgpu_ras_init(adev);
2528         if (r)
2529                 return r;
2530
2531         for (i = 0; i < adev->num_ip_blocks; i++) {
2532                 if (!adev->ip_blocks[i].status.valid)
2533                         continue;
2534                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2535                 if (r) {
2536                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2537                                   adev->ip_blocks[i].version->funcs->name, r);
2538                         goto init_failed;
2539                 }
2540                 adev->ip_blocks[i].status.sw = true;
2541
2542                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2543                         /* need to do common hw init early so everything is set up for gmc */
2544                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2545                         if (r) {
2546                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2547                                 goto init_failed;
2548                         }
2549                         adev->ip_blocks[i].status.hw = true;
2550                 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2551                         /* need to do gmc hw init early so we can allocate gpu mem */
2552                         /* Try to reserve bad pages early */
2553                         if (amdgpu_sriov_vf(adev))
2554                                 amdgpu_virt_exchange_data(adev);
2555
2556                         r = amdgpu_device_mem_scratch_init(adev);
2557                         if (r) {
2558                                 DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r);
2559                                 goto init_failed;
2560                         }
2561                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2562                         if (r) {
2563                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2564                                 goto init_failed;
2565                         }
2566                         r = amdgpu_device_wb_init(adev);
2567                         if (r) {
2568                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2569                                 goto init_failed;
2570                         }
2571                         adev->ip_blocks[i].status.hw = true;
2572
2573                         /* right after GMC hw init, we create CSA */
2574                         if (adev->gfx.mcbp) {
2575                                 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2576                                                                AMDGPU_GEM_DOMAIN_VRAM |
2577                                                                AMDGPU_GEM_DOMAIN_GTT,
2578                                                                AMDGPU_CSA_SIZE);
2579                                 if (r) {
2580                                         DRM_ERROR("allocate CSA failed %d\n", r);
2581                                         goto init_failed;
2582                                 }
2583                         }
2584                 }
2585         }
2586
2587         if (amdgpu_sriov_vf(adev))
2588                 amdgpu_virt_init_data_exchange(adev);
2589
2590         r = amdgpu_ib_pool_init(adev);
2591         if (r) {
2592                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2593                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2594                 goto init_failed;
2595         }
2596
2597         r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2598         if (r)
2599                 goto init_failed;
2600
2601         r = amdgpu_device_ip_hw_init_phase1(adev);
2602         if (r)
2603                 goto init_failed;
2604
2605         r = amdgpu_device_fw_loading(adev);
2606         if (r)
2607                 goto init_failed;
2608
2609         r = amdgpu_device_ip_hw_init_phase2(adev);
2610         if (r)
2611                 goto init_failed;
2612
2613         /*
2614          * retired pages will be loaded from eeprom and reserved here,
2615          * it should be called after amdgpu_device_ip_hw_init_phase2  since
2616          * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2617          * for I2C communication which only true at this point.
2618          *
2619          * amdgpu_ras_recovery_init may fail, but the upper only cares the
2620          * failure from bad gpu situation and stop amdgpu init process
2621          * accordingly. For other failed cases, it will still release all
2622          * the resource and print error message, rather than returning one
2623          * negative value to upper level.
2624          *
2625          * Note: theoretically, this should be called before all vram allocations
2626          * to protect retired page from abusing
2627          */
2628         r = amdgpu_ras_recovery_init(adev);
2629         if (r)
2630                 goto init_failed;
2631
2632         /**
2633          * In case of XGMI grab extra reference for reset domain for this device
2634          */
2635         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2636                 if (amdgpu_xgmi_add_device(adev) == 0) {
2637                         if (!amdgpu_sriov_vf(adev)) {
2638                                 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2639
2640                                 if (WARN_ON(!hive)) {
2641                                         r = -ENOENT;
2642                                         goto init_failed;
2643                                 }
2644
2645                                 if (!hive->reset_domain ||
2646                                     !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
2647                                         r = -ENOENT;
2648                                         amdgpu_put_xgmi_hive(hive);
2649                                         goto init_failed;
2650                                 }
2651
2652                                 /* Drop the early temporary reset domain we created for device */
2653                                 amdgpu_reset_put_reset_domain(adev->reset_domain);
2654                                 adev->reset_domain = hive->reset_domain;
2655                                 amdgpu_put_xgmi_hive(hive);
2656                         }
2657                 }
2658         }
2659
2660         r = amdgpu_device_init_schedulers(adev);
2661         if (r)
2662                 goto init_failed;
2663
2664         /* Don't init kfd if whole hive need to be reset during init */
2665         if (!adev->gmc.xgmi.pending_reset) {
2666                 kgd2kfd_init_zone_device(adev);
2667                 amdgpu_amdkfd_device_init(adev);
2668         }
2669
2670         amdgpu_fru_get_product_info(adev);
2671
2672 init_failed:
2673
2674         return r;
2675 }
2676
2677 /**
2678  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2679  *
2680  * @adev: amdgpu_device pointer
2681  *
2682  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2683  * this function before a GPU reset.  If the value is retained after a
2684  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2685  */
2686 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2687 {
2688         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2689 }
2690
2691 /**
2692  * amdgpu_device_check_vram_lost - check if vram is valid
2693  *
2694  * @adev: amdgpu_device pointer
2695  *
2696  * Checks the reset magic value written to the gart pointer in VRAM.
2697  * The driver calls this after a GPU reset to see if the contents of
2698  * VRAM is lost or now.
2699  * returns true if vram is lost, false if not.
2700  */
2701 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2702 {
2703         if (memcmp(adev->gart.ptr, adev->reset_magic,
2704                         AMDGPU_RESET_MAGIC_NUM))
2705                 return true;
2706
2707         if (!amdgpu_in_reset(adev))
2708                 return false;
2709
2710         /*
2711          * For all ASICs with baco/mode1 reset, the VRAM is
2712          * always assumed to be lost.
2713          */
2714         switch (amdgpu_asic_reset_method(adev)) {
2715         case AMD_RESET_METHOD_BACO:
2716         case AMD_RESET_METHOD_MODE1:
2717                 return true;
2718         default:
2719                 return false;
2720         }
2721 }
2722
2723 /**
2724  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2725  *
2726  * @adev: amdgpu_device pointer
2727  * @state: clockgating state (gate or ungate)
2728  *
2729  * The list of all the hardware IPs that make up the asic is walked and the
2730  * set_clockgating_state callbacks are run.
2731  * Late initialization pass enabling clockgating for hardware IPs.
2732  * Fini or suspend, pass disabling clockgating for hardware IPs.
2733  * Returns 0 on success, negative error code on failure.
2734  */
2735
2736 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2737                                enum amd_clockgating_state state)
2738 {
2739         int i, j, r;
2740
2741         if (amdgpu_emu_mode == 1)
2742                 return 0;
2743
2744         for (j = 0; j < adev->num_ip_blocks; j++) {
2745                 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2746                 if (!adev->ip_blocks[i].status.late_initialized)
2747                         continue;
2748                 /* skip CG for GFX, SDMA on S0ix */
2749                 if (adev->in_s0ix &&
2750                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2751                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2752                         continue;
2753                 /* skip CG for VCE/UVD, it's handled specially */
2754                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2755                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2756                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2757                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2758                     adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2759                         /* enable clockgating to save power */
2760                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2761                                                                                      state);
2762                         if (r) {
2763                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2764                                           adev->ip_blocks[i].version->funcs->name, r);
2765                                 return r;
2766                         }
2767                 }
2768         }
2769
2770         return 0;
2771 }
2772
2773 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2774                                enum amd_powergating_state state)
2775 {
2776         int i, j, r;
2777
2778         if (amdgpu_emu_mode == 1)
2779                 return 0;
2780
2781         for (j = 0; j < adev->num_ip_blocks; j++) {
2782                 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2783                 if (!adev->ip_blocks[i].status.late_initialized)
2784                         continue;
2785                 /* skip PG for GFX, SDMA on S0ix */
2786                 if (adev->in_s0ix &&
2787                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2788                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2789                         continue;
2790                 /* skip CG for VCE/UVD, it's handled specially */
2791                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2792                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2793                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2794                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2795                     adev->ip_blocks[i].version->funcs->set_powergating_state) {
2796                         /* enable powergating to save power */
2797                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2798                                                                                         state);
2799                         if (r) {
2800                                 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2801                                           adev->ip_blocks[i].version->funcs->name, r);
2802                                 return r;
2803                         }
2804                 }
2805         }
2806         return 0;
2807 }
2808
2809 static int amdgpu_device_enable_mgpu_fan_boost(void)
2810 {
2811         struct amdgpu_gpu_instance *gpu_ins;
2812         struct amdgpu_device *adev;
2813         int i, ret = 0;
2814
2815         mutex_lock(&mgpu_info.mutex);
2816
2817         /*
2818          * MGPU fan boost feature should be enabled
2819          * only when there are two or more dGPUs in
2820          * the system
2821          */
2822         if (mgpu_info.num_dgpu < 2)
2823                 goto out;
2824
2825         for (i = 0; i < mgpu_info.num_dgpu; i++) {
2826                 gpu_ins = &(mgpu_info.gpu_ins[i]);
2827                 adev = gpu_ins->adev;
2828                 if (!(adev->flags & AMD_IS_APU) &&
2829                     !gpu_ins->mgpu_fan_enabled) {
2830                         ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2831                         if (ret)
2832                                 break;
2833
2834                         gpu_ins->mgpu_fan_enabled = 1;
2835                 }
2836         }
2837
2838 out:
2839         mutex_unlock(&mgpu_info.mutex);
2840
2841         return ret;
2842 }
2843
2844 /**
2845  * amdgpu_device_ip_late_init - run late init for hardware IPs
2846  *
2847  * @adev: amdgpu_device pointer
2848  *
2849  * Late initialization pass for hardware IPs.  The list of all the hardware
2850  * IPs that make up the asic is walked and the late_init callbacks are run.
2851  * late_init covers any special initialization that an IP requires
2852  * after all of the have been initialized or something that needs to happen
2853  * late in the init process.
2854  * Returns 0 on success, negative error code on failure.
2855  */
2856 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2857 {
2858         struct amdgpu_gpu_instance *gpu_instance;
2859         int i = 0, r;
2860
2861         for (i = 0; i < adev->num_ip_blocks; i++) {
2862                 if (!adev->ip_blocks[i].status.hw)
2863                         continue;
2864                 if (adev->ip_blocks[i].version->funcs->late_init) {
2865                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2866                         if (r) {
2867                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2868                                           adev->ip_blocks[i].version->funcs->name, r);
2869                                 return r;
2870                         }
2871                 }
2872                 adev->ip_blocks[i].status.late_initialized = true;
2873         }
2874
2875         r = amdgpu_ras_late_init(adev);
2876         if (r) {
2877                 DRM_ERROR("amdgpu_ras_late_init failed %d", r);
2878                 return r;
2879         }
2880
2881         amdgpu_ras_set_error_query_ready(adev, true);
2882
2883         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2884         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2885
2886         amdgpu_device_fill_reset_magic(adev);
2887
2888         r = amdgpu_device_enable_mgpu_fan_boost();
2889         if (r)
2890                 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2891
2892         /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
2893         if (amdgpu_passthrough(adev) &&
2894             ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) ||
2895              adev->asic_type == CHIP_ALDEBARAN))
2896                 amdgpu_dpm_handle_passthrough_sbr(adev, true);
2897
2898         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2899                 mutex_lock(&mgpu_info.mutex);
2900
2901                 /*
2902                  * Reset device p-state to low as this was booted with high.
2903                  *
2904                  * This should be performed only after all devices from the same
2905                  * hive get initialized.
2906                  *
2907                  * However, it's unknown how many device in the hive in advance.
2908                  * As this is counted one by one during devices initializations.
2909                  *
2910                  * So, we wait for all XGMI interlinked devices initialized.
2911                  * This may bring some delays as those devices may come from
2912                  * different hives. But that should be OK.
2913                  */
2914                 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2915                         for (i = 0; i < mgpu_info.num_gpu; i++) {
2916                                 gpu_instance = &(mgpu_info.gpu_ins[i]);
2917                                 if (gpu_instance->adev->flags & AMD_IS_APU)
2918                                         continue;
2919
2920                                 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2921                                                 AMDGPU_XGMI_PSTATE_MIN);
2922                                 if (r) {
2923                                         DRM_ERROR("pstate setting failed (%d).\n", r);
2924                                         break;
2925                                 }
2926                         }
2927                 }
2928
2929                 mutex_unlock(&mgpu_info.mutex);
2930         }
2931
2932         return 0;
2933 }
2934
2935 /**
2936  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
2937  *
2938  * @adev: amdgpu_device pointer
2939  *
2940  * For ASICs need to disable SMC first
2941  */
2942 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
2943 {
2944         int i, r;
2945
2946         if (adev->ip_versions[GC_HWIP][0] > IP_VERSION(9, 0, 0))
2947                 return;
2948
2949         for (i = 0; i < adev->num_ip_blocks; i++) {
2950                 if (!adev->ip_blocks[i].status.hw)
2951                         continue;
2952                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2953                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2954                         /* XXX handle errors */
2955                         if (r) {
2956                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2957                                           adev->ip_blocks[i].version->funcs->name, r);
2958                         }
2959                         adev->ip_blocks[i].status.hw = false;
2960                         break;
2961                 }
2962         }
2963 }
2964
2965 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2966 {
2967         int i, r;
2968
2969         for (i = 0; i < adev->num_ip_blocks; i++) {
2970                 if (!adev->ip_blocks[i].version->funcs->early_fini)
2971                         continue;
2972
2973                 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
2974                 if (r) {
2975                         DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
2976                                   adev->ip_blocks[i].version->funcs->name, r);
2977                 }
2978         }
2979
2980         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2981         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2982
2983         amdgpu_amdkfd_suspend(adev, false);
2984
2985         /* Workaroud for ASICs need to disable SMC first */
2986         amdgpu_device_smu_fini_early(adev);
2987
2988         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2989                 if (!adev->ip_blocks[i].status.hw)
2990                         continue;
2991
2992                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2993                 /* XXX handle errors */
2994                 if (r) {
2995                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2996                                   adev->ip_blocks[i].version->funcs->name, r);
2997                 }
2998
2999                 adev->ip_blocks[i].status.hw = false;
3000         }
3001
3002         if (amdgpu_sriov_vf(adev)) {
3003                 if (amdgpu_virt_release_full_gpu(adev, false))
3004                         DRM_ERROR("failed to release exclusive mode on fini\n");
3005         }
3006
3007         return 0;
3008 }
3009
3010 /**
3011  * amdgpu_device_ip_fini - run fini for hardware IPs
3012  *
3013  * @adev: amdgpu_device pointer
3014  *
3015  * Main teardown pass for hardware IPs.  The list of all the hardware
3016  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
3017  * are run.  hw_fini tears down the hardware associated with each IP
3018  * and sw_fini tears down any software state associated with each IP.
3019  * Returns 0 on success, negative error code on failure.
3020  */
3021 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
3022 {
3023         int i, r;
3024
3025         if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
3026                 amdgpu_virt_release_ras_err_handler_data(adev);
3027
3028         if (adev->gmc.xgmi.num_physical_nodes > 1)
3029                 amdgpu_xgmi_remove_device(adev);
3030
3031         amdgpu_amdkfd_device_fini_sw(adev);
3032
3033         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3034                 if (!adev->ip_blocks[i].status.sw)
3035                         continue;
3036
3037                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
3038                         amdgpu_ucode_free_bo(adev);
3039                         amdgpu_free_static_csa(&adev->virt.csa_obj);
3040                         amdgpu_device_wb_fini(adev);
3041                         amdgpu_device_mem_scratch_fini(adev);
3042                         amdgpu_ib_pool_fini(adev);
3043                 }
3044
3045                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
3046                 /* XXX handle errors */
3047                 if (r) {
3048                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
3049                                   adev->ip_blocks[i].version->funcs->name, r);
3050                 }
3051                 adev->ip_blocks[i].status.sw = false;
3052                 adev->ip_blocks[i].status.valid = false;
3053         }
3054
3055         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3056                 if (!adev->ip_blocks[i].status.late_initialized)
3057                         continue;
3058                 if (adev->ip_blocks[i].version->funcs->late_fini)
3059                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
3060                 adev->ip_blocks[i].status.late_initialized = false;
3061         }
3062
3063         amdgpu_ras_fini(adev);
3064
3065         return 0;
3066 }
3067
3068 /**
3069  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
3070  *
3071  * @work: work_struct.
3072  */
3073 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
3074 {
3075         struct amdgpu_device *adev =
3076                 container_of(work, struct amdgpu_device, delayed_init_work.work);
3077         int r;
3078
3079         r = amdgpu_ib_ring_tests(adev);
3080         if (r)
3081                 DRM_ERROR("ib ring test failed (%d).\n", r);
3082 }
3083
3084 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
3085 {
3086         struct amdgpu_device *adev =
3087                 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
3088
3089         WARN_ON_ONCE(adev->gfx.gfx_off_state);
3090         WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
3091
3092         if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
3093                 adev->gfx.gfx_off_state = true;
3094 }
3095
3096 /**
3097  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3098  *
3099  * @adev: amdgpu_device pointer
3100  *
3101  * Main suspend function for hardware IPs.  The list of all the hardware
3102  * IPs that make up the asic is walked, clockgating is disabled and the
3103  * suspend callbacks are run.  suspend puts the hardware and software state
3104  * in each IP into a state suitable for suspend.
3105  * Returns 0 on success, negative error code on failure.
3106  */
3107 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
3108 {
3109         int i, r;
3110
3111         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3112         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3113
3114         /*
3115          * Per PMFW team's suggestion, driver needs to handle gfxoff
3116          * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
3117          * scenario. Add the missing df cstate disablement here.
3118          */
3119         if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
3120                 dev_warn(adev->dev, "Failed to disallow df cstate");
3121
3122         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3123                 if (!adev->ip_blocks[i].status.valid)
3124                         continue;
3125
3126                 /* displays are handled separately */
3127                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
3128                         continue;
3129
3130                 /* XXX handle errors */
3131                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3132                 /* XXX handle errors */
3133                 if (r) {
3134                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
3135                                   adev->ip_blocks[i].version->funcs->name, r);
3136                         return r;
3137                 }
3138
3139                 adev->ip_blocks[i].status.hw = false;
3140         }
3141
3142         return 0;
3143 }
3144
3145 /**
3146  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3147  *
3148  * @adev: amdgpu_device pointer
3149  *
3150  * Main suspend function for hardware IPs.  The list of all the hardware
3151  * IPs that make up the asic is walked, clockgating is disabled and the
3152  * suspend callbacks are run.  suspend puts the hardware and software state
3153  * in each IP into a state suitable for suspend.
3154  * Returns 0 on success, negative error code on failure.
3155  */
3156 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3157 {
3158         int i, r;
3159
3160         if (adev->in_s0ix)
3161                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
3162
3163         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3164                 if (!adev->ip_blocks[i].status.valid)
3165                         continue;
3166                 /* displays are handled in phase1 */
3167                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3168                         continue;
3169                 /* PSP lost connection when err_event_athub occurs */
3170                 if (amdgpu_ras_intr_triggered() &&
3171                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3172                         adev->ip_blocks[i].status.hw = false;
3173                         continue;
3174                 }
3175
3176                 /* skip unnecessary suspend if we do not initialize them yet */
3177                 if (adev->gmc.xgmi.pending_reset &&
3178                     !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3179                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
3180                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3181                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
3182                         adev->ip_blocks[i].status.hw = false;
3183                         continue;
3184                 }
3185
3186                 /* skip suspend of gfx/mes and psp for S0ix
3187                  * gfx is in gfxoff state, so on resume it will exit gfxoff just
3188                  * like at runtime. PSP is also part of the always on hardware
3189                  * so no need to suspend it.
3190                  */
3191                 if (adev->in_s0ix &&
3192                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3193                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3194                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3195                         continue;
3196
3197                 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3198                 if (adev->in_s0ix &&
3199                     (adev->ip_versions[SDMA0_HWIP][0] >= IP_VERSION(5, 0, 0)) &&
3200                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
3201                         continue;
3202
3203                 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3204                  * These are in TMR, hence are expected to be reused by PSP-TOS to reload
3205                  * from this location and RLC Autoload automatically also gets loaded
3206                  * from here based on PMFW -> PSP message during re-init sequence.
3207                  * Therefore, the psp suspend & resume should be skipped to avoid destroy
3208                  * the TMR and reload FWs again for IMU enabled APU ASICs.
3209                  */
3210                 if (amdgpu_in_reset(adev) &&
3211                     (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3212                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3213                         continue;
3214
3215                 /* XXX handle errors */
3216                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3217                 /* XXX handle errors */
3218                 if (r) {
3219                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
3220                                   adev->ip_blocks[i].version->funcs->name, r);
3221                 }
3222                 adev->ip_blocks[i].status.hw = false;
3223                 /* handle putting the SMC in the appropriate state */
3224                 if (!amdgpu_sriov_vf(adev)) {
3225                         if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3226                                 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3227                                 if (r) {
3228                                         DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3229                                                         adev->mp1_state, r);
3230                                         return r;
3231                                 }
3232                         }
3233                 }
3234         }
3235
3236         return 0;
3237 }
3238
3239 /**
3240  * amdgpu_device_ip_suspend - run suspend for hardware IPs
3241  *
3242  * @adev: amdgpu_device pointer
3243  *
3244  * Main suspend function for hardware IPs.  The list of all the hardware
3245  * IPs that make up the asic is walked, clockgating is disabled and the
3246  * suspend callbacks are run.  suspend puts the hardware and software state
3247  * in each IP into a state suitable for suspend.
3248  * Returns 0 on success, negative error code on failure.
3249  */
3250 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3251 {
3252         int r;
3253
3254         if (amdgpu_sriov_vf(adev)) {
3255                 amdgpu_virt_fini_data_exchange(adev);
3256                 amdgpu_virt_request_full_gpu(adev, false);
3257         }
3258
3259         r = amdgpu_device_ip_suspend_phase1(adev);
3260         if (r)
3261                 return r;
3262         r = amdgpu_device_ip_suspend_phase2(adev);
3263
3264         if (amdgpu_sriov_vf(adev))
3265                 amdgpu_virt_release_full_gpu(adev, false);
3266
3267         return r;
3268 }
3269
3270 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3271 {
3272         int i, r;
3273
3274         static enum amd_ip_block_type ip_order[] = {
3275                 AMD_IP_BLOCK_TYPE_COMMON,
3276                 AMD_IP_BLOCK_TYPE_GMC,
3277                 AMD_IP_BLOCK_TYPE_PSP,
3278                 AMD_IP_BLOCK_TYPE_IH,
3279         };
3280
3281         for (i = 0; i < adev->num_ip_blocks; i++) {
3282                 int j;
3283                 struct amdgpu_ip_block *block;
3284
3285                 block = &adev->ip_blocks[i];
3286                 block->status.hw = false;
3287
3288                 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3289
3290                         if (block->version->type != ip_order[j] ||
3291                                 !block->status.valid)
3292                                 continue;
3293
3294                         r = block->version->funcs->hw_init(adev);
3295                         DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3296                         if (r)
3297                                 return r;
3298                         block->status.hw = true;
3299                 }
3300         }
3301
3302         return 0;
3303 }
3304
3305 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3306 {
3307         int i, r;
3308
3309         static enum amd_ip_block_type ip_order[] = {
3310                 AMD_IP_BLOCK_TYPE_SMC,
3311                 AMD_IP_BLOCK_TYPE_DCE,
3312                 AMD_IP_BLOCK_TYPE_GFX,
3313                 AMD_IP_BLOCK_TYPE_SDMA,
3314                 AMD_IP_BLOCK_TYPE_MES,
3315                 AMD_IP_BLOCK_TYPE_UVD,
3316                 AMD_IP_BLOCK_TYPE_VCE,
3317                 AMD_IP_BLOCK_TYPE_VCN,
3318                 AMD_IP_BLOCK_TYPE_JPEG
3319         };
3320
3321         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3322                 int j;
3323                 struct amdgpu_ip_block *block;
3324
3325                 for (j = 0; j < adev->num_ip_blocks; j++) {
3326                         block = &adev->ip_blocks[j];
3327
3328                         if (block->version->type != ip_order[i] ||
3329                                 !block->status.valid ||
3330                                 block->status.hw)
3331                                 continue;
3332
3333                         if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3334                                 r = block->version->funcs->resume(adev);
3335                         else
3336                                 r = block->version->funcs->hw_init(adev);
3337
3338                         DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3339                         if (r)
3340                                 return r;
3341                         block->status.hw = true;
3342                 }
3343         }
3344
3345         return 0;
3346 }
3347
3348 /**
3349  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3350  *
3351  * @adev: amdgpu_device pointer
3352  *
3353  * First resume function for hardware IPs.  The list of all the hardware
3354  * IPs that make up the asic is walked and the resume callbacks are run for
3355  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3356  * after a suspend and updates the software state as necessary.  This
3357  * function is also used for restoring the GPU after a GPU reset.
3358  * Returns 0 on success, negative error code on failure.
3359  */
3360 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3361 {
3362         int i, r;
3363
3364         for (i = 0; i < adev->num_ip_blocks; i++) {
3365                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3366                         continue;
3367                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3368                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3369                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3370                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3371
3372                         r = adev->ip_blocks[i].version->funcs->resume(adev);
3373                         if (r) {
3374                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
3375                                           adev->ip_blocks[i].version->funcs->name, r);
3376                                 return r;
3377                         }
3378                         adev->ip_blocks[i].status.hw = true;
3379                 }
3380         }
3381
3382         return 0;
3383 }
3384
3385 /**
3386  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3387  *
3388  * @adev: amdgpu_device pointer
3389  *
3390  * First resume function for hardware IPs.  The list of all the hardware
3391  * IPs that make up the asic is walked and the resume callbacks are run for
3392  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3393  * functional state after a suspend and updates the software state as
3394  * necessary.  This function is also used for restoring the GPU after a GPU
3395  * reset.
3396  * Returns 0 on success, negative error code on failure.
3397  */
3398 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3399 {
3400         int i, r;
3401
3402         for (i = 0; i < adev->num_ip_blocks; i++) {
3403                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3404                         continue;
3405                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3406                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3407                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3408                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3409                         continue;
3410                 r = adev->ip_blocks[i].version->funcs->resume(adev);
3411                 if (r) {
3412                         DRM_ERROR("resume of IP block <%s> failed %d\n",
3413                                   adev->ip_blocks[i].version->funcs->name, r);
3414                         return r;
3415                 }
3416                 adev->ip_blocks[i].status.hw = true;
3417         }
3418
3419         return 0;
3420 }
3421
3422 /**
3423  * amdgpu_device_ip_resume - run resume for hardware IPs
3424  *
3425  * @adev: amdgpu_device pointer
3426  *
3427  * Main resume function for hardware IPs.  The hardware IPs
3428  * are split into two resume functions because they are
3429  * are also used in in recovering from a GPU reset and some additional
3430  * steps need to be take between them.  In this case (S3/S4) they are
3431  * run sequentially.
3432  * Returns 0 on success, negative error code on failure.
3433  */
3434 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3435 {
3436         int r;
3437
3438         if (!adev->in_s0ix) {
3439                 r = amdgpu_amdkfd_resume_iommu(adev);
3440                 if (r)
3441                         return r;
3442         }
3443
3444         r = amdgpu_device_ip_resume_phase1(adev);
3445         if (r)
3446                 return r;
3447
3448         r = amdgpu_device_fw_loading(adev);
3449         if (r)
3450                 return r;
3451
3452         r = amdgpu_device_ip_resume_phase2(adev);
3453
3454         return r;
3455 }
3456
3457 /**
3458  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3459  *
3460  * @adev: amdgpu_device pointer
3461  *
3462  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3463  */
3464 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3465 {
3466         if (amdgpu_sriov_vf(adev)) {
3467                 if (adev->is_atom_fw) {
3468                         if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3469                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3470                 } else {
3471                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3472                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3473                 }
3474
3475                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3476                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3477         }
3478 }
3479
3480 /**
3481  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3482  *
3483  * @asic_type: AMD asic type
3484  *
3485  * Check if there is DC (new modesetting infrastructre) support for an asic.
3486  * returns true if DC has support, false if not.
3487  */
3488 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3489 {
3490         switch (asic_type) {
3491 #ifdef CONFIG_DRM_AMDGPU_SI
3492         case CHIP_HAINAN:
3493 #endif
3494         case CHIP_TOPAZ:
3495                 /* chips with no display hardware */
3496                 return false;
3497 #if defined(CONFIG_DRM_AMD_DC)
3498         case CHIP_TAHITI:
3499         case CHIP_PITCAIRN:
3500         case CHIP_VERDE:
3501         case CHIP_OLAND:
3502                 /*
3503                  * We have systems in the wild with these ASICs that require
3504                  * LVDS and VGA support which is not supported with DC.
3505                  *
3506                  * Fallback to the non-DC driver here by default so as not to
3507                  * cause regressions.
3508                  */
3509 #if defined(CONFIG_DRM_AMD_DC_SI)
3510                 return amdgpu_dc > 0;
3511 #else
3512                 return false;
3513 #endif
3514         case CHIP_BONAIRE:
3515         case CHIP_KAVERI:
3516         case CHIP_KABINI:
3517         case CHIP_MULLINS:
3518                 /*
3519                  * We have systems in the wild with these ASICs that require
3520                  * VGA support which is not supported with DC.
3521                  *
3522                  * Fallback to the non-DC driver here by default so as not to
3523                  * cause regressions.
3524                  */
3525                 return amdgpu_dc > 0;
3526         default:
3527                 return amdgpu_dc != 0;
3528 #else
3529         default:
3530                 if (amdgpu_dc > 0)
3531                         DRM_INFO_ONCE("Display Core has been requested via kernel parameter "
3532                                          "but isn't supported by ASIC, ignoring\n");
3533                 return false;
3534 #endif
3535         }
3536 }
3537
3538 /**
3539  * amdgpu_device_has_dc_support - check if dc is supported
3540  *
3541  * @adev: amdgpu_device pointer
3542  *
3543  * Returns true for supported, false for not supported
3544  */
3545 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3546 {
3547         if (adev->enable_virtual_display ||
3548             (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3549                 return false;
3550
3551         return amdgpu_device_asic_has_dc_support(adev->asic_type);
3552 }
3553
3554 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3555 {
3556         struct amdgpu_device *adev =
3557                 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3558         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3559
3560         /* It's a bug to not have a hive within this function */
3561         if (WARN_ON(!hive))
3562                 return;
3563
3564         /*
3565          * Use task barrier to synchronize all xgmi reset works across the
3566          * hive. task_barrier_enter and task_barrier_exit will block
3567          * until all the threads running the xgmi reset works reach
3568          * those points. task_barrier_full will do both blocks.
3569          */
3570         if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3571
3572                 task_barrier_enter(&hive->tb);
3573                 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3574
3575                 if (adev->asic_reset_res)
3576                         goto fail;
3577
3578                 task_barrier_exit(&hive->tb);
3579                 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3580
3581                 if (adev->asic_reset_res)
3582                         goto fail;
3583
3584                 if (adev->mmhub.ras && adev->mmhub.ras->ras_block.hw_ops &&
3585                     adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
3586                         adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(adev);
3587         } else {
3588
3589                 task_barrier_full(&hive->tb);
3590                 adev->asic_reset_res =  amdgpu_asic_reset(adev);
3591         }
3592
3593 fail:
3594         if (adev->asic_reset_res)
3595                 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3596                          adev->asic_reset_res, adev_to_drm(adev)->unique);
3597         amdgpu_put_xgmi_hive(hive);
3598 }
3599
3600 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3601 {
3602         char *input = amdgpu_lockup_timeout;
3603         char *timeout_setting = NULL;
3604         int index = 0;
3605         long timeout;
3606         int ret = 0;
3607
3608         /*
3609          * By default timeout for non compute jobs is 10000
3610          * and 60000 for compute jobs.
3611          * In SR-IOV or passthrough mode, timeout for compute
3612          * jobs are 60000 by default.
3613          */
3614         adev->gfx_timeout = msecs_to_jiffies(10000);
3615         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3616         if (amdgpu_sriov_vf(adev))
3617                 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3618                                         msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3619         else
3620                 adev->compute_timeout =  msecs_to_jiffies(60000);
3621
3622         if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3623                 while ((timeout_setting = strsep(&input, ",")) &&
3624                                 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3625                         ret = kstrtol(timeout_setting, 0, &timeout);
3626                         if (ret)
3627                                 return ret;
3628
3629                         if (timeout == 0) {
3630                                 index++;
3631                                 continue;
3632                         } else if (timeout < 0) {
3633                                 timeout = MAX_SCHEDULE_TIMEOUT;
3634                                 dev_warn(adev->dev, "lockup timeout disabled");
3635                                 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3636                         } else {
3637                                 timeout = msecs_to_jiffies(timeout);
3638                         }
3639
3640                         switch (index++) {
3641                         case 0:
3642                                 adev->gfx_timeout = timeout;
3643                                 break;
3644                         case 1:
3645                                 adev->compute_timeout = timeout;
3646                                 break;
3647                         case 2:
3648                                 adev->sdma_timeout = timeout;
3649                                 break;
3650                         case 3:
3651                                 adev->video_timeout = timeout;
3652                                 break;
3653                         default:
3654                                 break;
3655                         }
3656                 }
3657                 /*
3658                  * There is only one value specified and
3659                  * it should apply to all non-compute jobs.
3660                  */
3661                 if (index == 1) {
3662                         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3663                         if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3664                                 adev->compute_timeout = adev->gfx_timeout;
3665                 }
3666         }
3667
3668         return ret;
3669 }
3670
3671 /**
3672  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3673  *
3674  * @adev: amdgpu_device pointer
3675  *
3676  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3677  */
3678 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3679 {
3680         struct iommu_domain *domain;
3681
3682         domain = iommu_get_domain_for_dev(adev->dev);
3683         if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3684                 adev->ram_is_direct_mapped = true;
3685 }
3686
3687 static const struct attribute *amdgpu_dev_attributes[] = {
3688         &dev_attr_product_name.attr,
3689         &dev_attr_product_number.attr,
3690         &dev_attr_serial_number.attr,
3691         &dev_attr_pcie_replay_count.attr,
3692         NULL
3693 };
3694
3695 static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)
3696 {
3697         if (amdgpu_mcbp == 1)
3698                 adev->gfx.mcbp = true;
3699
3700         if ((adev->ip_versions[GC_HWIP][0] >= IP_VERSION(9, 0, 0)) &&
3701             (adev->ip_versions[GC_HWIP][0] < IP_VERSION(10, 0, 0)) &&
3702             adev->gfx.num_gfx_rings)
3703                 adev->gfx.mcbp = true;
3704
3705         if (amdgpu_sriov_vf(adev))
3706                 adev->gfx.mcbp = true;
3707
3708         if (adev->gfx.mcbp)
3709                 DRM_INFO("MCBP is enabled\n");
3710 }
3711
3712 /**
3713  * amdgpu_device_init - initialize the driver
3714  *
3715  * @adev: amdgpu_device pointer
3716  * @flags: driver flags
3717  *
3718  * Initializes the driver info and hw (all asics).
3719  * Returns 0 for success or an error on failure.
3720  * Called at driver startup.
3721  */
3722 int amdgpu_device_init(struct amdgpu_device *adev,
3723                        uint32_t flags)
3724 {
3725         struct drm_device *ddev = adev_to_drm(adev);
3726         struct pci_dev *pdev = adev->pdev;
3727         int r, i;
3728         bool px = false;
3729         u32 max_MBps;
3730         int tmp;
3731
3732         adev->shutdown = false;
3733         adev->flags = flags;
3734
3735         if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3736                 adev->asic_type = amdgpu_force_asic_type;
3737         else
3738                 adev->asic_type = flags & AMD_ASIC_MASK;
3739
3740         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3741         if (amdgpu_emu_mode == 1)
3742                 adev->usec_timeout *= 10;
3743         adev->gmc.gart_size = 512 * 1024 * 1024;
3744         adev->accel_working = false;
3745         adev->num_rings = 0;
3746         RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3747         adev->mman.buffer_funcs = NULL;
3748         adev->mman.buffer_funcs_ring = NULL;
3749         adev->vm_manager.vm_pte_funcs = NULL;
3750         adev->vm_manager.vm_pte_num_scheds = 0;
3751         adev->gmc.gmc_funcs = NULL;
3752         adev->harvest_ip_mask = 0x0;
3753         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3754         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3755
3756         adev->smc_rreg = &amdgpu_invalid_rreg;
3757         adev->smc_wreg = &amdgpu_invalid_wreg;
3758         adev->pcie_rreg = &amdgpu_invalid_rreg;
3759         adev->pcie_wreg = &amdgpu_invalid_wreg;
3760         adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext;
3761         adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext;
3762         adev->pciep_rreg = &amdgpu_invalid_rreg;
3763         adev->pciep_wreg = &amdgpu_invalid_wreg;
3764         adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3765         adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3766         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3767         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3768         adev->didt_rreg = &amdgpu_invalid_rreg;
3769         adev->didt_wreg = &amdgpu_invalid_wreg;
3770         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3771         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3772         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3773         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3774
3775         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3776                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3777                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3778
3779         /* mutex initialization are all done here so we
3780          * can recall function without having locking issues */
3781         mutex_init(&adev->firmware.mutex);
3782         mutex_init(&adev->pm.mutex);
3783         mutex_init(&adev->gfx.gpu_clock_mutex);
3784         mutex_init(&adev->srbm_mutex);
3785         mutex_init(&adev->gfx.pipe_reserve_mutex);
3786         mutex_init(&adev->gfx.gfx_off_mutex);
3787         mutex_init(&adev->gfx.partition_mutex);
3788         mutex_init(&adev->grbm_idx_mutex);
3789         mutex_init(&adev->mn_lock);
3790         mutex_init(&adev->virt.vf_errors.lock);
3791         hash_init(adev->mn_hash);
3792         mutex_init(&adev->psp.mutex);
3793         mutex_init(&adev->notifier_lock);
3794         mutex_init(&adev->pm.stable_pstate_ctx_lock);
3795         mutex_init(&adev->benchmark_mutex);
3796
3797         amdgpu_device_init_apu_flags(adev);
3798
3799         r = amdgpu_device_check_arguments(adev);
3800         if (r)
3801                 return r;
3802
3803         spin_lock_init(&adev->mmio_idx_lock);
3804         spin_lock_init(&adev->smc_idx_lock);
3805         spin_lock_init(&adev->pcie_idx_lock);
3806         spin_lock_init(&adev->uvd_ctx_idx_lock);
3807         spin_lock_init(&adev->didt_idx_lock);
3808         spin_lock_init(&adev->gc_cac_idx_lock);
3809         spin_lock_init(&adev->se_cac_idx_lock);
3810         spin_lock_init(&adev->audio_endpt_idx_lock);
3811         spin_lock_init(&adev->mm_stats.lock);
3812
3813         INIT_LIST_HEAD(&adev->shadow_list);
3814         mutex_init(&adev->shadow_list_lock);
3815
3816         INIT_LIST_HEAD(&adev->reset_list);
3817
3818         INIT_LIST_HEAD(&adev->ras_list);
3819
3820         INIT_DELAYED_WORK(&adev->delayed_init_work,
3821                           amdgpu_device_delayed_init_work_handler);
3822         INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3823                           amdgpu_device_delay_enable_gfx_off);
3824
3825         INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3826
3827         adev->gfx.gfx_off_req_count = 1;
3828         adev->gfx.gfx_off_residency = 0;
3829         adev->gfx.gfx_off_entrycount = 0;
3830         adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3831
3832         atomic_set(&adev->throttling_logging_enabled, 1);
3833         /*
3834          * If throttling continues, logging will be performed every minute
3835          * to avoid log flooding. "-1" is subtracted since the thermal
3836          * throttling interrupt comes every second. Thus, the total logging
3837          * interval is 59 seconds(retelimited printk interval) + 1(waiting
3838          * for throttling interrupt) = 60 seconds.
3839          */
3840         ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3841         ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3842
3843         /* Registers mapping */
3844         /* TODO: block userspace mapping of io register */
3845         if (adev->asic_type >= CHIP_BONAIRE) {
3846                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3847                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3848         } else {
3849                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3850                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3851         }
3852
3853         for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
3854                 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
3855
3856         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3857         if (adev->rmmio == NULL) {
3858                 return -ENOMEM;
3859         }
3860         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3861         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
3862
3863         /*
3864          * Reset domain needs to be present early, before XGMI hive discovered
3865          * (if any) and intitialized to use reset sem and in_gpu reset flag
3866          * early on during init and before calling to RREG32.
3867          */
3868         adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
3869         if (!adev->reset_domain)
3870                 return -ENOMEM;
3871
3872         /* detect hw virtualization here */
3873         amdgpu_detect_virtualization(adev);
3874
3875         amdgpu_device_get_pcie_info(adev);
3876
3877         r = amdgpu_device_get_job_timeout_settings(adev);
3878         if (r) {
3879                 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3880                 return r;
3881         }
3882
3883         /* early init functions */
3884         r = amdgpu_device_ip_early_init(adev);
3885         if (r)
3886                 return r;
3887
3888         amdgpu_device_set_mcbp(adev);
3889
3890         /* Get rid of things like offb */
3891         r = drm_aperture_remove_conflicting_pci_framebuffers(adev->pdev, &amdgpu_kms_driver);
3892         if (r)
3893                 return r;
3894
3895         /* Enable TMZ based on IP_VERSION */
3896         amdgpu_gmc_tmz_set(adev);
3897
3898         amdgpu_gmc_noretry_set(adev);
3899         /* Need to get xgmi info early to decide the reset behavior*/
3900         if (adev->gmc.xgmi.supported) {
3901                 r = adev->gfxhub.funcs->get_xgmi_info(adev);
3902                 if (r)
3903                         return r;
3904         }
3905
3906         /* enable PCIE atomic ops */
3907         if (amdgpu_sriov_vf(adev)) {
3908                 if (adev->virt.fw_reserve.p_pf2vf)
3909                         adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
3910                                                       adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
3911                                 (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3912         /* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
3913          * internal path natively support atomics, set have_atomics_support to true.
3914          */
3915         } else if ((adev->flags & AMD_IS_APU) &&
3916                    (adev->ip_versions[GC_HWIP][0] > IP_VERSION(9, 0, 0))) {
3917                 adev->have_atomics_support = true;
3918         } else {
3919                 adev->have_atomics_support =
3920                         !pci_enable_atomic_ops_to_root(adev->pdev,
3921                                           PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3922                                           PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3923         }
3924
3925         if (!adev->have_atomics_support)
3926                 dev_info(adev->dev, "PCIE atomic ops is not supported\n");
3927
3928         /* doorbell bar mapping and doorbell index init*/
3929         amdgpu_device_doorbell_init(adev);
3930
3931         if (amdgpu_emu_mode == 1) {
3932                 /* post the asic on emulation mode */
3933                 emu_soc_asic_init(adev);
3934                 goto fence_driver_init;
3935         }
3936
3937         amdgpu_reset_init(adev);
3938
3939         /* detect if we are with an SRIOV vbios */
3940         if (adev->bios)
3941                 amdgpu_device_detect_sriov_bios(adev);
3942
3943         /* check if we need to reset the asic
3944          *  E.g., driver was not cleanly unloaded previously, etc.
3945          */
3946         if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3947                 if (adev->gmc.xgmi.num_physical_nodes) {
3948                         dev_info(adev->dev, "Pending hive reset.\n");
3949                         adev->gmc.xgmi.pending_reset = true;
3950                         /* Only need to init necessary block for SMU to handle the reset */
3951                         for (i = 0; i < adev->num_ip_blocks; i++) {
3952                                 if (!adev->ip_blocks[i].status.valid)
3953                                         continue;
3954                                 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3955                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3956                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3957                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3958                                         DRM_DEBUG("IP %s disabled for hw_init.\n",
3959                                                 adev->ip_blocks[i].version->funcs->name);
3960                                         adev->ip_blocks[i].status.hw = true;
3961                                 }
3962                         }
3963                 } else {
3964                         tmp = amdgpu_reset_method;
3965                         /* It should do a default reset when loading or reloading the driver,
3966                          * regardless of the module parameter reset_method.
3967                          */
3968                         amdgpu_reset_method = AMD_RESET_METHOD_NONE;
3969                         r = amdgpu_asic_reset(adev);
3970                         amdgpu_reset_method = tmp;
3971                         if (r) {
3972                                 dev_err(adev->dev, "asic reset on init failed\n");
3973                                 goto failed;
3974                         }
3975                 }
3976         }
3977
3978         /* Post card if necessary */
3979         if (amdgpu_device_need_post(adev)) {
3980                 if (!adev->bios) {
3981                         dev_err(adev->dev, "no vBIOS found\n");
3982                         r = -EINVAL;
3983                         goto failed;
3984                 }
3985                 DRM_INFO("GPU posting now...\n");
3986                 r = amdgpu_device_asic_init(adev);
3987                 if (r) {
3988                         dev_err(adev->dev, "gpu post error!\n");
3989                         goto failed;
3990                 }
3991         }
3992
3993         if (adev->bios) {
3994                 if (adev->is_atom_fw) {
3995                         /* Initialize clocks */
3996                         r = amdgpu_atomfirmware_get_clock_info(adev);
3997                         if (r) {
3998                                 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
3999                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4000                                 goto failed;
4001                         }
4002                 } else {
4003                         /* Initialize clocks */
4004                         r = amdgpu_atombios_get_clock_info(adev);
4005                         if (r) {
4006                                 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
4007                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4008                                 goto failed;
4009                         }
4010                         /* init i2c buses */
4011                         if (!amdgpu_device_has_dc_support(adev))
4012                                 amdgpu_atombios_i2c_init(adev);
4013                 }
4014         }
4015
4016 fence_driver_init:
4017         /* Fence driver */
4018         r = amdgpu_fence_driver_sw_init(adev);
4019         if (r) {
4020                 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
4021                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
4022                 goto failed;
4023         }
4024
4025         /* init the mode config */
4026         drm_mode_config_init(adev_to_drm(adev));
4027
4028         r = amdgpu_device_ip_init(adev);
4029         if (r) {
4030                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
4031                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
4032                 goto release_ras_con;
4033         }
4034
4035         amdgpu_fence_driver_hw_init(adev);
4036
4037         dev_info(adev->dev,
4038                 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
4039                         adev->gfx.config.max_shader_engines,
4040                         adev->gfx.config.max_sh_per_se,
4041                         adev->gfx.config.max_cu_per_sh,
4042                         adev->gfx.cu_info.number);
4043
4044         adev->accel_working = true;
4045
4046         amdgpu_vm_check_compute_bug(adev);
4047
4048         /* Initialize the buffer migration limit. */
4049         if (amdgpu_moverate >= 0)
4050                 max_MBps = amdgpu_moverate;
4051         else
4052                 max_MBps = 8; /* Allow 8 MB/s. */
4053         /* Get a log2 for easy divisions. */
4054         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
4055
4056         r = amdgpu_atombios_sysfs_init(adev);
4057         if (r)
4058                 drm_err(&adev->ddev,
4059                         "registering atombios sysfs failed (%d).\n", r);
4060
4061         r = amdgpu_pm_sysfs_init(adev);
4062         if (r)
4063                 DRM_ERROR("registering pm sysfs failed (%d).\n", r);
4064
4065         r = amdgpu_ucode_sysfs_init(adev);
4066         if (r) {
4067                 adev->ucode_sysfs_en = false;
4068                 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
4069         } else
4070                 adev->ucode_sysfs_en = true;
4071
4072         r = amdgpu_psp_sysfs_init(adev);
4073         if (r) {
4074                 adev->psp_sysfs_en = false;
4075                 if (!amdgpu_sriov_vf(adev))
4076                         DRM_ERROR("Creating psp sysfs failed\n");
4077         } else
4078                 adev->psp_sysfs_en = true;
4079
4080         /*
4081          * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
4082          * Otherwise the mgpu fan boost feature will be skipped due to the
4083          * gpu instance is counted less.
4084          */
4085         amdgpu_register_gpu_instance(adev);
4086
4087         /* enable clockgating, etc. after ib tests, etc. since some blocks require
4088          * explicit gating rather than handling it automatically.
4089          */
4090         if (!adev->gmc.xgmi.pending_reset) {
4091                 r = amdgpu_device_ip_late_init(adev);
4092                 if (r) {
4093                         dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
4094                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
4095                         goto release_ras_con;
4096                 }
4097                 /* must succeed. */
4098                 amdgpu_ras_resume(adev);
4099                 queue_delayed_work(system_wq, &adev->delayed_init_work,
4100                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
4101         }
4102
4103         if (amdgpu_sriov_vf(adev)) {
4104                 amdgpu_virt_release_full_gpu(adev, true);
4105                 flush_delayed_work(&adev->delayed_init_work);
4106         }
4107
4108         r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
4109         if (r)
4110                 dev_err(adev->dev, "Could not create amdgpu device attr\n");
4111
4112         if (IS_ENABLED(CONFIG_PERF_EVENTS))
4113                 r = amdgpu_pmu_init(adev);
4114         if (r)
4115                 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
4116
4117         /* Have stored pci confspace at hand for restore in sudden PCI error */
4118         if (amdgpu_device_cache_pci_state(adev->pdev))
4119                 pci_restore_state(pdev);
4120
4121         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
4122         /* this will fail for cards that aren't VGA class devices, just
4123          * ignore it */
4124         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4125                 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
4126
4127         px = amdgpu_device_supports_px(ddev);
4128
4129         if (px || (!pci_is_thunderbolt_attached(adev->pdev) &&
4130                                 apple_gmux_detect(NULL, NULL)))
4131                 vga_switcheroo_register_client(adev->pdev,
4132                                                &amdgpu_switcheroo_ops, px);
4133
4134         if (px)
4135                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
4136
4137         if (adev->gmc.xgmi.pending_reset)
4138                 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
4139                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
4140
4141         amdgpu_device_check_iommu_direct_map(adev);
4142
4143         return 0;
4144
4145 release_ras_con:
4146         if (amdgpu_sriov_vf(adev))
4147                 amdgpu_virt_release_full_gpu(adev, true);
4148
4149         /* failed in exclusive mode due to timeout */
4150         if (amdgpu_sriov_vf(adev) &&
4151                 !amdgpu_sriov_runtime(adev) &&
4152                 amdgpu_virt_mmio_blocked(adev) &&
4153                 !amdgpu_virt_wait_reset(adev)) {
4154                 dev_err(adev->dev, "VF exclusive mode timeout\n");
4155                 /* Don't send request since VF is inactive. */
4156                 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
4157                 adev->virt.ops = NULL;
4158                 r = -EAGAIN;
4159         }
4160         amdgpu_release_ras_context(adev);
4161
4162 failed:
4163         amdgpu_vf_error_trans_all(adev);
4164
4165         return r;
4166 }
4167
4168 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
4169 {
4170
4171         /* Clear all CPU mappings pointing to this device */
4172         unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
4173
4174         /* Unmap all mapped bars - Doorbell, registers and VRAM */
4175         amdgpu_device_doorbell_fini(adev);
4176
4177         iounmap(adev->rmmio);
4178         adev->rmmio = NULL;
4179         if (adev->mman.aper_base_kaddr)
4180                 iounmap(adev->mman.aper_base_kaddr);
4181         adev->mman.aper_base_kaddr = NULL;
4182
4183         /* Memory manager related */
4184         if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
4185                 arch_phys_wc_del(adev->gmc.vram_mtrr);
4186                 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
4187         }
4188 }
4189
4190 /**
4191  * amdgpu_device_fini_hw - tear down the driver
4192  *
4193  * @adev: amdgpu_device pointer
4194  *
4195  * Tear down the driver info (all asics).
4196  * Called at driver shutdown.
4197  */
4198 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4199 {
4200         dev_info(adev->dev, "amdgpu: finishing device.\n");
4201         flush_delayed_work(&adev->delayed_init_work);
4202         adev->shutdown = true;
4203
4204         /* make sure IB test finished before entering exclusive mode
4205          * to avoid preemption on IB test
4206          * */
4207         if (amdgpu_sriov_vf(adev)) {
4208                 amdgpu_virt_request_full_gpu(adev, false);
4209                 amdgpu_virt_fini_data_exchange(adev);
4210         }
4211
4212         /* disable all interrupts */
4213         amdgpu_irq_disable_all(adev);
4214         if (adev->mode_info.mode_config_initialized) {
4215                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
4216                         drm_helper_force_disable_all(adev_to_drm(adev));
4217                 else
4218                         drm_atomic_helper_shutdown(adev_to_drm(adev));
4219         }
4220         amdgpu_fence_driver_hw_fini(adev);
4221
4222         if (adev->mman.initialized)
4223                 drain_workqueue(adev->mman.bdev.wq);
4224
4225         if (adev->pm.sysfs_initialized)
4226                 amdgpu_pm_sysfs_fini(adev);
4227         if (adev->ucode_sysfs_en)
4228                 amdgpu_ucode_sysfs_fini(adev);
4229         if (adev->psp_sysfs_en)
4230                 amdgpu_psp_sysfs_fini(adev);
4231         sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4232
4233         /* disable ras feature must before hw fini */
4234         amdgpu_ras_pre_fini(adev);
4235
4236         amdgpu_device_ip_fini_early(adev);
4237
4238         amdgpu_irq_fini_hw(adev);
4239
4240         if (adev->mman.initialized)
4241                 ttm_device_clear_dma_mappings(&adev->mman.bdev);
4242
4243         amdgpu_gart_dummy_page_fini(adev);
4244
4245         if (drm_dev_is_unplugged(adev_to_drm(adev)))
4246                 amdgpu_device_unmap_mmio(adev);
4247
4248 }
4249
4250 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4251 {
4252         int idx;
4253         bool px;
4254
4255         amdgpu_fence_driver_sw_fini(adev);
4256         amdgpu_device_ip_fini(adev);
4257         amdgpu_ucode_release(&adev->firmware.gpu_info_fw);
4258         adev->accel_working = false;
4259         dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4260
4261         amdgpu_reset_fini(adev);
4262
4263         /* free i2c buses */
4264         if (!amdgpu_device_has_dc_support(adev))
4265                 amdgpu_i2c_fini(adev);
4266
4267         if (amdgpu_emu_mode != 1)
4268                 amdgpu_atombios_fini(adev);
4269
4270         kfree(adev->bios);
4271         adev->bios = NULL;
4272
4273         px = amdgpu_device_supports_px(adev_to_drm(adev));
4274
4275         if (px || (!pci_is_thunderbolt_attached(adev->pdev) &&
4276                                 apple_gmux_detect(NULL, NULL)))
4277                 vga_switcheroo_unregister_client(adev->pdev);
4278
4279         if (px)
4280                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
4281
4282         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4283                 vga_client_unregister(adev->pdev);
4284
4285         if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4286
4287                 iounmap(adev->rmmio);
4288                 adev->rmmio = NULL;
4289                 amdgpu_device_doorbell_fini(adev);
4290                 drm_dev_exit(idx);
4291         }
4292
4293         if (IS_ENABLED(CONFIG_PERF_EVENTS))
4294                 amdgpu_pmu_fini(adev);
4295         if (adev->mman.discovery_bin)
4296                 amdgpu_discovery_fini(adev);
4297
4298         amdgpu_reset_put_reset_domain(adev->reset_domain);
4299         adev->reset_domain = NULL;
4300
4301         kfree(adev->pci_state);
4302
4303 }
4304
4305 /**
4306  * amdgpu_device_evict_resources - evict device resources
4307  * @adev: amdgpu device object
4308  *
4309  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4310  * of the vram memory type. Mainly used for evicting device resources
4311  * at suspend time.
4312  *
4313  */
4314 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4315 {
4316         int ret;
4317
4318         /* No need to evict vram on APUs for suspend to ram or s2idle */
4319         if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4320                 return 0;
4321
4322         ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4323         if (ret)
4324                 DRM_WARN("evicting device resources failed\n");
4325         return ret;
4326 }
4327
4328 /*
4329  * Suspend & resume.
4330  */
4331 /**
4332  * amdgpu_device_suspend - initiate device suspend
4333  *
4334  * @dev: drm dev pointer
4335  * @fbcon : notify the fbdev of suspend
4336  *
4337  * Puts the hw in the suspend state (all asics).
4338  * Returns 0 for success or an error on failure.
4339  * Called at driver suspend.
4340  */
4341 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
4342 {
4343         struct amdgpu_device *adev = drm_to_adev(dev);
4344         int r = 0;
4345
4346         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4347                 return 0;
4348
4349         adev->in_suspend = true;
4350
4351         /* Evict the majority of BOs before grabbing the full access */
4352         r = amdgpu_device_evict_resources(adev);
4353         if (r)
4354                 return r;
4355
4356         if (amdgpu_sriov_vf(adev)) {
4357                 amdgpu_virt_fini_data_exchange(adev);
4358                 r = amdgpu_virt_request_full_gpu(adev, false);
4359                 if (r)
4360                         return r;
4361         }
4362
4363         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4364                 DRM_WARN("smart shift update failed\n");
4365
4366         if (fbcon)
4367                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
4368
4369         cancel_delayed_work_sync(&adev->delayed_init_work);
4370
4371         amdgpu_ras_suspend(adev);
4372
4373         amdgpu_device_ip_suspend_phase1(adev);
4374
4375         if (!adev->in_s0ix)
4376                 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4377
4378         r = amdgpu_device_evict_resources(adev);
4379         if (r)
4380                 return r;
4381
4382         amdgpu_fence_driver_hw_fini(adev);
4383
4384         amdgpu_device_ip_suspend_phase2(adev);
4385
4386         if (amdgpu_sriov_vf(adev))
4387                 amdgpu_virt_release_full_gpu(adev, false);
4388
4389         return 0;
4390 }
4391
4392 /**
4393  * amdgpu_device_resume - initiate device resume
4394  *
4395  * @dev: drm dev pointer
4396  * @fbcon : notify the fbdev of resume
4397  *
4398  * Bring the hw back to operating state (all asics).
4399  * Returns 0 for success or an error on failure.
4400  * Called at driver resume.
4401  */
4402 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4403 {
4404         struct amdgpu_device *adev = drm_to_adev(dev);
4405         int r = 0;
4406
4407         if (amdgpu_sriov_vf(adev)) {
4408                 r = amdgpu_virt_request_full_gpu(adev, true);
4409                 if (r)
4410                         return r;
4411         }
4412
4413         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4414                 return 0;
4415
4416         if (adev->in_s0ix)
4417                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4418
4419         /* post card */
4420         if (amdgpu_device_need_post(adev)) {
4421                 r = amdgpu_device_asic_init(adev);
4422                 if (r)
4423                         dev_err(adev->dev, "amdgpu asic init failed\n");
4424         }
4425
4426         r = amdgpu_device_ip_resume(adev);
4427
4428         if (r) {
4429                 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4430                 goto exit;
4431         }
4432         amdgpu_fence_driver_hw_init(adev);
4433
4434         r = amdgpu_device_ip_late_init(adev);
4435         if (r)
4436                 goto exit;
4437
4438         queue_delayed_work(system_wq, &adev->delayed_init_work,
4439                            msecs_to_jiffies(AMDGPU_RESUME_MS));
4440
4441         if (!adev->in_s0ix) {
4442                 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4443                 if (r)
4444                         goto exit;
4445         }
4446
4447 exit:
4448         if (amdgpu_sriov_vf(adev)) {
4449                 amdgpu_virt_init_data_exchange(adev);
4450                 amdgpu_virt_release_full_gpu(adev, true);
4451         }
4452
4453         if (r)
4454                 return r;
4455
4456         /* Make sure IB tests flushed */
4457         flush_delayed_work(&adev->delayed_init_work);
4458
4459         if (fbcon)
4460                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
4461
4462         amdgpu_ras_resume(adev);
4463
4464         if (adev->mode_info.num_crtc) {
4465                 /*
4466                  * Most of the connector probing functions try to acquire runtime pm
4467                  * refs to ensure that the GPU is powered on when connector polling is
4468                  * performed. Since we're calling this from a runtime PM callback,
4469                  * trying to acquire rpm refs will cause us to deadlock.
4470                  *
4471                  * Since we're guaranteed to be holding the rpm lock, it's safe to
4472                  * temporarily disable the rpm helpers so this doesn't deadlock us.
4473                  */
4474 #ifdef CONFIG_PM
4475                 dev->dev->power.disable_depth++;
4476 #endif
4477                 if (!adev->dc_enabled)
4478                         drm_helper_hpd_irq_event(dev);
4479                 else
4480                         drm_kms_helper_hotplug_event(dev);
4481 #ifdef CONFIG_PM
4482                 dev->dev->power.disable_depth--;
4483 #endif
4484         }
4485         adev->in_suspend = false;
4486
4487         if (adev->enable_mes)
4488                 amdgpu_mes_self_test(adev);
4489
4490         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
4491                 DRM_WARN("smart shift update failed\n");
4492
4493         return 0;
4494 }
4495
4496 /**
4497  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4498  *
4499  * @adev: amdgpu_device pointer
4500  *
4501  * The list of all the hardware IPs that make up the asic is walked and
4502  * the check_soft_reset callbacks are run.  check_soft_reset determines
4503  * if the asic is still hung or not.
4504  * Returns true if any of the IPs are still in a hung state, false if not.
4505  */
4506 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4507 {
4508         int i;
4509         bool asic_hang = false;
4510
4511         if (amdgpu_sriov_vf(adev))
4512                 return true;
4513
4514         if (amdgpu_asic_need_full_reset(adev))
4515                 return true;
4516
4517         for (i = 0; i < adev->num_ip_blocks; i++) {
4518                 if (!adev->ip_blocks[i].status.valid)
4519                         continue;
4520                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4521                         adev->ip_blocks[i].status.hang =
4522                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4523                 if (adev->ip_blocks[i].status.hang) {
4524                         dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4525                         asic_hang = true;
4526                 }
4527         }
4528         return asic_hang;
4529 }
4530
4531 /**
4532  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4533  *
4534  * @adev: amdgpu_device pointer
4535  *
4536  * The list of all the hardware IPs that make up the asic is walked and the
4537  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
4538  * handles any IP specific hardware or software state changes that are
4539  * necessary for a soft reset to succeed.
4540  * Returns 0 on success, negative error code on failure.
4541  */
4542 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4543 {
4544         int i, r = 0;
4545
4546         for (i = 0; i < adev->num_ip_blocks; i++) {
4547                 if (!adev->ip_blocks[i].status.valid)
4548                         continue;
4549                 if (adev->ip_blocks[i].status.hang &&
4550                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4551                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4552                         if (r)
4553                                 return r;
4554                 }
4555         }
4556
4557         return 0;
4558 }
4559
4560 /**
4561  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4562  *
4563  * @adev: amdgpu_device pointer
4564  *
4565  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
4566  * reset is necessary to recover.
4567  * Returns true if a full asic reset is required, false if not.
4568  */
4569 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4570 {
4571         int i;
4572
4573         if (amdgpu_asic_need_full_reset(adev))
4574                 return true;
4575
4576         for (i = 0; i < adev->num_ip_blocks; i++) {
4577                 if (!adev->ip_blocks[i].status.valid)
4578                         continue;
4579                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4580                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4581                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4582                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4583                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4584                         if (adev->ip_blocks[i].status.hang) {
4585                                 dev_info(adev->dev, "Some block need full reset!\n");
4586                                 return true;
4587                         }
4588                 }
4589         }
4590         return false;
4591 }
4592
4593 /**
4594  * amdgpu_device_ip_soft_reset - do a soft reset
4595  *
4596  * @adev: amdgpu_device pointer
4597  *
4598  * The list of all the hardware IPs that make up the asic is walked and the
4599  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
4600  * IP specific hardware or software state changes that are necessary to soft
4601  * reset the IP.
4602  * Returns 0 on success, negative error code on failure.
4603  */
4604 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4605 {
4606         int i, r = 0;
4607
4608         for (i = 0; i < adev->num_ip_blocks; i++) {
4609                 if (!adev->ip_blocks[i].status.valid)
4610                         continue;
4611                 if (adev->ip_blocks[i].status.hang &&
4612                     adev->ip_blocks[i].version->funcs->soft_reset) {
4613                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4614                         if (r)
4615                                 return r;
4616                 }
4617         }
4618
4619         return 0;
4620 }
4621
4622 /**
4623  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4624  *
4625  * @adev: amdgpu_device pointer
4626  *
4627  * The list of all the hardware IPs that make up the asic is walked and the
4628  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4629  * handles any IP specific hardware or software state changes that are
4630  * necessary after the IP has been soft reset.
4631  * Returns 0 on success, negative error code on failure.
4632  */
4633 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4634 {
4635         int i, r = 0;
4636
4637         for (i = 0; i < adev->num_ip_blocks; i++) {
4638                 if (!adev->ip_blocks[i].status.valid)
4639                         continue;
4640                 if (adev->ip_blocks[i].status.hang &&
4641                     adev->ip_blocks[i].version->funcs->post_soft_reset)
4642                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4643                 if (r)
4644                         return r;
4645         }
4646
4647         return 0;
4648 }
4649
4650 /**
4651  * amdgpu_device_recover_vram - Recover some VRAM contents
4652  *
4653  * @adev: amdgpu_device pointer
4654  *
4655  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4656  * restore things like GPUVM page tables after a GPU reset where
4657  * the contents of VRAM might be lost.
4658  *
4659  * Returns:
4660  * 0 on success, negative error code on failure.
4661  */
4662 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4663 {
4664         struct dma_fence *fence = NULL, *next = NULL;
4665         struct amdgpu_bo *shadow;
4666         struct amdgpu_bo_vm *vmbo;
4667         long r = 1, tmo;
4668
4669         if (amdgpu_sriov_runtime(adev))
4670                 tmo = msecs_to_jiffies(8000);
4671         else
4672                 tmo = msecs_to_jiffies(100);
4673
4674         dev_info(adev->dev, "recover vram bo from shadow start\n");
4675         mutex_lock(&adev->shadow_list_lock);
4676         list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4677                 /* If vm is compute context or adev is APU, shadow will be NULL */
4678                 if (!vmbo->shadow)
4679                         continue;
4680                 shadow = vmbo->shadow;
4681
4682                 /* No need to recover an evicted BO */
4683                 if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4684                     shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4685                     shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4686                         continue;
4687
4688                 r = amdgpu_bo_restore_shadow(shadow, &next);
4689                 if (r)
4690                         break;
4691
4692                 if (fence) {
4693                         tmo = dma_fence_wait_timeout(fence, false, tmo);
4694                         dma_fence_put(fence);
4695                         fence = next;
4696                         if (tmo == 0) {
4697                                 r = -ETIMEDOUT;
4698                                 break;
4699                         } else if (tmo < 0) {
4700                                 r = tmo;
4701                                 break;
4702                         }
4703                 } else {
4704                         fence = next;
4705                 }
4706         }
4707         mutex_unlock(&adev->shadow_list_lock);
4708
4709         if (fence)
4710                 tmo = dma_fence_wait_timeout(fence, false, tmo);
4711         dma_fence_put(fence);
4712
4713         if (r < 0 || tmo <= 0) {
4714                 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4715                 return -EIO;
4716         }
4717
4718         dev_info(adev->dev, "recover vram bo from shadow done\n");
4719         return 0;
4720 }
4721
4722
4723 /**
4724  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4725  *
4726  * @adev: amdgpu_device pointer
4727  * @from_hypervisor: request from hypervisor
4728  *
4729  * do VF FLR and reinitialize Asic
4730  * return 0 means succeeded otherwise failed
4731  */
4732 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4733                                      bool from_hypervisor)
4734 {
4735         int r;
4736         struct amdgpu_hive_info *hive = NULL;
4737         int retry_limit = 0;
4738
4739 retry:
4740         amdgpu_amdkfd_pre_reset(adev);
4741
4742         if (from_hypervisor)
4743                 r = amdgpu_virt_request_full_gpu(adev, true);
4744         else
4745                 r = amdgpu_virt_reset_gpu(adev);
4746         if (r)
4747                 return r;
4748
4749         /* Resume IP prior to SMC */
4750         r = amdgpu_device_ip_reinit_early_sriov(adev);
4751         if (r)
4752                 goto error;
4753
4754         amdgpu_virt_init_data_exchange(adev);
4755
4756         r = amdgpu_device_fw_loading(adev);
4757         if (r)
4758                 return r;
4759
4760         /* now we are okay to resume SMC/CP/SDMA */
4761         r = amdgpu_device_ip_reinit_late_sriov(adev);
4762         if (r)
4763                 goto error;
4764
4765         hive = amdgpu_get_xgmi_hive(adev);
4766         /* Update PSP FW topology after reset */
4767         if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
4768                 r = amdgpu_xgmi_update_topology(hive, adev);
4769
4770         if (hive)
4771                 amdgpu_put_xgmi_hive(hive);
4772
4773         if (!r) {
4774                 amdgpu_irq_gpu_reset_resume_helper(adev);
4775                 r = amdgpu_ib_ring_tests(adev);
4776
4777                 amdgpu_amdkfd_post_reset(adev);
4778         }
4779
4780 error:
4781         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4782                 amdgpu_inc_vram_lost(adev);
4783                 r = amdgpu_device_recover_vram(adev);
4784         }
4785         amdgpu_virt_release_full_gpu(adev, true);
4786
4787         if (AMDGPU_RETRY_SRIOV_RESET(r)) {
4788                 if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) {
4789                         retry_limit++;
4790                         goto retry;
4791                 } else
4792                         DRM_ERROR("GPU reset retry is beyond the retry limit\n");
4793         }
4794
4795         return r;
4796 }
4797
4798 /**
4799  * amdgpu_device_has_job_running - check if there is any job in mirror list
4800  *
4801  * @adev: amdgpu_device pointer
4802  *
4803  * check if there is any job in mirror list
4804  */
4805 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4806 {
4807         int i;
4808         struct drm_sched_job *job;
4809
4810         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4811                 struct amdgpu_ring *ring = adev->rings[i];
4812
4813                 if (!ring || !ring->sched.thread)
4814                         continue;
4815
4816                 spin_lock(&ring->sched.job_list_lock);
4817                 job = list_first_entry_or_null(&ring->sched.pending_list,
4818                                                struct drm_sched_job, list);
4819                 spin_unlock(&ring->sched.job_list_lock);
4820                 if (job)
4821                         return true;
4822         }
4823         return false;
4824 }
4825
4826 /**
4827  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4828  *
4829  * @adev: amdgpu_device pointer
4830  *
4831  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4832  * a hung GPU.
4833  */
4834 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4835 {
4836
4837         if (amdgpu_gpu_recovery == 0)
4838                 goto disabled;
4839
4840         /* Skip soft reset check in fatal error mode */
4841         if (!amdgpu_ras_is_poison_mode_supported(adev))
4842                 return true;
4843
4844         if (amdgpu_sriov_vf(adev))
4845                 return true;
4846
4847         if (amdgpu_gpu_recovery == -1) {
4848                 switch (adev->asic_type) {
4849 #ifdef CONFIG_DRM_AMDGPU_SI
4850                 case CHIP_VERDE:
4851                 case CHIP_TAHITI:
4852                 case CHIP_PITCAIRN:
4853                 case CHIP_OLAND:
4854                 case CHIP_HAINAN:
4855 #endif
4856 #ifdef CONFIG_DRM_AMDGPU_CIK
4857                 case CHIP_KAVERI:
4858                 case CHIP_KABINI:
4859                 case CHIP_MULLINS:
4860 #endif
4861                 case CHIP_CARRIZO:
4862                 case CHIP_STONEY:
4863                 case CHIP_CYAN_SKILLFISH:
4864                         goto disabled;
4865                 default:
4866                         break;
4867                 }
4868         }
4869
4870         return true;
4871
4872 disabled:
4873                 dev_info(adev->dev, "GPU recovery disabled.\n");
4874                 return false;
4875 }
4876
4877 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4878 {
4879         u32 i;
4880         int ret = 0;
4881
4882         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4883
4884         dev_info(adev->dev, "GPU mode1 reset\n");
4885
4886         /* disable BM */
4887         pci_clear_master(adev->pdev);
4888
4889         amdgpu_device_cache_pci_state(adev->pdev);
4890
4891         if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4892                 dev_info(adev->dev, "GPU smu mode1 reset\n");
4893                 ret = amdgpu_dpm_mode1_reset(adev);
4894         } else {
4895                 dev_info(adev->dev, "GPU psp mode1 reset\n");
4896                 ret = psp_gpu_reset(adev);
4897         }
4898
4899         if (ret)
4900                 dev_err(adev->dev, "GPU mode1 reset failed\n");
4901
4902         amdgpu_device_load_pci_state(adev->pdev);
4903
4904         /* wait for asic to come out of reset */
4905         for (i = 0; i < adev->usec_timeout; i++) {
4906                 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4907
4908                 if (memsize != 0xffffffff)
4909                         break;
4910                 udelay(1);
4911         }
4912
4913         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4914         return ret;
4915 }
4916
4917 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4918                                  struct amdgpu_reset_context *reset_context)
4919 {
4920         int i, r = 0;
4921         struct amdgpu_job *job = NULL;
4922         bool need_full_reset =
4923                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4924
4925         if (reset_context->reset_req_dev == adev)
4926                 job = reset_context->job;
4927
4928         if (amdgpu_sriov_vf(adev)) {
4929                 /* stop the data exchange thread */
4930                 amdgpu_virt_fini_data_exchange(adev);
4931         }
4932
4933         amdgpu_fence_driver_isr_toggle(adev, true);
4934
4935         /* block all schedulers and reset given job's ring */
4936         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4937                 struct amdgpu_ring *ring = adev->rings[i];
4938
4939                 if (!ring || !ring->sched.thread)
4940                         continue;
4941
4942                 /*clear job fence from fence drv to avoid force_completion
4943                  *leave NULL and vm flush fence in fence drv */
4944                 amdgpu_fence_driver_clear_job_fences(ring);
4945
4946                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
4947                 amdgpu_fence_driver_force_completion(ring);
4948         }
4949
4950         amdgpu_fence_driver_isr_toggle(adev, false);
4951
4952         if (job && job->vm)
4953                 drm_sched_increase_karma(&job->base);
4954
4955         r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
4956         /* If reset handler not implemented, continue; otherwise return */
4957         if (r == -ENOSYS)
4958                 r = 0;
4959         else
4960                 return r;
4961
4962         /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
4963         if (!amdgpu_sriov_vf(adev)) {
4964
4965                 if (!need_full_reset)
4966                         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
4967
4968                 if (!need_full_reset && amdgpu_gpu_recovery &&
4969                     amdgpu_device_ip_check_soft_reset(adev)) {
4970                         amdgpu_device_ip_pre_soft_reset(adev);
4971                         r = amdgpu_device_ip_soft_reset(adev);
4972                         amdgpu_device_ip_post_soft_reset(adev);
4973                         if (r || amdgpu_device_ip_check_soft_reset(adev)) {
4974                                 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
4975                                 need_full_reset = true;
4976                         }
4977                 }
4978
4979                 if (need_full_reset)
4980                         r = amdgpu_device_ip_suspend(adev);
4981                 if (need_full_reset)
4982                         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4983                 else
4984                         clear_bit(AMDGPU_NEED_FULL_RESET,
4985                                   &reset_context->flags);
4986         }
4987
4988         return r;
4989 }
4990
4991 static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev)
4992 {
4993         int i;
4994
4995         lockdep_assert_held(&adev->reset_domain->sem);
4996
4997         for (i = 0; i < adev->num_regs; i++) {
4998                 adev->reset_dump_reg_value[i] = RREG32(adev->reset_dump_reg_list[i]);
4999                 trace_amdgpu_reset_reg_dumps(adev->reset_dump_reg_list[i],
5000                                              adev->reset_dump_reg_value[i]);
5001         }
5002
5003         return 0;
5004 }
5005
5006 #ifdef CONFIG_DEV_COREDUMP
5007 static ssize_t amdgpu_devcoredump_read(char *buffer, loff_t offset,
5008                 size_t count, void *data, size_t datalen)
5009 {
5010         struct drm_printer p;
5011         struct amdgpu_device *adev = data;
5012         struct drm_print_iterator iter;
5013         int i;
5014
5015         iter.data = buffer;
5016         iter.offset = 0;
5017         iter.start = offset;
5018         iter.remain = count;
5019
5020         p = drm_coredump_printer(&iter);
5021
5022         drm_printf(&p, "**** AMDGPU Device Coredump ****\n");
5023         drm_printf(&p, "kernel: " UTS_RELEASE "\n");
5024         drm_printf(&p, "module: " KBUILD_MODNAME "\n");
5025         drm_printf(&p, "time: %lld.%09ld\n", adev->reset_time.tv_sec, adev->reset_time.tv_nsec);
5026         if (adev->reset_task_info.pid)
5027                 drm_printf(&p, "process_name: %s PID: %d\n",
5028                            adev->reset_task_info.process_name,
5029                            adev->reset_task_info.pid);
5030
5031         if (adev->reset_vram_lost)
5032                 drm_printf(&p, "VRAM is lost due to GPU reset!\n");
5033         if (adev->num_regs) {
5034                 drm_printf(&p, "AMDGPU register dumps:\nOffset:     Value:\n");
5035
5036                 for (i = 0; i < adev->num_regs; i++)
5037                         drm_printf(&p, "0x%08x: 0x%08x\n",
5038                                    adev->reset_dump_reg_list[i],
5039                                    adev->reset_dump_reg_value[i]);
5040         }
5041
5042         return count - iter.remain;
5043 }
5044
5045 static void amdgpu_devcoredump_free(void *data)
5046 {
5047 }
5048
5049 static void amdgpu_reset_capture_coredumpm(struct amdgpu_device *adev)
5050 {
5051         struct drm_device *dev = adev_to_drm(adev);
5052
5053         ktime_get_ts64(&adev->reset_time);
5054         dev_coredumpm(dev->dev, THIS_MODULE, adev, 0, GFP_KERNEL,
5055                       amdgpu_devcoredump_read, amdgpu_devcoredump_free);
5056 }
5057 #endif
5058
5059 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
5060                          struct amdgpu_reset_context *reset_context)
5061 {
5062         struct amdgpu_device *tmp_adev = NULL;
5063         bool need_full_reset, skip_hw_reset, vram_lost = false;
5064         int r = 0;
5065         bool gpu_reset_for_dev_remove = 0;
5066
5067         /* Try reset handler method first */
5068         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5069                                     reset_list);
5070         amdgpu_reset_reg_dumps(tmp_adev);
5071
5072         reset_context->reset_device_list = device_list_handle;
5073         r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
5074         /* If reset handler not implemented, continue; otherwise return */
5075         if (r == -ENOSYS)
5076                 r = 0;
5077         else
5078                 return r;
5079
5080         /* Reset handler not implemented, use the default method */
5081         need_full_reset =
5082                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5083         skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
5084
5085         gpu_reset_for_dev_remove =
5086                 test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5087                         test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5088
5089         /*
5090          * ASIC reset has to be done on all XGMI hive nodes ASAP
5091          * to allow proper links negotiation in FW (within 1 sec)
5092          */
5093         if (!skip_hw_reset && need_full_reset) {
5094                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5095                         /* For XGMI run all resets in parallel to speed up the process */
5096                         if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5097                                 tmp_adev->gmc.xgmi.pending_reset = false;
5098                                 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
5099                                         r = -EALREADY;
5100                         } else
5101                                 r = amdgpu_asic_reset(tmp_adev);
5102
5103                         if (r) {
5104                                 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
5105                                          r, adev_to_drm(tmp_adev)->unique);
5106                                 break;
5107                         }
5108                 }
5109
5110                 /* For XGMI wait for all resets to complete before proceed */
5111                 if (!r) {
5112                         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5113                                 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5114                                         flush_work(&tmp_adev->xgmi_reset_work);
5115                                         r = tmp_adev->asic_reset_res;
5116                                         if (r)
5117                                                 break;
5118                                 }
5119                         }
5120                 }
5121         }
5122
5123         if (!r && amdgpu_ras_intr_triggered()) {
5124                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5125                         if (tmp_adev->mmhub.ras && tmp_adev->mmhub.ras->ras_block.hw_ops &&
5126                             tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
5127                                 tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(tmp_adev);
5128                 }
5129
5130                 amdgpu_ras_intr_cleared();
5131         }
5132
5133         /* Since the mode1 reset affects base ip blocks, the
5134          * phase1 ip blocks need to be resumed. Otherwise there
5135          * will be a BIOS signature error and the psp bootloader
5136          * can't load kdb on the next amdgpu install.
5137          */
5138         if (gpu_reset_for_dev_remove) {
5139                 list_for_each_entry(tmp_adev, device_list_handle, reset_list)
5140                         amdgpu_device_ip_resume_phase1(tmp_adev);
5141
5142                 goto end;
5143         }
5144
5145         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5146                 if (need_full_reset) {
5147                         /* post card */
5148                         r = amdgpu_device_asic_init(tmp_adev);
5149                         if (r) {
5150                                 dev_warn(tmp_adev->dev, "asic atom init failed!");
5151                         } else {
5152                                 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
5153                                 r = amdgpu_amdkfd_resume_iommu(tmp_adev);
5154                                 if (r)
5155                                         goto out;
5156
5157                                 r = amdgpu_device_ip_resume_phase1(tmp_adev);
5158                                 if (r)
5159                                         goto out;
5160
5161                                 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
5162 #ifdef CONFIG_DEV_COREDUMP
5163                                 tmp_adev->reset_vram_lost = vram_lost;
5164                                 memset(&tmp_adev->reset_task_info, 0,
5165                                                 sizeof(tmp_adev->reset_task_info));
5166                                 if (reset_context->job && reset_context->job->vm)
5167                                         tmp_adev->reset_task_info =
5168                                                 reset_context->job->vm->task_info;
5169                                 amdgpu_reset_capture_coredumpm(tmp_adev);
5170 #endif
5171                                 if (vram_lost) {
5172                                         DRM_INFO("VRAM is lost due to GPU reset!\n");
5173                                         amdgpu_inc_vram_lost(tmp_adev);
5174                                 }
5175
5176                                 r = amdgpu_device_fw_loading(tmp_adev);
5177                                 if (r)
5178                                         return r;
5179
5180                                 r = amdgpu_device_ip_resume_phase2(tmp_adev);
5181                                 if (r)
5182                                         goto out;
5183
5184                                 if (vram_lost)
5185                                         amdgpu_device_fill_reset_magic(tmp_adev);
5186
5187                                 /*
5188                                  * Add this ASIC as tracked as reset was already
5189                                  * complete successfully.
5190                                  */
5191                                 amdgpu_register_gpu_instance(tmp_adev);
5192
5193                                 if (!reset_context->hive &&
5194                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5195                                         amdgpu_xgmi_add_device(tmp_adev);
5196
5197                                 r = amdgpu_device_ip_late_init(tmp_adev);
5198                                 if (r)
5199                                         goto out;
5200
5201                                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
5202
5203                                 /*
5204                                  * The GPU enters bad state once faulty pages
5205                                  * by ECC has reached the threshold, and ras
5206                                  * recovery is scheduled next. So add one check
5207                                  * here to break recovery if it indeed exceeds
5208                                  * bad page threshold, and remind user to
5209                                  * retire this GPU or setting one bigger
5210                                  * bad_page_threshold value to fix this once
5211                                  * probing driver again.
5212                                  */
5213                                 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
5214                                         /* must succeed. */
5215                                         amdgpu_ras_resume(tmp_adev);
5216                                 } else {
5217                                         r = -EINVAL;
5218                                         goto out;
5219                                 }
5220
5221                                 /* Update PSP FW topology after reset */
5222                                 if (reset_context->hive &&
5223                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5224                                         r = amdgpu_xgmi_update_topology(
5225                                                 reset_context->hive, tmp_adev);
5226                         }
5227                 }
5228
5229 out:
5230                 if (!r) {
5231                         amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5232                         r = amdgpu_ib_ring_tests(tmp_adev);
5233                         if (r) {
5234                                 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5235                                 need_full_reset = true;
5236                                 r = -EAGAIN;
5237                                 goto end;
5238                         }
5239                 }
5240
5241                 if (!r)
5242                         r = amdgpu_device_recover_vram(tmp_adev);
5243                 else
5244                         tmp_adev->asic_reset_res = r;
5245         }
5246
5247 end:
5248         if (need_full_reset)
5249                 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5250         else
5251                 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5252         return r;
5253 }
5254
5255 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5256 {
5257
5258         switch (amdgpu_asic_reset_method(adev)) {
5259         case AMD_RESET_METHOD_MODE1:
5260                 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5261                 break;
5262         case AMD_RESET_METHOD_MODE2:
5263                 adev->mp1_state = PP_MP1_STATE_RESET;
5264                 break;
5265         default:
5266                 adev->mp1_state = PP_MP1_STATE_NONE;
5267                 break;
5268         }
5269 }
5270
5271 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5272 {
5273         amdgpu_vf_error_trans_all(adev);
5274         adev->mp1_state = PP_MP1_STATE_NONE;
5275 }
5276
5277 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5278 {
5279         struct pci_dev *p = NULL;
5280
5281         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5282                         adev->pdev->bus->number, 1);
5283         if (p) {
5284                 pm_runtime_enable(&(p->dev));
5285                 pm_runtime_resume(&(p->dev));
5286         }
5287
5288         pci_dev_put(p);
5289 }
5290
5291 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5292 {
5293         enum amd_reset_method reset_method;
5294         struct pci_dev *p = NULL;
5295         u64 expires;
5296
5297         /*
5298          * For now, only BACO and mode1 reset are confirmed
5299          * to suffer the audio issue without proper suspended.
5300          */
5301         reset_method = amdgpu_asic_reset_method(adev);
5302         if ((reset_method != AMD_RESET_METHOD_BACO) &&
5303              (reset_method != AMD_RESET_METHOD_MODE1))
5304                 return -EINVAL;
5305
5306         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5307                         adev->pdev->bus->number, 1);
5308         if (!p)
5309                 return -ENODEV;
5310
5311         expires = pm_runtime_autosuspend_expiration(&(p->dev));
5312         if (!expires)
5313                 /*
5314                  * If we cannot get the audio device autosuspend delay,
5315                  * a fixed 4S interval will be used. Considering 3S is
5316                  * the audio controller default autosuspend delay setting.
5317                  * 4S used here is guaranteed to cover that.
5318                  */
5319                 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5320
5321         while (!pm_runtime_status_suspended(&(p->dev))) {
5322                 if (!pm_runtime_suspend(&(p->dev)))
5323                         break;
5324
5325                 if (expires < ktime_get_mono_fast_ns()) {
5326                         dev_warn(adev->dev, "failed to suspend display audio\n");
5327                         pci_dev_put(p);
5328                         /* TODO: abort the succeeding gpu reset? */
5329                         return -ETIMEDOUT;
5330                 }
5331         }
5332
5333         pm_runtime_disable(&(p->dev));
5334
5335         pci_dev_put(p);
5336         return 0;
5337 }
5338
5339 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5340 {
5341         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5342
5343 #if defined(CONFIG_DEBUG_FS)
5344         if (!amdgpu_sriov_vf(adev))
5345                 cancel_work(&adev->reset_work);
5346 #endif
5347
5348         if (adev->kfd.dev)
5349                 cancel_work(&adev->kfd.reset_work);
5350
5351         if (amdgpu_sriov_vf(adev))
5352                 cancel_work(&adev->virt.flr_work);
5353
5354         if (con && adev->ras_enabled)
5355                 cancel_work(&con->recovery_work);
5356
5357 }
5358
5359 /**
5360  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5361  *
5362  * @adev: amdgpu_device pointer
5363  * @job: which job trigger hang
5364  * @reset_context: amdgpu reset context pointer
5365  *
5366  * Attempt to reset the GPU if it has hung (all asics).
5367  * Attempt to do soft-reset or full-reset and reinitialize Asic
5368  * Returns 0 for success or an error on failure.
5369  */
5370
5371 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5372                               struct amdgpu_job *job,
5373                               struct amdgpu_reset_context *reset_context)
5374 {
5375         struct list_head device_list, *device_list_handle =  NULL;
5376         bool job_signaled = false;
5377         struct amdgpu_hive_info *hive = NULL;
5378         struct amdgpu_device *tmp_adev = NULL;
5379         int i, r = 0;
5380         bool need_emergency_restart = false;
5381         bool audio_suspended = false;
5382         bool gpu_reset_for_dev_remove = false;
5383
5384         gpu_reset_for_dev_remove =
5385                         test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5386                                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5387
5388         /*
5389          * Special case: RAS triggered and full reset isn't supported
5390          */
5391         need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5392
5393         /*
5394          * Flush RAM to disk so that after reboot
5395          * the user can read log and see why the system rebooted.
5396          */
5397         if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {
5398                 DRM_WARN("Emergency reboot.");
5399
5400                 ksys_sync_helper();
5401                 emergency_restart();
5402         }
5403
5404         dev_info(adev->dev, "GPU %s begin!\n",
5405                 need_emergency_restart ? "jobs stop":"reset");
5406
5407         if (!amdgpu_sriov_vf(adev))
5408                 hive = amdgpu_get_xgmi_hive(adev);
5409         if (hive)
5410                 mutex_lock(&hive->hive_lock);
5411
5412         reset_context->job = job;
5413         reset_context->hive = hive;
5414         /*
5415          * Build list of devices to reset.
5416          * In case we are in XGMI hive mode, resort the device list
5417          * to put adev in the 1st position.
5418          */
5419         INIT_LIST_HEAD(&device_list);
5420         if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5421                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5422                         list_add_tail(&tmp_adev->reset_list, &device_list);
5423                         if (gpu_reset_for_dev_remove && adev->shutdown)
5424                                 tmp_adev->shutdown = true;
5425                 }
5426                 if (!list_is_first(&adev->reset_list, &device_list))
5427                         list_rotate_to_front(&adev->reset_list, &device_list);
5428                 device_list_handle = &device_list;
5429         } else {
5430                 list_add_tail(&adev->reset_list, &device_list);
5431                 device_list_handle = &device_list;
5432         }
5433
5434         /* We need to lock reset domain only once both for XGMI and single device */
5435         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5436                                     reset_list);
5437         amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5438
5439         /* block all schedulers and reset given job's ring */
5440         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5441
5442                 amdgpu_device_set_mp1_state(tmp_adev);
5443
5444                 /*
5445                  * Try to put the audio codec into suspend state
5446                  * before gpu reset started.
5447                  *
5448                  * Due to the power domain of the graphics device
5449                  * is shared with AZ power domain. Without this,
5450                  * we may change the audio hardware from behind
5451                  * the audio driver's back. That will trigger
5452                  * some audio codec errors.
5453                  */
5454                 if (!amdgpu_device_suspend_display_audio(tmp_adev))
5455                         audio_suspended = true;
5456
5457                 amdgpu_ras_set_error_query_ready(tmp_adev, false);
5458
5459                 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5460
5461                 if (!amdgpu_sriov_vf(tmp_adev))
5462                         amdgpu_amdkfd_pre_reset(tmp_adev);
5463
5464                 /*
5465                  * Mark these ASICs to be reseted as untracked first
5466                  * And add them back after reset completed
5467                  */
5468                 amdgpu_unregister_gpu_instance(tmp_adev);
5469
5470                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true);
5471
5472                 /* disable ras on ALL IPs */
5473                 if (!need_emergency_restart &&
5474                       amdgpu_device_ip_need_full_reset(tmp_adev))
5475                         amdgpu_ras_suspend(tmp_adev);
5476
5477                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5478                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5479
5480                         if (!ring || !ring->sched.thread)
5481                                 continue;
5482
5483                         drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5484
5485                         if (need_emergency_restart)
5486                                 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5487                 }
5488                 atomic_inc(&tmp_adev->gpu_reset_counter);
5489         }
5490
5491         if (need_emergency_restart)
5492                 goto skip_sched_resume;
5493
5494         /*
5495          * Must check guilty signal here since after this point all old
5496          * HW fences are force signaled.
5497          *
5498          * job->base holds a reference to parent fence
5499          */
5500         if (job && dma_fence_is_signaled(&job->hw_fence)) {
5501                 job_signaled = true;
5502                 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5503                 goto skip_hw_reset;
5504         }
5505
5506 retry:  /* Rest of adevs pre asic reset from XGMI hive. */
5507         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5508                 if (gpu_reset_for_dev_remove) {
5509                         /* Workaroud for ASICs need to disable SMC first */
5510                         amdgpu_device_smu_fini_early(tmp_adev);
5511                 }
5512                 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5513                 /*TODO Should we stop ?*/
5514                 if (r) {
5515                         dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5516                                   r, adev_to_drm(tmp_adev)->unique);
5517                         tmp_adev->asic_reset_res = r;
5518                 }
5519
5520                 /*
5521                  * Drop all pending non scheduler resets. Scheduler resets
5522                  * were already dropped during drm_sched_stop
5523                  */
5524                 amdgpu_device_stop_pending_resets(tmp_adev);
5525         }
5526
5527         /* Actual ASIC resets if needed.*/
5528         /* Host driver will handle XGMI hive reset for SRIOV */
5529         if (amdgpu_sriov_vf(adev)) {
5530                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
5531                 if (r)
5532                         adev->asic_reset_res = r;
5533
5534                 /* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
5535                 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
5536                     adev->ip_versions[GC_HWIP][0] == IP_VERSION(11, 0, 3))
5537                         amdgpu_ras_resume(adev);
5538         } else {
5539                 r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5540                 if (r && r == -EAGAIN)
5541                         goto retry;
5542
5543                 if (!r && gpu_reset_for_dev_remove)
5544                         goto recover_end;
5545         }
5546
5547 skip_hw_reset:
5548
5549         /* Post ASIC reset for all devs .*/
5550         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5551
5552                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5553                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5554
5555                         if (!ring || !ring->sched.thread)
5556                                 continue;
5557
5558                         drm_sched_start(&ring->sched, true);
5559                 }
5560
5561                 if (adev->enable_mes && adev->ip_versions[GC_HWIP][0] != IP_VERSION(11, 0, 3))
5562                         amdgpu_mes_self_test(tmp_adev);
5563
5564                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) {
5565                         drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5566                 }
5567
5568                 if (tmp_adev->asic_reset_res)
5569                         r = tmp_adev->asic_reset_res;
5570
5571                 tmp_adev->asic_reset_res = 0;
5572
5573                 if (r) {
5574                         /* bad news, how to tell it to userspace ? */
5575                         dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5576                         amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5577                 } else {
5578                         dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5579                         if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5580                                 DRM_WARN("smart shift update failed\n");
5581                 }
5582         }
5583
5584 skip_sched_resume:
5585         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5586                 /* unlock kfd: SRIOV would do it separately */
5587                 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5588                         amdgpu_amdkfd_post_reset(tmp_adev);
5589
5590                 /* kfd_post_reset will do nothing if kfd device is not initialized,
5591                  * need to bring up kfd here if it's not be initialized before
5592                  */
5593                 if (!adev->kfd.init_complete)
5594                         amdgpu_amdkfd_device_init(adev);
5595
5596                 if (audio_suspended)
5597                         amdgpu_device_resume_display_audio(tmp_adev);
5598
5599                 amdgpu_device_unset_mp1_state(tmp_adev);
5600
5601                 amdgpu_ras_set_error_query_ready(tmp_adev, true);
5602         }
5603
5604 recover_end:
5605         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5606                                             reset_list);
5607         amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
5608
5609         if (hive) {
5610                 mutex_unlock(&hive->hive_lock);
5611                 amdgpu_put_xgmi_hive(hive);
5612         }
5613
5614         if (r)
5615                 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5616
5617         atomic_set(&adev->reset_domain->reset_res, r);
5618         return r;
5619 }
5620
5621 /**
5622  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5623  *
5624  * @adev: amdgpu_device pointer
5625  *
5626  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5627  * and lanes) of the slot the device is in. Handles APUs and
5628  * virtualized environments where PCIE config space may not be available.
5629  */
5630 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5631 {
5632         struct pci_dev *pdev;
5633         enum pci_bus_speed speed_cap, platform_speed_cap;
5634         enum pcie_link_width platform_link_width;
5635
5636         if (amdgpu_pcie_gen_cap)
5637                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5638
5639         if (amdgpu_pcie_lane_cap)
5640                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5641
5642         /* covers APUs as well */
5643         if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) {
5644                 if (adev->pm.pcie_gen_mask == 0)
5645                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5646                 if (adev->pm.pcie_mlw_mask == 0)
5647                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5648                 return;
5649         }
5650
5651         if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5652                 return;
5653
5654         pcie_bandwidth_available(adev->pdev, NULL,
5655                                  &platform_speed_cap, &platform_link_width);
5656
5657         if (adev->pm.pcie_gen_mask == 0) {
5658                 /* asic caps */
5659                 pdev = adev->pdev;
5660                 speed_cap = pcie_get_speed_cap(pdev);
5661                 if (speed_cap == PCI_SPEED_UNKNOWN) {
5662                         adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5663                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5664                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5665                 } else {
5666                         if (speed_cap == PCIE_SPEED_32_0GT)
5667                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5668                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5669                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5670                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5671                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5672                         else if (speed_cap == PCIE_SPEED_16_0GT)
5673                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5674                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5675                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5676                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5677                         else if (speed_cap == PCIE_SPEED_8_0GT)
5678                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5679                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5680                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5681                         else if (speed_cap == PCIE_SPEED_5_0GT)
5682                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5683                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5684                         else
5685                                 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5686                 }
5687                 /* platform caps */
5688                 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5689                         adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5690                                                    CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5691                 } else {
5692                         if (platform_speed_cap == PCIE_SPEED_32_0GT)
5693                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5694                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5695                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5696                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5697                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5698                         else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5699                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5700                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5701                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5702                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5703                         else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5704                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5705                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5706                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5707                         else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5708                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5709                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5710                         else
5711                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5712
5713                 }
5714         }
5715         if (adev->pm.pcie_mlw_mask == 0) {
5716                 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5717                         adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5718                 } else {
5719                         switch (platform_link_width) {
5720                         case PCIE_LNK_X32:
5721                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5722                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5723                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5724                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5725                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5726                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5727                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5728                                 break;
5729                         case PCIE_LNK_X16:
5730                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5731                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5732                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5733                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5734                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5735                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5736                                 break;
5737                         case PCIE_LNK_X12:
5738                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5739                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5740                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5741                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5742                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5743                                 break;
5744                         case PCIE_LNK_X8:
5745                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5746                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5747                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5748                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5749                                 break;
5750                         case PCIE_LNK_X4:
5751                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5752                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5753                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5754                                 break;
5755                         case PCIE_LNK_X2:
5756                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5757                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5758                                 break;
5759                         case PCIE_LNK_X1:
5760                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5761                                 break;
5762                         default:
5763                                 break;
5764                         }
5765                 }
5766         }
5767 }
5768
5769 /**
5770  * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
5771  *
5772  * @adev: amdgpu_device pointer
5773  * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
5774  *
5775  * Return true if @peer_adev can access (DMA) @adev through the PCIe
5776  * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
5777  * @peer_adev.
5778  */
5779 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
5780                                       struct amdgpu_device *peer_adev)
5781 {
5782 #ifdef CONFIG_HSA_AMD_P2P
5783         uint64_t address_mask = peer_adev->dev->dma_mask ?
5784                 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
5785         resource_size_t aper_limit =
5786                 adev->gmc.aper_base + adev->gmc.aper_size - 1;
5787         bool p2p_access =
5788                 !adev->gmc.xgmi.connected_to_cpu &&
5789                 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
5790
5791         return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
5792                 adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
5793                 !(adev->gmc.aper_base & address_mask ||
5794                   aper_limit & address_mask));
5795 #else
5796         return false;
5797 #endif
5798 }
5799
5800 int amdgpu_device_baco_enter(struct drm_device *dev)
5801 {
5802         struct amdgpu_device *adev = drm_to_adev(dev);
5803         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5804
5805         if (!amdgpu_device_supports_baco(dev))
5806                 return -ENOTSUPP;
5807
5808         if (ras && adev->ras_enabled &&
5809             adev->nbio.funcs->enable_doorbell_interrupt)
5810                 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5811
5812         return amdgpu_dpm_baco_enter(adev);
5813 }
5814
5815 int amdgpu_device_baco_exit(struct drm_device *dev)
5816 {
5817         struct amdgpu_device *adev = drm_to_adev(dev);
5818         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5819         int ret = 0;
5820
5821         if (!amdgpu_device_supports_baco(dev))
5822                 return -ENOTSUPP;
5823
5824         ret = amdgpu_dpm_baco_exit(adev);
5825         if (ret)
5826                 return ret;
5827
5828         if (ras && adev->ras_enabled &&
5829             adev->nbio.funcs->enable_doorbell_interrupt)
5830                 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5831
5832         if (amdgpu_passthrough(adev) &&
5833             adev->nbio.funcs->clear_doorbell_interrupt)
5834                 adev->nbio.funcs->clear_doorbell_interrupt(adev);
5835
5836         return 0;
5837 }
5838
5839 /**
5840  * amdgpu_pci_error_detected - Called when a PCI error is detected.
5841  * @pdev: PCI device struct
5842  * @state: PCI channel state
5843  *
5844  * Description: Called when a PCI error is detected.
5845  *
5846  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5847  */
5848 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5849 {
5850         struct drm_device *dev = pci_get_drvdata(pdev);
5851         struct amdgpu_device *adev = drm_to_adev(dev);
5852         int i;
5853
5854         DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5855
5856         if (adev->gmc.xgmi.num_physical_nodes > 1) {
5857                 DRM_WARN("No support for XGMI hive yet...");
5858                 return PCI_ERS_RESULT_DISCONNECT;
5859         }
5860
5861         adev->pci_channel_state = state;
5862
5863         switch (state) {
5864         case pci_channel_io_normal:
5865                 return PCI_ERS_RESULT_CAN_RECOVER;
5866         /* Fatal error, prepare for slot reset */
5867         case pci_channel_io_frozen:
5868                 /*
5869                  * Locking adev->reset_domain->sem will prevent any external access
5870                  * to GPU during PCI error recovery
5871                  */
5872                 amdgpu_device_lock_reset_domain(adev->reset_domain);
5873                 amdgpu_device_set_mp1_state(adev);
5874
5875                 /*
5876                  * Block any work scheduling as we do for regular GPU reset
5877                  * for the duration of the recovery
5878                  */
5879                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5880                         struct amdgpu_ring *ring = adev->rings[i];
5881
5882                         if (!ring || !ring->sched.thread)
5883                                 continue;
5884
5885                         drm_sched_stop(&ring->sched, NULL);
5886                 }
5887                 atomic_inc(&adev->gpu_reset_counter);
5888                 return PCI_ERS_RESULT_NEED_RESET;
5889         case pci_channel_io_perm_failure:
5890                 /* Permanent error, prepare for device removal */
5891                 return PCI_ERS_RESULT_DISCONNECT;
5892         }
5893
5894         return PCI_ERS_RESULT_NEED_RESET;
5895 }
5896
5897 /**
5898  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5899  * @pdev: pointer to PCI device
5900  */
5901 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5902 {
5903
5904         DRM_INFO("PCI error: mmio enabled callback!!\n");
5905
5906         /* TODO - dump whatever for debugging purposes */
5907
5908         /* This called only if amdgpu_pci_error_detected returns
5909          * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5910          * works, no need to reset slot.
5911          */
5912
5913         return PCI_ERS_RESULT_RECOVERED;
5914 }
5915
5916 /**
5917  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5918  * @pdev: PCI device struct
5919  *
5920  * Description: This routine is called by the pci error recovery
5921  * code after the PCI slot has been reset, just before we
5922  * should resume normal operations.
5923  */
5924 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5925 {
5926         struct drm_device *dev = pci_get_drvdata(pdev);
5927         struct amdgpu_device *adev = drm_to_adev(dev);
5928         int r, i;
5929         struct amdgpu_reset_context reset_context;
5930         u32 memsize;
5931         struct list_head device_list;
5932
5933         DRM_INFO("PCI error: slot reset callback!!\n");
5934
5935         memset(&reset_context, 0, sizeof(reset_context));
5936
5937         INIT_LIST_HEAD(&device_list);
5938         list_add_tail(&adev->reset_list, &device_list);
5939
5940         /* wait for asic to come out of reset */
5941         msleep(500);
5942
5943         /* Restore PCI confspace */
5944         amdgpu_device_load_pci_state(pdev);
5945
5946         /* confirm  ASIC came out of reset */
5947         for (i = 0; i < adev->usec_timeout; i++) {
5948                 memsize = amdgpu_asic_get_config_memsize(adev);
5949
5950                 if (memsize != 0xffffffff)
5951                         break;
5952                 udelay(1);
5953         }
5954         if (memsize == 0xffffffff) {
5955                 r = -ETIME;
5956                 goto out;
5957         }
5958
5959         reset_context.method = AMD_RESET_METHOD_NONE;
5960         reset_context.reset_req_dev = adev;
5961         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5962         set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
5963
5964         adev->no_hw_access = true;
5965         r = amdgpu_device_pre_asic_reset(adev, &reset_context);
5966         adev->no_hw_access = false;
5967         if (r)
5968                 goto out;
5969
5970         r = amdgpu_do_asic_reset(&device_list, &reset_context);
5971
5972 out:
5973         if (!r) {
5974                 if (amdgpu_device_cache_pci_state(adev->pdev))
5975                         pci_restore_state(adev->pdev);
5976
5977                 DRM_INFO("PCIe error recovery succeeded\n");
5978         } else {
5979                 DRM_ERROR("PCIe error recovery failed, err:%d", r);
5980                 amdgpu_device_unset_mp1_state(adev);
5981                 amdgpu_device_unlock_reset_domain(adev->reset_domain);
5982         }
5983
5984         return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
5985 }
5986
5987 /**
5988  * amdgpu_pci_resume() - resume normal ops after PCI reset
5989  * @pdev: pointer to PCI device
5990  *
5991  * Called when the error recovery driver tells us that its
5992  * OK to resume normal operation.
5993  */
5994 void amdgpu_pci_resume(struct pci_dev *pdev)
5995 {
5996         struct drm_device *dev = pci_get_drvdata(pdev);
5997         struct amdgpu_device *adev = drm_to_adev(dev);
5998         int i;
5999
6000
6001         DRM_INFO("PCI error: resume callback!!\n");
6002
6003         /* Only continue execution for the case of pci_channel_io_frozen */
6004         if (adev->pci_channel_state != pci_channel_io_frozen)
6005                 return;
6006
6007         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6008                 struct amdgpu_ring *ring = adev->rings[i];
6009
6010                 if (!ring || !ring->sched.thread)
6011                         continue;
6012
6013                 drm_sched_start(&ring->sched, true);
6014         }
6015
6016         amdgpu_device_unset_mp1_state(adev);
6017         amdgpu_device_unlock_reset_domain(adev->reset_domain);
6018 }
6019
6020 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
6021 {
6022         struct drm_device *dev = pci_get_drvdata(pdev);
6023         struct amdgpu_device *adev = drm_to_adev(dev);
6024         int r;
6025
6026         r = pci_save_state(pdev);
6027         if (!r) {
6028                 kfree(adev->pci_state);
6029
6030                 adev->pci_state = pci_store_saved_state(pdev);
6031
6032                 if (!adev->pci_state) {
6033                         DRM_ERROR("Failed to store PCI saved state");
6034                         return false;
6035                 }
6036         } else {
6037                 DRM_WARN("Failed to save PCI state, err:%d\n", r);
6038                 return false;
6039         }
6040
6041         return true;
6042 }
6043
6044 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
6045 {
6046         struct drm_device *dev = pci_get_drvdata(pdev);
6047         struct amdgpu_device *adev = drm_to_adev(dev);
6048         int r;
6049
6050         if (!adev->pci_state)
6051                 return false;
6052
6053         r = pci_load_saved_state(pdev, adev->pci_state);
6054
6055         if (!r) {
6056                 pci_restore_state(pdev);
6057         } else {
6058                 DRM_WARN("Failed to load PCI state, err:%d\n", r);
6059                 return false;
6060         }
6061
6062         return true;
6063 }
6064
6065 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
6066                 struct amdgpu_ring *ring)
6067 {
6068 #ifdef CONFIG_X86_64
6069         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6070                 return;
6071 #endif
6072         if (adev->gmc.xgmi.connected_to_cpu)
6073                 return;
6074
6075         if (ring && ring->funcs->emit_hdp_flush)
6076                 amdgpu_ring_emit_hdp_flush(ring);
6077         else
6078                 amdgpu_asic_flush_hdp(adev, ring);
6079 }
6080
6081 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
6082                 struct amdgpu_ring *ring)
6083 {
6084 #ifdef CONFIG_X86_64
6085         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6086                 return;
6087 #endif
6088         if (adev->gmc.xgmi.connected_to_cpu)
6089                 return;
6090
6091         amdgpu_asic_invalidate_hdp(adev, ring);
6092 }
6093
6094 int amdgpu_in_reset(struct amdgpu_device *adev)
6095 {
6096         return atomic_read(&adev->reset_domain->in_gpu_reset);
6097 }
6098
6099 /**
6100  * amdgpu_device_halt() - bring hardware to some kind of halt state
6101  *
6102  * @adev: amdgpu_device pointer
6103  *
6104  * Bring hardware to some kind of halt state so that no one can touch it
6105  * any more. It will help to maintain error context when error occurred.
6106  * Compare to a simple hang, the system will keep stable at least for SSH
6107  * access. Then it should be trivial to inspect the hardware state and
6108  * see what's going on. Implemented as following:
6109  *
6110  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
6111  *    clears all CPU mappings to device, disallows remappings through page faults
6112  * 2. amdgpu_irq_disable_all() disables all interrupts
6113  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
6114  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
6115  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
6116  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
6117  *    flush any in flight DMA operations
6118  */
6119 void amdgpu_device_halt(struct amdgpu_device *adev)
6120 {
6121         struct pci_dev *pdev = adev->pdev;
6122         struct drm_device *ddev = adev_to_drm(adev);
6123
6124         amdgpu_xcp_dev_unplug(adev);
6125         drm_dev_unplug(ddev);
6126
6127         amdgpu_irq_disable_all(adev);
6128
6129         amdgpu_fence_driver_hw_fini(adev);
6130
6131         adev->no_hw_access = true;
6132
6133         amdgpu_device_unmap_mmio(adev);
6134
6135         pci_disable_device(pdev);
6136         pci_wait_for_pending_transaction(pdev);
6137 }
6138
6139 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
6140                                 u32 reg)
6141 {
6142         unsigned long flags, address, data;
6143         u32 r;
6144
6145         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6146         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6147
6148         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6149         WREG32(address, reg * 4);
6150         (void)RREG32(address);
6151         r = RREG32(data);
6152         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6153         return r;
6154 }
6155
6156 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
6157                                 u32 reg, u32 v)
6158 {
6159         unsigned long flags, address, data;
6160
6161         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6162         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6163
6164         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6165         WREG32(address, reg * 4);
6166         (void)RREG32(address);
6167         WREG32(data, v);
6168         (void)RREG32(data);
6169         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6170 }
6171
6172 /**
6173  * amdgpu_device_switch_gang - switch to a new gang
6174  * @adev: amdgpu_device pointer
6175  * @gang: the gang to switch to
6176  *
6177  * Try to switch to a new gang.
6178  * Returns: NULL if we switched to the new gang or a reference to the current
6179  * gang leader.
6180  */
6181 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6182                                             struct dma_fence *gang)
6183 {
6184         struct dma_fence *old = NULL;
6185
6186         do {
6187                 dma_fence_put(old);
6188                 rcu_read_lock();
6189                 old = dma_fence_get_rcu_safe(&adev->gang_submit);
6190                 rcu_read_unlock();
6191
6192                 if (old == gang)
6193                         break;
6194
6195                 if (!dma_fence_is_signaled(old))
6196                         return old;
6197
6198         } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6199                          old, gang) != old);
6200
6201         dma_fence_put(old);
6202         return NULL;
6203 }
6204
6205 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6206 {
6207         switch (adev->asic_type) {
6208 #ifdef CONFIG_DRM_AMDGPU_SI
6209         case CHIP_HAINAN:
6210 #endif
6211         case CHIP_TOPAZ:
6212                 /* chips with no display hardware */
6213                 return false;
6214 #ifdef CONFIG_DRM_AMDGPU_SI
6215         case CHIP_TAHITI:
6216         case CHIP_PITCAIRN:
6217         case CHIP_VERDE:
6218         case CHIP_OLAND:
6219 #endif
6220 #ifdef CONFIG_DRM_AMDGPU_CIK
6221         case CHIP_BONAIRE:
6222         case CHIP_HAWAII:
6223         case CHIP_KAVERI:
6224         case CHIP_KABINI:
6225         case CHIP_MULLINS:
6226 #endif
6227         case CHIP_TONGA:
6228         case CHIP_FIJI:
6229         case CHIP_POLARIS10:
6230         case CHIP_POLARIS11:
6231         case CHIP_POLARIS12:
6232         case CHIP_VEGAM:
6233         case CHIP_CARRIZO:
6234         case CHIP_STONEY:
6235                 /* chips with display hardware */
6236                 return true;
6237         default:
6238                 /* IP discovery */
6239                 if (!adev->ip_versions[DCE_HWIP][0] ||
6240                     (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6241                         return false;
6242                 return true;
6243         }
6244 }
6245
6246 uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,
6247                 uint32_t inst, uint32_t reg_addr, char reg_name[],
6248                 uint32_t expected_value, uint32_t mask)
6249 {
6250         uint32_t ret = 0;
6251         uint32_t old_ = 0;
6252         uint32_t tmp_ = RREG32(reg_addr);
6253         uint32_t loop = adev->usec_timeout;
6254
6255         while ((tmp_ & (mask)) != (expected_value)) {
6256                 if (old_ != tmp_) {
6257                         loop = adev->usec_timeout;
6258                         old_ = tmp_;
6259                 } else
6260                         udelay(1);
6261                 tmp_ = RREG32(reg_addr);
6262                 loop--;
6263                 if (!loop) {
6264                         DRM_WARN("Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn",
6265                                   inst, reg_name, (uint32_t)expected_value,
6266                                   (uint32_t)(tmp_ & (mask)));
6267                         ret = -ETIMEDOUT;
6268                         break;
6269                 }
6270         }
6271         return ret;
6272 }