drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/power_supply.h>
  29 #include <linux/kthread.h>
  30 #include <linux/module.h>
  31 #include <linux/console.h>
  32 #include <linux/slab.h>
  33 #include <linux/iommu.h>
  34 #include <linux/pci.h>
  35 #include <linux/pci-p2pdma.h>
  36 #include <linux/apple-gmux.h>
  37
  38 #include <drm/drm_aperture.h>
  39 #include <drm/drm_atomic_helper.h>
  40 #include <drm/drm_crtc_helper.h>
  41 #include <drm/drm_fb_helper.h>
  42 #include <drm/drm_probe_helper.h>
  43 #include <drm/amdgpu_drm.h>
  44 #include <linux/device.h>
  45 #include <linux/vgaarb.h>
  46 #include <linux/vga_switcheroo.h>
  47 #include <linux/efi.h>
  48 #include "amdgpu.h"
  49 #include "amdgpu_trace.h"
  50 #include "amdgpu_i2c.h"
  51 #include "atom.h"
  52 #include "amdgpu_atombios.h"
  53 #include "amdgpu_atomfirmware.h"
  54 #include "amd_pcie.h"
  55 #ifdef CONFIG_DRM_AMDGPU_SI
  56 #include "si.h"
  57 #endif
  58 #ifdef CONFIG_DRM_AMDGPU_CIK
  59 #include "cik.h"
  60 #endif
  61 #include "vi.h"
  62 #include "soc15.h"
  63 #include "nv.h"
  64 #include "bif/bif_4_1_d.h"
  65 #include <linux/firmware.h>
  66 #include "amdgpu_vf_error.h"
  67
  68 #include "amdgpu_amdkfd.h"
  69 #include "amdgpu_pm.h"
  70
  71 #include "amdgpu_xgmi.h"
  72 #include "amdgpu_ras.h"
  73 #include "amdgpu_pmu.h"
  74 #include "amdgpu_fru_eeprom.h"
  75 #include "amdgpu_reset.h"
  76 #include "amdgpu_virt.h"
  77
  78 #include <linux/suspend.h>
  79 #include <drm/task_barrier.h>
  80 #include <linux/pm_runtime.h>
  81
  82 #include <drm/drm_drv.h>
  83
  84 #if IS_ENABLED(CONFIG_X86)
  85 #include <asm/intel-family.h>
  86 #endif
  87
  88 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  89 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
  90 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  91 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
  92 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
  93 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
  94 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
  95
  96 #define AMDGPU_RESUME_MS                2000
  97 #define AMDGPU_MAX_RETRY_LIMIT          2
  98 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
  99 #define AMDGPU_PCIE_INDEX_FALLBACK (0x38 >> 2)
 100 #define AMDGPU_PCIE_INDEX_HI_FALLBACK (0x44 >> 2)
 101 #define AMDGPU_PCIE_DATA_FALLBACK (0x3C >> 2)
 102
 103 static const struct drm_driver amdgpu_kms_driver;
 104
 105 const char *amdgpu_asic_name[] = {
 106         "TAHITI",
 107         "PITCAIRN",
 108         "VERDE",
 109         "OLAND",
 110         "HAINAN",
 111         "BONAIRE",
 112         "KAVERI",
 113         "KABINI",
 114         "HAWAII",
 115         "MULLINS",
 116         "TOPAZ",
 117         "TONGA",
 118         "FIJI",
 119         "CARRIZO",
 120         "STONEY",
 121         "POLARIS10",
 122         "POLARIS11",
 123         "POLARIS12",
 124         "VEGAM",
 125         "VEGA10",
 126         "VEGA12",
 127         "VEGA20",
 128         "RAVEN",
 129         "ARCTURUS",
 130         "RENOIR",
 131         "ALDEBARAN",
 132         "NAVI10",
 133         "CYAN_SKILLFISH",
 134         "NAVI14",
 135         "NAVI12",
 136         "SIENNA_CICHLID",
 137         "NAVY_FLOUNDER",
 138         "VANGOGH",
 139         "DIMGREY_CAVEFISH",
 140         "BEIGE_GOBY",
 141         "YELLOW_CARP",
 142         "IP DISCOVERY",
 143         "LAST",
 144 };
 145
 146 /**
 147  * DOC: pcie_replay_count
 148  *
 149  * The amdgpu driver provides a sysfs API for reporting the total number
 150  * of PCIe replays (NAKs)
 151  * The file pcie_replay_count is used for this and returns the total
 152  * number of replays as a sum of the NAKs generated and NAKs received
 153  */
 154
 155 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
 156                 struct device_attribute *attr, char *buf)
 157 {
 158         struct drm_device *ddev = dev_get_drvdata(dev);
 159         struct amdgpu_device *adev = drm_to_adev(ddev);
 160         uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
 161
 162         return sysfs_emit(buf, "%llu\n", cnt);
 163 }
 164
 165 static DEVICE_ATTR(pcie_replay_count, 0444,
 166                 amdgpu_device_get_pcie_replay_count, NULL);
 167
 168 static ssize_t amdgpu_sysfs_reg_state_get(struct file *f, struct kobject *kobj,
 169                                           struct bin_attribute *attr, char *buf,
 170                                           loff_t ppos, size_t count)
 171 {
 172         struct device *dev = kobj_to_dev(kobj);
 173         struct drm_device *ddev = dev_get_drvdata(dev);
 174         struct amdgpu_device *adev = drm_to_adev(ddev);
 175         ssize_t bytes_read;
 176
 177         switch (ppos) {
 178         case AMDGPU_SYS_REG_STATE_XGMI:
 179                 bytes_read = amdgpu_asic_get_reg_state(
 180                         adev, AMDGPU_REG_STATE_TYPE_XGMI, buf, count);
 181                 break;
 182         case AMDGPU_SYS_REG_STATE_WAFL:
 183                 bytes_read = amdgpu_asic_get_reg_state(
 184                         adev, AMDGPU_REG_STATE_TYPE_WAFL, buf, count);
 185                 break;
 186         case AMDGPU_SYS_REG_STATE_PCIE:
 187                 bytes_read = amdgpu_asic_get_reg_state(
 188                         adev, AMDGPU_REG_STATE_TYPE_PCIE, buf, count);
 189                 break;
 190         case AMDGPU_SYS_REG_STATE_USR:
 191                 bytes_read = amdgpu_asic_get_reg_state(
 192                         adev, AMDGPU_REG_STATE_TYPE_USR, buf, count);
 193                 break;
 194         case AMDGPU_SYS_REG_STATE_USR_1:
 195                 bytes_read = amdgpu_asic_get_reg_state(
 196                         adev, AMDGPU_REG_STATE_TYPE_USR_1, buf, count);
 197                 break;
 198         default:
 199                 return -EINVAL;
 200         }
 201
 202         return bytes_read;
 203 }
 204
 205 BIN_ATTR(reg_state, 0444, amdgpu_sysfs_reg_state_get, NULL,
 206          AMDGPU_SYS_REG_STATE_END);
 207
 208 int amdgpu_reg_state_sysfs_init(struct amdgpu_device *adev)
 209 {
 210         int ret;
 211
 212         if (!amdgpu_asic_get_reg_state_supported(adev))
 213                 return 0;
 214
 215         ret = sysfs_create_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
 216
 217         return ret;
 218 }
 219
 220 void amdgpu_reg_state_sysfs_fini(struct amdgpu_device *adev)
 221 {
 222         if (!amdgpu_asic_get_reg_state_supported(adev))
 223                 return;
 224         sysfs_remove_bin_file(&adev->dev->kobj, &bin_attr_reg_state);
 225 }
 226
 227 /**
 228  * DOC: board_info
 229  *
 230  * The amdgpu driver provides a sysfs API for giving board related information.
 231  * It provides the form factor information in the format
 232  *
 233  *   type : form factor
 234  *
 235  * Possible form factor values
 236  *
 237  * - "cem"              - PCIE CEM card
 238  * - "oam"              - Open Compute Accelerator Module
 239  * - "unknown"  - Not known
 240  *
 241  */
 242
 243 static ssize_t amdgpu_device_get_board_info(struct device *dev,
 244                                             struct device_attribute *attr,
 245                                             char *buf)
 246 {
 247         struct drm_device *ddev = dev_get_drvdata(dev);
 248         struct amdgpu_device *adev = drm_to_adev(ddev);
 249         enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM;
 250         const char *pkg;
 251
 252         if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type)
 253                 pkg_type = adev->smuio.funcs->get_pkg_type(adev);
 254
 255         switch (pkg_type) {
 256         case AMDGPU_PKG_TYPE_CEM:
 257                 pkg = "cem";
 258                 break;
 259         case AMDGPU_PKG_TYPE_OAM:
 260                 pkg = "oam";
 261                 break;
 262         default:
 263                 pkg = "unknown";
 264                 break;
 265         }
 266
 267         return sysfs_emit(buf, "%s : %s\n", "type", pkg);
 268 }
 269
 270 static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL);
 271
 272 static struct attribute *amdgpu_board_attrs[] = {
 273         &dev_attr_board_info.attr,
 274         NULL,
 275 };
 276
 277 static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj,
 278                                              struct attribute *attr, int n)
 279 {
 280         struct device *dev = kobj_to_dev(kobj);
 281         struct drm_device *ddev = dev_get_drvdata(dev);
 282         struct amdgpu_device *adev = drm_to_adev(ddev);
 283
 284         if (adev->flags & AMD_IS_APU)
 285                 return 0;
 286
 287         return attr->mode;
 288 }
 289
 290 static const struct attribute_group amdgpu_board_attrs_group = {
 291         .attrs = amdgpu_board_attrs,
 292         .is_visible = amdgpu_board_attrs_is_visible
 293 };
 294
 295 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 296
 297
 298 /**
 299  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
 300  *
 301  * @dev: drm_device pointer
 302  *
 303  * Returns true if the device is a dGPU with ATPX power control,
 304  * otherwise return false.
 305  */
 306 bool amdgpu_device_supports_px(struct drm_device *dev)
 307 {
 308         struct amdgpu_device *adev = drm_to_adev(dev);
 309
 310         if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
 311                 return true;
 312         return false;
 313 }
 314
 315 /**
 316  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
 317  *
 318  * @dev: drm_device pointer
 319  *
 320  * Returns true if the device is a dGPU with ACPI power control,
 321  * otherwise return false.
 322  */
 323 bool amdgpu_device_supports_boco(struct drm_device *dev)
 324 {
 325         struct amdgpu_device *adev = drm_to_adev(dev);
 326
 327         if (adev->has_pr3 ||
 328             ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
 329                 return true;
 330         return false;
 331 }
 332
 333 /**
 334  * amdgpu_device_supports_baco - Does the device support BACO
 335  *
 336  * @dev: drm_device pointer
 337  *
 338  * Returns true if the device supporte BACO,
 339  * otherwise return false.
 340  */
 341 bool amdgpu_device_supports_baco(struct drm_device *dev)
 342 {
 343         struct amdgpu_device *adev = drm_to_adev(dev);
 344
 345         return amdgpu_asic_supports_baco(adev);
 346 }
 347
 348 /**
 349  * amdgpu_device_supports_smart_shift - Is the device dGPU with
 350  * smart shift support
 351  *
 352  * @dev: drm_device pointer
 353  *
 354  * Returns true if the device is a dGPU with Smart Shift support,
 355  * otherwise returns false.
 356  */
 357 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
 358 {
 359         return (amdgpu_device_supports_boco(dev) &&
 360                 amdgpu_acpi_is_power_shift_control_supported());
 361 }
 362
 363 /*
 364  * VRAM access helper functions
 365  */
 366
 367 /**
 368  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
 369  *
 370  * @adev: amdgpu_device pointer
 371  * @pos: offset of the buffer in vram
 372  * @buf: virtual address of the buffer in system memory
 373  * @size: read/write size, sizeof(@buf) must > @size
 374  * @write: true - write to vram, otherwise - read from vram
 375  */
 376 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
 377                              void *buf, size_t size, bool write)
 378 {
 379         unsigned long flags;
 380         uint32_t hi = ~0, tmp = 0;
 381         uint32_t *data = buf;
 382         uint64_t last;
 383         int idx;
 384
 385         if (!drm_dev_enter(adev_to_drm(adev), &idx))
 386                 return;
 387
 388         BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
 389
 390         spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 391         for (last = pos + size; pos < last; pos += 4) {
 392                 tmp = pos >> 31;
 393
 394                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
 395                 if (tmp != hi) {
 396                         WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
 397                         hi = tmp;
 398                 }
 399                 if (write)
 400                         WREG32_NO_KIQ(mmMM_DATA, *data++);
 401                 else
 402                         *data++ = RREG32_NO_KIQ(mmMM_DATA);
 403         }
 404
 405         spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 406         drm_dev_exit(idx);
 407 }
 408
 409 /**
 410  * amdgpu_device_aper_access - access vram by vram aperature
 411  *
 412  * @adev: amdgpu_device pointer
 413  * @pos: offset of the buffer in vram
 414  * @buf: virtual address of the buffer in system memory
 415  * @size: read/write size, sizeof(@buf) must > @size
 416  * @write: true - write to vram, otherwise - read from vram
 417  *
 418  * The return value means how many bytes have been transferred.
 419  */
 420 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
 421                                  void *buf, size_t size, bool write)
 422 {
 423 #ifdef CONFIG_64BIT
 424         void __iomem *addr;
 425         size_t count = 0;
 426         uint64_t last;
 427
 428         if (!adev->mman.aper_base_kaddr)
 429                 return 0;
 430
 431         last = min(pos + size, adev->gmc.visible_vram_size);
 432         if (last > pos) {
 433                 addr = adev->mman.aper_base_kaddr + pos;
 434                 count = last - pos;
 435
 436                 if (write) {
 437                         memcpy_toio(addr, buf, count);
 438                         /* Make sure HDP write cache flush happens without any reordering
 439                          * after the system memory contents are sent over PCIe device
 440                          */
 441                         mb();
 442                         amdgpu_device_flush_hdp(adev, NULL);
 443                 } else {
 444                         amdgpu_device_invalidate_hdp(adev, NULL);
 445                         /* Make sure HDP read cache is invalidated before issuing a read
 446                          * to the PCIe device
 447                          */
 448                         mb();
 449                         memcpy_fromio(buf, addr, count);
 450                 }
 451
 452         }
 453
 454         return count;
 455 #else
 456         return 0;
 457 #endif
 458 }
 459
 460 /**
 461  * amdgpu_device_vram_access - read/write a buffer in vram
 462  *
 463  * @adev: amdgpu_device pointer
 464  * @pos: offset of the buffer in vram
 465  * @buf: virtual address of the buffer in system memory
 466  * @size: read/write size, sizeof(@buf) must > @size
 467  * @write: true - write to vram, otherwise - read from vram
 468  */
 469 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
 470                                void *buf, size_t size, bool write)
 471 {
 472         size_t count;
 473
 474         /* try to using vram apreature to access vram first */
 475         count = amdgpu_device_aper_access(adev, pos, buf, size, write);
 476         size -= count;
 477         if (size) {
 478                 /* using MM to access rest vram */
 479                 pos += count;
 480                 buf += count;
 481                 amdgpu_device_mm_access(adev, pos, buf, size, write);
 482         }
 483 }
 484
 485 /*
 486  * register access helper functions.
 487  */
 488
 489 /* Check if hw access should be skipped because of hotplug or device error */
 490 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
 491 {
 492         if (adev->no_hw_access)
 493                 return true;
 494
 495 #ifdef CONFIG_LOCKDEP
 496         /*
 497          * This is a bit complicated to understand, so worth a comment. What we assert
 498          * here is that the GPU reset is not running on another thread in parallel.
 499          *
 500          * For this we trylock the read side of the reset semaphore, if that succeeds
 501          * we know that the reset is not running in paralell.
 502          *
 503          * If the trylock fails we assert that we are either already holding the read
 504          * side of the lock or are the reset thread itself and hold the write side of
 505          * the lock.
 506          */
 507         if (in_task()) {
 508                 if (down_read_trylock(&adev->reset_domain->sem))
 509                         up_read(&adev->reset_domain->sem);
 510                 else
 511                         lockdep_assert_held(&adev->reset_domain->sem);
 512         }
 513 #endif
 514         return false;
 515 }
 516
 517 /**
 518  * amdgpu_device_rreg - read a memory mapped IO or indirect register
 519  *
 520  * @adev: amdgpu_device pointer
 521  * @reg: dword aligned register offset
 522  * @acc_flags: access flags which require special behavior
 523  *
 524  * Returns the 32 bit value from the offset specified.
 525  */
 526 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 527                             uint32_t reg, uint32_t acc_flags)
 528 {
 529         uint32_t ret;
 530
 531         if (amdgpu_device_skip_hw_access(adev))
 532                 return 0;
 533
 534         if ((reg * 4) < adev->rmmio_size) {
 535                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 536                     amdgpu_sriov_runtime(adev) &&
 537                     down_read_trylock(&adev->reset_domain->sem)) {
 538                         ret = amdgpu_kiq_rreg(adev, reg, 0);
 539                         up_read(&adev->reset_domain->sem);
 540                 } else {
 541                         ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 542                 }
 543         } else {
 544                 ret = adev->pcie_rreg(adev, reg * 4);
 545         }
 546
 547         trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
 548
 549         return ret;
 550 }
 551
 552 /*
 553  * MMIO register read with bytes helper functions
 554  * @offset:bytes offset from MMIO start
 555  */
 556
 557 /**
 558  * amdgpu_mm_rreg8 - read a memory mapped IO register
 559  *
 560  * @adev: amdgpu_device pointer
 561  * @offset: byte aligned register offset
 562  *
 563  * Returns the 8 bit value from the offset specified.
 564  */
 565 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
 566 {
 567         if (amdgpu_device_skip_hw_access(adev))
 568                 return 0;
 569
 570         if (offset < adev->rmmio_size)
 571                 return (readb(adev->rmmio + offset));
 572         BUG();
 573 }
 574
 575
 576 /**
 577  * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC
 578  *
 579  * @adev: amdgpu_device pointer
 580  * @reg: dword aligned register offset
 581  * @acc_flags: access flags which require special behavior
 582  * @xcc_id: xcc accelerated compute core id
 583  *
 584  * Returns the 32 bit value from the offset specified.
 585  */
 586 uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,
 587                                 uint32_t reg, uint32_t acc_flags,
 588                                 uint32_t xcc_id)
 589 {
 590         uint32_t ret, rlcg_flag;
 591
 592         if (amdgpu_device_skip_hw_access(adev))
 593                 return 0;
 594
 595         if ((reg * 4) < adev->rmmio_size) {
 596                 if (amdgpu_sriov_vf(adev) &&
 597                     !amdgpu_sriov_runtime(adev) &&
 598                     adev->gfx.rlc.rlcg_reg_access_supported &&
 599                     amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
 600                                                          GC_HWIP, false,
 601                                                          &rlcg_flag)) {
 602                         ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, xcc_id);
 603                 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 604                     amdgpu_sriov_runtime(adev) &&
 605                     down_read_trylock(&adev->reset_domain->sem)) {
 606                         ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
 607                         up_read(&adev->reset_domain->sem);
 608                 } else {
 609                         ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 610                 }
 611         } else {
 612                 ret = adev->pcie_rreg(adev, reg * 4);
 613         }
 614
 615         return ret;
 616 }
 617
 618 /*
 619  * MMIO register write with bytes helper functions
 620  * @offset:bytes offset from MMIO start
 621  * @value: the value want to be written to the register
 622  */
 623
 624 /**
 625  * amdgpu_mm_wreg8 - read a memory mapped IO register
 626  *
 627  * @adev: amdgpu_device pointer
 628  * @offset: byte aligned register offset
 629  * @value: 8 bit value to write
 630  *
 631  * Writes the value specified to the offset specified.
 632  */
 633 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
 634 {
 635         if (amdgpu_device_skip_hw_access(adev))
 636                 return;
 637
 638         if (offset < adev->rmmio_size)
 639                 writeb(value, adev->rmmio + offset);
 640         else
 641                 BUG();
 642 }
 643
 644 /**
 645  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
 646  *
 647  * @adev: amdgpu_device pointer
 648  * @reg: dword aligned register offset
 649  * @v: 32 bit value to write to the register
 650  * @acc_flags: access flags which require special behavior
 651  *
 652  * Writes the value specified to the offset specified.
 653  */
 654 void amdgpu_device_wreg(struct amdgpu_device *adev,
 655                         uint32_t reg, uint32_t v,
 656                         uint32_t acc_flags)
 657 {
 658         if (amdgpu_device_skip_hw_access(adev))
 659                 return;
 660
 661         if ((reg * 4) < adev->rmmio_size) {
 662                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 663                     amdgpu_sriov_runtime(adev) &&
 664                     down_read_trylock(&adev->reset_domain->sem)) {
 665                         amdgpu_kiq_wreg(adev, reg, v, 0);
 666                         up_read(&adev->reset_domain->sem);
 667                 } else {
 668                         writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 669                 }
 670         } else {
 671                 adev->pcie_wreg(adev, reg * 4, v);
 672         }
 673
 674         trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
 675 }
 676
 677 /**
 678  * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
 679  *
 680  * @adev: amdgpu_device pointer
 681  * @reg: mmio/rlc register
 682  * @v: value to write
 683  * @xcc_id: xcc accelerated compute core id
 684  *
 685  * this function is invoked only for the debugfs register access
 686  */
 687 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
 688                              uint32_t reg, uint32_t v,
 689                              uint32_t xcc_id)
 690 {
 691         if (amdgpu_device_skip_hw_access(adev))
 692                 return;
 693
 694         if (amdgpu_sriov_fullaccess(adev) &&
 695             adev->gfx.rlc.funcs &&
 696             adev->gfx.rlc.funcs->is_rlcg_access_range) {
 697                 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
 698                         return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id);
 699         } else if ((reg * 4) >= adev->rmmio_size) {
 700                 adev->pcie_wreg(adev, reg * 4, v);
 701         } else {
 702                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 703         }
 704 }
 705
 706 /**
 707  * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC
 708  *
 709  * @adev: amdgpu_device pointer
 710  * @reg: dword aligned register offset
 711  * @v: 32 bit value to write to the register
 712  * @acc_flags: access flags which require special behavior
 713  * @xcc_id: xcc accelerated compute core id
 714  *
 715  * Writes the value specified to the offset specified.
 716  */
 717 void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,
 718                         uint32_t reg, uint32_t v,
 719                         uint32_t acc_flags, uint32_t xcc_id)
 720 {
 721         uint32_t rlcg_flag;
 722
 723         if (amdgpu_device_skip_hw_access(adev))
 724                 return;
 725
 726         if ((reg * 4) < adev->rmmio_size) {
 727                 if (amdgpu_sriov_vf(adev) &&
 728                     !amdgpu_sriov_runtime(adev) &&
 729                     adev->gfx.rlc.rlcg_reg_access_supported &&
 730                     amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
 731                                                          GC_HWIP, true,
 732                                                          &rlcg_flag)) {
 733                         amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, xcc_id);
 734                 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 735                     amdgpu_sriov_runtime(adev) &&
 736                     down_read_trylock(&adev->reset_domain->sem)) {
 737                         amdgpu_kiq_wreg(adev, reg, v, xcc_id);
 738                         up_read(&adev->reset_domain->sem);
 739                 } else {
 740                         writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 741                 }
 742         } else {
 743                 adev->pcie_wreg(adev, reg * 4, v);
 744         }
 745 }
 746
 747 /**
 748  * amdgpu_device_indirect_rreg - read an indirect register
 749  *
 750  * @adev: amdgpu_device pointer
 751  * @reg_addr: indirect register address to read from
 752  *
 753  * Returns the value of indirect register @reg_addr
 754  */
 755 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
 756                                 u32 reg_addr)
 757 {
 758         unsigned long flags, pcie_index, pcie_data;
 759         void __iomem *pcie_index_offset;
 760         void __iomem *pcie_data_offset;
 761         u32 r;
 762
 763         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 764         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 765
 766         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 767         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 768         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 769
 770         writel(reg_addr, pcie_index_offset);
 771         readl(pcie_index_offset);
 772         r = readl(pcie_data_offset);
 773         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 774
 775         return r;
 776 }
 777
 778 u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,
 779                                     u64 reg_addr)
 780 {
 781         unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
 782         u32 r;
 783         void __iomem *pcie_index_offset;
 784         void __iomem *pcie_index_hi_offset;
 785         void __iomem *pcie_data_offset;
 786
 787         if (unlikely(!adev->nbio.funcs)) {
 788                 pcie_index = AMDGPU_PCIE_INDEX_FALLBACK;
 789                 pcie_data = AMDGPU_PCIE_DATA_FALLBACK;
 790         } else {
 791                 pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 792                 pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 793         }
 794
 795         if (reg_addr >> 32) {
 796                 if (unlikely(!adev->nbio.funcs))
 797                         pcie_index_hi = AMDGPU_PCIE_INDEX_HI_FALLBACK;
 798                 else
 799                         pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
 800         } else {
 801                 pcie_index_hi = 0;
 802         }
 803
 804         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 805         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 806         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 807         if (pcie_index_hi != 0)
 808                 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
 809                                 pcie_index_hi * 4;
 810
 811         writel(reg_addr, pcie_index_offset);
 812         readl(pcie_index_offset);
 813         if (pcie_index_hi != 0) {
 814                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 815                 readl(pcie_index_hi_offset);
 816         }
 817         r = readl(pcie_data_offset);
 818
 819         /* clear the high bits */
 820         if (pcie_index_hi != 0) {
 821                 writel(0, pcie_index_hi_offset);
 822                 readl(pcie_index_hi_offset);
 823         }
 824
 825         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 826
 827         return r;
 828 }
 829
 830 /**
 831  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
 832  *
 833  * @adev: amdgpu_device pointer
 834  * @reg_addr: indirect register address to read from
 835  *
 836  * Returns the value of indirect register @reg_addr
 837  */
 838 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
 839                                   u32 reg_addr)
 840 {
 841         unsigned long flags, pcie_index, pcie_data;
 842         void __iomem *pcie_index_offset;
 843         void __iomem *pcie_data_offset;
 844         u64 r;
 845
 846         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 847         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 848
 849         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 850         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 851         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 852
 853         /* read low 32 bits */
 854         writel(reg_addr, pcie_index_offset);
 855         readl(pcie_index_offset);
 856         r = readl(pcie_data_offset);
 857         /* read high 32 bits */
 858         writel(reg_addr + 4, pcie_index_offset);
 859         readl(pcie_index_offset);
 860         r |= ((u64)readl(pcie_data_offset) << 32);
 861         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 862
 863         return r;
 864 }
 865
 866 u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev,
 867                                   u64 reg_addr)
 868 {
 869         unsigned long flags, pcie_index, pcie_data;
 870         unsigned long pcie_index_hi = 0;
 871         void __iomem *pcie_index_offset;
 872         void __iomem *pcie_index_hi_offset;
 873         void __iomem *pcie_data_offset;
 874         u64 r;
 875
 876         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 877         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 878         if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
 879                 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
 880
 881         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 882         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 883         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 884         if (pcie_index_hi != 0)
 885                 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
 886                         pcie_index_hi * 4;
 887
 888         /* read low 32 bits */
 889         writel(reg_addr, pcie_index_offset);
 890         readl(pcie_index_offset);
 891         if (pcie_index_hi != 0) {
 892                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 893                 readl(pcie_index_hi_offset);
 894         }
 895         r = readl(pcie_data_offset);
 896         /* read high 32 bits */
 897         writel(reg_addr + 4, pcie_index_offset);
 898         readl(pcie_index_offset);
 899         if (pcie_index_hi != 0) {
 900                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 901                 readl(pcie_index_hi_offset);
 902         }
 903         r |= ((u64)readl(pcie_data_offset) << 32);
 904
 905         /* clear the high bits */
 906         if (pcie_index_hi != 0) {
 907                 writel(0, pcie_index_hi_offset);
 908                 readl(pcie_index_hi_offset);
 909         }
 910
 911         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 912
 913         return r;
 914 }
 915
 916 /**
 917  * amdgpu_device_indirect_wreg - write an indirect register address
 918  *
 919  * @adev: amdgpu_device pointer
 920  * @reg_addr: indirect register offset
 921  * @reg_data: indirect register data
 922  *
 923  */
 924 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
 925                                  u32 reg_addr, u32 reg_data)
 926 {
 927         unsigned long flags, pcie_index, pcie_data;
 928         void __iomem *pcie_index_offset;
 929         void __iomem *pcie_data_offset;
 930
 931         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 932         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 933
 934         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 935         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 936         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 937
 938         writel(reg_addr, pcie_index_offset);
 939         readl(pcie_index_offset);
 940         writel(reg_data, pcie_data_offset);
 941         readl(pcie_data_offset);
 942         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 943 }
 944
 945 void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,
 946                                      u64 reg_addr, u32 reg_data)
 947 {
 948         unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
 949         void __iomem *pcie_index_offset;
 950         void __iomem *pcie_index_hi_offset;
 951         void __iomem *pcie_data_offset;
 952
 953         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 954         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 955         if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
 956                 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
 957         else
 958                 pcie_index_hi = 0;
 959
 960         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 961         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 962         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 963         if (pcie_index_hi != 0)
 964                 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
 965                                 pcie_index_hi * 4;
 966
 967         writel(reg_addr, pcie_index_offset);
 968         readl(pcie_index_offset);
 969         if (pcie_index_hi != 0) {
 970                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 971                 readl(pcie_index_hi_offset);
 972         }
 973         writel(reg_data, pcie_data_offset);
 974         readl(pcie_data_offset);
 975
 976         /* clear the high bits */
 977         if (pcie_index_hi != 0) {
 978                 writel(0, pcie_index_hi_offset);
 979                 readl(pcie_index_hi_offset);
 980         }
 981
 982         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 983 }
 984
 985 /**
 986  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
 987  *
 988  * @adev: amdgpu_device pointer
 989  * @reg_addr: indirect register offset
 990  * @reg_data: indirect register data
 991  *
 992  */
 993 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
 994                                    u32 reg_addr, u64 reg_data)
 995 {
 996         unsigned long flags, pcie_index, pcie_data;
 997         void __iomem *pcie_index_offset;
 998         void __iomem *pcie_data_offset;
 999
1000         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1001         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1002
1003         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1004         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1005         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1006
1007         /* write low 32 bits */
1008         writel(reg_addr, pcie_index_offset);
1009         readl(pcie_index_offset);
1010         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
1011         readl(pcie_data_offset);
1012         /* write high 32 bits */
1013         writel(reg_addr + 4, pcie_index_offset);
1014         readl(pcie_index_offset);
1015         writel((u32)(reg_data >> 32), pcie_data_offset);
1016         readl(pcie_data_offset);
1017         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1018 }
1019
1020 void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev,
1021                                    u64 reg_addr, u64 reg_data)
1022 {
1023         unsigned long flags, pcie_index, pcie_data;
1024         unsigned long pcie_index_hi = 0;
1025         void __iomem *pcie_index_offset;
1026         void __iomem *pcie_index_hi_offset;
1027         void __iomem *pcie_data_offset;
1028
1029         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
1030         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
1031         if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
1032                 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
1033
1034         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1035         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
1036         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
1037         if (pcie_index_hi != 0)
1038                 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
1039                                 pcie_index_hi * 4;
1040
1041         /* write low 32 bits */
1042         writel(reg_addr, pcie_index_offset);
1043         readl(pcie_index_offset);
1044         if (pcie_index_hi != 0) {
1045                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1046                 readl(pcie_index_hi_offset);
1047         }
1048         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
1049         readl(pcie_data_offset);
1050         /* write high 32 bits */
1051         writel(reg_addr + 4, pcie_index_offset);
1052         readl(pcie_index_offset);
1053         if (pcie_index_hi != 0) {
1054                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
1055                 readl(pcie_index_hi_offset);
1056         }
1057         writel((u32)(reg_data >> 32), pcie_data_offset);
1058         readl(pcie_data_offset);
1059
1060         /* clear the high bits */
1061         if (pcie_index_hi != 0) {
1062                 writel(0, pcie_index_hi_offset);
1063                 readl(pcie_index_hi_offset);
1064         }
1065
1066         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1067 }
1068
1069 /**
1070  * amdgpu_device_get_rev_id - query device rev_id
1071  *
1072  * @adev: amdgpu_device pointer
1073  *
1074  * Return device rev_id
1075  */
1076 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
1077 {
1078         return adev->nbio.funcs->get_rev_id(adev);
1079 }
1080
1081 /**
1082  * amdgpu_invalid_rreg - dummy reg read function
1083  *
1084  * @adev: amdgpu_device pointer
1085  * @reg: offset of register
1086  *
1087  * Dummy register read function.  Used for register blocks
1088  * that certain asics don't have (all asics).
1089  * Returns the value in the register.
1090  */
1091 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
1092 {
1093         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
1094         BUG();
1095         return 0;
1096 }
1097
1098 static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg)
1099 {
1100         DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1101         BUG();
1102         return 0;
1103 }
1104
1105 /**
1106  * amdgpu_invalid_wreg - dummy reg write function
1107  *
1108  * @adev: amdgpu_device pointer
1109  * @reg: offset of register
1110  * @v: value to write to the register
1111  *
1112  * Dummy register read function.  Used for register blocks
1113  * that certain asics don't have (all asics).
1114  */
1115 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
1116 {
1117         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
1118                   reg, v);
1119         BUG();
1120 }
1121
1122 static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v)
1123 {
1124         DRM_ERROR("Invalid callback to write register 0x%llX with 0x%08X\n",
1125                   reg, v);
1126         BUG();
1127 }
1128
1129 /**
1130  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
1131  *
1132  * @adev: amdgpu_device pointer
1133  * @reg: offset of register
1134  *
1135  * Dummy register read function.  Used for register blocks
1136  * that certain asics don't have (all asics).
1137  * Returns the value in the register.
1138  */
1139 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
1140 {
1141         DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
1142         BUG();
1143         return 0;
1144 }
1145
1146 static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg)
1147 {
1148         DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1149         BUG();
1150         return 0;
1151 }
1152
1153 /**
1154  * amdgpu_invalid_wreg64 - dummy reg write function
1155  *
1156  * @adev: amdgpu_device pointer
1157  * @reg: offset of register
1158  * @v: value to write to the register
1159  *
1160  * Dummy register read function.  Used for register blocks
1161  * that certain asics don't have (all asics).
1162  */
1163 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
1164 {
1165         DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
1166                   reg, v);
1167         BUG();
1168 }
1169
1170 static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v)
1171 {
1172         DRM_ERROR("Invalid callback to write 64 bit register 0x%llX with 0x%08llX\n",
1173                   reg, v);
1174         BUG();
1175 }
1176
1177 /**
1178  * amdgpu_block_invalid_rreg - dummy reg read function
1179  *
1180  * @adev: amdgpu_device pointer
1181  * @block: offset of instance
1182  * @reg: offset of register
1183  *
1184  * Dummy register read function.  Used for register blocks
1185  * that certain asics don't have (all asics).
1186  * Returns the value in the register.
1187  */
1188 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
1189                                           uint32_t block, uint32_t reg)
1190 {
1191         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
1192                   reg, block);
1193         BUG();
1194         return 0;
1195 }
1196
1197 /**
1198  * amdgpu_block_invalid_wreg - dummy reg write function
1199  *
1200  * @adev: amdgpu_device pointer
1201  * @block: offset of instance
1202  * @reg: offset of register
1203  * @v: value to write to the register
1204  *
1205  * Dummy register read function.  Used for register blocks
1206  * that certain asics don't have (all asics).
1207  */
1208 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
1209                                       uint32_t block,
1210                                       uint32_t reg, uint32_t v)
1211 {
1212         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
1213                   reg, block, v);
1214         BUG();
1215 }
1216
1217 /**
1218  * amdgpu_device_asic_init - Wrapper for atom asic_init
1219  *
1220  * @adev: amdgpu_device pointer
1221  *
1222  * Does any asic specific work and then calls atom asic init.
1223  */
1224 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
1225 {
1226         int ret;
1227
1228         amdgpu_asic_pre_asic_init(adev);
1229
1230         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
1231             amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
1232                 amdgpu_psp_wait_for_bootloader(adev);
1233                 ret = amdgpu_atomfirmware_asic_init(adev, true);
1234                 return ret;
1235         } else {
1236                 return amdgpu_atom_asic_init(adev->mode_info.atom_context);
1237         }
1238
1239         return 0;
1240 }
1241
1242 /**
1243  * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
1244  *
1245  * @adev: amdgpu_device pointer
1246  *
1247  * Allocates a scratch page of VRAM for use by various things in the
1248  * driver.
1249  */
1250 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
1251 {
1252         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
1253                                        AMDGPU_GEM_DOMAIN_VRAM |
1254                                        AMDGPU_GEM_DOMAIN_GTT,
1255                                        &adev->mem_scratch.robj,
1256                                        &adev->mem_scratch.gpu_addr,
1257                                        (void **)&adev->mem_scratch.ptr);
1258 }
1259
1260 /**
1261  * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1262  *
1263  * @adev: amdgpu_device pointer
1264  *
1265  * Frees the VRAM scratch page.
1266  */
1267 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
1268 {
1269         amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
1270 }
1271
1272 /**
1273  * amdgpu_device_program_register_sequence - program an array of registers.
1274  *
1275  * @adev: amdgpu_device pointer
1276  * @registers: pointer to the register array
1277  * @array_size: size of the register array
1278  *
1279  * Programs an array or registers with and or masks.
1280  * This is a helper for setting golden registers.
1281  */
1282 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1283                                              const u32 *registers,
1284                                              const u32 array_size)
1285 {
1286         u32 tmp, reg, and_mask, or_mask;
1287         int i;
1288
1289         if (array_size % 3)
1290                 return;
1291
1292         for (i = 0; i < array_size; i += 3) {
1293                 reg = registers[i + 0];
1294                 and_mask = registers[i + 1];
1295                 or_mask = registers[i + 2];
1296
1297                 if (and_mask == 0xffffffff) {
1298                         tmp = or_mask;
1299                 } else {
1300                         tmp = RREG32(reg);
1301                         tmp &= ~and_mask;
1302                         if (adev->family >= AMDGPU_FAMILY_AI)
1303                                 tmp |= (or_mask & and_mask);
1304                         else
1305                                 tmp |= or_mask;
1306                 }
1307                 WREG32(reg, tmp);
1308         }
1309 }
1310
1311 /**
1312  * amdgpu_device_pci_config_reset - reset the GPU
1313  *
1314  * @adev: amdgpu_device pointer
1315  *
1316  * Resets the GPU using the pci config reset sequence.
1317  * Only applicable to asics prior to vega10.
1318  */
1319 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1320 {
1321         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1322 }
1323
1324 /**
1325  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1326  *
1327  * @adev: amdgpu_device pointer
1328  *
1329  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1330  */
1331 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1332 {
1333         return pci_reset_function(adev->pdev);
1334 }
1335
1336 /*
1337  * amdgpu_device_wb_*()
1338  * Writeback is the method by which the GPU updates special pages in memory
1339  * with the status of certain GPU events (fences, ring pointers,etc.).
1340  */
1341
1342 /**
1343  * amdgpu_device_wb_fini - Disable Writeback and free memory
1344  *
1345  * @adev: amdgpu_device pointer
1346  *
1347  * Disables Writeback and frees the Writeback memory (all asics).
1348  * Used at driver shutdown.
1349  */
1350 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1351 {
1352         if (adev->wb.wb_obj) {
1353                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1354                                       &adev->wb.gpu_addr,
1355                                       (void **)&adev->wb.wb);
1356                 adev->wb.wb_obj = NULL;
1357         }
1358 }
1359
1360 /**
1361  * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1362  *
1363  * @adev: amdgpu_device pointer
1364  *
1365  * Initializes writeback and allocates writeback memory (all asics).
1366  * Used at driver startup.
1367  * Returns 0 on success or an -error on failure.
1368  */
1369 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1370 {
1371         int r;
1372
1373         if (adev->wb.wb_obj == NULL) {
1374                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1375                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1376                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1377                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
1378                                             (void **)&adev->wb.wb);
1379                 if (r) {
1380                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1381                         return r;
1382                 }
1383
1384                 adev->wb.num_wb = AMDGPU_MAX_WB;
1385                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1386
1387                 /* clear wb memory */
1388                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1389         }
1390
1391         return 0;
1392 }
1393
1394 /**
1395  * amdgpu_device_wb_get - Allocate a wb entry
1396  *
1397  * @adev: amdgpu_device pointer
1398  * @wb: wb index
1399  *
1400  * Allocate a wb slot for use by the driver (all asics).
1401  * Returns 0 on success or -EINVAL on failure.
1402  */
1403 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1404 {
1405         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1406
1407         if (offset < adev->wb.num_wb) {
1408                 __set_bit(offset, adev->wb.used);
1409                 *wb = offset << 3; /* convert to dw offset */
1410                 return 0;
1411         } else {
1412                 return -EINVAL;
1413         }
1414 }
1415
1416 /**
1417  * amdgpu_device_wb_free - Free a wb entry
1418  *
1419  * @adev: amdgpu_device pointer
1420  * @wb: wb index
1421  *
1422  * Free a wb slot allocated for use by the driver (all asics)
1423  */
1424 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1425 {
1426         wb >>= 3;
1427         if (wb < adev->wb.num_wb)
1428                 __clear_bit(wb, adev->wb.used);
1429 }
1430
1431 /**
1432  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1433  *
1434  * @adev: amdgpu_device pointer
1435  *
1436  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1437  * to fail, but if any of the BARs is not accessible after the size we abort
1438  * driver loading by returning -ENODEV.
1439  */
1440 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1441 {
1442         int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1443         struct pci_bus *root;
1444         struct resource *res;
1445         unsigned int i;
1446         u16 cmd;
1447         int r;
1448
1449         if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
1450                 return 0;
1451
1452         /* Bypass for VF */
1453         if (amdgpu_sriov_vf(adev))
1454                 return 0;
1455
1456         /* skip if the bios has already enabled large BAR */
1457         if (adev->gmc.real_vram_size &&
1458             (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1459                 return 0;
1460
1461         /* Check if the root BUS has 64bit memory resources */
1462         root = adev->pdev->bus;
1463         while (root->parent)
1464                 root = root->parent;
1465
1466         pci_bus_for_each_resource(root, res, i) {
1467                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1468                     res->start > 0x100000000ull)
1469                         break;
1470         }
1471
1472         /* Trying to resize is pointless without a root hub window above 4GB */
1473         if (!res)
1474                 return 0;
1475
1476         /* Limit the BAR size to what is available */
1477         rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1478                         rbar_size);
1479
1480         /* Disable memory decoding while we change the BAR addresses and size */
1481         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1482         pci_write_config_word(adev->pdev, PCI_COMMAND,
1483                               cmd & ~PCI_COMMAND_MEMORY);
1484
1485         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1486         amdgpu_doorbell_fini(adev);
1487         if (adev->asic_type >= CHIP_BONAIRE)
1488                 pci_release_resource(adev->pdev, 2);
1489
1490         pci_release_resource(adev->pdev, 0);
1491
1492         r = pci_resize_resource(adev->pdev, 0, rbar_size);
1493         if (r == -ENOSPC)
1494                 DRM_INFO("Not enough PCI address space for a large BAR.");
1495         else if (r && r != -ENOTSUPP)
1496                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1497
1498         pci_assign_unassigned_bus_resources(adev->pdev->bus);
1499
1500         /* When the doorbell or fb BAR isn't available we have no chance of
1501          * using the device.
1502          */
1503         r = amdgpu_doorbell_init(adev);
1504         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1505                 return -ENODEV;
1506
1507         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1508
1509         return 0;
1510 }
1511
1512 static bool amdgpu_device_read_bios(struct amdgpu_device *adev)
1513 {
1514         if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))
1515                 return false;
1516
1517         return true;
1518 }
1519
1520 /*
1521  * GPU helpers function.
1522  */
1523 /**
1524  * amdgpu_device_need_post - check if the hw need post or not
1525  *
1526  * @adev: amdgpu_device pointer
1527  *
1528  * Check if the asic has been initialized (all asics) at driver startup
1529  * or post is needed if  hw reset is performed.
1530  * Returns true if need or false if not.
1531  */
1532 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1533 {
1534         uint32_t reg;
1535
1536         if (amdgpu_sriov_vf(adev))
1537                 return false;
1538
1539         if (!amdgpu_device_read_bios(adev))
1540                 return false;
1541
1542         if (amdgpu_passthrough(adev)) {
1543                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1544                  * some old smc fw still need driver do vPost otherwise gpu hang, while
1545                  * those smc fw version above 22.15 doesn't have this flaw, so we force
1546                  * vpost executed for smc version below 22.15
1547                  */
1548                 if (adev->asic_type == CHIP_FIJI) {
1549                         int err;
1550                         uint32_t fw_ver;
1551
1552                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1553                         /* force vPost if error occured */
1554                         if (err)
1555                                 return true;
1556
1557                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1558                         release_firmware(adev->pm.fw);
1559                         if (fw_ver < 0x00160e00)
1560                                 return true;
1561                 }
1562         }
1563
1564         /* Don't post if we need to reset whole hive on init */
1565         if (adev->gmc.xgmi.pending_reset)
1566                 return false;
1567
1568         if (adev->has_hw_reset) {
1569                 adev->has_hw_reset = false;
1570                 return true;
1571         }
1572
1573         /* bios scratch used on CIK+ */
1574         if (adev->asic_type >= CHIP_BONAIRE)
1575                 return amdgpu_atombios_scratch_need_asic_init(adev);
1576
1577         /* check MEM_SIZE for older asics */
1578         reg = amdgpu_asic_get_config_memsize(adev);
1579
1580         if ((reg != 0) && (reg != 0xffffffff))
1581                 return false;
1582
1583         return true;
1584 }
1585
1586 /*
1587  * Check whether seamless boot is supported.
1588  *
1589  * So far we only support seamless boot on DCE 3.0 or later.
1590  * If users report that it works on older ASICS as well, we may
1591  * loosen this.
1592  */
1593 bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)
1594 {
1595         switch (amdgpu_seamless) {
1596         case -1:
1597                 break;
1598         case 1:
1599                 return true;
1600         case 0:
1601                 return false;
1602         default:
1603                 DRM_ERROR("Invalid value for amdgpu.seamless: %d\n",
1604                           amdgpu_seamless);
1605                 return false;
1606         }
1607
1608         if (!(adev->flags & AMD_IS_APU))
1609                 return false;
1610
1611         if (adev->mman.keep_stolen_vga_memory)
1612                 return false;
1613
1614         return amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0);
1615 }
1616
1617 /*
1618  * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids
1619  * don't support dynamic speed switching. Until we have confirmation from Intel
1620  * that a specific host supports it, it's safer that we keep it disabled for all.
1621  *
1622  * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
1623  * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1624  */
1625 static bool amdgpu_device_pcie_dynamic_switching_supported(struct amdgpu_device *adev)
1626 {
1627 #if IS_ENABLED(CONFIG_X86)
1628         struct cpuinfo_x86 *c = &cpu_data(0);
1629
1630         /* eGPU change speeds based on USB4 fabric conditions */
1631         if (dev_is_removable(adev->dev))
1632                 return true;
1633
1634         if (c->x86_vendor == X86_VENDOR_INTEL)
1635                 return false;
1636 #endif
1637         return true;
1638 }
1639
1640 /**
1641  * amdgpu_device_should_use_aspm - check if the device should program ASPM
1642  *
1643  * @adev: amdgpu_device pointer
1644  *
1645  * Confirm whether the module parameter and pcie bridge agree that ASPM should
1646  * be set for this device.
1647  *
1648  * Returns true if it should be used or false if not.
1649  */
1650 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1651 {
1652         switch (amdgpu_aspm) {
1653         case -1:
1654                 break;
1655         case 0:
1656                 return false;
1657         case 1:
1658                 return true;
1659         default:
1660                 return false;
1661         }
1662         if (adev->flags & AMD_IS_APU)
1663                 return false;
1664         if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK))
1665                 return false;
1666         return pcie_aspm_enabled(adev->pdev);
1667 }
1668
1669 /* if we get transitioned to only one device, take VGA back */
1670 /**
1671  * amdgpu_device_vga_set_decode - enable/disable vga decode
1672  *
1673  * @pdev: PCI device pointer
1674  * @state: enable/disable vga decode
1675  *
1676  * Enable/disable vga decode (all asics).
1677  * Returns VGA resource flags.
1678  */
1679 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1680                 bool state)
1681 {
1682         struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1683
1684         amdgpu_asic_set_vga_state(adev, state);
1685         if (state)
1686                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1687                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1688         else
1689                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1690 }
1691
1692 /**
1693  * amdgpu_device_check_block_size - validate the vm block size
1694  *
1695  * @adev: amdgpu_device pointer
1696  *
1697  * Validates the vm block size specified via module parameter.
1698  * The vm block size defines number of bits in page table versus page directory,
1699  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1700  * page table and the remaining bits are in the page directory.
1701  */
1702 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1703 {
1704         /* defines number of bits in page table versus page directory,
1705          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1706          * page table and the remaining bits are in the page directory
1707          */
1708         if (amdgpu_vm_block_size == -1)
1709                 return;
1710
1711         if (amdgpu_vm_block_size < 9) {
1712                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1713                          amdgpu_vm_block_size);
1714                 amdgpu_vm_block_size = -1;
1715         }
1716 }
1717
1718 /**
1719  * amdgpu_device_check_vm_size - validate the vm size
1720  *
1721  * @adev: amdgpu_device pointer
1722  *
1723  * Validates the vm size in GB specified via module parameter.
1724  * The VM size is the size of the GPU virtual memory space in GB.
1725  */
1726 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1727 {
1728         /* no need to check the default value */
1729         if (amdgpu_vm_size == -1)
1730                 return;
1731
1732         if (amdgpu_vm_size < 1) {
1733                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1734                          amdgpu_vm_size);
1735                 amdgpu_vm_size = -1;
1736         }
1737 }
1738
1739 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1740 {
1741         struct sysinfo si;
1742         bool is_os_64 = (sizeof(void *) == 8);
1743         uint64_t total_memory;
1744         uint64_t dram_size_seven_GB = 0x1B8000000;
1745         uint64_t dram_size_three_GB = 0xB8000000;
1746
1747         if (amdgpu_smu_memory_pool_size == 0)
1748                 return;
1749
1750         if (!is_os_64) {
1751                 DRM_WARN("Not 64-bit OS, feature not supported\n");
1752                 goto def_value;
1753         }
1754         si_meminfo(&si);
1755         total_memory = (uint64_t)si.totalram * si.mem_unit;
1756
1757         if ((amdgpu_smu_memory_pool_size == 1) ||
1758                 (amdgpu_smu_memory_pool_size == 2)) {
1759                 if (total_memory < dram_size_three_GB)
1760                         goto def_value1;
1761         } else if ((amdgpu_smu_memory_pool_size == 4) ||
1762                 (amdgpu_smu_memory_pool_size == 8)) {
1763                 if (total_memory < dram_size_seven_GB)
1764                         goto def_value1;
1765         } else {
1766                 DRM_WARN("Smu memory pool size not supported\n");
1767                 goto def_value;
1768         }
1769         adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1770
1771         return;
1772
1773 def_value1:
1774         DRM_WARN("No enough system memory\n");
1775 def_value:
1776         adev->pm.smu_prv_buffer_size = 0;
1777 }
1778
1779 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1780 {
1781         if (!(adev->flags & AMD_IS_APU) ||
1782             adev->asic_type < CHIP_RAVEN)
1783                 return 0;
1784
1785         switch (adev->asic_type) {
1786         case CHIP_RAVEN:
1787                 if (adev->pdev->device == 0x15dd)
1788                         adev->apu_flags |= AMD_APU_IS_RAVEN;
1789                 if (adev->pdev->device == 0x15d8)
1790                         adev->apu_flags |= AMD_APU_IS_PICASSO;
1791                 break;
1792         case CHIP_RENOIR:
1793                 if ((adev->pdev->device == 0x1636) ||
1794                     (adev->pdev->device == 0x164c))
1795                         adev->apu_flags |= AMD_APU_IS_RENOIR;
1796                 else
1797                         adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1798                 break;
1799         case CHIP_VANGOGH:
1800                 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1801                 break;
1802         case CHIP_YELLOW_CARP:
1803                 break;
1804         case CHIP_CYAN_SKILLFISH:
1805                 if ((adev->pdev->device == 0x13FE) ||
1806                     (adev->pdev->device == 0x143F))
1807                         adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1808                 break;
1809         default:
1810                 break;
1811         }
1812
1813         return 0;
1814 }
1815
1816 /**
1817  * amdgpu_device_check_arguments - validate module params
1818  *
1819  * @adev: amdgpu_device pointer
1820  *
1821  * Validates certain module parameters and updates
1822  * the associated values used by the driver (all asics).
1823  */
1824 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1825 {
1826         if (amdgpu_sched_jobs < 4) {
1827                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1828                          amdgpu_sched_jobs);
1829                 amdgpu_sched_jobs = 4;
1830         } else if (!is_power_of_2(amdgpu_sched_jobs)) {
1831                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1832                          amdgpu_sched_jobs);
1833                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1834         }
1835
1836         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1837                 /* gart size must be greater or equal to 32M */
1838                 dev_warn(adev->dev, "gart size (%d) too small\n",
1839                          amdgpu_gart_size);
1840                 amdgpu_gart_size = -1;
1841         }
1842
1843         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1844                 /* gtt size must be greater or equal to 32M */
1845                 dev_warn(adev->dev, "gtt size (%d) too small\n",
1846                                  amdgpu_gtt_size);
1847                 amdgpu_gtt_size = -1;
1848         }
1849
1850         /* valid range is between 4 and 9 inclusive */
1851         if (amdgpu_vm_fragment_size != -1 &&
1852             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1853                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1854                 amdgpu_vm_fragment_size = -1;
1855         }
1856
1857         if (amdgpu_sched_hw_submission < 2) {
1858                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1859                          amdgpu_sched_hw_submission);
1860                 amdgpu_sched_hw_submission = 2;
1861         } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1862                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1863                          amdgpu_sched_hw_submission);
1864                 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1865         }
1866
1867         if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1868                 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1869                 amdgpu_reset_method = -1;
1870         }
1871
1872         amdgpu_device_check_smu_prv_buffer_size(adev);
1873
1874         amdgpu_device_check_vm_size(adev);
1875
1876         amdgpu_device_check_block_size(adev);
1877
1878         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1879
1880         return 0;
1881 }
1882
1883 /**
1884  * amdgpu_switcheroo_set_state - set switcheroo state
1885  *
1886  * @pdev: pci dev pointer
1887  * @state: vga_switcheroo state
1888  *
1889  * Callback for the switcheroo driver.  Suspends or resumes
1890  * the asics before or after it is powered up using ACPI methods.
1891  */
1892 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1893                                         enum vga_switcheroo_state state)
1894 {
1895         struct drm_device *dev = pci_get_drvdata(pdev);
1896         int r;
1897
1898         if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1899                 return;
1900
1901         if (state == VGA_SWITCHEROO_ON) {
1902                 pr_info("switched on\n");
1903                 /* don't suspend or resume card normally */
1904                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1905
1906                 pci_set_power_state(pdev, PCI_D0);
1907                 amdgpu_device_load_pci_state(pdev);
1908                 r = pci_enable_device(pdev);
1909                 if (r)
1910                         DRM_WARN("pci_enable_device failed (%d)\n", r);
1911                 amdgpu_device_resume(dev, true);
1912
1913                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1914         } else {
1915                 pr_info("switched off\n");
1916                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1917                 amdgpu_device_prepare(dev);
1918                 amdgpu_device_suspend(dev, true);
1919                 amdgpu_device_cache_pci_state(pdev);
1920                 /* Shut down the device */
1921                 pci_disable_device(pdev);
1922                 pci_set_power_state(pdev, PCI_D3cold);
1923                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1924         }
1925 }
1926
1927 /**
1928  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1929  *
1930  * @pdev: pci dev pointer
1931  *
1932  * Callback for the switcheroo driver.  Check of the switcheroo
1933  * state can be changed.
1934  * Returns true if the state can be changed, false if not.
1935  */
1936 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1937 {
1938         struct drm_device *dev = pci_get_drvdata(pdev);
1939
1940        /*
1941         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1942         * locking inversion with the driver load path. And the access here is
1943         * completely racy anyway. So don't bother with locking for now.
1944         */
1945         return atomic_read(&dev->open_count) == 0;
1946 }
1947
1948 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1949         .set_gpu_state = amdgpu_switcheroo_set_state,
1950         .reprobe = NULL,
1951         .can_switch = amdgpu_switcheroo_can_switch,
1952 };
1953
1954 /**
1955  * amdgpu_device_ip_set_clockgating_state - set the CG state
1956  *
1957  * @dev: amdgpu_device pointer
1958  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1959  * @state: clockgating state (gate or ungate)
1960  *
1961  * Sets the requested clockgating state for all instances of
1962  * the hardware IP specified.
1963  * Returns the error code from the last instance.
1964  */
1965 int amdgpu_device_ip_set_clockgating_state(void *dev,
1966                                            enum amd_ip_block_type block_type,
1967                                            enum amd_clockgating_state state)
1968 {
1969         struct amdgpu_device *adev = dev;
1970         int i, r = 0;
1971
1972         for (i = 0; i < adev->num_ip_blocks; i++) {
1973                 if (!adev->ip_blocks[i].status.valid)
1974                         continue;
1975                 if (adev->ip_blocks[i].version->type != block_type)
1976                         continue;
1977                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1978                         continue;
1979                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1980                         (void *)adev, state);
1981                 if (r)
1982                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1983                                   adev->ip_blocks[i].version->funcs->name, r);
1984         }
1985         return r;
1986 }
1987
1988 /**
1989  * amdgpu_device_ip_set_powergating_state - set the PG state
1990  *
1991  * @dev: amdgpu_device pointer
1992  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1993  * @state: powergating state (gate or ungate)
1994  *
1995  * Sets the requested powergating state for all instances of
1996  * the hardware IP specified.
1997  * Returns the error code from the last instance.
1998  */
1999 int amdgpu_device_ip_set_powergating_state(void *dev,
2000                                            enum amd_ip_block_type block_type,
2001                                            enum amd_powergating_state state)
2002 {
2003         struct amdgpu_device *adev = dev;
2004         int i, r = 0;
2005
2006         for (i = 0; i < adev->num_ip_blocks; i++) {
2007                 if (!adev->ip_blocks[i].status.valid)
2008                         continue;
2009                 if (adev->ip_blocks[i].version->type != block_type)
2010                         continue;
2011                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
2012                         continue;
2013                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
2014                         (void *)adev, state);
2015                 if (r)
2016                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
2017                                   adev->ip_blocks[i].version->funcs->name, r);
2018         }
2019         return r;
2020 }
2021
2022 /**
2023  * amdgpu_device_ip_get_clockgating_state - get the CG state
2024  *
2025  * @adev: amdgpu_device pointer
2026  * @flags: clockgating feature flags
2027  *
2028  * Walks the list of IPs on the device and updates the clockgating
2029  * flags for each IP.
2030  * Updates @flags with the feature flags for each hardware IP where
2031  * clockgating is enabled.
2032  */
2033 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
2034                                             u64 *flags)
2035 {
2036         int i;
2037
2038         for (i = 0; i < adev->num_ip_blocks; i++) {
2039                 if (!adev->ip_blocks[i].status.valid)
2040                         continue;
2041                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
2042                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
2043         }
2044 }
2045
2046 /**
2047  * amdgpu_device_ip_wait_for_idle - wait for idle
2048  *
2049  * @adev: amdgpu_device pointer
2050  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2051  *
2052  * Waits for the request hardware IP to be idle.
2053  * Returns 0 for success or a negative error code on failure.
2054  */
2055 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
2056                                    enum amd_ip_block_type block_type)
2057 {
2058         int i, r;
2059
2060         for (i = 0; i < adev->num_ip_blocks; i++) {
2061                 if (!adev->ip_blocks[i].status.valid)
2062                         continue;
2063                 if (adev->ip_blocks[i].version->type == block_type) {
2064                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
2065                         if (r)
2066                                 return r;
2067                         break;
2068                 }
2069         }
2070         return 0;
2071
2072 }
2073
2074 /**
2075  * amdgpu_device_ip_is_idle - is the hardware IP idle
2076  *
2077  * @adev: amdgpu_device pointer
2078  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2079  *
2080  * Check if the hardware IP is idle or not.
2081  * Returns true if it the IP is idle, false if not.
2082  */
2083 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
2084                               enum amd_ip_block_type block_type)
2085 {
2086         int i;
2087
2088         for (i = 0; i < adev->num_ip_blocks; i++) {
2089                 if (!adev->ip_blocks[i].status.valid)
2090                         continue;
2091                 if (adev->ip_blocks[i].version->type == block_type)
2092                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
2093         }
2094         return true;
2095
2096 }
2097
2098 /**
2099  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
2100  *
2101  * @adev: amdgpu_device pointer
2102  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
2103  *
2104  * Returns a pointer to the hardware IP block structure
2105  * if it exists for the asic, otherwise NULL.
2106  */
2107 struct amdgpu_ip_block *
2108 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
2109                               enum amd_ip_block_type type)
2110 {
2111         int i;
2112
2113         for (i = 0; i < adev->num_ip_blocks; i++)
2114                 if (adev->ip_blocks[i].version->type == type)
2115                         return &adev->ip_blocks[i];
2116
2117         return NULL;
2118 }
2119
2120 /**
2121  * amdgpu_device_ip_block_version_cmp
2122  *
2123  * @adev: amdgpu_device pointer
2124  * @type: enum amd_ip_block_type
2125  * @major: major version
2126  * @minor: minor version
2127  *
2128  * return 0 if equal or greater
2129  * return 1 if smaller or the ip_block doesn't exist
2130  */
2131 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
2132                                        enum amd_ip_block_type type,
2133                                        u32 major, u32 minor)
2134 {
2135         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
2136
2137         if (ip_block && ((ip_block->version->major > major) ||
2138                         ((ip_block->version->major == major) &&
2139                         (ip_block->version->minor >= minor))))
2140                 return 0;
2141
2142         return 1;
2143 }
2144
2145 /**
2146  * amdgpu_device_ip_block_add
2147  *
2148  * @adev: amdgpu_device pointer
2149  * @ip_block_version: pointer to the IP to add
2150  *
2151  * Adds the IP block driver information to the collection of IPs
2152  * on the asic.
2153  */
2154 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
2155                                const struct amdgpu_ip_block_version *ip_block_version)
2156 {
2157         if (!ip_block_version)
2158                 return -EINVAL;
2159
2160         switch (ip_block_version->type) {
2161         case AMD_IP_BLOCK_TYPE_VCN:
2162                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
2163                         return 0;
2164                 break;
2165         case AMD_IP_BLOCK_TYPE_JPEG:
2166                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
2167                         return 0;
2168                 break;
2169         default:
2170                 break;
2171         }
2172
2173         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
2174                   ip_block_version->funcs->name);
2175
2176         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
2177
2178         return 0;
2179 }
2180
2181 /**
2182  * amdgpu_device_enable_virtual_display - enable virtual display feature
2183  *
2184  * @adev: amdgpu_device pointer
2185  *
2186  * Enabled the virtual display feature if the user has enabled it via
2187  * the module parameter virtual_display.  This feature provides a virtual
2188  * display hardware on headless boards or in virtualized environments.
2189  * This function parses and validates the configuration string specified by
2190  * the user and configues the virtual display configuration (number of
2191  * virtual connectors, crtcs, etc.) specified.
2192  */
2193 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
2194 {
2195         adev->enable_virtual_display = false;
2196
2197         if (amdgpu_virtual_display) {
2198                 const char *pci_address_name = pci_name(adev->pdev);
2199                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
2200
2201                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
2202                 pciaddstr_tmp = pciaddstr;
2203                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
2204                         pciaddname = strsep(&pciaddname_tmp, ",");
2205                         if (!strcmp("all", pciaddname)
2206                             || !strcmp(pci_address_name, pciaddname)) {
2207                                 long num_crtc;
2208                                 int res = -1;
2209
2210                                 adev->enable_virtual_display = true;
2211
2212                                 if (pciaddname_tmp)
2213                                         res = kstrtol(pciaddname_tmp, 10,
2214                                                       &num_crtc);
2215
2216                                 if (!res) {
2217                                         if (num_crtc < 1)
2218                                                 num_crtc = 1;
2219                                         if (num_crtc > 6)
2220                                                 num_crtc = 6;
2221                                         adev->mode_info.num_crtc = num_crtc;
2222                                 } else {
2223                                         adev->mode_info.num_crtc = 1;
2224                                 }
2225                                 break;
2226                         }
2227                 }
2228
2229                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
2230                          amdgpu_virtual_display, pci_address_name,
2231                          adev->enable_virtual_display, adev->mode_info.num_crtc);
2232
2233                 kfree(pciaddstr);
2234         }
2235 }
2236
2237 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
2238 {
2239         if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
2240                 adev->mode_info.num_crtc = 1;
2241                 adev->enable_virtual_display = true;
2242                 DRM_INFO("virtual_display:%d, num_crtc:%d\n",
2243                          adev->enable_virtual_display, adev->mode_info.num_crtc);
2244         }
2245 }
2246
2247 /**
2248  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
2249  *
2250  * @adev: amdgpu_device pointer
2251  *
2252  * Parses the asic configuration parameters specified in the gpu info
2253  * firmware and makes them availale to the driver for use in configuring
2254  * the asic.
2255  * Returns 0 on success, -EINVAL on failure.
2256  */
2257 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
2258 {
2259         const char *chip_name;
2260         char fw_name[40];
2261         int err;
2262         const struct gpu_info_firmware_header_v1_0 *hdr;
2263
2264         adev->firmware.gpu_info_fw = NULL;
2265
2266         if (adev->mman.discovery_bin)
2267                 return 0;
2268
2269         switch (adev->asic_type) {
2270         default:
2271                 return 0;
2272         case CHIP_VEGA10:
2273                 chip_name = "vega10";
2274                 break;
2275         case CHIP_VEGA12:
2276                 chip_name = "vega12";
2277                 break;
2278         case CHIP_RAVEN:
2279                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2280                         chip_name = "raven2";
2281                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
2282                         chip_name = "picasso";
2283                 else
2284                         chip_name = "raven";
2285                 break;
2286         case CHIP_ARCTURUS:
2287                 chip_name = "arcturus";
2288                 break;
2289         case CHIP_NAVI12:
2290                 chip_name = "navi12";
2291                 break;
2292         }
2293
2294         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
2295         err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, fw_name);
2296         if (err) {
2297                 dev_err(adev->dev,
2298                         "Failed to get gpu_info firmware \"%s\"\n",
2299                         fw_name);
2300                 goto out;
2301         }
2302
2303         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2304         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2305
2306         switch (hdr->version_major) {
2307         case 1:
2308         {
2309                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2310                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2311                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2312
2313                 /*
2314                  * Should be droped when DAL no longer needs it.
2315                  */
2316                 if (adev->asic_type == CHIP_NAVI12)
2317                         goto parse_soc_bounding_box;
2318
2319                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2320                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2321                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2322                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2323                 adev->gfx.config.max_texture_channel_caches =
2324                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
2325                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2326                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2327                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2328                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2329                 adev->gfx.config.double_offchip_lds_buf =
2330                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2331                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2332                 adev->gfx.cu_info.max_waves_per_simd =
2333                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2334                 adev->gfx.cu_info.max_scratch_slots_per_cu =
2335                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2336                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2337                 if (hdr->version_minor >= 1) {
2338                         const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2339                                 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2340                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2341                         adev->gfx.config.num_sc_per_sh =
2342                                 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2343                         adev->gfx.config.num_packer_per_sc =
2344                                 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2345                 }
2346
2347 parse_soc_bounding_box:
2348                 /*
2349                  * soc bounding box info is not integrated in disocovery table,
2350                  * we always need to parse it from gpu info firmware if needed.
2351                  */
2352                 if (hdr->version_minor == 2) {
2353                         const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2354                                 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2355                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2356                         adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2357                 }
2358                 break;
2359         }
2360         default:
2361                 dev_err(adev->dev,
2362                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2363                 err = -EINVAL;
2364                 goto out;
2365         }
2366 out:
2367         return err;
2368 }
2369
2370 /**
2371  * amdgpu_device_ip_early_init - run early init for hardware IPs
2372  *
2373  * @adev: amdgpu_device pointer
2374  *
2375  * Early initialization pass for hardware IPs.  The hardware IPs that make
2376  * up each asic are discovered each IP's early_init callback is run.  This
2377  * is the first stage in initializing the asic.
2378  * Returns 0 on success, negative error code on failure.
2379  */
2380 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2381 {
2382         struct pci_dev *parent;
2383         int i, r;
2384         bool total;
2385
2386         amdgpu_device_enable_virtual_display(adev);
2387
2388         if (amdgpu_sriov_vf(adev)) {
2389                 r = amdgpu_virt_request_full_gpu(adev, true);
2390                 if (r)
2391                         return r;
2392         }
2393
2394         switch (adev->asic_type) {
2395 #ifdef CONFIG_DRM_AMDGPU_SI
2396         case CHIP_VERDE:
2397         case CHIP_TAHITI:
2398         case CHIP_PITCAIRN:
2399         case CHIP_OLAND:
2400         case CHIP_HAINAN:
2401                 adev->family = AMDGPU_FAMILY_SI;
2402                 r = si_set_ip_blocks(adev);
2403                 if (r)
2404                         return r;
2405                 break;
2406 #endif
2407 #ifdef CONFIG_DRM_AMDGPU_CIK
2408         case CHIP_BONAIRE:
2409         case CHIP_HAWAII:
2410         case CHIP_KAVERI:
2411         case CHIP_KABINI:
2412         case CHIP_MULLINS:
2413                 if (adev->flags & AMD_IS_APU)
2414                         adev->family = AMDGPU_FAMILY_KV;
2415                 else
2416                         adev->family = AMDGPU_FAMILY_CI;
2417
2418                 r = cik_set_ip_blocks(adev);
2419                 if (r)
2420                         return r;
2421                 break;
2422 #endif
2423         case CHIP_TOPAZ:
2424         case CHIP_TONGA:
2425         case CHIP_FIJI:
2426         case CHIP_POLARIS10:
2427         case CHIP_POLARIS11:
2428         case CHIP_POLARIS12:
2429         case CHIP_VEGAM:
2430         case CHIP_CARRIZO:
2431         case CHIP_STONEY:
2432                 if (adev->flags & AMD_IS_APU)
2433                         adev->family = AMDGPU_FAMILY_CZ;
2434                 else
2435                         adev->family = AMDGPU_FAMILY_VI;
2436
2437                 r = vi_set_ip_blocks(adev);
2438                 if (r)
2439                         return r;
2440                 break;
2441         default:
2442                 r = amdgpu_discovery_set_ip_blocks(adev);
2443                 if (r)
2444                         return r;
2445                 break;
2446         }
2447
2448         if (amdgpu_has_atpx() &&
2449             (amdgpu_is_atpx_hybrid() ||
2450              amdgpu_has_atpx_dgpu_power_cntl()) &&
2451             ((adev->flags & AMD_IS_APU) == 0) &&
2452             !dev_is_removable(&adev->pdev->dev))
2453                 adev->flags |= AMD_IS_PX;
2454
2455         if (!(adev->flags & AMD_IS_APU)) {
2456                 parent = pcie_find_root_port(adev->pdev);
2457                 adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2458         }
2459
2460
2461         adev->pm.pp_feature = amdgpu_pp_feature_mask;
2462         if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2463                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2464         if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2465                 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2466         if (!amdgpu_device_pcie_dynamic_switching_supported(adev))
2467                 adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK;
2468
2469         total = true;
2470         for (i = 0; i < adev->num_ip_blocks; i++) {
2471                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2472                         DRM_WARN("disabled ip block: %d <%s>\n",
2473                                   i, adev->ip_blocks[i].version->funcs->name);
2474                         adev->ip_blocks[i].status.valid = false;
2475                 } else {
2476                         if (adev->ip_blocks[i].version->funcs->early_init) {
2477                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2478                                 if (r == -ENOENT) {
2479                                         adev->ip_blocks[i].status.valid = false;
2480                                 } else if (r) {
2481                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
2482                                                   adev->ip_blocks[i].version->funcs->name, r);
2483                                         total = false;
2484                                 } else {
2485                                         adev->ip_blocks[i].status.valid = true;
2486                                 }
2487                         } else {
2488                                 adev->ip_blocks[i].status.valid = true;
2489                         }
2490                 }
2491                 /* get the vbios after the asic_funcs are set up */
2492                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2493                         r = amdgpu_device_parse_gpu_info_fw(adev);
2494                         if (r)
2495                                 return r;
2496
2497                         /* Read BIOS */
2498                         if (amdgpu_device_read_bios(adev)) {
2499                                 if (!amdgpu_get_bios(adev))
2500                                         return -EINVAL;
2501
2502                                 r = amdgpu_atombios_init(adev);
2503                                 if (r) {
2504                                         dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2505                                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2506                                         return r;
2507                                 }
2508                         }
2509
2510                         /*get pf2vf msg info at it's earliest time*/
2511                         if (amdgpu_sriov_vf(adev))
2512                                 amdgpu_virt_init_data_exchange(adev);
2513
2514                 }
2515         }
2516         if (!total)
2517                 return -ENODEV;
2518
2519         amdgpu_amdkfd_device_probe(adev);
2520         adev->cg_flags &= amdgpu_cg_mask;
2521         adev->pg_flags &= amdgpu_pg_mask;
2522
2523         return 0;
2524 }
2525
2526 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2527 {
2528         int i, r;
2529
2530         for (i = 0; i < adev->num_ip_blocks; i++) {
2531                 if (!adev->ip_blocks[i].status.sw)
2532                         continue;
2533                 if (adev->ip_blocks[i].status.hw)
2534                         continue;
2535                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2536                     (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2537                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2538                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2539                         if (r) {
2540                                 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2541                                           adev->ip_blocks[i].version->funcs->name, r);
2542                                 return r;
2543                         }
2544                         adev->ip_blocks[i].status.hw = true;
2545                 }
2546         }
2547
2548         return 0;
2549 }
2550
2551 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2552 {
2553         int i, r;
2554
2555         for (i = 0; i < adev->num_ip_blocks; i++) {
2556                 if (!adev->ip_blocks[i].status.sw)
2557                         continue;
2558                 if (adev->ip_blocks[i].status.hw)
2559                         continue;
2560                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2561                 if (r) {
2562                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2563                                   adev->ip_blocks[i].version->funcs->name, r);
2564                         return r;
2565                 }
2566                 adev->ip_blocks[i].status.hw = true;
2567         }
2568
2569         return 0;
2570 }
2571
2572 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2573 {
2574         int r = 0;
2575         int i;
2576         uint32_t smu_version;
2577
2578         if (adev->asic_type >= CHIP_VEGA10) {
2579                 for (i = 0; i < adev->num_ip_blocks; i++) {
2580                         if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2581                                 continue;
2582
2583                         if (!adev->ip_blocks[i].status.sw)
2584                                 continue;
2585
2586                         /* no need to do the fw loading again if already done*/
2587                         if (adev->ip_blocks[i].status.hw == true)
2588                                 break;
2589
2590                         if (amdgpu_in_reset(adev) || adev->in_suspend) {
2591                                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2592                                 if (r) {
2593                                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2594                                                           adev->ip_blocks[i].version->funcs->name, r);
2595                                         return r;
2596                                 }
2597                         } else {
2598                                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2599                                 if (r) {
2600                                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2601                                                           adev->ip_blocks[i].version->funcs->name, r);
2602                                         return r;
2603                                 }
2604                         }
2605
2606                         adev->ip_blocks[i].status.hw = true;
2607                         break;
2608                 }
2609         }
2610
2611         if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2612                 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2613
2614         return r;
2615 }
2616
2617 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2618 {
2619         long timeout;
2620         int r, i;
2621
2622         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2623                 struct amdgpu_ring *ring = adev->rings[i];
2624
2625                 /* No need to setup the GPU scheduler for rings that don't need it */
2626                 if (!ring || ring->no_scheduler)
2627                         continue;
2628
2629                 switch (ring->funcs->type) {
2630                 case AMDGPU_RING_TYPE_GFX:
2631                         timeout = adev->gfx_timeout;
2632                         break;
2633                 case AMDGPU_RING_TYPE_COMPUTE:
2634                         timeout = adev->compute_timeout;
2635                         break;
2636                 case AMDGPU_RING_TYPE_SDMA:
2637                         timeout = adev->sdma_timeout;
2638                         break;
2639                 default:
2640                         timeout = adev->video_timeout;
2641                         break;
2642                 }
2643
2644                 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, NULL,
2645                                    DRM_SCHED_PRIORITY_COUNT,
2646                                    ring->num_hw_submission, 0,
2647                                    timeout, adev->reset_domain->wq,
2648                                    ring->sched_score, ring->name,
2649                                    adev->dev);
2650                 if (r) {
2651                         DRM_ERROR("Failed to create scheduler on ring %s.\n",
2652                                   ring->name);
2653                         return r;
2654                 }
2655                 r = amdgpu_uvd_entity_init(adev, ring);
2656                 if (r) {
2657                         DRM_ERROR("Failed to create UVD scheduling entity on ring %s.\n",
2658                                   ring->name);
2659                         return r;
2660                 }
2661                 r = amdgpu_vce_entity_init(adev, ring);
2662                 if (r) {
2663                         DRM_ERROR("Failed to create VCE scheduling entity on ring %s.\n",
2664                                   ring->name);
2665                         return r;
2666                 }
2667         }
2668
2669         amdgpu_xcp_update_partition_sched_list(adev);
2670
2671         return 0;
2672 }
2673
2674
2675 /**
2676  * amdgpu_device_ip_init - run init for hardware IPs
2677  *
2678  * @adev: amdgpu_device pointer
2679  *
2680  * Main initialization pass for hardware IPs.  The list of all the hardware
2681  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2682  * are run.  sw_init initializes the software state associated with each IP
2683  * and hw_init initializes the hardware associated with each IP.
2684  * Returns 0 on success, negative error code on failure.
2685  */
2686 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2687 {
2688         int i, r;
2689
2690         r = amdgpu_ras_init(adev);
2691         if (r)
2692                 return r;
2693
2694         for (i = 0; i < adev->num_ip_blocks; i++) {
2695                 if (!adev->ip_blocks[i].status.valid)
2696                         continue;
2697                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2698                 if (r) {
2699                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2700                                   adev->ip_blocks[i].version->funcs->name, r);
2701                         goto init_failed;
2702                 }
2703                 adev->ip_blocks[i].status.sw = true;
2704
2705                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2706                         /* need to do common hw init early so everything is set up for gmc */
2707                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2708                         if (r) {
2709                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2710                                 goto init_failed;
2711                         }
2712                         adev->ip_blocks[i].status.hw = true;
2713                 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2714                         /* need to do gmc hw init early so we can allocate gpu mem */
2715                         /* Try to reserve bad pages early */
2716                         if (amdgpu_sriov_vf(adev))
2717                                 amdgpu_virt_exchange_data(adev);
2718
2719                         r = amdgpu_device_mem_scratch_init(adev);
2720                         if (r) {
2721                                 DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r);
2722                                 goto init_failed;
2723                         }
2724                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2725                         if (r) {
2726                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2727                                 goto init_failed;
2728                         }
2729                         r = amdgpu_device_wb_init(adev);
2730                         if (r) {
2731                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2732                                 goto init_failed;
2733                         }
2734                         adev->ip_blocks[i].status.hw = true;
2735
2736                         /* right after GMC hw init, we create CSA */
2737                         if (adev->gfx.mcbp) {
2738                                 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2739                                                                AMDGPU_GEM_DOMAIN_VRAM |
2740                                                                AMDGPU_GEM_DOMAIN_GTT,
2741                                                                AMDGPU_CSA_SIZE);
2742                                 if (r) {
2743                                         DRM_ERROR("allocate CSA failed %d\n", r);
2744                                         goto init_failed;
2745                                 }
2746                         }
2747
2748                         r = amdgpu_seq64_init(adev);
2749                         if (r) {
2750                                 DRM_ERROR("allocate seq64 failed %d\n", r);
2751                                 goto init_failed;
2752                         }
2753                 }
2754         }
2755
2756         if (amdgpu_sriov_vf(adev))
2757                 amdgpu_virt_init_data_exchange(adev);
2758
2759         r = amdgpu_ib_pool_init(adev);
2760         if (r) {
2761                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2762                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2763                 goto init_failed;
2764         }
2765
2766         r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2767         if (r)
2768                 goto init_failed;
2769
2770         r = amdgpu_device_ip_hw_init_phase1(adev);
2771         if (r)
2772                 goto init_failed;
2773
2774         r = amdgpu_device_fw_loading(adev);
2775         if (r)
2776                 goto init_failed;
2777
2778         r = amdgpu_device_ip_hw_init_phase2(adev);
2779         if (r)
2780                 goto init_failed;
2781
2782         /*
2783          * retired pages will be loaded from eeprom and reserved here,
2784          * it should be called after amdgpu_device_ip_hw_init_phase2  since
2785          * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2786          * for I2C communication which only true at this point.
2787          *
2788          * amdgpu_ras_recovery_init may fail, but the upper only cares the
2789          * failure from bad gpu situation and stop amdgpu init process
2790          * accordingly. For other failed cases, it will still release all
2791          * the resource and print error message, rather than returning one
2792          * negative value to upper level.
2793          *
2794          * Note: theoretically, this should be called before all vram allocations
2795          * to protect retired page from abusing
2796          */
2797         r = amdgpu_ras_recovery_init(adev);
2798         if (r)
2799                 goto init_failed;
2800
2801         /**
2802          * In case of XGMI grab extra reference for reset domain for this device
2803          */
2804         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2805                 if (amdgpu_xgmi_add_device(adev) == 0) {
2806                         if (!amdgpu_sriov_vf(adev)) {
2807                                 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2808
2809                                 if (WARN_ON(!hive)) {
2810                                         r = -ENOENT;
2811                                         goto init_failed;
2812                                 }
2813
2814                                 if (!hive->reset_domain ||
2815                                     !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
2816                                         r = -ENOENT;
2817                                         amdgpu_put_xgmi_hive(hive);
2818                                         goto init_failed;
2819                                 }
2820
2821                                 /* Drop the early temporary reset domain we created for device */
2822                                 amdgpu_reset_put_reset_domain(adev->reset_domain);
2823                                 adev->reset_domain = hive->reset_domain;
2824                                 amdgpu_put_xgmi_hive(hive);
2825                         }
2826                 }
2827         }
2828
2829         r = amdgpu_device_init_schedulers(adev);
2830         if (r)
2831                 goto init_failed;
2832
2833         if (adev->mman.buffer_funcs_ring->sched.ready)
2834                 amdgpu_ttm_set_buffer_funcs_status(adev, true);
2835
2836         /* Don't init kfd if whole hive need to be reset during init */
2837         if (!adev->gmc.xgmi.pending_reset) {
2838                 kgd2kfd_init_zone_device(adev);
2839                 amdgpu_amdkfd_device_init(adev);
2840         }
2841
2842         amdgpu_fru_get_product_info(adev);
2843
2844 init_failed:
2845
2846         return r;
2847 }
2848
2849 /**
2850  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2851  *
2852  * @adev: amdgpu_device pointer
2853  *
2854  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2855  * this function before a GPU reset.  If the value is retained after a
2856  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2857  */
2858 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2859 {
2860         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2861 }
2862
2863 /**
2864  * amdgpu_device_check_vram_lost - check if vram is valid
2865  *
2866  * @adev: amdgpu_device pointer
2867  *
2868  * Checks the reset magic value written to the gart pointer in VRAM.
2869  * The driver calls this after a GPU reset to see if the contents of
2870  * VRAM is lost or now.
2871  * returns true if vram is lost, false if not.
2872  */
2873 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2874 {
2875         if (memcmp(adev->gart.ptr, adev->reset_magic,
2876                         AMDGPU_RESET_MAGIC_NUM))
2877                 return true;
2878
2879         if (!amdgpu_in_reset(adev))
2880                 return false;
2881
2882         /*
2883          * For all ASICs with baco/mode1 reset, the VRAM is
2884          * always assumed to be lost.
2885          */
2886         switch (amdgpu_asic_reset_method(adev)) {
2887         case AMD_RESET_METHOD_BACO:
2888         case AMD_RESET_METHOD_MODE1:
2889                 return true;
2890         default:
2891                 return false;
2892         }
2893 }
2894
2895 /**
2896  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2897  *
2898  * @adev: amdgpu_device pointer
2899  * @state: clockgating state (gate or ungate)
2900  *
2901  * The list of all the hardware IPs that make up the asic is walked and the
2902  * set_clockgating_state callbacks are run.
2903  * Late initialization pass enabling clockgating for hardware IPs.
2904  * Fini or suspend, pass disabling clockgating for hardware IPs.
2905  * Returns 0 on success, negative error code on failure.
2906  */
2907
2908 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2909                                enum amd_clockgating_state state)
2910 {
2911         int i, j, r;
2912
2913         if (amdgpu_emu_mode == 1)
2914                 return 0;
2915
2916         for (j = 0; j < adev->num_ip_blocks; j++) {
2917                 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2918                 if (!adev->ip_blocks[i].status.late_initialized)
2919                         continue;
2920                 /* skip CG for GFX, SDMA on S0ix */
2921                 if (adev->in_s0ix &&
2922                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2923                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2924                         continue;
2925                 /* skip CG for VCE/UVD, it's handled specially */
2926                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2927                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2928                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2929                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2930                     adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2931                         /* enable clockgating to save power */
2932                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2933                                                                                      state);
2934                         if (r) {
2935                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2936                                           adev->ip_blocks[i].version->funcs->name, r);
2937                                 return r;
2938                         }
2939                 }
2940         }
2941
2942         return 0;
2943 }
2944
2945 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2946                                enum amd_powergating_state state)
2947 {
2948         int i, j, r;
2949
2950         if (amdgpu_emu_mode == 1)
2951                 return 0;
2952
2953         for (j = 0; j < adev->num_ip_blocks; j++) {
2954                 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2955                 if (!adev->ip_blocks[i].status.late_initialized)
2956                         continue;
2957                 /* skip PG for GFX, SDMA on S0ix */
2958                 if (adev->in_s0ix &&
2959                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2960                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2961                         continue;
2962                 /* skip CG for VCE/UVD, it's handled specially */
2963                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2964                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2965                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2966                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2967                     adev->ip_blocks[i].version->funcs->set_powergating_state) {
2968                         /* enable powergating to save power */
2969                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2970                                                                                         state);
2971                         if (r) {
2972                                 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2973                                           adev->ip_blocks[i].version->funcs->name, r);
2974                                 return r;
2975                         }
2976                 }
2977         }
2978         return 0;
2979 }
2980
2981 static int amdgpu_device_enable_mgpu_fan_boost(void)
2982 {
2983         struct amdgpu_gpu_instance *gpu_ins;
2984         struct amdgpu_device *adev;
2985         int i, ret = 0;
2986
2987         mutex_lock(&mgpu_info.mutex);
2988
2989         /*
2990          * MGPU fan boost feature should be enabled
2991          * only when there are two or more dGPUs in
2992          * the system
2993          */
2994         if (mgpu_info.num_dgpu < 2)
2995                 goto out;
2996
2997         for (i = 0; i < mgpu_info.num_dgpu; i++) {
2998                 gpu_ins = &(mgpu_info.gpu_ins[i]);
2999                 adev = gpu_ins->adev;
3000                 if (!(adev->flags & AMD_IS_APU) &&
3001                     !gpu_ins->mgpu_fan_enabled) {
3002                         ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
3003                         if (ret)
3004                                 break;
3005
3006                         gpu_ins->mgpu_fan_enabled = 1;
3007                 }
3008         }
3009
3010 out:
3011         mutex_unlock(&mgpu_info.mutex);
3012
3013         return ret;
3014 }
3015
3016 /**
3017  * amdgpu_device_ip_late_init - run late init for hardware IPs
3018  *
3019  * @adev: amdgpu_device pointer
3020  *
3021  * Late initialization pass for hardware IPs.  The list of all the hardware
3022  * IPs that make up the asic is walked and the late_init callbacks are run.
3023  * late_init covers any special initialization that an IP requires
3024  * after all of the have been initialized or something that needs to happen
3025  * late in the init process.
3026  * Returns 0 on success, negative error code on failure.
3027  */
3028 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
3029 {
3030         struct amdgpu_gpu_instance *gpu_instance;
3031         int i = 0, r;
3032
3033         for (i = 0; i < adev->num_ip_blocks; i++) {
3034                 if (!adev->ip_blocks[i].status.hw)
3035                         continue;
3036                 if (adev->ip_blocks[i].version->funcs->late_init) {
3037                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
3038                         if (r) {
3039                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
3040                                           adev->ip_blocks[i].version->funcs->name, r);
3041                                 return r;
3042                         }
3043                 }
3044                 adev->ip_blocks[i].status.late_initialized = true;
3045         }
3046
3047         r = amdgpu_ras_late_init(adev);
3048         if (r) {
3049                 DRM_ERROR("amdgpu_ras_late_init failed %d", r);
3050                 return r;
3051         }
3052
3053         amdgpu_ras_set_error_query_ready(adev, true);
3054
3055         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
3056         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
3057
3058         amdgpu_device_fill_reset_magic(adev);
3059
3060         r = amdgpu_device_enable_mgpu_fan_boost();
3061         if (r)
3062                 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
3063
3064         /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
3065         if (amdgpu_passthrough(adev) &&
3066             ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) ||
3067              adev->asic_type == CHIP_ALDEBARAN))
3068                 amdgpu_dpm_handle_passthrough_sbr(adev, true);
3069
3070         if (adev->gmc.xgmi.num_physical_nodes > 1) {
3071                 mutex_lock(&mgpu_info.mutex);
3072
3073                 /*
3074                  * Reset device p-state to low as this was booted with high.
3075                  *
3076                  * This should be performed only after all devices from the same
3077                  * hive get initialized.
3078                  *
3079                  * However, it's unknown how many device in the hive in advance.
3080                  * As this is counted one by one during devices initializations.
3081                  *
3082                  * So, we wait for all XGMI interlinked devices initialized.
3083                  * This may bring some delays as those devices may come from
3084                  * different hives. But that should be OK.
3085                  */
3086                 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
3087                         for (i = 0; i < mgpu_info.num_gpu; i++) {
3088                                 gpu_instance = &(mgpu_info.gpu_ins[i]);
3089                                 if (gpu_instance->adev->flags & AMD_IS_APU)
3090                                         continue;
3091
3092                                 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
3093                                                 AMDGPU_XGMI_PSTATE_MIN);
3094                                 if (r) {
3095                                         DRM_ERROR("pstate setting failed (%d).\n", r);
3096                                         break;
3097                                 }
3098                         }
3099                 }
3100
3101                 mutex_unlock(&mgpu_info.mutex);
3102         }
3103
3104         return 0;
3105 }
3106
3107 /**
3108  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
3109  *
3110  * @adev: amdgpu_device pointer
3111  *
3112  * For ASICs need to disable SMC first
3113  */
3114 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
3115 {
3116         int i, r;
3117
3118         if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0))
3119                 return;
3120
3121         for (i = 0; i < adev->num_ip_blocks; i++) {
3122                 if (!adev->ip_blocks[i].status.hw)
3123                         continue;
3124                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3125                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
3126                         /* XXX handle errors */
3127                         if (r) {
3128                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
3129                                           adev->ip_blocks[i].version->funcs->name, r);
3130                         }
3131                         adev->ip_blocks[i].status.hw = false;
3132                         break;
3133                 }
3134         }
3135 }
3136
3137 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
3138 {
3139         int i, r;
3140
3141         for (i = 0; i < adev->num_ip_blocks; i++) {
3142                 if (!adev->ip_blocks[i].version->funcs->early_fini)
3143                         continue;
3144
3145                 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
3146                 if (r) {
3147                         DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
3148                                   adev->ip_blocks[i].version->funcs->name, r);
3149                 }
3150         }
3151
3152         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3153         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3154
3155         amdgpu_amdkfd_suspend(adev, false);
3156
3157         /* Workaroud for ASICs need to disable SMC first */
3158         amdgpu_device_smu_fini_early(adev);
3159
3160         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3161                 if (!adev->ip_blocks[i].status.hw)
3162                         continue;
3163
3164                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
3165                 /* XXX handle errors */
3166                 if (r) {
3167                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
3168                                   adev->ip_blocks[i].version->funcs->name, r);
3169                 }
3170
3171                 adev->ip_blocks[i].status.hw = false;
3172         }
3173
3174         if (amdgpu_sriov_vf(adev)) {
3175                 if (amdgpu_virt_release_full_gpu(adev, false))
3176                         DRM_ERROR("failed to release exclusive mode on fini\n");
3177         }
3178
3179         return 0;
3180 }
3181
3182 /**
3183  * amdgpu_device_ip_fini - run fini for hardware IPs
3184  *
3185  * @adev: amdgpu_device pointer
3186  *
3187  * Main teardown pass for hardware IPs.  The list of all the hardware
3188  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
3189  * are run.  hw_fini tears down the hardware associated with each IP
3190  * and sw_fini tears down any software state associated with each IP.
3191  * Returns 0 on success, negative error code on failure.
3192  */
3193 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
3194 {
3195         int i, r;
3196
3197         if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
3198                 amdgpu_virt_release_ras_err_handler_data(adev);
3199
3200         if (adev->gmc.xgmi.num_physical_nodes > 1)
3201                 amdgpu_xgmi_remove_device(adev);
3202
3203         amdgpu_amdkfd_device_fini_sw(adev);
3204
3205         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3206                 if (!adev->ip_blocks[i].status.sw)
3207                         continue;
3208
3209                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
3210                         amdgpu_ucode_free_bo(adev);
3211                         amdgpu_free_static_csa(&adev->virt.csa_obj);
3212                         amdgpu_device_wb_fini(adev);
3213                         amdgpu_device_mem_scratch_fini(adev);
3214                         amdgpu_ib_pool_fini(adev);
3215                         amdgpu_seq64_fini(adev);
3216                 }
3217
3218                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
3219                 /* XXX handle errors */
3220                 if (r) {
3221                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
3222                                   adev->ip_blocks[i].version->funcs->name, r);
3223                 }
3224                 adev->ip_blocks[i].status.sw = false;
3225                 adev->ip_blocks[i].status.valid = false;
3226         }
3227
3228         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3229                 if (!adev->ip_blocks[i].status.late_initialized)
3230                         continue;
3231                 if (adev->ip_blocks[i].version->funcs->late_fini)
3232                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
3233                 adev->ip_blocks[i].status.late_initialized = false;
3234         }
3235
3236         amdgpu_ras_fini(adev);
3237
3238         return 0;
3239 }
3240
3241 /**
3242  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
3243  *
3244  * @work: work_struct.
3245  */
3246 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
3247 {
3248         struct amdgpu_device *adev =
3249                 container_of(work, struct amdgpu_device, delayed_init_work.work);
3250         int r;
3251
3252         r = amdgpu_ib_ring_tests(adev);
3253         if (r)
3254                 DRM_ERROR("ib ring test failed (%d).\n", r);
3255 }
3256
3257 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
3258 {
3259         struct amdgpu_device *adev =
3260                 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
3261
3262         WARN_ON_ONCE(adev->gfx.gfx_off_state);
3263         WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
3264
3265         if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
3266                 adev->gfx.gfx_off_state = true;
3267 }
3268
3269 /**
3270  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3271  *
3272  * @adev: amdgpu_device pointer
3273  *
3274  * Main suspend function for hardware IPs.  The list of all the hardware
3275  * IPs that make up the asic is walked, clockgating is disabled and the
3276  * suspend callbacks are run.  suspend puts the hardware and software state
3277  * in each IP into a state suitable for suspend.
3278  * Returns 0 on success, negative error code on failure.
3279  */
3280 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
3281 {
3282         int i, r;
3283
3284         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3285         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3286
3287         /*
3288          * Per PMFW team's suggestion, driver needs to handle gfxoff
3289          * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
3290          * scenario. Add the missing df cstate disablement here.
3291          */
3292         if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
3293                 dev_warn(adev->dev, "Failed to disallow df cstate");
3294
3295         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3296                 if (!adev->ip_blocks[i].status.valid)
3297                         continue;
3298
3299                 /* displays are handled separately */
3300                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
3301                         continue;
3302
3303                 /* XXX handle errors */
3304                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3305                 /* XXX handle errors */
3306                 if (r) {
3307                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
3308                                   adev->ip_blocks[i].version->funcs->name, r);
3309                         return r;
3310                 }
3311
3312                 adev->ip_blocks[i].status.hw = false;
3313         }
3314
3315         return 0;
3316 }
3317
3318 /**
3319  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3320  *
3321  * @adev: amdgpu_device pointer
3322  *
3323  * Main suspend function for hardware IPs.  The list of all the hardware
3324  * IPs that make up the asic is walked, clockgating is disabled and the
3325  * suspend callbacks are run.  suspend puts the hardware and software state
3326  * in each IP into a state suitable for suspend.
3327  * Returns 0 on success, negative error code on failure.
3328  */
3329 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3330 {
3331         int i, r;
3332
3333         if (adev->in_s0ix)
3334                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
3335
3336         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3337                 if (!adev->ip_blocks[i].status.valid)
3338                         continue;
3339                 /* displays are handled in phase1 */
3340                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3341                         continue;
3342                 /* PSP lost connection when err_event_athub occurs */
3343                 if (amdgpu_ras_intr_triggered() &&
3344                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3345                         adev->ip_blocks[i].status.hw = false;
3346                         continue;
3347                 }
3348
3349                 /* skip unnecessary suspend if we do not initialize them yet */
3350                 if (adev->gmc.xgmi.pending_reset &&
3351                     !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3352                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
3353                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3354                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
3355                         adev->ip_blocks[i].status.hw = false;
3356                         continue;
3357                 }
3358
3359                 /* skip suspend of gfx/mes and psp for S0ix
3360                  * gfx is in gfxoff state, so on resume it will exit gfxoff just
3361                  * like at runtime. PSP is also part of the always on hardware
3362                  * so no need to suspend it.
3363                  */
3364                 if (adev->in_s0ix &&
3365                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3366                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3367                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3368                         continue;
3369
3370                 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3371                 if (adev->in_s0ix &&
3372                     (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=
3373                      IP_VERSION(5, 0, 0)) &&
3374                     (adev->ip_blocks[i].version->type ==
3375                      AMD_IP_BLOCK_TYPE_SDMA))
3376                         continue;
3377
3378                 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3379                  * These are in TMR, hence are expected to be reused by PSP-TOS to reload
3380                  * from this location and RLC Autoload automatically also gets loaded
3381                  * from here based on PMFW -> PSP message during re-init sequence.
3382                  * Therefore, the psp suspend & resume should be skipped to avoid destroy
3383                  * the TMR and reload FWs again for IMU enabled APU ASICs.
3384                  */
3385                 if (amdgpu_in_reset(adev) &&
3386                     (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3387                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3388                         continue;
3389
3390                 /* XXX handle errors */
3391                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3392                 /* XXX handle errors */
3393                 if (r) {
3394                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
3395                                   adev->ip_blocks[i].version->funcs->name, r);
3396                 }
3397                 adev->ip_blocks[i].status.hw = false;
3398                 /* handle putting the SMC in the appropriate state */
3399                 if (!amdgpu_sriov_vf(adev)) {
3400                         if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3401                                 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3402                                 if (r) {
3403                                         DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3404                                                         adev->mp1_state, r);
3405                                         return r;
3406                                 }
3407                         }
3408                 }
3409         }
3410
3411         return 0;
3412 }
3413
3414 /**
3415  * amdgpu_device_ip_suspend - run suspend for hardware IPs
3416  *
3417  * @adev: amdgpu_device pointer
3418  *
3419  * Main suspend function for hardware IPs.  The list of all the hardware
3420  * IPs that make up the asic is walked, clockgating is disabled and the
3421  * suspend callbacks are run.  suspend puts the hardware and software state
3422  * in each IP into a state suitable for suspend.
3423  * Returns 0 on success, negative error code on failure.
3424  */
3425 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3426 {
3427         int r;
3428
3429         if (amdgpu_sriov_vf(adev)) {
3430                 amdgpu_virt_fini_data_exchange(adev);
3431                 amdgpu_virt_request_full_gpu(adev, false);
3432         }
3433
3434         amdgpu_ttm_set_buffer_funcs_status(adev, false);
3435
3436         r = amdgpu_device_ip_suspend_phase1(adev);
3437         if (r)
3438                 return r;
3439         r = amdgpu_device_ip_suspend_phase2(adev);
3440
3441         if (amdgpu_sriov_vf(adev))
3442                 amdgpu_virt_release_full_gpu(adev, false);
3443
3444         return r;
3445 }
3446
3447 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3448 {
3449         int i, r;
3450
3451         static enum amd_ip_block_type ip_order[] = {
3452                 AMD_IP_BLOCK_TYPE_COMMON,
3453                 AMD_IP_BLOCK_TYPE_GMC,
3454                 AMD_IP_BLOCK_TYPE_PSP,
3455                 AMD_IP_BLOCK_TYPE_IH,
3456         };
3457
3458         for (i = 0; i < adev->num_ip_blocks; i++) {
3459                 int j;
3460                 struct amdgpu_ip_block *block;
3461
3462                 block = &adev->ip_blocks[i];
3463                 block->status.hw = false;
3464
3465                 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3466
3467                         if (block->version->type != ip_order[j] ||
3468                                 !block->status.valid)
3469                                 continue;
3470
3471                         r = block->version->funcs->hw_init(adev);
3472                         DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3473                         if (r)
3474                                 return r;
3475                         block->status.hw = true;
3476                 }
3477         }
3478
3479         return 0;
3480 }
3481
3482 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3483 {
3484         int i, r;
3485
3486         static enum amd_ip_block_type ip_order[] = {
3487                 AMD_IP_BLOCK_TYPE_SMC,
3488                 AMD_IP_BLOCK_TYPE_DCE,
3489                 AMD_IP_BLOCK_TYPE_GFX,
3490                 AMD_IP_BLOCK_TYPE_SDMA,
3491                 AMD_IP_BLOCK_TYPE_MES,
3492                 AMD_IP_BLOCK_TYPE_UVD,
3493                 AMD_IP_BLOCK_TYPE_VCE,
3494                 AMD_IP_BLOCK_TYPE_VCN,
3495                 AMD_IP_BLOCK_TYPE_JPEG
3496         };
3497
3498         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3499                 int j;
3500                 struct amdgpu_ip_block *block;
3501
3502                 for (j = 0; j < adev->num_ip_blocks; j++) {
3503                         block = &adev->ip_blocks[j];
3504
3505                         if (block->version->type != ip_order[i] ||
3506                                 !block->status.valid ||
3507                                 block->status.hw)
3508                                 continue;
3509
3510                         if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3511                                 r = block->version->funcs->resume(adev);
3512                         else
3513                                 r = block->version->funcs->hw_init(adev);
3514
3515                         DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3516                         if (r)
3517                                 return r;
3518                         block->status.hw = true;
3519                 }
3520         }
3521
3522         return 0;
3523 }
3524
3525 /**
3526  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3527  *
3528  * @adev: amdgpu_device pointer
3529  *
3530  * First resume function for hardware IPs.  The list of all the hardware
3531  * IPs that make up the asic is walked and the resume callbacks are run for
3532  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3533  * after a suspend and updates the software state as necessary.  This
3534  * function is also used for restoring the GPU after a GPU reset.
3535  * Returns 0 on success, negative error code on failure.
3536  */
3537 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3538 {
3539         int i, r;
3540
3541         for (i = 0; i < adev->num_ip_blocks; i++) {
3542                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3543                         continue;
3544                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3545                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3546                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3547                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3548
3549                         r = adev->ip_blocks[i].version->funcs->resume(adev);
3550                         if (r) {
3551                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
3552                                           adev->ip_blocks[i].version->funcs->name, r);
3553                                 return r;
3554                         }
3555                         adev->ip_blocks[i].status.hw = true;
3556                 }
3557         }
3558
3559         return 0;
3560 }
3561
3562 /**
3563  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3564  *
3565  * @adev: amdgpu_device pointer
3566  *
3567  * First resume function for hardware IPs.  The list of all the hardware
3568  * IPs that make up the asic is walked and the resume callbacks are run for
3569  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3570  * functional state after a suspend and updates the software state as
3571  * necessary.  This function is also used for restoring the GPU after a GPU
3572  * reset.
3573  * Returns 0 on success, negative error code on failure.
3574  */
3575 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3576 {
3577         int i, r;
3578
3579         for (i = 0; i < adev->num_ip_blocks; i++) {
3580                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3581                         continue;
3582                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3583                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3584                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3585                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3586                         continue;
3587                 r = adev->ip_blocks[i].version->funcs->resume(adev);
3588                 if (r) {
3589                         DRM_ERROR("resume of IP block <%s> failed %d\n",
3590                                   adev->ip_blocks[i].version->funcs->name, r);
3591                         return r;
3592                 }
3593                 adev->ip_blocks[i].status.hw = true;
3594         }
3595
3596         return 0;
3597 }
3598
3599 /**
3600  * amdgpu_device_ip_resume - run resume for hardware IPs
3601  *
3602  * @adev: amdgpu_device pointer
3603  *
3604  * Main resume function for hardware IPs.  The hardware IPs
3605  * are split into two resume functions because they are
3606  * also used in recovering from a GPU reset and some additional
3607  * steps need to be take between them.  In this case (S3/S4) they are
3608  * run sequentially.
3609  * Returns 0 on success, negative error code on failure.
3610  */
3611 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3612 {
3613         int r;
3614
3615         r = amdgpu_device_ip_resume_phase1(adev);
3616         if (r)
3617                 return r;
3618
3619         r = amdgpu_device_fw_loading(adev);
3620         if (r)
3621                 return r;
3622
3623         r = amdgpu_device_ip_resume_phase2(adev);
3624
3625         if (adev->mman.buffer_funcs_ring->sched.ready)
3626                 amdgpu_ttm_set_buffer_funcs_status(adev, true);
3627
3628         return r;
3629 }
3630
3631 /**
3632  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3633  *
3634  * @adev: amdgpu_device pointer
3635  *
3636  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3637  */
3638 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3639 {
3640         if (amdgpu_sriov_vf(adev)) {
3641                 if (adev->is_atom_fw) {
3642                         if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3643                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3644                 } else {
3645                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3646                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3647                 }
3648
3649                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3650                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3651         }
3652 }
3653
3654 /**
3655  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3656  *
3657  * @asic_type: AMD asic type
3658  *
3659  * Check if there is DC (new modesetting infrastructre) support for an asic.
3660  * returns true if DC has support, false if not.
3661  */
3662 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3663 {
3664         switch (asic_type) {
3665 #ifdef CONFIG_DRM_AMDGPU_SI
3666         case CHIP_HAINAN:
3667 #endif
3668         case CHIP_TOPAZ:
3669                 /* chips with no display hardware */
3670                 return false;
3671 #if defined(CONFIG_DRM_AMD_DC)
3672         case CHIP_TAHITI:
3673         case CHIP_PITCAIRN:
3674         case CHIP_VERDE:
3675         case CHIP_OLAND:
3676                 /*
3677                  * We have systems in the wild with these ASICs that require
3678                  * LVDS and VGA support which is not supported with DC.
3679                  *
3680                  * Fallback to the non-DC driver here by default so as not to
3681                  * cause regressions.
3682                  */
3683 #if defined(CONFIG_DRM_AMD_DC_SI)
3684                 return amdgpu_dc > 0;
3685 #else
3686                 return false;
3687 #endif
3688         case CHIP_BONAIRE:
3689         case CHIP_KAVERI:
3690         case CHIP_KABINI:
3691         case CHIP_MULLINS:
3692                 /*
3693                  * We have systems in the wild with these ASICs that require
3694                  * VGA support which is not supported with DC.
3695                  *
3696                  * Fallback to the non-DC driver here by default so as not to
3697                  * cause regressions.
3698                  */
3699                 return amdgpu_dc > 0;
3700         default:
3701                 return amdgpu_dc != 0;
3702 #else
3703         default:
3704                 if (amdgpu_dc > 0)
3705                         DRM_INFO_ONCE("Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n");
3706                 return false;
3707 #endif
3708         }
3709 }
3710
3711 /**
3712  * amdgpu_device_has_dc_support - check if dc is supported
3713  *
3714  * @adev: amdgpu_device pointer
3715  *
3716  * Returns true for supported, false for not supported
3717  */
3718 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3719 {
3720         if (adev->enable_virtual_display ||
3721             (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3722                 return false;
3723
3724         return amdgpu_device_asic_has_dc_support(adev->asic_type);
3725 }
3726
3727 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3728 {
3729         struct amdgpu_device *adev =
3730                 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3731         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3732
3733         /* It's a bug to not have a hive within this function */
3734         if (WARN_ON(!hive))
3735                 return;
3736
3737         /*
3738          * Use task barrier to synchronize all xgmi reset works across the
3739          * hive. task_barrier_enter and task_barrier_exit will block
3740          * until all the threads running the xgmi reset works reach
3741          * those points. task_barrier_full will do both blocks.
3742          */
3743         if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3744
3745                 task_barrier_enter(&hive->tb);
3746                 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3747
3748                 if (adev->asic_reset_res)
3749                         goto fail;
3750
3751                 task_barrier_exit(&hive->tb);
3752                 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3753
3754                 if (adev->asic_reset_res)
3755                         goto fail;
3756
3757                 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB);
3758         } else {
3759
3760                 task_barrier_full(&hive->tb);
3761                 adev->asic_reset_res =  amdgpu_asic_reset(adev);
3762         }
3763
3764 fail:
3765         if (adev->asic_reset_res)
3766                 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3767                          adev->asic_reset_res, adev_to_drm(adev)->unique);
3768         amdgpu_put_xgmi_hive(hive);
3769 }
3770
3771 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3772 {
3773         char *input = amdgpu_lockup_timeout;
3774         char *timeout_setting = NULL;
3775         int index = 0;
3776         long timeout;
3777         int ret = 0;
3778
3779         /*
3780          * By default timeout for non compute jobs is 10000
3781          * and 60000 for compute jobs.
3782          * In SR-IOV or passthrough mode, timeout for compute
3783          * jobs are 60000 by default.
3784          */
3785         adev->gfx_timeout = msecs_to_jiffies(10000);
3786         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3787         if (amdgpu_sriov_vf(adev))
3788                 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3789                                         msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3790         else
3791                 adev->compute_timeout =  msecs_to_jiffies(60000);
3792
3793         if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3794                 while ((timeout_setting = strsep(&input, ",")) &&
3795                                 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3796                         ret = kstrtol(timeout_setting, 0, &timeout);
3797                         if (ret)
3798                                 return ret;
3799
3800                         if (timeout == 0) {
3801                                 index++;
3802                                 continue;
3803                         } else if (timeout < 0) {
3804                                 timeout = MAX_SCHEDULE_TIMEOUT;
3805                                 dev_warn(adev->dev, "lockup timeout disabled");
3806                                 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3807                         } else {
3808                                 timeout = msecs_to_jiffies(timeout);
3809                         }
3810
3811                         switch (index++) {
3812                         case 0:
3813                                 adev->gfx_timeout = timeout;
3814                                 break;
3815                         case 1:
3816                                 adev->compute_timeout = timeout;
3817                                 break;
3818                         case 2:
3819                                 adev->sdma_timeout = timeout;
3820                                 break;
3821                         case 3:
3822                                 adev->video_timeout = timeout;
3823                                 break;
3824                         default:
3825                                 break;
3826                         }
3827                 }
3828                 /*
3829                  * There is only one value specified and
3830                  * it should apply to all non-compute jobs.
3831                  */
3832                 if (index == 1) {
3833                         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3834                         if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3835                                 adev->compute_timeout = adev->gfx_timeout;
3836                 }
3837         }
3838
3839         return ret;
3840 }
3841
3842 /**
3843  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3844  *
3845  * @adev: amdgpu_device pointer
3846  *
3847  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3848  */
3849 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3850 {
3851         struct iommu_domain *domain;
3852
3853         domain = iommu_get_domain_for_dev(adev->dev);
3854         if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3855                 adev->ram_is_direct_mapped = true;
3856 }
3857
3858 static const struct attribute *amdgpu_dev_attributes[] = {
3859         &dev_attr_pcie_replay_count.attr,
3860         NULL
3861 };
3862
3863 static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)
3864 {
3865         if (amdgpu_mcbp == 1)
3866                 adev->gfx.mcbp = true;
3867         else if (amdgpu_mcbp == 0)
3868                 adev->gfx.mcbp = false;
3869
3870         if (amdgpu_sriov_vf(adev))
3871                 adev->gfx.mcbp = true;
3872
3873         if (adev->gfx.mcbp)
3874                 DRM_INFO("MCBP is enabled\n");
3875 }
3876
3877 /**
3878  * amdgpu_device_init - initialize the driver
3879  *
3880  * @adev: amdgpu_device pointer
3881  * @flags: driver flags
3882  *
3883  * Initializes the driver info and hw (all asics).
3884  * Returns 0 for success or an error on failure.
3885  * Called at driver startup.
3886  */
3887 int amdgpu_device_init(struct amdgpu_device *adev,
3888                        uint32_t flags)
3889 {
3890         struct drm_device *ddev = adev_to_drm(adev);
3891         struct pci_dev *pdev = adev->pdev;
3892         int r, i;
3893         bool px = false;
3894         u32 max_MBps;
3895         int tmp;
3896
3897         adev->shutdown = false;
3898         adev->flags = flags;
3899
3900         if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3901                 adev->asic_type = amdgpu_force_asic_type;
3902         else
3903                 adev->asic_type = flags & AMD_ASIC_MASK;
3904
3905         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3906         if (amdgpu_emu_mode == 1)
3907                 adev->usec_timeout *= 10;
3908         adev->gmc.gart_size = 512 * 1024 * 1024;
3909         adev->accel_working = false;
3910         adev->num_rings = 0;
3911         RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3912         adev->mman.buffer_funcs = NULL;
3913         adev->mman.buffer_funcs_ring = NULL;
3914         adev->vm_manager.vm_pte_funcs = NULL;
3915         adev->vm_manager.vm_pte_num_scheds = 0;
3916         adev->gmc.gmc_funcs = NULL;
3917         adev->harvest_ip_mask = 0x0;
3918         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3919         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3920
3921         adev->smc_rreg = &amdgpu_invalid_rreg;
3922         adev->smc_wreg = &amdgpu_invalid_wreg;
3923         adev->pcie_rreg = &amdgpu_invalid_rreg;
3924         adev->pcie_wreg = &amdgpu_invalid_wreg;
3925         adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext;
3926         adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext;
3927         adev->pciep_rreg = &amdgpu_invalid_rreg;
3928         adev->pciep_wreg = &amdgpu_invalid_wreg;
3929         adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3930         adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3931         adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext;
3932         adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext;
3933         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3934         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3935         adev->didt_rreg = &amdgpu_invalid_rreg;
3936         adev->didt_wreg = &amdgpu_invalid_wreg;
3937         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3938         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3939         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3940         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3941
3942         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3943                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3944                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3945
3946         /* mutex initialization are all done here so we
3947          * can recall function without having locking issues
3948          */
3949         mutex_init(&adev->firmware.mutex);
3950         mutex_init(&adev->pm.mutex);
3951         mutex_init(&adev->gfx.gpu_clock_mutex);
3952         mutex_init(&adev->srbm_mutex);
3953         mutex_init(&adev->gfx.pipe_reserve_mutex);
3954         mutex_init(&adev->gfx.gfx_off_mutex);
3955         mutex_init(&adev->gfx.partition_mutex);
3956         mutex_init(&adev->grbm_idx_mutex);
3957         mutex_init(&adev->mn_lock);
3958         mutex_init(&adev->virt.vf_errors.lock);
3959         hash_init(adev->mn_hash);
3960         mutex_init(&adev->psp.mutex);
3961         mutex_init(&adev->notifier_lock);
3962         mutex_init(&adev->pm.stable_pstate_ctx_lock);
3963         mutex_init(&adev->benchmark_mutex);
3964
3965         amdgpu_device_init_apu_flags(adev);
3966
3967         r = amdgpu_device_check_arguments(adev);
3968         if (r)
3969                 return r;
3970
3971         spin_lock_init(&adev->mmio_idx_lock);
3972         spin_lock_init(&adev->smc_idx_lock);
3973         spin_lock_init(&adev->pcie_idx_lock);
3974         spin_lock_init(&adev->uvd_ctx_idx_lock);
3975         spin_lock_init(&adev->didt_idx_lock);
3976         spin_lock_init(&adev->gc_cac_idx_lock);
3977         spin_lock_init(&adev->se_cac_idx_lock);
3978         spin_lock_init(&adev->audio_endpt_idx_lock);
3979         spin_lock_init(&adev->mm_stats.lock);
3980
3981         INIT_LIST_HEAD(&adev->shadow_list);
3982         mutex_init(&adev->shadow_list_lock);
3983
3984         INIT_LIST_HEAD(&adev->reset_list);
3985
3986         INIT_LIST_HEAD(&adev->ras_list);
3987
3988         INIT_LIST_HEAD(&adev->pm.od_kobj_list);
3989
3990         INIT_DELAYED_WORK(&adev->delayed_init_work,
3991                           amdgpu_device_delayed_init_work_handler);
3992         INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3993                           amdgpu_device_delay_enable_gfx_off);
3994
3995         INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3996
3997         adev->gfx.gfx_off_req_count = 1;
3998         adev->gfx.gfx_off_residency = 0;
3999         adev->gfx.gfx_off_entrycount = 0;
4000         adev->pm.ac_power = power_supply_is_system_supplied() > 0;
4001
4002         atomic_set(&adev->throttling_logging_enabled, 1);
4003         /*
4004          * If throttling continues, logging will be performed every minute
4005          * to avoid log flooding. "-1" is subtracted since the thermal
4006          * throttling interrupt comes every second. Thus, the total logging
4007          * interval is 59 seconds(retelimited printk interval) + 1(waiting
4008          * for throttling interrupt) = 60 seconds.
4009          */
4010         ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
4011         ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
4012
4013         /* Registers mapping */
4014         /* TODO: block userspace mapping of io register */
4015         if (adev->asic_type >= CHIP_BONAIRE) {
4016                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
4017                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
4018         } else {
4019                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
4020                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
4021         }
4022
4023         for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
4024                 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
4025
4026         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
4027         if (!adev->rmmio)
4028                 return -ENOMEM;
4029
4030         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
4031         DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size);
4032
4033         /*
4034          * Reset domain needs to be present early, before XGMI hive discovered
4035          * (if any) and intitialized to use reset sem and in_gpu reset flag
4036          * early on during init and before calling to RREG32.
4037          */
4038         adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
4039         if (!adev->reset_domain)
4040                 return -ENOMEM;
4041
4042         /* detect hw virtualization here */
4043         amdgpu_detect_virtualization(adev);
4044
4045         amdgpu_device_get_pcie_info(adev);
4046
4047         r = amdgpu_aca_init(adev);
4048         if (r)
4049                 return r;
4050
4051         r = amdgpu_device_get_job_timeout_settings(adev);
4052         if (r) {
4053                 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
4054                 return r;
4055         }
4056
4057         /* early init functions */
4058         r = amdgpu_device_ip_early_init(adev);
4059         if (r)
4060                 return r;
4061
4062         amdgpu_device_set_mcbp(adev);
4063
4064         /* Get rid of things like offb */
4065         r = drm_aperture_remove_conflicting_pci_framebuffers(adev->pdev, &amdgpu_kms_driver);
4066         if (r)
4067                 return r;
4068
4069         /* Enable TMZ based on IP_VERSION */
4070         amdgpu_gmc_tmz_set(adev);
4071
4072         amdgpu_gmc_noretry_set(adev);
4073         /* Need to get xgmi info early to decide the reset behavior*/
4074         if (adev->gmc.xgmi.supported) {
4075                 r = adev->gfxhub.funcs->get_xgmi_info(adev);
4076                 if (r)
4077                         return r;
4078         }
4079
4080         /* enable PCIE atomic ops */
4081         if (amdgpu_sriov_vf(adev)) {
4082                 if (adev->virt.fw_reserve.p_pf2vf)
4083                         adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
4084                                                       adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
4085                                 (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4086         /* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
4087          * internal path natively support atomics, set have_atomics_support to true.
4088          */
4089         } else if ((adev->flags & AMD_IS_APU) &&
4090                    (amdgpu_ip_version(adev, GC_HWIP, 0) >
4091                     IP_VERSION(9, 0, 0))) {
4092                 adev->have_atomics_support = true;
4093         } else {
4094                 adev->have_atomics_support =
4095                         !pci_enable_atomic_ops_to_root(adev->pdev,
4096                                           PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
4097                                           PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4098         }
4099
4100         if (!adev->have_atomics_support)
4101                 dev_info(adev->dev, "PCIE atomic ops is not supported\n");
4102
4103         /* doorbell bar mapping and doorbell index init*/
4104         amdgpu_doorbell_init(adev);
4105
4106         if (amdgpu_emu_mode == 1) {
4107                 /* post the asic on emulation mode */
4108                 emu_soc_asic_init(adev);
4109                 goto fence_driver_init;
4110         }
4111
4112         amdgpu_reset_init(adev);
4113
4114         /* detect if we are with an SRIOV vbios */
4115         if (adev->bios)
4116                 amdgpu_device_detect_sriov_bios(adev);
4117
4118         /* check if we need to reset the asic
4119          *  E.g., driver was not cleanly unloaded previously, etc.
4120          */
4121         if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
4122                 if (adev->gmc.xgmi.num_physical_nodes) {
4123                         dev_info(adev->dev, "Pending hive reset.\n");
4124                         adev->gmc.xgmi.pending_reset = true;
4125                         /* Only need to init necessary block for SMU to handle the reset */
4126                         for (i = 0; i < adev->num_ip_blocks; i++) {
4127                                 if (!adev->ip_blocks[i].status.valid)
4128                                         continue;
4129                                 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
4130                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
4131                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
4132                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
4133                                         DRM_DEBUG("IP %s disabled for hw_init.\n",
4134                                                 adev->ip_blocks[i].version->funcs->name);
4135                                         adev->ip_blocks[i].status.hw = true;
4136                                 }
4137                         }
4138                 } else {
4139                         switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) {
4140                         case IP_VERSION(13, 0, 0):
4141                         case IP_VERSION(13, 0, 7):
4142                         case IP_VERSION(13, 0, 10):
4143                                 r = psp_gpu_reset(adev);
4144                                 break;
4145                         default:
4146                                 tmp = amdgpu_reset_method;
4147                                 /* It should do a default reset when loading or reloading the driver,
4148                                  * regardless of the module parameter reset_method.
4149                                  */
4150                                 amdgpu_reset_method = AMD_RESET_METHOD_NONE;
4151                                 r = amdgpu_asic_reset(adev);
4152                                 amdgpu_reset_method = tmp;
4153                                 break;
4154                         }
4155
4156                         if (r) {
4157                                 dev_err(adev->dev, "asic reset on init failed\n");
4158                                 goto failed;
4159                         }
4160                 }
4161         }
4162
4163         /* Post card if necessary */
4164         if (amdgpu_device_need_post(adev)) {
4165                 if (!adev->bios) {
4166                         dev_err(adev->dev, "no vBIOS found\n");
4167                         r = -EINVAL;
4168                         goto failed;
4169                 }
4170                 DRM_INFO("GPU posting now...\n");
4171                 r = amdgpu_device_asic_init(adev);
4172                 if (r) {
4173                         dev_err(adev->dev, "gpu post error!\n");
4174                         goto failed;
4175                 }
4176         }
4177
4178         if (adev->bios) {
4179                 if (adev->is_atom_fw) {
4180                         /* Initialize clocks */
4181                         r = amdgpu_atomfirmware_get_clock_info(adev);
4182                         if (r) {
4183                                 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
4184                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4185                                 goto failed;
4186                         }
4187                 } else {
4188                         /* Initialize clocks */
4189                         r = amdgpu_atombios_get_clock_info(adev);
4190                         if (r) {
4191                                 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
4192                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4193                                 goto failed;
4194                         }
4195                         /* init i2c buses */
4196                         if (!amdgpu_device_has_dc_support(adev))
4197                                 amdgpu_atombios_i2c_init(adev);
4198                 }
4199         }
4200
4201 fence_driver_init:
4202         /* Fence driver */
4203         r = amdgpu_fence_driver_sw_init(adev);
4204         if (r) {
4205                 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
4206                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
4207                 goto failed;
4208         }
4209
4210         /* init the mode config */
4211         drm_mode_config_init(adev_to_drm(adev));
4212
4213         r = amdgpu_device_ip_init(adev);
4214         if (r) {
4215                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
4216                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
4217                 goto release_ras_con;
4218         }
4219
4220         amdgpu_fence_driver_hw_init(adev);
4221
4222         dev_info(adev->dev,
4223                 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
4224                         adev->gfx.config.max_shader_engines,
4225                         adev->gfx.config.max_sh_per_se,
4226                         adev->gfx.config.max_cu_per_sh,
4227                         adev->gfx.cu_info.number);
4228
4229         adev->accel_working = true;
4230
4231         amdgpu_vm_check_compute_bug(adev);
4232
4233         /* Initialize the buffer migration limit. */
4234         if (amdgpu_moverate >= 0)
4235                 max_MBps = amdgpu_moverate;
4236         else
4237                 max_MBps = 8; /* Allow 8 MB/s. */
4238         /* Get a log2 for easy divisions. */
4239         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
4240
4241         /*
4242          * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
4243          * Otherwise the mgpu fan boost feature will be skipped due to the
4244          * gpu instance is counted less.
4245          */
4246         amdgpu_register_gpu_instance(adev);
4247
4248         /* enable clockgating, etc. after ib tests, etc. since some blocks require
4249          * explicit gating rather than handling it automatically.
4250          */
4251         if (!adev->gmc.xgmi.pending_reset) {
4252                 r = amdgpu_device_ip_late_init(adev);
4253                 if (r) {
4254                         dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
4255                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
4256                         goto release_ras_con;
4257                 }
4258                 /* must succeed. */
4259                 amdgpu_ras_resume(adev);
4260                 queue_delayed_work(system_wq, &adev->delayed_init_work,
4261                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
4262         }
4263
4264         if (amdgpu_sriov_vf(adev)) {
4265                 amdgpu_virt_release_full_gpu(adev, true);
4266                 flush_delayed_work(&adev->delayed_init_work);
4267         }
4268
4269         /*
4270          * Place those sysfs registering after `late_init`. As some of those
4271          * operations performed in `late_init` might affect the sysfs
4272          * interfaces creating.
4273          */
4274         r = amdgpu_atombios_sysfs_init(adev);
4275         if (r)
4276                 drm_err(&adev->ddev,
4277                         "registering atombios sysfs failed (%d).\n", r);
4278
4279         r = amdgpu_pm_sysfs_init(adev);
4280         if (r)
4281                 DRM_ERROR("registering pm sysfs failed (%d).\n", r);
4282
4283         r = amdgpu_ucode_sysfs_init(adev);
4284         if (r) {
4285                 adev->ucode_sysfs_en = false;
4286                 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
4287         } else
4288                 adev->ucode_sysfs_en = true;
4289
4290         r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
4291         if (r)
4292                 dev_err(adev->dev, "Could not create amdgpu device attr\n");
4293
4294         r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group);
4295         if (r)
4296                 dev_err(adev->dev,
4297                         "Could not create amdgpu board attributes\n");
4298
4299         amdgpu_fru_sysfs_init(adev);
4300         amdgpu_reg_state_sysfs_init(adev);
4301
4302         if (IS_ENABLED(CONFIG_PERF_EVENTS))
4303                 r = amdgpu_pmu_init(adev);
4304         if (r)
4305                 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
4306
4307         /* Have stored pci confspace at hand for restore in sudden PCI error */
4308         if (amdgpu_device_cache_pci_state(adev->pdev))
4309                 pci_restore_state(pdev);
4310
4311         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
4312         /* this will fail for cards that aren't VGA class devices, just
4313          * ignore it
4314          */
4315         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4316                 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
4317
4318         px = amdgpu_device_supports_px(ddev);
4319
4320         if (px || (!dev_is_removable(&adev->pdev->dev) &&
4321                                 apple_gmux_detect(NULL, NULL)))
4322                 vga_switcheroo_register_client(adev->pdev,
4323                                                &amdgpu_switcheroo_ops, px);
4324
4325         if (px)
4326                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
4327
4328         if (adev->gmc.xgmi.pending_reset)
4329                 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
4330                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
4331
4332         amdgpu_device_check_iommu_direct_map(adev);
4333
4334         return 0;
4335
4336 release_ras_con:
4337         if (amdgpu_sriov_vf(adev))
4338                 amdgpu_virt_release_full_gpu(adev, true);
4339
4340         /* failed in exclusive mode due to timeout */
4341         if (amdgpu_sriov_vf(adev) &&
4342                 !amdgpu_sriov_runtime(adev) &&
4343                 amdgpu_virt_mmio_blocked(adev) &&
4344                 !amdgpu_virt_wait_reset(adev)) {
4345                 dev_err(adev->dev, "VF exclusive mode timeout\n");
4346                 /* Don't send request since VF is inactive. */
4347                 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
4348                 adev->virt.ops = NULL;
4349                 r = -EAGAIN;
4350         }
4351         amdgpu_release_ras_context(adev);
4352
4353 failed:
4354         amdgpu_vf_error_trans_all(adev);
4355
4356         return r;
4357 }
4358
4359 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
4360 {
4361
4362         /* Clear all CPU mappings pointing to this device */
4363         unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
4364
4365         /* Unmap all mapped bars - Doorbell, registers and VRAM */
4366         amdgpu_doorbell_fini(adev);
4367
4368         iounmap(adev->rmmio);
4369         adev->rmmio = NULL;
4370         if (adev->mman.aper_base_kaddr)
4371                 iounmap(adev->mman.aper_base_kaddr);
4372         adev->mman.aper_base_kaddr = NULL;
4373
4374         /* Memory manager related */
4375         if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
4376                 arch_phys_wc_del(adev->gmc.vram_mtrr);
4377                 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
4378         }
4379 }
4380
4381 /**
4382  * amdgpu_device_fini_hw - tear down the driver
4383  *
4384  * @adev: amdgpu_device pointer
4385  *
4386  * Tear down the driver info (all asics).
4387  * Called at driver shutdown.
4388  */
4389 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4390 {
4391         dev_info(adev->dev, "amdgpu: finishing device.\n");
4392         flush_delayed_work(&adev->delayed_init_work);
4393         adev->shutdown = true;
4394
4395         /* make sure IB test finished before entering exclusive mode
4396          * to avoid preemption on IB test
4397          */
4398         if (amdgpu_sriov_vf(adev)) {
4399                 amdgpu_virt_request_full_gpu(adev, false);
4400                 amdgpu_virt_fini_data_exchange(adev);
4401         }
4402
4403         /* disable all interrupts */
4404         amdgpu_irq_disable_all(adev);
4405         if (adev->mode_info.mode_config_initialized) {
4406                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
4407                         drm_helper_force_disable_all(adev_to_drm(adev));
4408                 else
4409                         drm_atomic_helper_shutdown(adev_to_drm(adev));
4410         }
4411         amdgpu_fence_driver_hw_fini(adev);
4412
4413         if (adev->mman.initialized)
4414                 drain_workqueue(adev->mman.bdev.wq);
4415
4416         if (adev->pm.sysfs_initialized)
4417                 amdgpu_pm_sysfs_fini(adev);
4418         if (adev->ucode_sysfs_en)
4419                 amdgpu_ucode_sysfs_fini(adev);
4420         sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4421         amdgpu_fru_sysfs_fini(adev);
4422
4423         amdgpu_reg_state_sysfs_fini(adev);
4424
4425         /* disable ras feature must before hw fini */
4426         amdgpu_ras_pre_fini(adev);
4427
4428         amdgpu_ttm_set_buffer_funcs_status(adev, false);
4429
4430         amdgpu_device_ip_fini_early(adev);
4431
4432         amdgpu_irq_fini_hw(adev);
4433
4434         if (adev->mman.initialized)
4435                 ttm_device_clear_dma_mappings(&adev->mman.bdev);
4436
4437         amdgpu_gart_dummy_page_fini(adev);
4438
4439         if (drm_dev_is_unplugged(adev_to_drm(adev)))
4440                 amdgpu_device_unmap_mmio(adev);
4441
4442 }
4443
4444 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4445 {
4446         int idx;
4447         bool px;
4448
4449         amdgpu_fence_driver_sw_fini(adev);
4450         amdgpu_device_ip_fini(adev);
4451         amdgpu_ucode_release(&adev->firmware.gpu_info_fw);
4452         adev->accel_working = false;
4453         dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4454
4455         amdgpu_reset_fini(adev);
4456
4457         amdgpu_aca_fini(adev);
4458
4459         /* free i2c buses */
4460         if (!amdgpu_device_has_dc_support(adev))
4461                 amdgpu_i2c_fini(adev);
4462
4463         if (amdgpu_emu_mode != 1)
4464                 amdgpu_atombios_fini(adev);
4465
4466         kfree(adev->bios);
4467         adev->bios = NULL;
4468
4469         kfree(adev->fru_info);
4470         adev->fru_info = NULL;
4471
4472         px = amdgpu_device_supports_px(adev_to_drm(adev));
4473
4474         if (px || (!dev_is_removable(&adev->pdev->dev) &&
4475                                 apple_gmux_detect(NULL, NULL)))
4476                 vga_switcheroo_unregister_client(adev->pdev);
4477
4478         if (px)
4479                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
4480
4481         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4482                 vga_client_unregister(adev->pdev);
4483
4484         if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4485
4486                 iounmap(adev->rmmio);
4487                 adev->rmmio = NULL;
4488                 amdgpu_doorbell_fini(adev);
4489                 drm_dev_exit(idx);
4490         }
4491
4492         if (IS_ENABLED(CONFIG_PERF_EVENTS))
4493                 amdgpu_pmu_fini(adev);
4494         if (adev->mman.discovery_bin)
4495                 amdgpu_discovery_fini(adev);
4496
4497         amdgpu_reset_put_reset_domain(adev->reset_domain);
4498         adev->reset_domain = NULL;
4499
4500         kfree(adev->pci_state);
4501
4502 }
4503
4504 /**
4505  * amdgpu_device_evict_resources - evict device resources
4506  * @adev: amdgpu device object
4507  *
4508  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4509  * of the vram memory type. Mainly used for evicting device resources
4510  * at suspend time.
4511  *
4512  */
4513 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4514 {
4515         int ret;
4516
4517         /* No need to evict vram on APUs for suspend to ram or s2idle */
4518         if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4519                 return 0;
4520
4521         ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4522         if (ret)
4523                 DRM_WARN("evicting device resources failed\n");
4524         return ret;
4525 }
4526
4527 /*
4528  * Suspend & resume.
4529  */
4530 /**
4531  * amdgpu_device_prepare - prepare for device suspend
4532  *
4533  * @dev: drm dev pointer
4534  *
4535  * Prepare to put the hw in the suspend state (all asics).
4536  * Returns 0 for success or an error on failure.
4537  * Called at driver suspend.
4538  */
4539 int amdgpu_device_prepare(struct drm_device *dev)
4540 {
4541         struct amdgpu_device *adev = drm_to_adev(dev);
4542         int i, r;
4543
4544         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4545                 return 0;
4546
4547         /* Evict the majority of BOs before starting suspend sequence */
4548         r = amdgpu_device_evict_resources(adev);
4549         if (r)
4550                 return r;
4551
4552         for (i = 0; i < adev->num_ip_blocks; i++) {
4553                 if (!adev->ip_blocks[i].status.valid)
4554                         continue;
4555                 if (!adev->ip_blocks[i].version->funcs->prepare_suspend)
4556                         continue;
4557                 r = adev->ip_blocks[i].version->funcs->prepare_suspend((void *)adev);
4558                 if (r)
4559                         return r;
4560         }
4561
4562         return 0;
4563 }
4564
4565 /**
4566  * amdgpu_device_suspend - initiate device suspend
4567  *
4568  * @dev: drm dev pointer
4569  * @fbcon : notify the fbdev of suspend
4570  *
4571  * Puts the hw in the suspend state (all asics).
4572  * Returns 0 for success or an error on failure.
4573  * Called at driver suspend.
4574  */
4575 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
4576 {
4577         struct amdgpu_device *adev = drm_to_adev(dev);
4578         int r = 0;
4579
4580         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4581                 return 0;
4582
4583         adev->in_suspend = true;
4584
4585         if (amdgpu_sriov_vf(adev)) {
4586                 amdgpu_virt_fini_data_exchange(adev);
4587                 r = amdgpu_virt_request_full_gpu(adev, false);
4588                 if (r)
4589                         return r;
4590         }
4591
4592         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4593                 DRM_WARN("smart shift update failed\n");
4594
4595         if (fbcon)
4596                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
4597
4598         cancel_delayed_work_sync(&adev->delayed_init_work);
4599         flush_delayed_work(&adev->gfx.gfx_off_delay_work);
4600
4601         amdgpu_ras_suspend(adev);
4602
4603         amdgpu_device_ip_suspend_phase1(adev);
4604
4605         if (!adev->in_s0ix)
4606                 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4607
4608         r = amdgpu_device_evict_resources(adev);
4609         if (r)
4610                 return r;
4611
4612         amdgpu_ttm_set_buffer_funcs_status(adev, false);
4613
4614         amdgpu_fence_driver_hw_fini(adev);
4615
4616         amdgpu_device_ip_suspend_phase2(adev);
4617
4618         if (amdgpu_sriov_vf(adev))
4619                 amdgpu_virt_release_full_gpu(adev, false);
4620
4621         r = amdgpu_dpm_notify_rlc_state(adev, false);
4622         if (r)
4623                 return r;
4624
4625         return 0;
4626 }
4627
4628 /**
4629  * amdgpu_device_resume - initiate device resume
4630  *
4631  * @dev: drm dev pointer
4632  * @fbcon : notify the fbdev of resume
4633  *
4634  * Bring the hw back to operating state (all asics).
4635  * Returns 0 for success or an error on failure.
4636  * Called at driver resume.
4637  */
4638 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4639 {
4640         struct amdgpu_device *adev = drm_to_adev(dev);
4641         int r = 0;
4642
4643         if (amdgpu_sriov_vf(adev)) {
4644                 r = amdgpu_virt_request_full_gpu(adev, true);
4645                 if (r)
4646                         return r;
4647         }
4648
4649         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4650                 return 0;
4651
4652         if (adev->in_s0ix)
4653                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4654
4655         /* post card */
4656         if (amdgpu_device_need_post(adev)) {
4657                 r = amdgpu_device_asic_init(adev);
4658                 if (r)
4659                         dev_err(adev->dev, "amdgpu asic init failed\n");
4660         }
4661
4662         r = amdgpu_device_ip_resume(adev);
4663
4664         if (r) {
4665                 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4666                 goto exit;
4667         }
4668         amdgpu_fence_driver_hw_init(adev);
4669
4670         if (!adev->in_s0ix) {
4671                 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4672                 if (r)
4673                         goto exit;
4674         }
4675
4676         r = amdgpu_device_ip_late_init(adev);
4677         if (r)
4678                 goto exit;
4679
4680         queue_delayed_work(system_wq, &adev->delayed_init_work,
4681                            msecs_to_jiffies(AMDGPU_RESUME_MS));
4682 exit:
4683         if (amdgpu_sriov_vf(adev)) {
4684                 amdgpu_virt_init_data_exchange(adev);
4685                 amdgpu_virt_release_full_gpu(adev, true);
4686         }
4687
4688         if (r)
4689                 return r;
4690
4691         /* Make sure IB tests flushed */
4692         flush_delayed_work(&adev->delayed_init_work);
4693
4694         if (fbcon)
4695                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
4696
4697         amdgpu_ras_resume(adev);
4698
4699         if (adev->mode_info.num_crtc) {
4700                 /*
4701                  * Most of the connector probing functions try to acquire runtime pm
4702                  * refs to ensure that the GPU is powered on when connector polling is
4703                  * performed. Since we're calling this from a runtime PM callback,
4704                  * trying to acquire rpm refs will cause us to deadlock.
4705                  *
4706                  * Since we're guaranteed to be holding the rpm lock, it's safe to
4707                  * temporarily disable the rpm helpers so this doesn't deadlock us.
4708                  */
4709 #ifdef CONFIG_PM
4710                 dev->dev->power.disable_depth++;
4711 #endif
4712                 if (!adev->dc_enabled)
4713                         drm_helper_hpd_irq_event(dev);
4714                 else
4715                         drm_kms_helper_hotplug_event(dev);
4716 #ifdef CONFIG_PM
4717                 dev->dev->power.disable_depth--;
4718 #endif
4719         }
4720         adev->in_suspend = false;
4721
4722         if (adev->enable_mes)
4723                 amdgpu_mes_self_test(adev);
4724
4725         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
4726                 DRM_WARN("smart shift update failed\n");
4727
4728         return 0;
4729 }
4730
4731 /**
4732  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4733  *
4734  * @adev: amdgpu_device pointer
4735  *
4736  * The list of all the hardware IPs that make up the asic is walked and
4737  * the check_soft_reset callbacks are run.  check_soft_reset determines
4738  * if the asic is still hung or not.
4739  * Returns true if any of the IPs are still in a hung state, false if not.
4740  */
4741 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4742 {
4743         int i;
4744         bool asic_hang = false;
4745
4746         if (amdgpu_sriov_vf(adev))
4747                 return true;
4748
4749         if (amdgpu_asic_need_full_reset(adev))
4750                 return true;
4751
4752         for (i = 0; i < adev->num_ip_blocks; i++) {
4753                 if (!adev->ip_blocks[i].status.valid)
4754                         continue;
4755                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4756                         adev->ip_blocks[i].status.hang =
4757                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4758                 if (adev->ip_blocks[i].status.hang) {
4759                         dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4760                         asic_hang = true;
4761                 }
4762         }
4763         return asic_hang;
4764 }
4765
4766 /**
4767  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4768  *
4769  * @adev: amdgpu_device pointer
4770  *
4771  * The list of all the hardware IPs that make up the asic is walked and the
4772  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
4773  * handles any IP specific hardware or software state changes that are
4774  * necessary for a soft reset to succeed.
4775  * Returns 0 on success, negative error code on failure.
4776  */
4777 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4778 {
4779         int i, r = 0;
4780
4781         for (i = 0; i < adev->num_ip_blocks; i++) {
4782                 if (!adev->ip_blocks[i].status.valid)
4783                         continue;
4784                 if (adev->ip_blocks[i].status.hang &&
4785                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4786                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4787                         if (r)
4788                                 return r;
4789                 }
4790         }
4791
4792         return 0;
4793 }
4794
4795 /**
4796  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4797  *
4798  * @adev: amdgpu_device pointer
4799  *
4800  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
4801  * reset is necessary to recover.
4802  * Returns true if a full asic reset is required, false if not.
4803  */
4804 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4805 {
4806         int i;
4807
4808         if (amdgpu_asic_need_full_reset(adev))
4809                 return true;
4810
4811         for (i = 0; i < adev->num_ip_blocks; i++) {
4812                 if (!adev->ip_blocks[i].status.valid)
4813                         continue;
4814                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4815                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4816                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4817                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4818                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4819                         if (adev->ip_blocks[i].status.hang) {
4820                                 dev_info(adev->dev, "Some block need full reset!\n");
4821                                 return true;
4822                         }
4823                 }
4824         }
4825         return false;
4826 }
4827
4828 /**
4829  * amdgpu_device_ip_soft_reset - do a soft reset
4830  *
4831  * @adev: amdgpu_device pointer
4832  *
4833  * The list of all the hardware IPs that make up the asic is walked and the
4834  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
4835  * IP specific hardware or software state changes that are necessary to soft
4836  * reset the IP.
4837  * Returns 0 on success, negative error code on failure.
4838  */
4839 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4840 {
4841         int i, r = 0;
4842
4843         for (i = 0; i < adev->num_ip_blocks; i++) {
4844                 if (!adev->ip_blocks[i].status.valid)
4845                         continue;
4846                 if (adev->ip_blocks[i].status.hang &&
4847                     adev->ip_blocks[i].version->funcs->soft_reset) {
4848                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4849                         if (r)
4850                                 return r;
4851                 }
4852         }
4853
4854         return 0;
4855 }
4856
4857 /**
4858  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4859  *
4860  * @adev: amdgpu_device pointer
4861  *
4862  * The list of all the hardware IPs that make up the asic is walked and the
4863  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4864  * handles any IP specific hardware or software state changes that are
4865  * necessary after the IP has been soft reset.
4866  * Returns 0 on success, negative error code on failure.
4867  */
4868 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4869 {
4870         int i, r = 0;
4871
4872         for (i = 0; i < adev->num_ip_blocks; i++) {
4873                 if (!adev->ip_blocks[i].status.valid)
4874                         continue;
4875                 if (adev->ip_blocks[i].status.hang &&
4876                     adev->ip_blocks[i].version->funcs->post_soft_reset)
4877                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4878                 if (r)
4879                         return r;
4880         }
4881
4882         return 0;
4883 }
4884
4885 /**
4886  * amdgpu_device_recover_vram - Recover some VRAM contents
4887  *
4888  * @adev: amdgpu_device pointer
4889  *
4890  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4891  * restore things like GPUVM page tables after a GPU reset where
4892  * the contents of VRAM might be lost.
4893  *
4894  * Returns:
4895  * 0 on success, negative error code on failure.
4896  */
4897 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4898 {
4899         struct dma_fence *fence = NULL, *next = NULL;
4900         struct amdgpu_bo *shadow;
4901         struct amdgpu_bo_vm *vmbo;
4902         long r = 1, tmo;
4903
4904         if (amdgpu_sriov_runtime(adev))
4905                 tmo = msecs_to_jiffies(8000);
4906         else
4907                 tmo = msecs_to_jiffies(100);
4908
4909         dev_info(adev->dev, "recover vram bo from shadow start\n");
4910         mutex_lock(&adev->shadow_list_lock);
4911         list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4912                 /* If vm is compute context or adev is APU, shadow will be NULL */
4913                 if (!vmbo->shadow)
4914                         continue;
4915                 shadow = vmbo->shadow;
4916
4917                 /* No need to recover an evicted BO */
4918                 if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4919                     shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4920                     shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4921                         continue;
4922
4923                 r = amdgpu_bo_restore_shadow(shadow, &next);
4924                 if (r)
4925                         break;
4926
4927                 if (fence) {
4928                         tmo = dma_fence_wait_timeout(fence, false, tmo);
4929                         dma_fence_put(fence);
4930                         fence = next;
4931                         if (tmo == 0) {
4932                                 r = -ETIMEDOUT;
4933                                 break;
4934                         } else if (tmo < 0) {
4935                                 r = tmo;
4936                                 break;
4937                         }
4938                 } else {
4939                         fence = next;
4940                 }
4941         }
4942         mutex_unlock(&adev->shadow_list_lock);
4943
4944         if (fence)
4945                 tmo = dma_fence_wait_timeout(fence, false, tmo);
4946         dma_fence_put(fence);
4947
4948         if (r < 0 || tmo <= 0) {
4949                 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4950                 return -EIO;
4951         }
4952
4953         dev_info(adev->dev, "recover vram bo from shadow done\n");
4954         return 0;
4955 }
4956
4957
4958 /**
4959  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4960  *
4961  * @adev: amdgpu_device pointer
4962  * @from_hypervisor: request from hypervisor
4963  *
4964  * do VF FLR and reinitialize Asic
4965  * return 0 means succeeded otherwise failed
4966  */
4967 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4968                                      bool from_hypervisor)
4969 {
4970         int r;
4971         struct amdgpu_hive_info *hive = NULL;
4972         int retry_limit = 0;
4973
4974 retry:
4975         amdgpu_amdkfd_pre_reset(adev);
4976
4977         if (from_hypervisor)
4978                 r = amdgpu_virt_request_full_gpu(adev, true);
4979         else
4980                 r = amdgpu_virt_reset_gpu(adev);
4981         if (r)
4982                 return r;
4983         amdgpu_irq_gpu_reset_resume_helper(adev);
4984
4985         /* some sw clean up VF needs to do before recover */
4986         amdgpu_virt_post_reset(adev);
4987
4988         /* Resume IP prior to SMC */
4989         r = amdgpu_device_ip_reinit_early_sriov(adev);
4990         if (r)
4991                 goto error;
4992
4993         amdgpu_virt_init_data_exchange(adev);
4994
4995         r = amdgpu_device_fw_loading(adev);
4996         if (r)
4997                 return r;
4998
4999         /* now we are okay to resume SMC/CP/SDMA */
5000         r = amdgpu_device_ip_reinit_late_sriov(adev);
5001         if (r)
5002                 goto error;
5003
5004         hive = amdgpu_get_xgmi_hive(adev);
5005         /* Update PSP FW topology after reset */
5006         if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
5007                 r = amdgpu_xgmi_update_topology(hive, adev);
5008
5009         if (hive)
5010                 amdgpu_put_xgmi_hive(hive);
5011
5012         if (!r) {
5013                 r = amdgpu_ib_ring_tests(adev);
5014
5015                 amdgpu_amdkfd_post_reset(adev);
5016         }
5017
5018 error:
5019         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
5020                 amdgpu_inc_vram_lost(adev);
5021                 r = amdgpu_device_recover_vram(adev);
5022         }
5023         amdgpu_virt_release_full_gpu(adev, true);
5024
5025         if (AMDGPU_RETRY_SRIOV_RESET(r)) {
5026                 if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) {
5027                         retry_limit++;
5028                         goto retry;
5029                 } else
5030                         DRM_ERROR("GPU reset retry is beyond the retry limit\n");
5031         }
5032
5033         return r;
5034 }
5035
5036 /**
5037  * amdgpu_device_has_job_running - check if there is any job in mirror list
5038  *
5039  * @adev: amdgpu_device pointer
5040  *
5041  * check if there is any job in mirror list
5042  */
5043 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
5044 {
5045         int i;
5046         struct drm_sched_job *job;
5047
5048         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5049                 struct amdgpu_ring *ring = adev->rings[i];
5050
5051                 if (!ring || !drm_sched_wqueue_ready(&ring->sched))
5052                         continue;
5053
5054                 spin_lock(&ring->sched.job_list_lock);
5055                 job = list_first_entry_or_null(&ring->sched.pending_list,
5056                                                struct drm_sched_job, list);
5057                 spin_unlock(&ring->sched.job_list_lock);
5058                 if (job)
5059                         return true;
5060         }
5061         return false;
5062 }
5063
5064 /**
5065  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
5066  *
5067  * @adev: amdgpu_device pointer
5068  *
5069  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
5070  * a hung GPU.
5071  */
5072 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
5073 {
5074
5075         if (amdgpu_gpu_recovery == 0)
5076                 goto disabled;
5077
5078         /* Skip soft reset check in fatal error mode */
5079         if (!amdgpu_ras_is_poison_mode_supported(adev))
5080                 return true;
5081
5082         if (amdgpu_sriov_vf(adev))
5083                 return true;
5084
5085         if (amdgpu_gpu_recovery == -1) {
5086                 switch (adev->asic_type) {
5087 #ifdef CONFIG_DRM_AMDGPU_SI
5088                 case CHIP_VERDE:
5089                 case CHIP_TAHITI:
5090                 case CHIP_PITCAIRN:
5091                 case CHIP_OLAND:
5092                 case CHIP_HAINAN:
5093 #endif
5094 #ifdef CONFIG_DRM_AMDGPU_CIK
5095                 case CHIP_KAVERI:
5096                 case CHIP_KABINI:
5097                 case CHIP_MULLINS:
5098 #endif
5099                 case CHIP_CARRIZO:
5100                 case CHIP_STONEY:
5101                 case CHIP_CYAN_SKILLFISH:
5102                         goto disabled;
5103                 default:
5104                         break;
5105                 }
5106         }
5107
5108         return true;
5109
5110 disabled:
5111                 dev_info(adev->dev, "GPU recovery disabled.\n");
5112                 return false;
5113 }
5114
5115 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
5116 {
5117         u32 i;
5118         int ret = 0;
5119
5120         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
5121
5122         dev_info(adev->dev, "GPU mode1 reset\n");
5123
5124         /* disable BM */
5125         pci_clear_master(adev->pdev);
5126
5127         amdgpu_device_cache_pci_state(adev->pdev);
5128
5129         if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
5130                 dev_info(adev->dev, "GPU smu mode1 reset\n");
5131                 ret = amdgpu_dpm_mode1_reset(adev);
5132         } else {
5133                 dev_info(adev->dev, "GPU psp mode1 reset\n");
5134                 ret = psp_gpu_reset(adev);
5135         }
5136
5137         if (ret)
5138                 goto mode1_reset_failed;
5139
5140         amdgpu_device_load_pci_state(adev->pdev);
5141         ret = amdgpu_psp_wait_for_bootloader(adev);
5142         if (ret)
5143                 goto mode1_reset_failed;
5144
5145         /* wait for asic to come out of reset */
5146         for (i = 0; i < adev->usec_timeout; i++) {
5147                 u32 memsize = adev->nbio.funcs->get_memsize(adev);
5148
5149                 if (memsize != 0xffffffff)
5150                         break;
5151                 udelay(1);
5152         }
5153
5154         if (i >= adev->usec_timeout) {
5155                 ret = -ETIMEDOUT;
5156                 goto mode1_reset_failed;
5157         }
5158
5159         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
5160
5161         return 0;
5162
5163 mode1_reset_failed:
5164         dev_err(adev->dev, "GPU mode1 reset failed\n");
5165         return ret;
5166 }
5167
5168 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
5169                                  struct amdgpu_reset_context *reset_context)
5170 {
5171         int i, r = 0;
5172         struct amdgpu_job *job = NULL;
5173         bool need_full_reset =
5174                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5175
5176         if (reset_context->reset_req_dev == adev)
5177                 job = reset_context->job;
5178
5179         if (amdgpu_sriov_vf(adev)) {
5180                 /* stop the data exchange thread */
5181                 amdgpu_virt_fini_data_exchange(adev);
5182         }
5183
5184         amdgpu_fence_driver_isr_toggle(adev, true);
5185
5186         /* block all schedulers and reset given job's ring */
5187         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5188                 struct amdgpu_ring *ring = adev->rings[i];
5189
5190                 if (!ring || !drm_sched_wqueue_ready(&ring->sched))
5191                         continue;
5192
5193                 /* Clear job fence from fence drv to avoid force_completion
5194                  * leave NULL and vm flush fence in fence drv
5195                  */
5196                 amdgpu_fence_driver_clear_job_fences(ring);
5197
5198                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
5199                 amdgpu_fence_driver_force_completion(ring);
5200         }
5201
5202         amdgpu_fence_driver_isr_toggle(adev, false);
5203
5204         if (job && job->vm)
5205                 drm_sched_increase_karma(&job->base);
5206
5207         r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
5208         /* If reset handler not implemented, continue; otherwise return */
5209         if (r == -EOPNOTSUPP)
5210                 r = 0;
5211         else
5212                 return r;
5213
5214         /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
5215         if (!amdgpu_sriov_vf(adev)) {
5216
5217                 if (!need_full_reset)
5218                         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
5219
5220                 if (!need_full_reset && amdgpu_gpu_recovery &&
5221                     amdgpu_device_ip_check_soft_reset(adev)) {
5222                         amdgpu_device_ip_pre_soft_reset(adev);
5223                         r = amdgpu_device_ip_soft_reset(adev);
5224                         amdgpu_device_ip_post_soft_reset(adev);
5225                         if (r || amdgpu_device_ip_check_soft_reset(adev)) {
5226                                 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
5227                                 need_full_reset = true;
5228                         }
5229                 }
5230
5231                 if (need_full_reset)
5232                         r = amdgpu_device_ip_suspend(adev);
5233                 if (need_full_reset)
5234                         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5235                 else
5236                         clear_bit(AMDGPU_NEED_FULL_RESET,
5237                                   &reset_context->flags);
5238         }
5239
5240         return r;
5241 }
5242
5243 static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev)
5244 {
5245         int i;
5246
5247         lockdep_assert_held(&adev->reset_domain->sem);
5248
5249         for (i = 0; i < adev->reset_info.num_regs; i++) {
5250                 adev->reset_info.reset_dump_reg_value[i] =
5251                         RREG32(adev->reset_info.reset_dump_reg_list[i]);
5252
5253                 trace_amdgpu_reset_reg_dumps(adev->reset_info.reset_dump_reg_list[i],
5254                                              adev->reset_info.reset_dump_reg_value[i]);
5255         }
5256
5257         return 0;
5258 }
5259
5260 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
5261                          struct amdgpu_reset_context *reset_context)
5262 {
5263         struct amdgpu_device *tmp_adev = NULL;
5264         bool need_full_reset, skip_hw_reset, vram_lost = false;
5265         int r = 0;
5266         bool gpu_reset_for_dev_remove = 0;
5267
5268         /* Try reset handler method first */
5269         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5270                                     reset_list);
5271         amdgpu_reset_reg_dumps(tmp_adev);
5272
5273         reset_context->reset_device_list = device_list_handle;
5274         r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
5275         /* If reset handler not implemented, continue; otherwise return */
5276         if (r == -EOPNOTSUPP)
5277                 r = 0;
5278         else
5279                 return r;
5280
5281         /* Reset handler not implemented, use the default method */
5282         need_full_reset =
5283                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5284         skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
5285
5286         gpu_reset_for_dev_remove =
5287                 test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5288                         test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5289
5290         /*
5291          * ASIC reset has to be done on all XGMI hive nodes ASAP
5292          * to allow proper links negotiation in FW (within 1 sec)
5293          */
5294         if (!skip_hw_reset && need_full_reset) {
5295                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5296                         /* For XGMI run all resets in parallel to speed up the process */
5297                         if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5298                                 tmp_adev->gmc.xgmi.pending_reset = false;
5299                                 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
5300                                         r = -EALREADY;
5301                         } else
5302                                 r = amdgpu_asic_reset(tmp_adev);
5303
5304                         if (r) {
5305                                 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
5306                                          r, adev_to_drm(tmp_adev)->unique);
5307                                 goto out;
5308                         }
5309                 }
5310
5311                 /* For XGMI wait for all resets to complete before proceed */
5312                 if (!r) {
5313                         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5314                                 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5315                                         flush_work(&tmp_adev->xgmi_reset_work);
5316                                         r = tmp_adev->asic_reset_res;
5317                                         if (r)
5318                                                 break;
5319                                 }
5320                         }
5321                 }
5322         }
5323
5324         if (!r && amdgpu_ras_intr_triggered()) {
5325                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5326                         amdgpu_ras_reset_error_count(tmp_adev, AMDGPU_RAS_BLOCK__MMHUB);
5327                 }
5328
5329                 amdgpu_ras_intr_cleared();
5330         }
5331
5332         /* Since the mode1 reset affects base ip blocks, the
5333          * phase1 ip blocks need to be resumed. Otherwise there
5334          * will be a BIOS signature error and the psp bootloader
5335          * can't load kdb on the next amdgpu install.
5336          */
5337         if (gpu_reset_for_dev_remove) {
5338                 list_for_each_entry(tmp_adev, device_list_handle, reset_list)
5339                         amdgpu_device_ip_resume_phase1(tmp_adev);
5340
5341                 goto end;
5342         }
5343
5344         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5345                 if (need_full_reset) {
5346                         /* post card */
5347                         r = amdgpu_device_asic_init(tmp_adev);
5348                         if (r) {
5349                                 dev_warn(tmp_adev->dev, "asic atom init failed!");
5350                         } else {
5351                                 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
5352
5353                                 r = amdgpu_device_ip_resume_phase1(tmp_adev);
5354                                 if (r)
5355                                         goto out;
5356
5357                                 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
5358
5359                                 amdgpu_coredump(tmp_adev, vram_lost, reset_context);
5360
5361                                 if (vram_lost) {
5362                                         DRM_INFO("VRAM is lost due to GPU reset!\n");
5363                                         amdgpu_inc_vram_lost(tmp_adev);
5364                                 }
5365
5366                                 r = amdgpu_device_fw_loading(tmp_adev);
5367                                 if (r)
5368                                         return r;
5369
5370                                 r = amdgpu_xcp_restore_partition_mode(
5371                                         tmp_adev->xcp_mgr);
5372                                 if (r)
5373                                         goto out;
5374
5375                                 r = amdgpu_device_ip_resume_phase2(tmp_adev);
5376                                 if (r)
5377                                         goto out;
5378
5379                                 if (tmp_adev->mman.buffer_funcs_ring->sched.ready)
5380                                         amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true);
5381
5382                                 if (vram_lost)
5383                                         amdgpu_device_fill_reset_magic(tmp_adev);
5384
5385                                 /*
5386                                  * Add this ASIC as tracked as reset was already
5387                                  * complete successfully.
5388                                  */
5389                                 amdgpu_register_gpu_instance(tmp_adev);
5390
5391                                 if (!reset_context->hive &&
5392                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5393                                         amdgpu_xgmi_add_device(tmp_adev);
5394
5395                                 r = amdgpu_device_ip_late_init(tmp_adev);
5396                                 if (r)
5397                                         goto out;
5398
5399                                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
5400
5401                                 /*
5402                                  * The GPU enters bad state once faulty pages
5403                                  * by ECC has reached the threshold, and ras
5404                                  * recovery is scheduled next. So add one check
5405                                  * here to break recovery if it indeed exceeds
5406                                  * bad page threshold, and remind user to
5407                                  * retire this GPU or setting one bigger
5408                                  * bad_page_threshold value to fix this once
5409                                  * probing driver again.
5410                                  */
5411                                 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
5412                                         /* must succeed. */
5413                                         amdgpu_ras_resume(tmp_adev);
5414                                 } else {
5415                                         r = -EINVAL;
5416                                         goto out;
5417                                 }
5418
5419                                 /* Update PSP FW topology after reset */
5420                                 if (reset_context->hive &&
5421                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5422                                         r = amdgpu_xgmi_update_topology(
5423                                                 reset_context->hive, tmp_adev);
5424                         }
5425                 }
5426
5427 out:
5428                 if (!r) {
5429                         amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5430                         r = amdgpu_ib_ring_tests(tmp_adev);
5431                         if (r) {
5432                                 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5433                                 need_full_reset = true;
5434                                 r = -EAGAIN;
5435                                 goto end;
5436                         }
5437                 }
5438
5439                 if (!r)
5440                         r = amdgpu_device_recover_vram(tmp_adev);
5441                 else
5442                         tmp_adev->asic_reset_res = r;
5443         }
5444
5445 end:
5446         if (need_full_reset)
5447                 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5448         else
5449                 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5450         return r;
5451 }
5452
5453 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5454 {
5455
5456         switch (amdgpu_asic_reset_method(adev)) {
5457         case AMD_RESET_METHOD_MODE1:
5458                 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5459                 break;
5460         case AMD_RESET_METHOD_MODE2:
5461                 adev->mp1_state = PP_MP1_STATE_RESET;
5462                 break;
5463         default:
5464                 adev->mp1_state = PP_MP1_STATE_NONE;
5465                 break;
5466         }
5467 }
5468
5469 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5470 {
5471         amdgpu_vf_error_trans_all(adev);
5472         adev->mp1_state = PP_MP1_STATE_NONE;
5473 }
5474
5475 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5476 {
5477         struct pci_dev *p = NULL;
5478
5479         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5480                         adev->pdev->bus->number, 1);
5481         if (p) {
5482                 pm_runtime_enable(&(p->dev));
5483                 pm_runtime_resume(&(p->dev));
5484         }
5485
5486         pci_dev_put(p);
5487 }
5488
5489 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5490 {
5491         enum amd_reset_method reset_method;
5492         struct pci_dev *p = NULL;
5493         u64 expires;
5494
5495         /*
5496          * For now, only BACO and mode1 reset are confirmed
5497          * to suffer the audio issue without proper suspended.
5498          */
5499         reset_method = amdgpu_asic_reset_method(adev);
5500         if ((reset_method != AMD_RESET_METHOD_BACO) &&
5501              (reset_method != AMD_RESET_METHOD_MODE1))
5502                 return -EINVAL;
5503
5504         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5505                         adev->pdev->bus->number, 1);
5506         if (!p)
5507                 return -ENODEV;
5508
5509         expires = pm_runtime_autosuspend_expiration(&(p->dev));
5510         if (!expires)
5511                 /*
5512                  * If we cannot get the audio device autosuspend delay,
5513                  * a fixed 4S interval will be used. Considering 3S is
5514                  * the audio controller default autosuspend delay setting.
5515                  * 4S used here is guaranteed to cover that.
5516                  */
5517                 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5518
5519         while (!pm_runtime_status_suspended(&(p->dev))) {
5520                 if (!pm_runtime_suspend(&(p->dev)))
5521                         break;
5522
5523                 if (expires < ktime_get_mono_fast_ns()) {
5524                         dev_warn(adev->dev, "failed to suspend display audio\n");
5525                         pci_dev_put(p);
5526                         /* TODO: abort the succeeding gpu reset? */
5527                         return -ETIMEDOUT;
5528                 }
5529         }
5530
5531         pm_runtime_disable(&(p->dev));
5532
5533         pci_dev_put(p);
5534         return 0;
5535 }
5536
5537 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5538 {
5539         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5540
5541 #if defined(CONFIG_DEBUG_FS)
5542         if (!amdgpu_sriov_vf(adev))
5543                 cancel_work(&adev->reset_work);
5544 #endif
5545
5546         if (adev->kfd.dev)
5547                 cancel_work(&adev->kfd.reset_work);
5548
5549         if (amdgpu_sriov_vf(adev))
5550                 cancel_work(&adev->virt.flr_work);
5551
5552         if (con && adev->ras_enabled)
5553                 cancel_work(&con->recovery_work);
5554
5555 }
5556
5557 /**
5558  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5559  *
5560  * @adev: amdgpu_device pointer
5561  * @job: which job trigger hang
5562  * @reset_context: amdgpu reset context pointer
5563  *
5564  * Attempt to reset the GPU if it has hung (all asics).
5565  * Attempt to do soft-reset or full-reset and reinitialize Asic
5566  * Returns 0 for success or an error on failure.
5567  */
5568
5569 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5570                               struct amdgpu_job *job,
5571                               struct amdgpu_reset_context *reset_context)
5572 {
5573         struct list_head device_list, *device_list_handle =  NULL;
5574         bool job_signaled = false;
5575         struct amdgpu_hive_info *hive = NULL;
5576         struct amdgpu_device *tmp_adev = NULL;
5577         int i, r = 0;
5578         bool need_emergency_restart = false;
5579         bool audio_suspended = false;
5580         bool gpu_reset_for_dev_remove = false;
5581
5582         gpu_reset_for_dev_remove =
5583                         test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5584                                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5585
5586         /*
5587          * Special case: RAS triggered and full reset isn't supported
5588          */
5589         need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5590
5591         /*
5592          * Flush RAM to disk so that after reboot
5593          * the user can read log and see why the system rebooted.
5594          */
5595         if (need_emergency_restart && amdgpu_ras_get_context(adev) &&
5596                 amdgpu_ras_get_context(adev)->reboot) {
5597                 DRM_WARN("Emergency reboot.");
5598
5599                 ksys_sync_helper();
5600                 emergency_restart();
5601         }
5602
5603         dev_info(adev->dev, "GPU %s begin!\n",
5604                 need_emergency_restart ? "jobs stop":"reset");
5605
5606         if (!amdgpu_sriov_vf(adev))
5607                 hive = amdgpu_get_xgmi_hive(adev);
5608         if (hive)
5609                 mutex_lock(&hive->hive_lock);
5610
5611         reset_context->job = job;
5612         reset_context->hive = hive;
5613         /*
5614          * Build list of devices to reset.
5615          * In case we are in XGMI hive mode, resort the device list
5616          * to put adev in the 1st position.
5617          */
5618         INIT_LIST_HEAD(&device_list);
5619         if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5620                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5621                         list_add_tail(&tmp_adev->reset_list, &device_list);
5622                         if (gpu_reset_for_dev_remove && adev->shutdown)
5623                                 tmp_adev->shutdown = true;
5624                 }
5625                 if (!list_is_first(&adev->reset_list, &device_list))
5626                         list_rotate_to_front(&adev->reset_list, &device_list);
5627                 device_list_handle = &device_list;
5628         } else {
5629                 list_add_tail(&adev->reset_list, &device_list);
5630                 device_list_handle = &device_list;
5631         }
5632
5633         /* We need to lock reset domain only once both for XGMI and single device */
5634         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5635                                     reset_list);
5636         amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5637
5638         /* block all schedulers and reset given job's ring */
5639         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5640
5641                 amdgpu_device_set_mp1_state(tmp_adev);
5642
5643                 /*
5644                  * Try to put the audio codec into suspend state
5645                  * before gpu reset started.
5646                  *
5647                  * Due to the power domain of the graphics device
5648                  * is shared with AZ power domain. Without this,
5649                  * we may change the audio hardware from behind
5650                  * the audio driver's back. That will trigger
5651                  * some audio codec errors.
5652                  */
5653                 if (!amdgpu_device_suspend_display_audio(tmp_adev))
5654                         audio_suspended = true;
5655
5656                 amdgpu_ras_set_error_query_ready(tmp_adev, false);
5657
5658                 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5659
5660                 if (!amdgpu_sriov_vf(tmp_adev))
5661                         amdgpu_amdkfd_pre_reset(tmp_adev);
5662
5663                 /*
5664                  * Mark these ASICs to be reseted as untracked first
5665                  * And add them back after reset completed
5666                  */
5667                 amdgpu_unregister_gpu_instance(tmp_adev);
5668
5669                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true);
5670
5671                 /* disable ras on ALL IPs */
5672                 if (!need_emergency_restart &&
5673                       amdgpu_device_ip_need_full_reset(tmp_adev))
5674                         amdgpu_ras_suspend(tmp_adev);
5675
5676                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5677                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5678
5679                         if (!ring || !drm_sched_wqueue_ready(&ring->sched))
5680                                 continue;
5681
5682                         drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5683
5684                         if (need_emergency_restart)
5685                                 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5686                 }
5687                 atomic_inc(&tmp_adev->gpu_reset_counter);
5688         }
5689
5690         if (need_emergency_restart)
5691                 goto skip_sched_resume;
5692
5693         /*
5694          * Must check guilty signal here since after this point all old
5695          * HW fences are force signaled.
5696          *
5697          * job->base holds a reference to parent fence
5698          */
5699         if (job && dma_fence_is_signaled(&job->hw_fence)) {
5700                 job_signaled = true;
5701                 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5702                 goto skip_hw_reset;
5703         }
5704
5705 retry:  /* Rest of adevs pre asic reset from XGMI hive. */
5706         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5707                 if (gpu_reset_for_dev_remove) {
5708                         /* Workaroud for ASICs need to disable SMC first */
5709                         amdgpu_device_smu_fini_early(tmp_adev);
5710                 }
5711                 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5712                 /*TODO Should we stop ?*/
5713                 if (r) {
5714                         dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5715                                   r, adev_to_drm(tmp_adev)->unique);
5716                         tmp_adev->asic_reset_res = r;
5717                 }
5718
5719                 /*
5720                  * Drop all pending non scheduler resets. Scheduler resets
5721                  * were already dropped during drm_sched_stop
5722                  */
5723                 amdgpu_device_stop_pending_resets(tmp_adev);
5724         }
5725
5726         /* Actual ASIC resets if needed.*/
5727         /* Host driver will handle XGMI hive reset for SRIOV */
5728         if (amdgpu_sriov_vf(adev)) {
5729                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
5730                 if (r)
5731                         adev->asic_reset_res = r;
5732
5733                 /* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
5734                 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
5735                             IP_VERSION(9, 4, 2) ||
5736                     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3))
5737                         amdgpu_ras_resume(adev);
5738         } else {
5739                 r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5740                 if (r && r == -EAGAIN)
5741                         goto retry;
5742
5743                 if (!r && gpu_reset_for_dev_remove)
5744                         goto recover_end;
5745         }
5746
5747 skip_hw_reset:
5748
5749         /* Post ASIC reset for all devs .*/
5750         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5751
5752                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5753                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5754
5755                         if (!ring || !drm_sched_wqueue_ready(&ring->sched))
5756                                 continue;
5757
5758                         drm_sched_start(&ring->sched, true);
5759                 }
5760
5761                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled)
5762                         drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5763
5764                 if (tmp_adev->asic_reset_res)
5765                         r = tmp_adev->asic_reset_res;
5766
5767                 tmp_adev->asic_reset_res = 0;
5768
5769                 if (r) {
5770                         /* bad news, how to tell it to userspace ? */
5771                         dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5772                         amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5773                 } else {
5774                         dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5775                         if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5776                                 DRM_WARN("smart shift update failed\n");
5777                 }
5778         }
5779
5780 skip_sched_resume:
5781         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5782                 /* unlock kfd: SRIOV would do it separately */
5783                 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5784                         amdgpu_amdkfd_post_reset(tmp_adev);
5785
5786                 /* kfd_post_reset will do nothing if kfd device is not initialized,
5787                  * need to bring up kfd here if it's not be initialized before
5788                  */
5789                 if (!adev->kfd.init_complete)
5790                         amdgpu_amdkfd_device_init(adev);
5791
5792                 if (audio_suspended)
5793                         amdgpu_device_resume_display_audio(tmp_adev);
5794
5795                 amdgpu_device_unset_mp1_state(tmp_adev);
5796
5797                 amdgpu_ras_set_error_query_ready(tmp_adev, true);
5798         }
5799
5800 recover_end:
5801         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5802                                             reset_list);
5803         amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
5804
5805         if (hive) {
5806                 mutex_unlock(&hive->hive_lock);
5807                 amdgpu_put_xgmi_hive(hive);
5808         }
5809
5810         if (r)
5811                 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5812
5813         atomic_set(&adev->reset_domain->reset_res, r);
5814         return r;
5815 }
5816
5817 /**
5818  * amdgpu_device_partner_bandwidth - find the bandwidth of appropriate partner
5819  *
5820  * @adev: amdgpu_device pointer
5821  * @speed: pointer to the speed of the link
5822  * @width: pointer to the width of the link
5823  *
5824  * Evaluate the hierarchy to find the speed and bandwidth capabilities of the
5825  * first physical partner to an AMD dGPU.
5826  * This will exclude any virtual switches and links.
5827  */
5828 static void amdgpu_device_partner_bandwidth(struct amdgpu_device *adev,
5829                                             enum pci_bus_speed *speed,
5830                                             enum pcie_link_width *width)
5831 {
5832         struct pci_dev *parent = adev->pdev;
5833
5834         if (!speed || !width)
5835                 return;
5836
5837         *speed = PCI_SPEED_UNKNOWN;
5838         *width = PCIE_LNK_WIDTH_UNKNOWN;
5839
5840         while ((parent = pci_upstream_bridge(parent))) {
5841                 /* skip upstream/downstream switches internal to dGPU*/
5842                 if (parent->vendor == PCI_VENDOR_ID_ATI)
5843                         continue;
5844                 *speed = pcie_get_speed_cap(parent);
5845                 *width = pcie_get_width_cap(parent);
5846                 break;
5847         }
5848 }
5849
5850 /**
5851  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5852  *
5853  * @adev: amdgpu_device pointer
5854  *
5855  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5856  * and lanes) of the slot the device is in. Handles APUs and
5857  * virtualized environments where PCIE config space may not be available.
5858  */
5859 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5860 {
5861         struct pci_dev *pdev;
5862         enum pci_bus_speed speed_cap, platform_speed_cap;
5863         enum pcie_link_width platform_link_width;
5864
5865         if (amdgpu_pcie_gen_cap)
5866                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5867
5868         if (amdgpu_pcie_lane_cap)
5869                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5870
5871         /* covers APUs as well */
5872         if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) {
5873                 if (adev->pm.pcie_gen_mask == 0)
5874                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5875                 if (adev->pm.pcie_mlw_mask == 0)
5876                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5877                 return;
5878         }
5879
5880         if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5881                 return;
5882
5883         amdgpu_device_partner_bandwidth(adev, &platform_speed_cap,
5884                                         &platform_link_width);
5885
5886         if (adev->pm.pcie_gen_mask == 0) {
5887                 /* asic caps */
5888                 pdev = adev->pdev;
5889                 speed_cap = pcie_get_speed_cap(pdev);
5890                 if (speed_cap == PCI_SPEED_UNKNOWN) {
5891                         adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5892                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5893                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5894                 } else {
5895                         if (speed_cap == PCIE_SPEED_32_0GT)
5896                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5897                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5898                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5899                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5900                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5901                         else if (speed_cap == PCIE_SPEED_16_0GT)
5902                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5903                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5904                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5905                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5906                         else if (speed_cap == PCIE_SPEED_8_0GT)
5907                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5908                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5909                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5910                         else if (speed_cap == PCIE_SPEED_5_0GT)
5911                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5912                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5913                         else
5914                                 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5915                 }
5916                 /* platform caps */
5917                 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5918                         adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5919                                                    CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5920                 } else {
5921                         if (platform_speed_cap == PCIE_SPEED_32_0GT)
5922                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5923                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5924                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5925                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5926                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5927                         else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5928                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5929                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5930                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5931                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5932                         else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5933                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5934                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5935                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5936                         else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5937                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5938                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5939                         else
5940                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5941
5942                 }
5943         }
5944         if (adev->pm.pcie_mlw_mask == 0) {
5945                 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5946                         adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5947                 } else {
5948                         switch (platform_link_width) {
5949                         case PCIE_LNK_X32:
5950                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5951                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5952                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5953                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5954                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5955                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5956                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5957                                 break;
5958                         case PCIE_LNK_X16:
5959                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5960                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5961                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5962                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5963                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5964                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5965                                 break;
5966                         case PCIE_LNK_X12:
5967                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5968                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5969                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5970                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5971                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5972                                 break;
5973                         case PCIE_LNK_X8:
5974                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5975                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5976                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5977                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5978                                 break;
5979                         case PCIE_LNK_X4:
5980                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5981                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5982                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5983                                 break;
5984                         case PCIE_LNK_X2:
5985                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5986                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5987                                 break;
5988                         case PCIE_LNK_X1:
5989                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5990                                 break;
5991                         default:
5992                                 break;
5993                         }
5994                 }
5995         }
5996 }
5997
5998 /**
5999  * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
6000  *
6001  * @adev: amdgpu_device pointer
6002  * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
6003  *
6004  * Return true if @peer_adev can access (DMA) @adev through the PCIe
6005  * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
6006  * @peer_adev.
6007  */
6008 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
6009                                       struct amdgpu_device *peer_adev)
6010 {
6011 #ifdef CONFIG_HSA_AMD_P2P
6012         uint64_t address_mask = peer_adev->dev->dma_mask ?
6013                 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
6014         resource_size_t aper_limit =
6015                 adev->gmc.aper_base + adev->gmc.aper_size - 1;
6016         bool p2p_access =
6017                 !adev->gmc.xgmi.connected_to_cpu &&
6018                 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
6019
6020         return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
6021                 adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
6022                 !(adev->gmc.aper_base & address_mask ||
6023                   aper_limit & address_mask));
6024 #else
6025         return false;
6026 #endif
6027 }
6028
6029 int amdgpu_device_baco_enter(struct drm_device *dev)
6030 {
6031         struct amdgpu_device *adev = drm_to_adev(dev);
6032         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6033
6034         if (!amdgpu_device_supports_baco(dev))
6035                 return -ENOTSUPP;
6036
6037         if (ras && adev->ras_enabled &&
6038             adev->nbio.funcs->enable_doorbell_interrupt)
6039                 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
6040
6041         return amdgpu_dpm_baco_enter(adev);
6042 }
6043
6044 int amdgpu_device_baco_exit(struct drm_device *dev)
6045 {
6046         struct amdgpu_device *adev = drm_to_adev(dev);
6047         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
6048         int ret = 0;
6049
6050         if (!amdgpu_device_supports_baco(dev))
6051                 return -ENOTSUPP;
6052
6053         ret = amdgpu_dpm_baco_exit(adev);
6054         if (ret)
6055                 return ret;
6056
6057         if (ras && adev->ras_enabled &&
6058             adev->nbio.funcs->enable_doorbell_interrupt)
6059                 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
6060
6061         if (amdgpu_passthrough(adev) &&
6062             adev->nbio.funcs->clear_doorbell_interrupt)
6063                 adev->nbio.funcs->clear_doorbell_interrupt(adev);
6064
6065         return 0;
6066 }
6067
6068 /**
6069  * amdgpu_pci_error_detected - Called when a PCI error is detected.
6070  * @pdev: PCI device struct
6071  * @state: PCI channel state
6072  *
6073  * Description: Called when a PCI error is detected.
6074  *
6075  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
6076  */
6077 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
6078 {
6079         struct drm_device *dev = pci_get_drvdata(pdev);
6080         struct amdgpu_device *adev = drm_to_adev(dev);
6081         int i;
6082
6083         DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
6084
6085         if (adev->gmc.xgmi.num_physical_nodes > 1) {
6086                 DRM_WARN("No support for XGMI hive yet...");
6087                 return PCI_ERS_RESULT_DISCONNECT;
6088         }
6089
6090         adev->pci_channel_state = state;
6091
6092         switch (state) {
6093         case pci_channel_io_normal:
6094                 return PCI_ERS_RESULT_CAN_RECOVER;
6095         /* Fatal error, prepare for slot reset */
6096         case pci_channel_io_frozen:
6097                 /*
6098                  * Locking adev->reset_domain->sem will prevent any external access
6099                  * to GPU during PCI error recovery
6100                  */
6101                 amdgpu_device_lock_reset_domain(adev->reset_domain);
6102                 amdgpu_device_set_mp1_state(adev);
6103
6104                 /*
6105                  * Block any work scheduling as we do for regular GPU reset
6106                  * for the duration of the recovery
6107                  */
6108                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6109                         struct amdgpu_ring *ring = adev->rings[i];
6110
6111                         if (!ring || !drm_sched_wqueue_ready(&ring->sched))
6112                                 continue;
6113
6114                         drm_sched_stop(&ring->sched, NULL);
6115                 }
6116                 atomic_inc(&adev->gpu_reset_counter);
6117                 return PCI_ERS_RESULT_NEED_RESET;
6118         case pci_channel_io_perm_failure:
6119                 /* Permanent error, prepare for device removal */
6120                 return PCI_ERS_RESULT_DISCONNECT;
6121         }
6122
6123         return PCI_ERS_RESULT_NEED_RESET;
6124 }
6125
6126 /**
6127  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
6128  * @pdev: pointer to PCI device
6129  */
6130 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
6131 {
6132
6133         DRM_INFO("PCI error: mmio enabled callback!!\n");
6134
6135         /* TODO - dump whatever for debugging purposes */
6136
6137         /* This called only if amdgpu_pci_error_detected returns
6138          * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
6139          * works, no need to reset slot.
6140          */
6141
6142         return PCI_ERS_RESULT_RECOVERED;
6143 }
6144
6145 /**
6146  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
6147  * @pdev: PCI device struct
6148  *
6149  * Description: This routine is called by the pci error recovery
6150  * code after the PCI slot has been reset, just before we
6151  * should resume normal operations.
6152  */
6153 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
6154 {
6155         struct drm_device *dev = pci_get_drvdata(pdev);
6156         struct amdgpu_device *adev = drm_to_adev(dev);
6157         int r, i;
6158         struct amdgpu_reset_context reset_context;
6159         u32 memsize;
6160         struct list_head device_list;
6161
6162         DRM_INFO("PCI error: slot reset callback!!\n");
6163
6164         memset(&reset_context, 0, sizeof(reset_context));
6165
6166         INIT_LIST_HEAD(&device_list);
6167         list_add_tail(&adev->reset_list, &device_list);
6168
6169         /* wait for asic to come out of reset */
6170         msleep(500);
6171
6172         /* Restore PCI confspace */
6173         amdgpu_device_load_pci_state(pdev);
6174
6175         /* confirm  ASIC came out of reset */
6176         for (i = 0; i < adev->usec_timeout; i++) {
6177                 memsize = amdgpu_asic_get_config_memsize(adev);
6178
6179                 if (memsize != 0xffffffff)
6180                         break;
6181                 udelay(1);
6182         }
6183         if (memsize == 0xffffffff) {
6184                 r = -ETIME;
6185                 goto out;
6186         }
6187
6188         reset_context.method = AMD_RESET_METHOD_NONE;
6189         reset_context.reset_req_dev = adev;
6190         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
6191         set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
6192
6193         adev->no_hw_access = true;
6194         r = amdgpu_device_pre_asic_reset(adev, &reset_context);
6195         adev->no_hw_access = false;
6196         if (r)
6197                 goto out;
6198
6199         r = amdgpu_do_asic_reset(&device_list, &reset_context);
6200
6201 out:
6202         if (!r) {
6203                 if (amdgpu_device_cache_pci_state(adev->pdev))
6204                         pci_restore_state(adev->pdev);
6205
6206                 DRM_INFO("PCIe error recovery succeeded\n");
6207         } else {
6208                 DRM_ERROR("PCIe error recovery failed, err:%d", r);
6209                 amdgpu_device_unset_mp1_state(adev);
6210                 amdgpu_device_unlock_reset_domain(adev->reset_domain);
6211         }
6212
6213         return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
6214 }
6215
6216 /**
6217  * amdgpu_pci_resume() - resume normal ops after PCI reset
6218  * @pdev: pointer to PCI device
6219  *
6220  * Called when the error recovery driver tells us that its
6221  * OK to resume normal operation.
6222  */
6223 void amdgpu_pci_resume(struct pci_dev *pdev)
6224 {
6225         struct drm_device *dev = pci_get_drvdata(pdev);
6226         struct amdgpu_device *adev = drm_to_adev(dev);
6227         int i;
6228
6229
6230         DRM_INFO("PCI error: resume callback!!\n");
6231
6232         /* Only continue execution for the case of pci_channel_io_frozen */
6233         if (adev->pci_channel_state != pci_channel_io_frozen)
6234                 return;
6235
6236         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6237                 struct amdgpu_ring *ring = adev->rings[i];
6238
6239                 if (!ring || !drm_sched_wqueue_ready(&ring->sched))
6240                         continue;
6241
6242                 drm_sched_start(&ring->sched, true);
6243         }
6244
6245         amdgpu_device_unset_mp1_state(adev);
6246         amdgpu_device_unlock_reset_domain(adev->reset_domain);
6247 }
6248
6249 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
6250 {
6251         struct drm_device *dev = pci_get_drvdata(pdev);
6252         struct amdgpu_device *adev = drm_to_adev(dev);
6253         int r;
6254
6255         r = pci_save_state(pdev);
6256         if (!r) {
6257                 kfree(adev->pci_state);
6258
6259                 adev->pci_state = pci_store_saved_state(pdev);
6260
6261                 if (!adev->pci_state) {
6262                         DRM_ERROR("Failed to store PCI saved state");
6263                         return false;
6264                 }
6265         } else {
6266                 DRM_WARN("Failed to save PCI state, err:%d\n", r);
6267                 return false;
6268         }
6269
6270         return true;
6271 }
6272
6273 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
6274 {
6275         struct drm_device *dev = pci_get_drvdata(pdev);
6276         struct amdgpu_device *adev = drm_to_adev(dev);
6277         int r;
6278
6279         if (!adev->pci_state)
6280                 return false;
6281
6282         r = pci_load_saved_state(pdev, adev->pci_state);
6283
6284         if (!r) {
6285                 pci_restore_state(pdev);
6286         } else {
6287                 DRM_WARN("Failed to load PCI state, err:%d\n", r);
6288                 return false;
6289         }
6290
6291         return true;
6292 }
6293
6294 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
6295                 struct amdgpu_ring *ring)
6296 {
6297 #ifdef CONFIG_X86_64
6298         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6299                 return;
6300 #endif
6301         if (adev->gmc.xgmi.connected_to_cpu)
6302                 return;
6303
6304         if (ring && ring->funcs->emit_hdp_flush)
6305                 amdgpu_ring_emit_hdp_flush(ring);
6306         else
6307                 amdgpu_asic_flush_hdp(adev, ring);
6308 }
6309
6310 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
6311                 struct amdgpu_ring *ring)
6312 {
6313 #ifdef CONFIG_X86_64
6314         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6315                 return;
6316 #endif
6317         if (adev->gmc.xgmi.connected_to_cpu)
6318                 return;
6319
6320         amdgpu_asic_invalidate_hdp(adev, ring);
6321 }
6322
6323 int amdgpu_in_reset(struct amdgpu_device *adev)
6324 {
6325         return atomic_read(&adev->reset_domain->in_gpu_reset);
6326 }
6327
6328 /**
6329  * amdgpu_device_halt() - bring hardware to some kind of halt state
6330  *
6331  * @adev: amdgpu_device pointer
6332  *
6333  * Bring hardware to some kind of halt state so that no one can touch it
6334  * any more. It will help to maintain error context when error occurred.
6335  * Compare to a simple hang, the system will keep stable at least for SSH
6336  * access. Then it should be trivial to inspect the hardware state and
6337  * see what's going on. Implemented as following:
6338  *
6339  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
6340  *    clears all CPU mappings to device, disallows remappings through page faults
6341  * 2. amdgpu_irq_disable_all() disables all interrupts
6342  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
6343  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
6344  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
6345  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
6346  *    flush any in flight DMA operations
6347  */
6348 void amdgpu_device_halt(struct amdgpu_device *adev)
6349 {
6350         struct pci_dev *pdev = adev->pdev;
6351         struct drm_device *ddev = adev_to_drm(adev);
6352
6353         amdgpu_xcp_dev_unplug(adev);
6354         drm_dev_unplug(ddev);
6355
6356         amdgpu_irq_disable_all(adev);
6357
6358         amdgpu_fence_driver_hw_fini(adev);
6359
6360         adev->no_hw_access = true;
6361
6362         amdgpu_device_unmap_mmio(adev);
6363
6364         pci_disable_device(pdev);
6365         pci_wait_for_pending_transaction(pdev);
6366 }
6367
6368 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
6369                                 u32 reg)
6370 {
6371         unsigned long flags, address, data;
6372         u32 r;
6373
6374         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6375         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6376
6377         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6378         WREG32(address, reg * 4);
6379         (void)RREG32(address);
6380         r = RREG32(data);
6381         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6382         return r;
6383 }
6384
6385 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
6386                                 u32 reg, u32 v)
6387 {
6388         unsigned long flags, address, data;
6389
6390         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6391         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6392
6393         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6394         WREG32(address, reg * 4);
6395         (void)RREG32(address);
6396         WREG32(data, v);
6397         (void)RREG32(data);
6398         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6399 }
6400
6401 /**
6402  * amdgpu_device_switch_gang - switch to a new gang
6403  * @adev: amdgpu_device pointer
6404  * @gang: the gang to switch to
6405  *
6406  * Try to switch to a new gang.
6407  * Returns: NULL if we switched to the new gang or a reference to the current
6408  * gang leader.
6409  */
6410 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6411                                             struct dma_fence *gang)
6412 {
6413         struct dma_fence *old = NULL;
6414
6415         do {
6416                 dma_fence_put(old);
6417                 rcu_read_lock();
6418                 old = dma_fence_get_rcu_safe(&adev->gang_submit);
6419                 rcu_read_unlock();
6420
6421                 if (old == gang)
6422                         break;
6423
6424                 if (!dma_fence_is_signaled(old))
6425                         return old;
6426
6427         } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6428                          old, gang) != old);
6429
6430         dma_fence_put(old);
6431         return NULL;
6432 }
6433
6434 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6435 {
6436         switch (adev->asic_type) {
6437 #ifdef CONFIG_DRM_AMDGPU_SI
6438         case CHIP_HAINAN:
6439 #endif
6440         case CHIP_TOPAZ:
6441                 /* chips with no display hardware */
6442                 return false;
6443 #ifdef CONFIG_DRM_AMDGPU_SI
6444         case CHIP_TAHITI:
6445         case CHIP_PITCAIRN:
6446         case CHIP_VERDE:
6447         case CHIP_OLAND:
6448 #endif
6449 #ifdef CONFIG_DRM_AMDGPU_CIK
6450         case CHIP_BONAIRE:
6451         case CHIP_HAWAII:
6452         case CHIP_KAVERI:
6453         case CHIP_KABINI:
6454         case CHIP_MULLINS:
6455 #endif
6456         case CHIP_TONGA:
6457         case CHIP_FIJI:
6458         case CHIP_POLARIS10:
6459         case CHIP_POLARIS11:
6460         case CHIP_POLARIS12:
6461         case CHIP_VEGAM:
6462         case CHIP_CARRIZO:
6463         case CHIP_STONEY:
6464                 /* chips with display hardware */
6465                 return true;
6466         default:
6467                 /* IP discovery */
6468                 if (!amdgpu_ip_version(adev, DCE_HWIP, 0) ||
6469                     (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6470                         return false;
6471                 return true;
6472         }
6473 }
6474
6475 uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,
6476                 uint32_t inst, uint32_t reg_addr, char reg_name[],
6477                 uint32_t expected_value, uint32_t mask)
6478 {
6479         uint32_t ret = 0;
6480         uint32_t old_ = 0;
6481         uint32_t tmp_ = RREG32(reg_addr);
6482         uint32_t loop = adev->usec_timeout;
6483
6484         while ((tmp_ & (mask)) != (expected_value)) {
6485                 if (old_ != tmp_) {
6486                         loop = adev->usec_timeout;
6487                         old_ = tmp_;
6488                 } else
6489                         udelay(1);
6490                 tmp_ = RREG32(reg_addr);
6491                 loop--;
6492                 if (!loop) {
6493                         DRM_WARN("Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn",
6494                                   inst, reg_name, (uint32_t)expected_value,
6495                                   (uint32_t)(tmp_ & (mask)));
6496                         ret = -ETIMEDOUT;
6497                         break;
6498                 }
6499         }
6500         return ret;
6501 }