drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/kthread.h>
  29 #include <linux/console.h>
  30 #include <linux/slab.h>
  31 #include <drm/drmP.h>
  32 #include <drm/drm_crtc_helper.h>
  33 #include <drm/drm_atomic_helper.h>
  34 #include <drm/amdgpu_drm.h>
  35 #include <linux/vgaarb.h>
  36 #include <linux/vga_switcheroo.h>
  37 #include <linux/efi.h>
  38 #include "amdgpu.h"
  39 #include "amdgpu_trace.h"
  40 #include "amdgpu_i2c.h"
  41 #include "atom.h"
  42 #include "amdgpu_atombios.h"
  43 #include "amdgpu_atomfirmware.h"
  44 #include "amd_pcie.h"
  45 #ifdef CONFIG_DRM_AMDGPU_SI
  46 #include "si.h"
  47 #endif
  48 #ifdef CONFIG_DRM_AMDGPU_CIK
  49 #include "cik.h"
  50 #endif
  51 #include "vi.h"
  52 #include "soc15.h"
  53 #include "bif/bif_4_1_d.h"
  54 #include <linux/pci.h>
  55 #include <linux/firmware.h>
  56 #include "amdgpu_vf_error.h"
  57
  58 #include "amdgpu_amdkfd.h"
  59 #include "amdgpu_pm.h"
  60
  61 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  62 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  63
  64 #define AMDGPU_RESUME_MS                2000
  65
  66 static const char *amdgpu_asic_name[] = {
  67         "TAHITI",
  68         "PITCAIRN",
  69         "VERDE",
  70         "OLAND",
  71         "HAINAN",
  72         "BONAIRE",
  73         "KAVERI",
  74         "KABINI",
  75         "HAWAII",
  76         "MULLINS",
  77         "TOPAZ",
  78         "TONGA",
  79         "FIJI",
  80         "CARRIZO",
  81         "STONEY",
  82         "POLARIS10",
  83         "POLARIS11",
  84         "POLARIS12",
  85         "VEGA10",
  86         "RAVEN",
  87         "LAST",
  88 };
  89
  90 bool amdgpu_device_is_px(struct drm_device *dev)
  91 {
  92         struct amdgpu_device *adev = dev->dev_private;
  93
  94         if (adev->flags & AMD_IS_PX)
  95                 return true;
  96         return false;
  97 }
  98
  99 /*
 100  * MMIO register access helper functions.
 101  */
 102 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
 103                         uint32_t acc_flags)
 104 {
 105         uint32_t ret;
 106
 107         if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
 108                 return amdgpu_virt_kiq_rreg(adev, reg);
 109
 110         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
 111                 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 112         else {
 113                 unsigned long flags;
 114
 115                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 116                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 117                 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 118                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 119         }
 120         trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
 121         return ret;
 122 }
 123
 124 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 125                     uint32_t acc_flags)
 126 {
 127         trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 128
 129         if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
 130                 adev->last_mm_index = v;
 131         }
 132
 133         if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
 134                 return amdgpu_virt_kiq_wreg(adev, reg, v);
 135
 136         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
 137                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 138         else {
 139                 unsigned long flags;
 140
 141                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 142                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 143                 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 144                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 145         }
 146
 147         if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
 148                 udelay(500);
 149         }
 150 }
 151
 152 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
 153 {
 154         if ((reg * 4) < adev->rio_mem_size)
 155                 return ioread32(adev->rio_mem + (reg * 4));
 156         else {
 157                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 158                 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
 159         }
 160 }
 161
 162 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 163 {
 164         if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
 165                 adev->last_mm_index = v;
 166         }
 167
 168         if ((reg * 4) < adev->rio_mem_size)
 169                 iowrite32(v, adev->rio_mem + (reg * 4));
 170         else {
 171                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 172                 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
 173         }
 174
 175         if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
 176                 udelay(500);
 177         }
 178 }
 179
 180 /**
 181  * amdgpu_mm_rdoorbell - read a doorbell dword
 182  *
 183  * @adev: amdgpu_device pointer
 184  * @index: doorbell index
 185  *
 186  * Returns the value in the doorbell aperture at the
 187  * requested doorbell index (CIK).
 188  */
 189 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 190 {
 191         if (index < adev->doorbell.num_doorbells) {
 192                 return readl(adev->doorbell.ptr + index);
 193         } else {
 194                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 195                 return 0;
 196         }
 197 }
 198
 199 /**
 200  * amdgpu_mm_wdoorbell - write a doorbell dword
 201  *
 202  * @adev: amdgpu_device pointer
 203  * @index: doorbell index
 204  * @v: value to write
 205  *
 206  * Writes @v to the doorbell aperture at the
 207  * requested doorbell index (CIK).
 208  */
 209 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 210 {
 211         if (index < adev->doorbell.num_doorbells) {
 212                 writel(v, adev->doorbell.ptr + index);
 213         } else {
 214                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 215         }
 216 }
 217
 218 /**
 219  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 220  *
 221  * @adev: amdgpu_device pointer
 222  * @index: doorbell index
 223  *
 224  * Returns the value in the doorbell aperture at the
 225  * requested doorbell index (VEGA10+).
 226  */
 227 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 228 {
 229         if (index < adev->doorbell.num_doorbells) {
 230                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 231         } else {
 232                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 233                 return 0;
 234         }
 235 }
 236
 237 /**
 238  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 239  *
 240  * @adev: amdgpu_device pointer
 241  * @index: doorbell index
 242  * @v: value to write
 243  *
 244  * Writes @v to the doorbell aperture at the
 245  * requested doorbell index (VEGA10+).
 246  */
 247 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 248 {
 249         if (index < adev->doorbell.num_doorbells) {
 250                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 251         } else {
 252                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 253         }
 254 }
 255
 256 /**
 257  * amdgpu_invalid_rreg - dummy reg read function
 258  *
 259  * @adev: amdgpu device pointer
 260  * @reg: offset of register
 261  *
 262  * Dummy register read function.  Used for register blocks
 263  * that certain asics don't have (all asics).
 264  * Returns the value in the register.
 265  */
 266 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 267 {
 268         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 269         BUG();
 270         return 0;
 271 }
 272
 273 /**
 274  * amdgpu_invalid_wreg - dummy reg write function
 275  *
 276  * @adev: amdgpu device pointer
 277  * @reg: offset of register
 278  * @v: value to write to the register
 279  *
 280  * Dummy register read function.  Used for register blocks
 281  * that certain asics don't have (all asics).
 282  */
 283 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 284 {
 285         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 286                   reg, v);
 287         BUG();
 288 }
 289
 290 /**
 291  * amdgpu_block_invalid_rreg - dummy reg read function
 292  *
 293  * @adev: amdgpu device pointer
 294  * @block: offset of instance
 295  * @reg: offset of register
 296  *
 297  * Dummy register read function.  Used for register blocks
 298  * that certain asics don't have (all asics).
 299  * Returns the value in the register.
 300  */
 301 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 302                                           uint32_t block, uint32_t reg)
 303 {
 304         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 305                   reg, block);
 306         BUG();
 307         return 0;
 308 }
 309
 310 /**
 311  * amdgpu_block_invalid_wreg - dummy reg write function
 312  *
 313  * @adev: amdgpu device pointer
 314  * @block: offset of instance
 315  * @reg: offset of register
 316  * @v: value to write to the register
 317  *
 318  * Dummy register read function.  Used for register blocks
 319  * that certain asics don't have (all asics).
 320  */
 321 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 322                                       uint32_t block,
 323                                       uint32_t reg, uint32_t v)
 324 {
 325         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 326                   reg, block, v);
 327         BUG();
 328 }
 329
 330 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
 331 {
 332         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
 333                                        PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 334                                        &adev->vram_scratch.robj,
 335                                        &adev->vram_scratch.gpu_addr,
 336                                        (void **)&adev->vram_scratch.ptr);
 337 }
 338
 339 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
 340 {
 341         amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
 342 }
 343
 344 /**
 345  * amdgpu_program_register_sequence - program an array of registers.
 346  *
 347  * @adev: amdgpu_device pointer
 348  * @registers: pointer to the register array
 349  * @array_size: size of the register array
 350  *
 351  * Programs an array or registers with and and or masks.
 352  * This is a helper for setting golden registers.
 353  */
 354 void amdgpu_program_register_sequence(struct amdgpu_device *adev,
 355                                       const u32 *registers,
 356                                       const u32 array_size)
 357 {
 358         u32 tmp, reg, and_mask, or_mask;
 359         int i;
 360
 361         if (array_size % 3)
 362                 return;
 363
 364         for (i = 0; i < array_size; i +=3) {
 365                 reg = registers[i + 0];
 366                 and_mask = registers[i + 1];
 367                 or_mask = registers[i + 2];
 368
 369                 if (and_mask == 0xffffffff) {
 370                         tmp = or_mask;
 371                 } else {
 372                         tmp = RREG32(reg);
 373                         tmp &= ~and_mask;
 374                         tmp |= or_mask;
 375                 }
 376                 WREG32(reg, tmp);
 377         }
 378 }
 379
 380 void amdgpu_pci_config_reset(struct amdgpu_device *adev)
 381 {
 382         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
 383 }
 384
 385 /*
 386  * GPU doorbell aperture helpers function.
 387  */
 388 /**
 389  * amdgpu_device_doorbell_init - Init doorbell driver information.
 390  *
 391  * @adev: amdgpu_device pointer
 392  *
 393  * Init doorbell driver information (CIK)
 394  * Returns 0 on success, error on failure.
 395  */
 396 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
 397 {
 398         /* No doorbell on SI hardware generation */
 399         if (adev->asic_type < CHIP_BONAIRE) {
 400                 adev->doorbell.base = 0;
 401                 adev->doorbell.size = 0;
 402                 adev->doorbell.num_doorbells = 0;
 403                 adev->doorbell.ptr = NULL;
 404                 return 0;
 405         }
 406
 407         if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
 408                 return -EINVAL;
 409
 410         /* doorbell bar mapping */
 411         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
 412         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
 413
 414         adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
 415                                              AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
 416         if (adev->doorbell.num_doorbells == 0)
 417                 return -EINVAL;
 418
 419         adev->doorbell.ptr = ioremap(adev->doorbell.base,
 420                                      adev->doorbell.num_doorbells *
 421                                      sizeof(u32));
 422         if (adev->doorbell.ptr == NULL)
 423                 return -ENOMEM;
 424
 425         return 0;
 426 }
 427
 428 /**
 429  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
 430  *
 431  * @adev: amdgpu_device pointer
 432  *
 433  * Tear down doorbell driver information (CIK)
 434  */
 435 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
 436 {
 437         iounmap(adev->doorbell.ptr);
 438         adev->doorbell.ptr = NULL;
 439 }
 440
 441 /**
 442  * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to
 443  *                                setup amdkfd
 444  *
 445  * @adev: amdgpu_device pointer
 446  * @aperture_base: output returning doorbell aperture base physical address
 447  * @aperture_size: output returning doorbell aperture size in bytes
 448  * @start_offset: output returning # of doorbell bytes reserved for amdgpu.
 449  *
 450  * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up,
 451  * takes doorbells required for its own rings and reports the setup to amdkfd.
 452  * amdgpu reserved doorbells are at the start of the doorbell aperture.
 453  */
 454 void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
 455                                 phys_addr_t *aperture_base,
 456                                 size_t *aperture_size,
 457                                 size_t *start_offset)
 458 {
 459         /*
 460          * The first num_doorbells are used by amdgpu.
 461          * amdkfd takes whatever's left in the aperture.
 462          */
 463         if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) {
 464                 *aperture_base = adev->doorbell.base;
 465                 *aperture_size = adev->doorbell.size;
 466                 *start_offset = adev->doorbell.num_doorbells * sizeof(u32);
 467         } else {
 468                 *aperture_base = 0;
 469                 *aperture_size = 0;
 470                 *start_offset = 0;
 471         }
 472 }
 473
 474 /*
 475  * amdgpu_device_wb_*()
 476  * Writeback is the method by which the GPU updates special pages in memory
 477  * with the status of certain GPU events (fences, ring pointers,etc.).
 478  */
 479
 480 /**
 481  * amdgpu_device_wb_fini - Disable Writeback and free memory
 482  *
 483  * @adev: amdgpu_device pointer
 484  *
 485  * Disables Writeback and frees the Writeback memory (all asics).
 486  * Used at driver shutdown.
 487  */
 488 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
 489 {
 490         if (adev->wb.wb_obj) {
 491                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
 492                                       &adev->wb.gpu_addr,
 493                                       (void **)&adev->wb.wb);
 494                 adev->wb.wb_obj = NULL;
 495         }
 496 }
 497
 498 /**
 499  * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
 500  *
 501  * @adev: amdgpu_device pointer
 502  *
 503  * Initializes writeback and allocates writeback memory (all asics).
 504  * Used at driver startup.
 505  * Returns 0 on success or an -error on failure.
 506  */
 507 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
 508 {
 509         int r;
 510
 511         if (adev->wb.wb_obj == NULL) {
 512                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
 513                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
 514                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 515                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
 516                                             (void **)&adev->wb.wb);
 517                 if (r) {
 518                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
 519                         return r;
 520                 }
 521
 522                 adev->wb.num_wb = AMDGPU_MAX_WB;
 523                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
 524
 525                 /* clear wb memory */
 526                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t));
 527         }
 528
 529         return 0;
 530 }
 531
 532 /**
 533  * amdgpu_device_wb_get - Allocate a wb entry
 534  *
 535  * @adev: amdgpu_device pointer
 536  * @wb: wb index
 537  *
 538  * Allocate a wb slot for use by the driver (all asics).
 539  * Returns 0 on success or -EINVAL on failure.
 540  */
 541 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
 542 {
 543         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
 544
 545         if (offset < adev->wb.num_wb) {
 546                 __set_bit(offset, adev->wb.used);
 547                 *wb = offset << 3; /* convert to dw offset */
 548                 return 0;
 549         } else {
 550                 return -EINVAL;
 551         }
 552 }
 553
 554 /**
 555  * amdgpu_device_wb_free - Free a wb entry
 556  *
 557  * @adev: amdgpu_device pointer
 558  * @wb: wb index
 559  *
 560  * Free a wb slot allocated for use by the driver (all asics)
 561  */
 562 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
 563 {
 564         if (wb < adev->wb.num_wb)
 565                 __clear_bit(wb >> 3, adev->wb.used);
 566 }
 567
 568 /**
 569  * amdgpu_vram_location - try to find VRAM location
 570  * @adev: amdgpu device structure holding all necessary informations
 571  * @mc: memory controller structure holding memory informations
 572  * @base: base address at which to put VRAM
 573  *
 574  * Function will try to place VRAM at base address provided
 575  * as parameter.
 576  */
 577 void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base)
 578 {
 579         uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
 580
 581         mc->vram_start = base;
 582         mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 583         if (limit && limit < mc->real_vram_size)
 584                 mc->real_vram_size = limit;
 585         dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
 586                         mc->mc_vram_size >> 20, mc->vram_start,
 587                         mc->vram_end, mc->real_vram_size >> 20);
 588 }
 589
 590 /**
 591  * amdgpu_gart_location - try to find GTT location
 592  * @adev: amdgpu device structure holding all necessary informations
 593  * @mc: memory controller structure holding memory informations
 594  *
 595  * Function will place try to place GTT before or after VRAM.
 596  *
 597  * If GTT size is bigger than space left then we ajust GTT size.
 598  * Thus function will never fails.
 599  *
 600  * FIXME: when reducing GTT size align new size on power of 2.
 601  */
 602 void amdgpu_gart_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
 603 {
 604         u64 size_af, size_bf;
 605
 606         size_af = adev->mc.mc_mask - mc->vram_end;
 607         size_bf = mc->vram_start;
 608         if (size_bf > size_af) {
 609                 if (mc->gart_size > size_bf) {
 610                         dev_warn(adev->dev, "limiting GTT\n");
 611                         mc->gart_size = size_bf;
 612                 }
 613                 mc->gart_start = 0;
 614         } else {
 615                 if (mc->gart_size > size_af) {
 616                         dev_warn(adev->dev, "limiting GTT\n");
 617                         mc->gart_size = size_af;
 618                 }
 619                 /* VCE doesn't like it when BOs cross a 4GB segment, so align
 620                  * the GART base on a 4GB boundary as well.
 621                  */
 622                 mc->gart_start = ALIGN(mc->vram_end + 1, 0x100000000ULL);
 623         }
 624         mc->gart_end = mc->gart_start + mc->gart_size - 1;
 625         dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
 626                         mc->gart_size >> 20, mc->gart_start, mc->gart_end);
 627 }
 628
 629 /*
 630  * Firmware Reservation functions
 631  */
 632 /**
 633  * amdgpu_fw_reserve_vram_fini - free fw reserved vram
 634  *
 635  * @adev: amdgpu_device pointer
 636  *
 637  * free fw reserved vram if it has been reserved.
 638  */
 639 void amdgpu_fw_reserve_vram_fini(struct amdgpu_device *adev)
 640 {
 641         amdgpu_bo_free_kernel(&adev->fw_vram_usage.reserved_bo,
 642                 NULL, &adev->fw_vram_usage.va);
 643 }
 644
 645 /**
 646  * amdgpu_fw_reserve_vram_init - create bo vram reservation from fw
 647  *
 648  * @adev: amdgpu_device pointer
 649  *
 650  * create bo vram reservation from fw.
 651  */
 652 int amdgpu_fw_reserve_vram_init(struct amdgpu_device *adev)
 653 {
 654         struct ttm_operation_ctx ctx = { false, false };
 655         int r = 0;
 656         int i;
 657         u64 vram_size = adev->mc.visible_vram_size;
 658         u64 offset = adev->fw_vram_usage.start_offset;
 659         u64 size = adev->fw_vram_usage.size;
 660         struct amdgpu_bo *bo;
 661
 662         adev->fw_vram_usage.va = NULL;
 663         adev->fw_vram_usage.reserved_bo = NULL;
 664
 665         if (adev->fw_vram_usage.size > 0 &&
 666                 adev->fw_vram_usage.size <= vram_size) {
 667
 668                 r = amdgpu_bo_create(adev, adev->fw_vram_usage.size,
 669                         PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
 670                         AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
 671                         AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS, NULL, NULL, 0,
 672                         &adev->fw_vram_usage.reserved_bo);
 673                 if (r)
 674                         goto error_create;
 675
 676                 r = amdgpu_bo_reserve(adev->fw_vram_usage.reserved_bo, false);
 677                 if (r)
 678                         goto error_reserve;
 679
 680                 /* remove the original mem node and create a new one at the
 681                  * request position
 682                  */
 683                 bo = adev->fw_vram_usage.reserved_bo;
 684                 offset = ALIGN(offset, PAGE_SIZE);
 685                 for (i = 0; i < bo->placement.num_placement; ++i) {
 686                         bo->placements[i].fpfn = offset >> PAGE_SHIFT;
 687                         bo->placements[i].lpfn = (offset + size) >> PAGE_SHIFT;
 688                 }
 689
 690                 ttm_bo_mem_put(&bo->tbo, &bo->tbo.mem);
 691                 r = ttm_bo_mem_space(&bo->tbo, &bo->placement,
 692                                      &bo->tbo.mem, &ctx);
 693                 if (r)
 694                         goto error_pin;
 695
 696                 r = amdgpu_bo_pin_restricted(adev->fw_vram_usage.reserved_bo,
 697                         AMDGPU_GEM_DOMAIN_VRAM,
 698                         adev->fw_vram_usage.start_offset,
 699                         (adev->fw_vram_usage.start_offset +
 700                         adev->fw_vram_usage.size), NULL);
 701                 if (r)
 702                         goto error_pin;
 703                 r = amdgpu_bo_kmap(adev->fw_vram_usage.reserved_bo,
 704                         &adev->fw_vram_usage.va);
 705                 if (r)
 706                         goto error_kmap;
 707
 708                 amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
 709         }
 710         return r;
 711
 712 error_kmap:
 713         amdgpu_bo_unpin(adev->fw_vram_usage.reserved_bo);
 714 error_pin:
 715         amdgpu_bo_unreserve(adev->fw_vram_usage.reserved_bo);
 716 error_reserve:
 717         amdgpu_bo_unref(&adev->fw_vram_usage.reserved_bo);
 718 error_create:
 719         adev->fw_vram_usage.va = NULL;
 720         adev->fw_vram_usage.reserved_bo = NULL;
 721         return r;
 722 }
 723
 724 /**
 725  * amdgpu_device_resize_fb_bar - try to resize FB BAR
 726  *
 727  * @adev: amdgpu_device pointer
 728  *
 729  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
 730  * to fail, but if any of the BARs is not accessible after the size we abort
 731  * driver loading by returning -ENODEV.
 732  */
 733 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
 734 {
 735         u64 space_needed = roundup_pow_of_two(adev->mc.real_vram_size);
 736         u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
 737         struct pci_bus *root;
 738         struct resource *res;
 739         unsigned i;
 740         u16 cmd;
 741         int r;
 742
 743         /* Bypass for VF */
 744         if (amdgpu_sriov_vf(adev))
 745                 return 0;
 746
 747         /* Check if the root BUS has 64bit memory resources */
 748         root = adev->pdev->bus;
 749         while (root->parent)
 750                 root = root->parent;
 751
 752         pci_bus_for_each_resource(root, res, i) {
 753                 if (res && res->flags & IORESOURCE_MEM_64 &&
 754                     res->start > 0x100000000ull)
 755                         break;
 756         }
 757
 758         /* Trying to resize is pointless without a root hub window above 4GB */
 759         if (!res)
 760                 return 0;
 761
 762         /* Disable memory decoding while we change the BAR addresses and size */
 763         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
 764         pci_write_config_word(adev->pdev, PCI_COMMAND,
 765                               cmd & ~PCI_COMMAND_MEMORY);
 766
 767         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
 768         amdgpu_device_doorbell_fini(adev);
 769         if (adev->asic_type >= CHIP_BONAIRE)
 770                 pci_release_resource(adev->pdev, 2);
 771
 772         pci_release_resource(adev->pdev, 0);
 773
 774         r = pci_resize_resource(adev->pdev, 0, rbar_size);
 775         if (r == -ENOSPC)
 776                 DRM_INFO("Not enough PCI address space for a large BAR.");
 777         else if (r && r != -ENOTSUPP)
 778                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
 779
 780         pci_assign_unassigned_bus_resources(adev->pdev->bus);
 781
 782         /* When the doorbell or fb BAR isn't available we have no chance of
 783          * using the device.
 784          */
 785         r = amdgpu_device_doorbell_init(adev);
 786         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
 787                 return -ENODEV;
 788
 789         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
 790
 791         return 0;
 792 }
 793
 794 /*
 795  * GPU helpers function.
 796  */
 797 /**
 798  * amdgpu_need_post - check if the hw need post or not
 799  *
 800  * @adev: amdgpu_device pointer
 801  *
 802  * Check if the asic has been initialized (all asics) at driver startup
 803  * or post is needed if  hw reset is performed.
 804  * Returns true if need or false if not.
 805  */
 806 bool amdgpu_need_post(struct amdgpu_device *adev)
 807 {
 808         uint32_t reg;
 809
 810         if (amdgpu_sriov_vf(adev))
 811                 return false;
 812
 813         if (amdgpu_passthrough(adev)) {
 814                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
 815                  * some old smc fw still need driver do vPost otherwise gpu hang, while
 816                  * those smc fw version above 22.15 doesn't have this flaw, so we force
 817                  * vpost executed for smc version below 22.15
 818                  */
 819                 if (adev->asic_type == CHIP_FIJI) {
 820                         int err;
 821                         uint32_t fw_ver;
 822                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
 823                         /* force vPost if error occured */
 824                         if (err)
 825                                 return true;
 826
 827                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
 828                         if (fw_ver < 0x00160e00)
 829                                 return true;
 830                 }
 831         }
 832
 833         if (adev->has_hw_reset) {
 834                 adev->has_hw_reset = false;
 835                 return true;
 836         }
 837
 838         /* bios scratch used on CIK+ */
 839         if (adev->asic_type >= CHIP_BONAIRE)
 840                 return amdgpu_atombios_scratch_need_asic_init(adev);
 841
 842         /* check MEM_SIZE for older asics */
 843         reg = amdgpu_asic_get_config_memsize(adev);
 844
 845         if ((reg != 0) && (reg != 0xffffffff))
 846                 return false;
 847
 848         return true;
 849 }
 850
 851 /**
 852  * amdgpu_dummy_page_init - init dummy page used by the driver
 853  *
 854  * @adev: amdgpu_device pointer
 855  *
 856  * Allocate the dummy page used by the driver (all asics).
 857  * This dummy page is used by the driver as a filler for gart entries
 858  * when pages are taken out of the GART
 859  * Returns 0 on sucess, -ENOMEM on failure.
 860  */
 861 int amdgpu_dummy_page_init(struct amdgpu_device *adev)
 862 {
 863         if (adev->dummy_page.page)
 864                 return 0;
 865         adev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
 866         if (adev->dummy_page.page == NULL)
 867                 return -ENOMEM;
 868         adev->dummy_page.addr = pci_map_page(adev->pdev, adev->dummy_page.page,
 869                                         0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 870         if (pci_dma_mapping_error(adev->pdev, adev->dummy_page.addr)) {
 871                 dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n");
 872                 __free_page(adev->dummy_page.page);
 873                 adev->dummy_page.page = NULL;
 874                 return -ENOMEM;
 875         }
 876         return 0;
 877 }
 878
 879 /**
 880  * amdgpu_dummy_page_fini - free dummy page used by the driver
 881  *
 882  * @adev: amdgpu_device pointer
 883  *
 884  * Frees the dummy page used by the driver (all asics).
 885  */
 886 void amdgpu_dummy_page_fini(struct amdgpu_device *adev)
 887 {
 888         if (adev->dummy_page.page == NULL)
 889                 return;
 890         pci_unmap_page(adev->pdev, adev->dummy_page.addr,
 891                         PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 892         __free_page(adev->dummy_page.page);
 893         adev->dummy_page.page = NULL;
 894 }
 895
 896 /* if we get transitioned to only one device, take VGA back */
 897 /**
 898  * amdgpu_device_vga_set_decode - enable/disable vga decode
 899  *
 900  * @cookie: amdgpu_device pointer
 901  * @state: enable/disable vga decode
 902  *
 903  * Enable/disable vga decode (all asics).
 904  * Returns VGA resource flags.
 905  */
 906 static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
 907 {
 908         struct amdgpu_device *adev = cookie;
 909         amdgpu_asic_set_vga_state(adev, state);
 910         if (state)
 911                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
 912                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 913         else
 914                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 915 }
 916
 917 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
 918 {
 919         /* defines number of bits in page table versus page directory,
 920          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
 921          * page table and the remaining bits are in the page directory */
 922         if (amdgpu_vm_block_size == -1)
 923                 return;
 924
 925         if (amdgpu_vm_block_size < 9) {
 926                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
 927                          amdgpu_vm_block_size);
 928                 amdgpu_vm_block_size = -1;
 929         }
 930 }
 931
 932 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
 933 {
 934         /* no need to check the default value */
 935         if (amdgpu_vm_size == -1)
 936                 return;
 937
 938         if (amdgpu_vm_size < 1) {
 939                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
 940                          amdgpu_vm_size);
 941                 amdgpu_vm_size = -1;
 942         }
 943 }
 944
 945 /**
 946  * amdgpu_device_check_arguments - validate module params
 947  *
 948  * @adev: amdgpu_device pointer
 949  *
 950  * Validates certain module parameters and updates
 951  * the associated values used by the driver (all asics).
 952  */
 953 static void amdgpu_device_check_arguments(struct amdgpu_device *adev)
 954 {
 955         if (amdgpu_sched_jobs < 4) {
 956                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
 957                          amdgpu_sched_jobs);
 958                 amdgpu_sched_jobs = 4;
 959         } else if (!is_power_of_2(amdgpu_sched_jobs)){
 960                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
 961                          amdgpu_sched_jobs);
 962                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
 963         }
 964
 965         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
 966                 /* gart size must be greater or equal to 32M */
 967                 dev_warn(adev->dev, "gart size (%d) too small\n",
 968                          amdgpu_gart_size);
 969                 amdgpu_gart_size = -1;
 970         }
 971
 972         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
 973                 /* gtt size must be greater or equal to 32M */
 974                 dev_warn(adev->dev, "gtt size (%d) too small\n",
 975                                  amdgpu_gtt_size);
 976                 amdgpu_gtt_size = -1;
 977         }
 978
 979         /* valid range is between 4 and 9 inclusive */
 980         if (amdgpu_vm_fragment_size != -1 &&
 981             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
 982                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
 983                 amdgpu_vm_fragment_size = -1;
 984         }
 985
 986         amdgpu_device_check_vm_size(adev);
 987
 988         amdgpu_device_check_block_size(adev);
 989
 990         if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
 991             !is_power_of_2(amdgpu_vram_page_split))) {
 992                 dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
 993                          amdgpu_vram_page_split);
 994                 amdgpu_vram_page_split = 1024;
 995         }
 996
 997         if (amdgpu_lockup_timeout == 0) {
 998                 dev_warn(adev->dev, "lockup_timeout msut be > 0, adjusting to 10000\n");
 999                 amdgpu_lockup_timeout = 10000;
1000         }
1001 }
1002
1003 /**
1004  * amdgpu_switcheroo_set_state - set switcheroo state
1005  *
1006  * @pdev: pci dev pointer
1007  * @state: vga_switcheroo state
1008  *
1009  * Callback for the switcheroo driver.  Suspends or resumes the
1010  * the asics before or after it is powered up using ACPI methods.
1011  */
1012 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1013 {
1014         struct drm_device *dev = pci_get_drvdata(pdev);
1015
1016         if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1017                 return;
1018
1019         if (state == VGA_SWITCHEROO_ON) {
1020                 pr_info("amdgpu: switched on\n");
1021                 /* don't suspend or resume card normally */
1022                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1023
1024                 amdgpu_device_resume(dev, true, true);
1025
1026                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1027                 drm_kms_helper_poll_enable(dev);
1028         } else {
1029                 pr_info("amdgpu: switched off\n");
1030                 drm_kms_helper_poll_disable(dev);
1031                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1032                 amdgpu_device_suspend(dev, true, true);
1033                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1034         }
1035 }
1036
1037 /**
1038  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1039  *
1040  * @pdev: pci dev pointer
1041  *
1042  * Callback for the switcheroo driver.  Check of the switcheroo
1043  * state can be changed.
1044  * Returns true if the state can be changed, false if not.
1045  */
1046 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1047 {
1048         struct drm_device *dev = pci_get_drvdata(pdev);
1049
1050         /*
1051         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1052         * locking inversion with the driver load path. And the access here is
1053         * completely racy anyway. So don't bother with locking for now.
1054         */
1055         return dev->open_count == 0;
1056 }
1057
1058 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1059         .set_gpu_state = amdgpu_switcheroo_set_state,
1060         .reprobe = NULL,
1061         .can_switch = amdgpu_switcheroo_can_switch,
1062 };
1063
1064 int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
1065                                   enum amd_ip_block_type block_type,
1066                                   enum amd_clockgating_state state)
1067 {
1068         int i, r = 0;
1069
1070         for (i = 0; i < adev->num_ip_blocks; i++) {
1071                 if (!adev->ip_blocks[i].status.valid)
1072                         continue;
1073                 if (adev->ip_blocks[i].version->type != block_type)
1074                         continue;
1075                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1076                         continue;
1077                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1078                         (void *)adev, state);
1079                 if (r)
1080                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1081                                   adev->ip_blocks[i].version->funcs->name, r);
1082         }
1083         return r;
1084 }
1085
1086 int amdgpu_set_powergating_state(struct amdgpu_device *adev,
1087                                   enum amd_ip_block_type block_type,
1088                                   enum amd_powergating_state state)
1089 {
1090         int i, r = 0;
1091
1092         for (i = 0; i < adev->num_ip_blocks; i++) {
1093                 if (!adev->ip_blocks[i].status.valid)
1094                         continue;
1095                 if (adev->ip_blocks[i].version->type != block_type)
1096                         continue;
1097                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1098                         continue;
1099                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1100                         (void *)adev, state);
1101                 if (r)
1102                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1103                                   adev->ip_blocks[i].version->funcs->name, r);
1104         }
1105         return r;
1106 }
1107
1108 void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
1109 {
1110         int i;
1111
1112         for (i = 0; i < adev->num_ip_blocks; i++) {
1113                 if (!adev->ip_blocks[i].status.valid)
1114                         continue;
1115                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1116                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1117         }
1118 }
1119
1120 int amdgpu_wait_for_idle(struct amdgpu_device *adev,
1121                          enum amd_ip_block_type block_type)
1122 {
1123         int i, r;
1124
1125         for (i = 0; i < adev->num_ip_blocks; i++) {
1126                 if (!adev->ip_blocks[i].status.valid)
1127                         continue;
1128                 if (adev->ip_blocks[i].version->type == block_type) {
1129                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1130                         if (r)
1131                                 return r;
1132                         break;
1133                 }
1134         }
1135         return 0;
1136
1137 }
1138
1139 bool amdgpu_is_idle(struct amdgpu_device *adev,
1140                     enum amd_ip_block_type block_type)
1141 {
1142         int i;
1143
1144         for (i = 0; i < adev->num_ip_blocks; i++) {
1145                 if (!adev->ip_blocks[i].status.valid)
1146                         continue;
1147                 if (adev->ip_blocks[i].version->type == block_type)
1148                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1149         }
1150         return true;
1151
1152 }
1153
1154 struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
1155                                              enum amd_ip_block_type type)
1156 {
1157         int i;
1158
1159         for (i = 0; i < adev->num_ip_blocks; i++)
1160                 if (adev->ip_blocks[i].version->type == type)
1161                         return &adev->ip_blocks[i];
1162
1163         return NULL;
1164 }
1165
1166 /**
1167  * amdgpu_ip_block_version_cmp
1168  *
1169  * @adev: amdgpu_device pointer
1170  * @type: enum amd_ip_block_type
1171  * @major: major version
1172  * @minor: minor version
1173  *
1174  * return 0 if equal or greater
1175  * return 1 if smaller or the ip_block doesn't exist
1176  */
1177 int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
1178                                 enum amd_ip_block_type type,
1179                                 u32 major, u32 minor)
1180 {
1181         struct amdgpu_ip_block *ip_block = amdgpu_get_ip_block(adev, type);
1182
1183         if (ip_block && ((ip_block->version->major > major) ||
1184                         ((ip_block->version->major == major) &&
1185                         (ip_block->version->minor >= minor))))
1186                 return 0;
1187
1188         return 1;
1189 }
1190
1191 /**
1192  * amdgpu_ip_block_add
1193  *
1194  * @adev: amdgpu_device pointer
1195  * @ip_block_version: pointer to the IP to add
1196  *
1197  * Adds the IP block driver information to the collection of IPs
1198  * on the asic.
1199  */
1200 int amdgpu_ip_block_add(struct amdgpu_device *adev,
1201                         const struct amdgpu_ip_block_version *ip_block_version)
1202 {
1203         if (!ip_block_version)
1204                 return -EINVAL;
1205
1206         DRM_DEBUG("add ip block number %d <%s>\n", adev->num_ip_blocks,
1207                   ip_block_version->funcs->name);
1208
1209         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1210
1211         return 0;
1212 }
1213
1214 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1215 {
1216         adev->enable_virtual_display = false;
1217
1218         if (amdgpu_virtual_display) {
1219                 struct drm_device *ddev = adev->ddev;
1220                 const char *pci_address_name = pci_name(ddev->pdev);
1221                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1222
1223                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1224                 pciaddstr_tmp = pciaddstr;
1225                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1226                         pciaddname = strsep(&pciaddname_tmp, ",");
1227                         if (!strcmp("all", pciaddname)
1228                             || !strcmp(pci_address_name, pciaddname)) {
1229                                 long num_crtc;
1230                                 int res = -1;
1231
1232                                 adev->enable_virtual_display = true;
1233
1234                                 if (pciaddname_tmp)
1235                                         res = kstrtol(pciaddname_tmp, 10,
1236                                                       &num_crtc);
1237
1238                                 if (!res) {
1239                                         if (num_crtc < 1)
1240                                                 num_crtc = 1;
1241                                         if (num_crtc > 6)
1242                                                 num_crtc = 6;
1243                                         adev->mode_info.num_crtc = num_crtc;
1244                                 } else {
1245                                         adev->mode_info.num_crtc = 1;
1246                                 }
1247                                 break;
1248                         }
1249                 }
1250
1251                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1252                          amdgpu_virtual_display, pci_address_name,
1253                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1254
1255                 kfree(pciaddstr);
1256         }
1257 }
1258
1259 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1260 {
1261         const char *chip_name;
1262         char fw_name[30];
1263         int err;
1264         const struct gpu_info_firmware_header_v1_0 *hdr;
1265
1266         adev->firmware.gpu_info_fw = NULL;
1267
1268         switch (adev->asic_type) {
1269         case CHIP_TOPAZ:
1270         case CHIP_TONGA:
1271         case CHIP_FIJI:
1272         case CHIP_POLARIS11:
1273         case CHIP_POLARIS10:
1274         case CHIP_POLARIS12:
1275         case CHIP_CARRIZO:
1276         case CHIP_STONEY:
1277 #ifdef CONFIG_DRM_AMDGPU_SI
1278         case CHIP_VERDE:
1279         case CHIP_TAHITI:
1280         case CHIP_PITCAIRN:
1281         case CHIP_OLAND:
1282         case CHIP_HAINAN:
1283 #endif
1284 #ifdef CONFIG_DRM_AMDGPU_CIK
1285         case CHIP_BONAIRE:
1286         case CHIP_HAWAII:
1287         case CHIP_KAVERI:
1288         case CHIP_KABINI:
1289         case CHIP_MULLINS:
1290 #endif
1291         default:
1292                 return 0;
1293         case CHIP_VEGA10:
1294                 chip_name = "vega10";
1295                 break;
1296         case CHIP_RAVEN:
1297                 chip_name = "raven";
1298                 break;
1299         }
1300
1301         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1302         err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1303         if (err) {
1304                 dev_err(adev->dev,
1305                         "Failed to load gpu_info firmware \"%s\"\n",
1306                         fw_name);
1307                 goto out;
1308         }
1309         err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1310         if (err) {
1311                 dev_err(adev->dev,
1312                         "Failed to validate gpu_info firmware \"%s\"\n",
1313                         fw_name);
1314                 goto out;
1315         }
1316
1317         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1318         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1319
1320         switch (hdr->version_major) {
1321         case 1:
1322         {
1323                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1324                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1325                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1326
1327                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1328                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1329                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1330                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1331                 adev->gfx.config.max_texture_channel_caches =
1332                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
1333                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1334                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1335                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1336                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1337                 adev->gfx.config.double_offchip_lds_buf =
1338                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1339                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1340                 adev->gfx.cu_info.max_waves_per_simd =
1341                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1342                 adev->gfx.cu_info.max_scratch_slots_per_cu =
1343                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1344                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1345                 break;
1346         }
1347         default:
1348                 dev_err(adev->dev,
1349                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1350                 err = -EINVAL;
1351                 goto out;
1352         }
1353 out:
1354         return err;
1355 }
1356
1357 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
1358 {
1359         int i, r;
1360
1361         amdgpu_device_enable_virtual_display(adev);
1362
1363         switch (adev->asic_type) {
1364         case CHIP_TOPAZ:
1365         case CHIP_TONGA:
1366         case CHIP_FIJI:
1367         case CHIP_POLARIS11:
1368         case CHIP_POLARIS10:
1369         case CHIP_POLARIS12:
1370         case CHIP_CARRIZO:
1371         case CHIP_STONEY:
1372                 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1373                         adev->family = AMDGPU_FAMILY_CZ;
1374                 else
1375                         adev->family = AMDGPU_FAMILY_VI;
1376
1377                 r = vi_set_ip_blocks(adev);
1378                 if (r)
1379                         return r;
1380                 break;
1381 #ifdef CONFIG_DRM_AMDGPU_SI
1382         case CHIP_VERDE:
1383         case CHIP_TAHITI:
1384         case CHIP_PITCAIRN:
1385         case CHIP_OLAND:
1386         case CHIP_HAINAN:
1387                 adev->family = AMDGPU_FAMILY_SI;
1388                 r = si_set_ip_blocks(adev);
1389                 if (r)
1390                         return r;
1391                 break;
1392 #endif
1393 #ifdef CONFIG_DRM_AMDGPU_CIK
1394         case CHIP_BONAIRE:
1395         case CHIP_HAWAII:
1396         case CHIP_KAVERI:
1397         case CHIP_KABINI:
1398         case CHIP_MULLINS:
1399                 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1400                         adev->family = AMDGPU_FAMILY_CI;
1401                 else
1402                         adev->family = AMDGPU_FAMILY_KV;
1403
1404                 r = cik_set_ip_blocks(adev);
1405                 if (r)
1406                         return r;
1407                 break;
1408 #endif
1409         case  CHIP_VEGA10:
1410         case  CHIP_RAVEN:
1411                 if (adev->asic_type == CHIP_RAVEN)
1412                         adev->family = AMDGPU_FAMILY_RV;
1413                 else
1414                         adev->family = AMDGPU_FAMILY_AI;
1415
1416                 r = soc15_set_ip_blocks(adev);
1417                 if (r)
1418                         return r;
1419                 break;
1420         default:
1421                 /* FIXME: not supported yet */
1422                 return -EINVAL;
1423         }
1424
1425         r = amdgpu_device_parse_gpu_info_fw(adev);
1426         if (r)
1427                 return r;
1428
1429         amdgpu_amdkfd_device_probe(adev);
1430
1431         if (amdgpu_sriov_vf(adev)) {
1432                 r = amdgpu_virt_request_full_gpu(adev, true);
1433                 if (r)
1434                         return -EAGAIN;
1435         }
1436
1437         for (i = 0; i < adev->num_ip_blocks; i++) {
1438                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1439                         DRM_ERROR("disabled ip block: %d <%s>\n",
1440                                   i, adev->ip_blocks[i].version->funcs->name);
1441                         adev->ip_blocks[i].status.valid = false;
1442                 } else {
1443                         if (adev->ip_blocks[i].version->funcs->early_init) {
1444                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1445                                 if (r == -ENOENT) {
1446                                         adev->ip_blocks[i].status.valid = false;
1447                                 } else if (r) {
1448                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
1449                                                   adev->ip_blocks[i].version->funcs->name, r);
1450                                         return r;
1451                                 } else {
1452                                         adev->ip_blocks[i].status.valid = true;
1453                                 }
1454                         } else {
1455                                 adev->ip_blocks[i].status.valid = true;
1456                         }
1457                 }
1458         }
1459
1460         adev->cg_flags &= amdgpu_cg_mask;
1461         adev->pg_flags &= amdgpu_pg_mask;
1462
1463         return 0;
1464 }
1465
1466 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
1467 {
1468         int i, r;
1469
1470         for (i = 0; i < adev->num_ip_blocks; i++) {
1471                 if (!adev->ip_blocks[i].status.valid)
1472                         continue;
1473                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1474                 if (r) {
1475                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1476                                   adev->ip_blocks[i].version->funcs->name, r);
1477                         return r;
1478                 }
1479                 adev->ip_blocks[i].status.sw = true;
1480                 /* need to do gmc hw init early so we can allocate gpu mem */
1481                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1482                         r = amdgpu_device_vram_scratch_init(adev);
1483                         if (r) {
1484                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1485                                 return r;
1486                         }
1487                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1488                         if (r) {
1489                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
1490                                 return r;
1491                         }
1492                         r = amdgpu_device_wb_init(adev);
1493                         if (r) {
1494                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
1495                                 return r;
1496                         }
1497                         adev->ip_blocks[i].status.hw = true;
1498
1499                         /* right after GMC hw init, we create CSA */
1500                         if (amdgpu_sriov_vf(adev)) {
1501                                 r = amdgpu_allocate_static_csa(adev);
1502                                 if (r) {
1503                                         DRM_ERROR("allocate CSA failed %d\n", r);
1504                                         return r;
1505                                 }
1506                         }
1507                 }
1508         }
1509
1510         for (i = 0; i < adev->num_ip_blocks; i++) {
1511                 if (!adev->ip_blocks[i].status.sw)
1512                         continue;
1513                 /* gmc hw init is done early */
1514                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC)
1515                         continue;
1516                 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1517                 if (r) {
1518                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1519                                   adev->ip_blocks[i].version->funcs->name, r);
1520                         return r;
1521                 }
1522                 adev->ip_blocks[i].status.hw = true;
1523         }
1524
1525         amdgpu_amdkfd_device_init(adev);
1526
1527         if (amdgpu_sriov_vf(adev))
1528                 amdgpu_virt_release_full_gpu(adev, true);
1529
1530         return 0;
1531 }
1532
1533 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
1534 {
1535         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
1536 }
1537
1538 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
1539 {
1540         return !!memcmp(adev->gart.ptr, adev->reset_magic,
1541                         AMDGPU_RESET_MAGIC_NUM);
1542 }
1543
1544 static int amdgpu_device_ip_late_set_cg_state(struct amdgpu_device *adev)
1545 {
1546         int i = 0, r;
1547
1548         for (i = 0; i < adev->num_ip_blocks; i++) {
1549                 if (!adev->ip_blocks[i].status.valid)
1550                         continue;
1551                 /* skip CG for VCE/UVD, it's handled specially */
1552                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1553                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1554                         /* enable clockgating to save power */
1555                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1556                                                                                      AMD_CG_STATE_GATE);
1557                         if (r) {
1558                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1559                                           adev->ip_blocks[i].version->funcs->name, r);
1560                                 return r;
1561                         }
1562                 }
1563         }
1564         return 0;
1565 }
1566
1567 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
1568 {
1569         int i = 0, r;
1570
1571         for (i = 0; i < adev->num_ip_blocks; i++) {
1572                 if (!adev->ip_blocks[i].status.valid)
1573                         continue;
1574                 if (adev->ip_blocks[i].version->funcs->late_init) {
1575                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
1576                         if (r) {
1577                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
1578                                           adev->ip_blocks[i].version->funcs->name, r);
1579                                 return r;
1580                         }
1581                         adev->ip_blocks[i].status.late_initialized = true;
1582                 }
1583         }
1584
1585         mod_delayed_work(system_wq, &adev->late_init_work,
1586                         msecs_to_jiffies(AMDGPU_RESUME_MS));
1587
1588         amdgpu_device_fill_reset_magic(adev);
1589
1590         return 0;
1591 }
1592
1593 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
1594 {
1595         int i, r;
1596
1597         amdgpu_amdkfd_device_fini(adev);
1598         /* need to disable SMC first */
1599         for (i = 0; i < adev->num_ip_blocks; i++) {
1600                 if (!adev->ip_blocks[i].status.hw)
1601                         continue;
1602                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
1603                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1604                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1605                                                                                      AMD_CG_STATE_UNGATE);
1606                         if (r) {
1607                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1608                                           adev->ip_blocks[i].version->funcs->name, r);
1609                                 return r;
1610                         }
1611                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1612                         /* XXX handle errors */
1613                         if (r) {
1614                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1615                                           adev->ip_blocks[i].version->funcs->name, r);
1616                         }
1617                         adev->ip_blocks[i].status.hw = false;
1618                         break;
1619                 }
1620         }
1621
1622         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1623                 if (!adev->ip_blocks[i].status.hw)
1624                         continue;
1625                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1626                         amdgpu_free_static_csa(adev);
1627                         amdgpu_device_wb_fini(adev);
1628                         amdgpu_device_vram_scratch_fini(adev);
1629                 }
1630
1631                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1632                         adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1633                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1634                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1635                                                                                      AMD_CG_STATE_UNGATE);
1636                         if (r) {
1637                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1638                                           adev->ip_blocks[i].version->funcs->name, r);
1639                                 return r;
1640                         }
1641                 }
1642
1643                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1644                 /* XXX handle errors */
1645                 if (r) {
1646                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1647                                   adev->ip_blocks[i].version->funcs->name, r);
1648                 }
1649
1650                 adev->ip_blocks[i].status.hw = false;
1651         }
1652
1653         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1654                 if (!adev->ip_blocks[i].status.sw)
1655                         continue;
1656                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
1657                 /* XXX handle errors */
1658                 if (r) {
1659                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
1660                                   adev->ip_blocks[i].version->funcs->name, r);
1661                 }
1662                 adev->ip_blocks[i].status.sw = false;
1663                 adev->ip_blocks[i].status.valid = false;
1664         }
1665
1666         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1667                 if (!adev->ip_blocks[i].status.late_initialized)
1668                         continue;
1669                 if (adev->ip_blocks[i].version->funcs->late_fini)
1670                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
1671                 adev->ip_blocks[i].status.late_initialized = false;
1672         }
1673
1674         if (amdgpu_sriov_vf(adev))
1675                 if (amdgpu_virt_release_full_gpu(adev, false))
1676                         DRM_ERROR("failed to release exclusive mode on fini\n");
1677
1678         return 0;
1679 }
1680
1681 static void amdgpu_device_ip_late_init_func_handler(struct work_struct *work)
1682 {
1683         struct amdgpu_device *adev =
1684                 container_of(work, struct amdgpu_device, late_init_work.work);
1685         amdgpu_device_ip_late_set_cg_state(adev);
1686 }
1687
1688 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
1689 {
1690         int i, r;
1691
1692         if (amdgpu_sriov_vf(adev))
1693                 amdgpu_virt_request_full_gpu(adev, false);
1694
1695         /* ungate SMC block first */
1696         r = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
1697                                          AMD_CG_STATE_UNGATE);
1698         if (r) {
1699                 DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n",r);
1700         }
1701
1702         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1703                 if (!adev->ip_blocks[i].status.valid)
1704                         continue;
1705                 /* ungate blocks so that suspend can properly shut them down */
1706                 if (i != AMD_IP_BLOCK_TYPE_SMC) {
1707                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1708                                                                                      AMD_CG_STATE_UNGATE);
1709                         if (r) {
1710                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1711                                           adev->ip_blocks[i].version->funcs->name, r);
1712                         }
1713                 }
1714                 /* XXX handle errors */
1715                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
1716                 /* XXX handle errors */
1717                 if (r) {
1718                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
1719                                   adev->ip_blocks[i].version->funcs->name, r);
1720                 }
1721         }
1722
1723         if (amdgpu_sriov_vf(adev))
1724                 amdgpu_virt_release_full_gpu(adev, false);
1725
1726         return 0;
1727 }
1728
1729 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
1730 {
1731         int i, r;
1732
1733         static enum amd_ip_block_type ip_order[] = {
1734                 AMD_IP_BLOCK_TYPE_GMC,
1735                 AMD_IP_BLOCK_TYPE_COMMON,
1736                 AMD_IP_BLOCK_TYPE_IH,
1737         };
1738
1739         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1740                 int j;
1741                 struct amdgpu_ip_block *block;
1742
1743                 for (j = 0; j < adev->num_ip_blocks; j++) {
1744                         block = &adev->ip_blocks[j];
1745
1746                         if (block->version->type != ip_order[i] ||
1747                                 !block->status.valid)
1748                                 continue;
1749
1750                         r = block->version->funcs->hw_init(adev);
1751                         DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
1752                 }
1753         }
1754
1755         return 0;
1756 }
1757
1758 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
1759 {
1760         int i, r;
1761
1762         static enum amd_ip_block_type ip_order[] = {
1763                 AMD_IP_BLOCK_TYPE_SMC,
1764                 AMD_IP_BLOCK_TYPE_PSP,
1765                 AMD_IP_BLOCK_TYPE_DCE,
1766                 AMD_IP_BLOCK_TYPE_GFX,
1767                 AMD_IP_BLOCK_TYPE_SDMA,
1768                 AMD_IP_BLOCK_TYPE_UVD,
1769                 AMD_IP_BLOCK_TYPE_VCE
1770         };
1771
1772         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1773                 int j;
1774                 struct amdgpu_ip_block *block;
1775
1776                 for (j = 0; j < adev->num_ip_blocks; j++) {
1777                         block = &adev->ip_blocks[j];
1778
1779                         if (block->version->type != ip_order[i] ||
1780                                 !block->status.valid)
1781                                 continue;
1782
1783                         r = block->version->funcs->hw_init(adev);
1784                         DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
1785                 }
1786         }
1787
1788         return 0;
1789 }
1790
1791 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
1792 {
1793         int i, r;
1794
1795         for (i = 0; i < adev->num_ip_blocks; i++) {
1796                 if (!adev->ip_blocks[i].status.valid)
1797                         continue;
1798                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1799                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
1800                                 adev->ip_blocks[i].version->type ==
1801                                 AMD_IP_BLOCK_TYPE_IH) {
1802                         r = adev->ip_blocks[i].version->funcs->resume(adev);
1803                         if (r) {
1804                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
1805                                           adev->ip_blocks[i].version->funcs->name, r);
1806                                 return r;
1807                         }
1808                 }
1809         }
1810
1811         return 0;
1812 }
1813
1814 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
1815 {
1816         int i, r;
1817
1818         for (i = 0; i < adev->num_ip_blocks; i++) {
1819                 if (!adev->ip_blocks[i].status.valid)
1820                         continue;
1821                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1822                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
1823                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH )
1824                         continue;
1825                 r = adev->ip_blocks[i].version->funcs->resume(adev);
1826                 if (r) {
1827                         DRM_ERROR("resume of IP block <%s> failed %d\n",
1828                                   adev->ip_blocks[i].version->funcs->name, r);
1829                         return r;
1830                 }
1831         }
1832
1833         return 0;
1834 }
1835
1836 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
1837 {
1838         int r;
1839
1840         r = amdgpu_device_ip_resume_phase1(adev);
1841         if (r)
1842                 return r;
1843         r = amdgpu_device_ip_resume_phase2(adev);
1844
1845         return r;
1846 }
1847
1848 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
1849 {
1850         if (amdgpu_sriov_vf(adev)) {
1851                 if (adev->is_atom_fw) {
1852                         if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
1853                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
1854                 } else {
1855                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
1856                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
1857                 }
1858
1859                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
1860                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
1861         }
1862 }
1863
1864 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
1865 {
1866         switch (asic_type) {
1867 #if defined(CONFIG_DRM_AMD_DC)
1868         case CHIP_BONAIRE:
1869         case CHIP_HAWAII:
1870         case CHIP_KAVERI:
1871         case CHIP_CARRIZO:
1872         case CHIP_STONEY:
1873         case CHIP_POLARIS11:
1874         case CHIP_POLARIS10:
1875         case CHIP_POLARIS12:
1876         case CHIP_TONGA:
1877         case CHIP_FIJI:
1878 #if defined(CONFIG_DRM_AMD_DC_PRE_VEGA)
1879                 return amdgpu_dc != 0;
1880 #endif
1881         case CHIP_KABINI:
1882         case CHIP_MULLINS:
1883                 return amdgpu_dc > 0;
1884         case CHIP_VEGA10:
1885 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
1886         case CHIP_RAVEN:
1887 #endif
1888                 return amdgpu_dc != 0;
1889 #endif
1890         default:
1891                 return false;
1892         }
1893 }
1894
1895 /**
1896  * amdgpu_device_has_dc_support - check if dc is supported
1897  *
1898  * @adev: amdgpu_device_pointer
1899  *
1900  * Returns true for supported, false for not supported
1901  */
1902 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
1903 {
1904         if (amdgpu_sriov_vf(adev))
1905                 return false;
1906
1907         return amdgpu_device_asic_has_dc_support(adev->asic_type);
1908 }
1909
1910 /**
1911  * amdgpu_device_init - initialize the driver
1912  *
1913  * @adev: amdgpu_device pointer
1914  * @pdev: drm dev pointer
1915  * @pdev: pci dev pointer
1916  * @flags: driver flags
1917  *
1918  * Initializes the driver info and hw (all asics).
1919  * Returns 0 for success or an error on failure.
1920  * Called at driver startup.
1921  */
1922 int amdgpu_device_init(struct amdgpu_device *adev,
1923                        struct drm_device *ddev,
1924                        struct pci_dev *pdev,
1925                        uint32_t flags)
1926 {
1927         int r, i;
1928         bool runtime = false;
1929         u32 max_MBps;
1930
1931         adev->shutdown = false;
1932         adev->dev = &pdev->dev;
1933         adev->ddev = ddev;
1934         adev->pdev = pdev;
1935         adev->flags = flags;
1936         adev->asic_type = flags & AMD_ASIC_MASK;
1937         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
1938         adev->mc.gart_size = 512 * 1024 * 1024;
1939         adev->accel_working = false;
1940         adev->num_rings = 0;
1941         adev->mman.buffer_funcs = NULL;
1942         adev->mman.buffer_funcs_ring = NULL;
1943         adev->vm_manager.vm_pte_funcs = NULL;
1944         adev->vm_manager.vm_pte_num_rings = 0;
1945         adev->gart.gart_funcs = NULL;
1946         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
1947         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1948
1949         adev->smc_rreg = &amdgpu_invalid_rreg;
1950         adev->smc_wreg = &amdgpu_invalid_wreg;
1951         adev->pcie_rreg = &amdgpu_invalid_rreg;
1952         adev->pcie_wreg = &amdgpu_invalid_wreg;
1953         adev->pciep_rreg = &amdgpu_invalid_rreg;
1954         adev->pciep_wreg = &amdgpu_invalid_wreg;
1955         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
1956         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
1957         adev->didt_rreg = &amdgpu_invalid_rreg;
1958         adev->didt_wreg = &amdgpu_invalid_wreg;
1959         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
1960         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
1961         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
1962         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
1963
1964         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
1965                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
1966                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
1967
1968         /* mutex initialization are all done here so we
1969          * can recall function without having locking issues */
1970         atomic_set(&adev->irq.ih.lock, 0);
1971         mutex_init(&adev->firmware.mutex);
1972         mutex_init(&adev->pm.mutex);
1973         mutex_init(&adev->gfx.gpu_clock_mutex);
1974         mutex_init(&adev->srbm_mutex);
1975         mutex_init(&adev->gfx.pipe_reserve_mutex);
1976         mutex_init(&adev->grbm_idx_mutex);
1977         mutex_init(&adev->mn_lock);
1978         mutex_init(&adev->virt.vf_errors.lock);
1979         hash_init(adev->mn_hash);
1980         mutex_init(&adev->lock_reset);
1981
1982         amdgpu_device_check_arguments(adev);
1983
1984         spin_lock_init(&adev->mmio_idx_lock);
1985         spin_lock_init(&adev->smc_idx_lock);
1986         spin_lock_init(&adev->pcie_idx_lock);
1987         spin_lock_init(&adev->uvd_ctx_idx_lock);
1988         spin_lock_init(&adev->didt_idx_lock);
1989         spin_lock_init(&adev->gc_cac_idx_lock);
1990         spin_lock_init(&adev->se_cac_idx_lock);
1991         spin_lock_init(&adev->audio_endpt_idx_lock);
1992         spin_lock_init(&adev->mm_stats.lock);
1993
1994         INIT_LIST_HEAD(&adev->shadow_list);
1995         mutex_init(&adev->shadow_list_lock);
1996
1997         INIT_LIST_HEAD(&adev->ring_lru_list);
1998         spin_lock_init(&adev->ring_lru_list_lock);
1999
2000         INIT_DELAYED_WORK(&adev->late_init_work,
2001                           amdgpu_device_ip_late_init_func_handler);
2002
2003         /* Registers mapping */
2004         /* TODO: block userspace mapping of io register */
2005         if (adev->asic_type >= CHIP_BONAIRE) {
2006                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
2007                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
2008         } else {
2009                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
2010                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
2011         }
2012
2013         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
2014         if (adev->rmmio == NULL) {
2015                 return -ENOMEM;
2016         }
2017         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
2018         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
2019
2020         /* doorbell bar mapping */
2021         amdgpu_device_doorbell_init(adev);
2022
2023         /* io port mapping */
2024         for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
2025                 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
2026                         adev->rio_mem_size = pci_resource_len(adev->pdev, i);
2027                         adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
2028                         break;
2029                 }
2030         }
2031         if (adev->rio_mem == NULL)
2032                 DRM_INFO("PCI I/O BAR is not found.\n");
2033
2034         /* early init functions */
2035         r = amdgpu_device_ip_early_init(adev);
2036         if (r)
2037                 return r;
2038
2039         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
2040         /* this will fail for cards that aren't VGA class devices, just
2041          * ignore it */
2042         vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
2043
2044         if (amdgpu_runtime_pm == 1)
2045                 runtime = true;
2046         if (amdgpu_device_is_px(ddev))
2047                 runtime = true;
2048         if (!pci_is_thunderbolt_attached(adev->pdev))
2049                 vga_switcheroo_register_client(adev->pdev,
2050                                                &amdgpu_switcheroo_ops, runtime);
2051         if (runtime)
2052                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
2053
2054         /* Read BIOS */
2055         if (!amdgpu_get_bios(adev)) {
2056                 r = -EINVAL;
2057                 goto failed;
2058         }
2059
2060         r = amdgpu_atombios_init(adev);
2061         if (r) {
2062                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2063                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2064                 goto failed;
2065         }
2066
2067         /* detect if we are with an SRIOV vbios */
2068         amdgpu_device_detect_sriov_bios(adev);
2069
2070         /* Post card if necessary */
2071         if (amdgpu_need_post(adev)) {
2072                 if (!adev->bios) {
2073                         dev_err(adev->dev, "no vBIOS found\n");
2074                         r = -EINVAL;
2075                         goto failed;
2076                 }
2077                 DRM_INFO("GPU posting now...\n");
2078                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2079                 if (r) {
2080                         dev_err(adev->dev, "gpu post error!\n");
2081                         goto failed;
2082                 }
2083         }
2084
2085         if (adev->is_atom_fw) {
2086                 /* Initialize clocks */
2087                 r = amdgpu_atomfirmware_get_clock_info(adev);
2088                 if (r) {
2089                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
2090                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2091                         goto failed;
2092                 }
2093         } else {
2094                 /* Initialize clocks */
2095                 r = amdgpu_atombios_get_clock_info(adev);
2096                 if (r) {
2097                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
2098                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
2099                         goto failed;
2100                 }
2101                 /* init i2c buses */
2102                 if (!amdgpu_device_has_dc_support(adev))
2103                         amdgpu_atombios_i2c_init(adev);
2104         }
2105
2106         /* Fence driver */
2107         r = amdgpu_fence_driver_init(adev);
2108         if (r) {
2109                 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
2110                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
2111                 goto failed;
2112         }
2113
2114         /* init the mode config */
2115         drm_mode_config_init(adev->ddev);
2116
2117         r = amdgpu_device_ip_init(adev);
2118         if (r) {
2119                 /* failed in exclusive mode due to timeout */
2120                 if (amdgpu_sriov_vf(adev) &&
2121                     !amdgpu_sriov_runtime(adev) &&
2122                     amdgpu_virt_mmio_blocked(adev) &&
2123                     !amdgpu_virt_wait_reset(adev)) {
2124                         dev_err(adev->dev, "VF exclusive mode timeout\n");
2125                         /* Don't send request since VF is inactive. */
2126                         adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
2127                         adev->virt.ops = NULL;
2128                         r = -EAGAIN;
2129                         goto failed;
2130                 }
2131                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
2132                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2133                 amdgpu_device_ip_fini(adev);
2134                 goto failed;
2135         }
2136
2137         adev->accel_working = true;
2138
2139         amdgpu_vm_check_compute_bug(adev);
2140
2141         /* Initialize the buffer migration limit. */
2142         if (amdgpu_moverate >= 0)
2143                 max_MBps = amdgpu_moverate;
2144         else
2145                 max_MBps = 8; /* Allow 8 MB/s. */
2146         /* Get a log2 for easy divisions. */
2147         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
2148
2149         r = amdgpu_ib_pool_init(adev);
2150         if (r) {
2151                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2152                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2153                 goto failed;
2154         }
2155
2156         r = amdgpu_ib_ring_tests(adev);
2157         if (r)
2158                 DRM_ERROR("ib ring test failed (%d).\n", r);
2159
2160         if (amdgpu_sriov_vf(adev))
2161                 amdgpu_virt_init_data_exchange(adev);
2162
2163         amdgpu_fbdev_init(adev);
2164
2165         r = amdgpu_pm_sysfs_init(adev);
2166         if (r)
2167                 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
2168
2169         r = amdgpu_debugfs_gem_init(adev);
2170         if (r)
2171                 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
2172
2173         r = amdgpu_debugfs_regs_init(adev);
2174         if (r)
2175                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2176
2177         r = amdgpu_debugfs_firmware_init(adev);
2178         if (r)
2179                 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
2180
2181         r = amdgpu_debugfs_init(adev);
2182         if (r)
2183                 DRM_ERROR("Creating debugfs files failed (%d).\n", r);
2184
2185         if ((amdgpu_testing & 1)) {
2186                 if (adev->accel_working)
2187                         amdgpu_test_moves(adev);
2188                 else
2189                         DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
2190         }
2191         if (amdgpu_benchmarking) {
2192                 if (adev->accel_working)
2193                         amdgpu_benchmark(adev, amdgpu_benchmarking);
2194                 else
2195                         DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
2196         }
2197
2198         /* enable clockgating, etc. after ib tests, etc. since some blocks require
2199          * explicit gating rather than handling it automatically.
2200          */
2201         r = amdgpu_device_ip_late_init(adev);
2202         if (r) {
2203                 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
2204                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
2205                 goto failed;
2206         }
2207
2208         return 0;
2209
2210 failed:
2211         amdgpu_vf_error_trans_all(adev);
2212         if (runtime)
2213                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2214
2215         return r;
2216 }
2217
2218 /**
2219  * amdgpu_device_fini - tear down the driver
2220  *
2221  * @adev: amdgpu_device pointer
2222  *
2223  * Tear down the driver info (all asics).
2224  * Called at driver shutdown.
2225  */
2226 void amdgpu_device_fini(struct amdgpu_device *adev)
2227 {
2228         int r;
2229
2230         DRM_INFO("amdgpu: finishing device.\n");
2231         adev->shutdown = true;
2232         if (adev->mode_info.mode_config_initialized)
2233                 drm_crtc_force_disable_all(adev->ddev);
2234
2235         amdgpu_ib_pool_fini(adev);
2236         amdgpu_fence_driver_fini(adev);
2237         amdgpu_fbdev_fini(adev);
2238         r = amdgpu_device_ip_fini(adev);
2239         if (adev->firmware.gpu_info_fw) {
2240                 release_firmware(adev->firmware.gpu_info_fw);
2241                 adev->firmware.gpu_info_fw = NULL;
2242         }
2243         adev->accel_working = false;
2244         cancel_delayed_work_sync(&adev->late_init_work);
2245         /* free i2c buses */
2246         if (!amdgpu_device_has_dc_support(adev))
2247                 amdgpu_i2c_fini(adev);
2248         amdgpu_atombios_fini(adev);
2249         kfree(adev->bios);
2250         adev->bios = NULL;
2251         if (!pci_is_thunderbolt_attached(adev->pdev))
2252                 vga_switcheroo_unregister_client(adev->pdev);
2253         if (adev->flags & AMD_IS_PX)
2254                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2255         vga_client_register(adev->pdev, NULL, NULL, NULL);
2256         if (adev->rio_mem)
2257                 pci_iounmap(adev->pdev, adev->rio_mem);
2258         adev->rio_mem = NULL;
2259         iounmap(adev->rmmio);
2260         adev->rmmio = NULL;
2261         amdgpu_device_doorbell_fini(adev);
2262         amdgpu_pm_sysfs_fini(adev);
2263         amdgpu_debugfs_regs_cleanup(adev);
2264 }
2265
2266
2267 /*
2268  * Suspend & resume.
2269  */
2270 /**
2271  * amdgpu_device_suspend - initiate device suspend
2272  *
2273  * @pdev: drm dev pointer
2274  * @state: suspend state
2275  *
2276  * Puts the hw in the suspend state (all asics).
2277  * Returns 0 for success or an error on failure.
2278  * Called at driver suspend.
2279  */
2280 int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
2281 {
2282         struct amdgpu_device *adev;
2283         struct drm_crtc *crtc;
2284         struct drm_connector *connector;
2285         int r;
2286
2287         if (dev == NULL || dev->dev_private == NULL) {
2288                 return -ENODEV;
2289         }
2290
2291         adev = dev->dev_private;
2292
2293         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2294                 return 0;
2295
2296         drm_kms_helper_poll_disable(dev);
2297
2298         if (!amdgpu_device_has_dc_support(adev)) {
2299                 /* turn off display hw */
2300                 drm_modeset_lock_all(dev);
2301                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2302                         drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
2303                 }
2304                 drm_modeset_unlock_all(dev);
2305         }
2306
2307         amdgpu_amdkfd_suspend(adev);
2308
2309         /* unpin the front buffers and cursors */
2310         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2311                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2312                 struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
2313                 struct amdgpu_bo *robj;
2314
2315                 if (amdgpu_crtc->cursor_bo) {
2316                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2317                         r = amdgpu_bo_reserve(aobj, true);
2318                         if (r == 0) {
2319                                 amdgpu_bo_unpin(aobj);
2320                                 amdgpu_bo_unreserve(aobj);
2321                         }
2322                 }
2323
2324                 if (rfb == NULL || rfb->obj == NULL) {
2325                         continue;
2326                 }
2327                 robj = gem_to_amdgpu_bo(rfb->obj);
2328                 /* don't unpin kernel fb objects */
2329                 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
2330                         r = amdgpu_bo_reserve(robj, true);
2331                         if (r == 0) {
2332                                 amdgpu_bo_unpin(robj);
2333                                 amdgpu_bo_unreserve(robj);
2334                         }
2335                 }
2336         }
2337         /* evict vram memory */
2338         amdgpu_bo_evict_vram(adev);
2339
2340         amdgpu_fence_driver_suspend(adev);
2341
2342         r = amdgpu_device_ip_suspend(adev);
2343
2344         /* evict remaining vram memory
2345          * This second call to evict vram is to evict the gart page table
2346          * using the CPU.
2347          */
2348         amdgpu_bo_evict_vram(adev);
2349
2350         pci_save_state(dev->pdev);
2351         if (suspend) {
2352                 /* Shut down the device */
2353                 pci_disable_device(dev->pdev);
2354                 pci_set_power_state(dev->pdev, PCI_D3hot);
2355         } else {
2356                 r = amdgpu_asic_reset(adev);
2357                 if (r)
2358                         DRM_ERROR("amdgpu asic reset failed\n");
2359         }
2360
2361         if (fbcon) {
2362                 console_lock();
2363                 amdgpu_fbdev_set_suspend(adev, 1);
2364                 console_unlock();
2365         }
2366         return 0;
2367 }
2368
2369 /**
2370  * amdgpu_device_resume - initiate device resume
2371  *
2372  * @pdev: drm dev pointer
2373  *
2374  * Bring the hw back to operating state (all asics).
2375  * Returns 0 for success or an error on failure.
2376  * Called at driver resume.
2377  */
2378 int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2379 {
2380         struct drm_connector *connector;
2381         struct amdgpu_device *adev = dev->dev_private;
2382         struct drm_crtc *crtc;
2383         int r = 0;
2384
2385         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2386                 return 0;
2387
2388         if (fbcon)
2389                 console_lock();
2390
2391         if (resume) {
2392                 pci_set_power_state(dev->pdev, PCI_D0);
2393                 pci_restore_state(dev->pdev);
2394                 r = pci_enable_device(dev->pdev);
2395                 if (r)
2396                         goto unlock;
2397         }
2398
2399         /* post card */
2400         if (amdgpu_need_post(adev)) {
2401                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2402                 if (r)
2403                         DRM_ERROR("amdgpu asic init failed\n");
2404         }
2405
2406         r = amdgpu_device_ip_resume(adev);
2407         if (r) {
2408                 DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r);
2409                 goto unlock;
2410         }
2411         amdgpu_fence_driver_resume(adev);
2412
2413         if (resume) {
2414                 r = amdgpu_ib_ring_tests(adev);
2415                 if (r)
2416                         DRM_ERROR("ib ring test failed (%d).\n", r);
2417         }
2418
2419         r = amdgpu_device_ip_late_init(adev);
2420         if (r)
2421                 goto unlock;
2422
2423         /* pin cursors */
2424         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2425                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2426
2427                 if (amdgpu_crtc->cursor_bo) {
2428                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2429                         r = amdgpu_bo_reserve(aobj, true);
2430                         if (r == 0) {
2431                                 r = amdgpu_bo_pin(aobj,
2432                                                   AMDGPU_GEM_DOMAIN_VRAM,
2433                                                   &amdgpu_crtc->cursor_addr);
2434                                 if (r != 0)
2435                                         DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
2436                                 amdgpu_bo_unreserve(aobj);
2437                         }
2438                 }
2439         }
2440         r = amdgpu_amdkfd_resume(adev);
2441         if (r)
2442                 return r;
2443
2444         /* blat the mode back in */
2445         if (fbcon) {
2446                 if (!amdgpu_device_has_dc_support(adev)) {
2447                         /* pre DCE11 */
2448                         drm_helper_resume_force_mode(dev);
2449
2450                         /* turn on display hw */
2451                         drm_modeset_lock_all(dev);
2452                         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2453                                 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
2454                         }
2455                         drm_modeset_unlock_all(dev);
2456                 } else {
2457                         /*
2458                          * There is no equivalent atomic helper to turn on
2459                          * display, so we defined our own function for this,
2460                          * once suspend resume is supported by the atomic
2461                          * framework this will be reworked
2462                          */
2463                         amdgpu_dm_display_resume(adev);
2464                 }
2465         }
2466
2467         drm_kms_helper_poll_enable(dev);
2468
2469         /*
2470          * Most of the connector probing functions try to acquire runtime pm
2471          * refs to ensure that the GPU is powered on when connector polling is
2472          * performed. Since we're calling this from a runtime PM callback,
2473          * trying to acquire rpm refs will cause us to deadlock.
2474          *
2475          * Since we're guaranteed to be holding the rpm lock, it's safe to
2476          * temporarily disable the rpm helpers so this doesn't deadlock us.
2477          */
2478 #ifdef CONFIG_PM
2479         dev->dev->power.disable_depth++;
2480 #endif
2481         if (!amdgpu_device_has_dc_support(adev))
2482                 drm_helper_hpd_irq_event(dev);
2483         else
2484                 drm_kms_helper_hotplug_event(dev);
2485 #ifdef CONFIG_PM
2486         dev->dev->power.disable_depth--;
2487 #endif
2488
2489         if (fbcon)
2490                 amdgpu_fbdev_set_suspend(adev, 0);
2491
2492 unlock:
2493         if (fbcon)
2494                 console_unlock();
2495
2496         return r;
2497 }
2498
2499 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
2500 {
2501         int i;
2502         bool asic_hang = false;
2503
2504         if (amdgpu_sriov_vf(adev))
2505                 return true;
2506
2507         for (i = 0; i < adev->num_ip_blocks; i++) {
2508                 if (!adev->ip_blocks[i].status.valid)
2509                         continue;
2510                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
2511                         adev->ip_blocks[i].status.hang =
2512                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
2513                 if (adev->ip_blocks[i].status.hang) {
2514                         DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
2515                         asic_hang = true;
2516                 }
2517         }
2518         return asic_hang;
2519 }
2520
2521 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
2522 {
2523         int i, r = 0;
2524
2525         for (i = 0; i < adev->num_ip_blocks; i++) {
2526                 if (!adev->ip_blocks[i].status.valid)
2527                         continue;
2528                 if (adev->ip_blocks[i].status.hang &&
2529                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
2530                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
2531                         if (r)
2532                                 return r;
2533                 }
2534         }
2535
2536         return 0;
2537 }
2538
2539 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
2540 {
2541         int i;
2542
2543         for (i = 0; i < adev->num_ip_blocks; i++) {
2544                 if (!adev->ip_blocks[i].status.valid)
2545                         continue;
2546                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
2547                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
2548                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
2549                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
2550                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2551                         if (adev->ip_blocks[i].status.hang) {
2552                                 DRM_INFO("Some block need full reset!\n");
2553                                 return true;
2554                         }
2555                 }
2556         }
2557         return false;
2558 }
2559
2560 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
2561 {
2562         int i, r = 0;
2563
2564         for (i = 0; i < adev->num_ip_blocks; i++) {
2565                 if (!adev->ip_blocks[i].status.valid)
2566                         continue;
2567                 if (adev->ip_blocks[i].status.hang &&
2568                     adev->ip_blocks[i].version->funcs->soft_reset) {
2569                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
2570                         if (r)
2571                                 return r;
2572                 }
2573         }
2574
2575         return 0;
2576 }
2577
2578 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
2579 {
2580         int i, r = 0;
2581
2582         for (i = 0; i < adev->num_ip_blocks; i++) {
2583                 if (!adev->ip_blocks[i].status.valid)
2584                         continue;
2585                 if (adev->ip_blocks[i].status.hang &&
2586                     adev->ip_blocks[i].version->funcs->post_soft_reset)
2587                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
2588                 if (r)
2589                         return r;
2590         }
2591
2592         return 0;
2593 }
2594
2595 bool amdgpu_need_backup(struct amdgpu_device *adev)
2596 {
2597         if (adev->flags & AMD_IS_APU)
2598                 return false;
2599
2600         return amdgpu_gpu_recovery;
2601 }
2602
2603 static int amdgpu_device_recover_vram_from_shadow(struct amdgpu_device *adev,
2604                                                   struct amdgpu_ring *ring,
2605                                                   struct amdgpu_bo *bo,
2606                                                   struct dma_fence **fence)
2607 {
2608         uint32_t domain;
2609         int r;
2610
2611         if (!bo->shadow)
2612                 return 0;
2613
2614         r = amdgpu_bo_reserve(bo, true);
2615         if (r)
2616                 return r;
2617         domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
2618         /* if bo has been evicted, then no need to recover */
2619         if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
2620                 r = amdgpu_bo_validate(bo->shadow);
2621                 if (r) {
2622                         DRM_ERROR("bo validate failed!\n");
2623                         goto err;
2624                 }
2625
2626                 r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
2627                                                  NULL, fence, true);
2628                 if (r) {
2629                         DRM_ERROR("recover page table failed!\n");
2630                         goto err;
2631                 }
2632         }
2633 err:
2634         amdgpu_bo_unreserve(bo);
2635         return r;
2636 }
2637
2638 /*
2639  * amdgpu_device_reset - reset ASIC/GPU for bare-metal or passthrough
2640  *
2641  * @adev: amdgpu device pointer
2642  * @reset_flags: output param tells caller the reset result
2643  *
2644  * attempt to do soft-reset or full-reset and reinitialize Asic
2645  * return 0 means successed otherwise failed
2646 */
2647 static int amdgpu_device_reset(struct amdgpu_device *adev,
2648                                uint64_t* reset_flags)
2649 {
2650         bool need_full_reset, vram_lost = 0;
2651         int r;
2652
2653         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
2654
2655         if (!need_full_reset) {
2656                 amdgpu_device_ip_pre_soft_reset(adev);
2657                 r = amdgpu_device_ip_soft_reset(adev);
2658                 amdgpu_device_ip_post_soft_reset(adev);
2659                 if (r || amdgpu_device_ip_check_soft_reset(adev)) {
2660                         DRM_INFO("soft reset failed, will fallback to full reset!\n");
2661                         need_full_reset = true;
2662                 }
2663
2664         }
2665
2666         if (need_full_reset) {
2667                 r = amdgpu_device_ip_suspend(adev);
2668
2669 retry:
2670                 r = amdgpu_asic_reset(adev);
2671                 /* post card */
2672                 amdgpu_atom_asic_init(adev->mode_info.atom_context);
2673
2674                 if (!r) {
2675                         dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
2676                         r = amdgpu_device_ip_resume_phase1(adev);
2677                         if (r)
2678                                 goto out;
2679
2680                         vram_lost = amdgpu_device_check_vram_lost(adev);
2681                         if (vram_lost) {
2682                                 DRM_ERROR("VRAM is lost!\n");
2683                                 atomic_inc(&adev->vram_lost_counter);
2684                         }
2685
2686                         r = amdgpu_gtt_mgr_recover(
2687                                 &adev->mman.bdev.man[TTM_PL_TT]);
2688                         if (r)
2689                                 goto out;
2690
2691                         r = amdgpu_device_ip_resume_phase2(adev);
2692                         if (r)
2693                                 goto out;
2694
2695                         if (vram_lost)
2696                                 amdgpu_device_fill_reset_magic(adev);
2697                 }
2698         }
2699
2700 out:
2701         if (!r) {
2702                 amdgpu_irq_gpu_reset_resume_helper(adev);
2703                 r = amdgpu_ib_ring_tests(adev);
2704                 if (r) {
2705                         dev_err(adev->dev, "ib ring test failed (%d).\n", r);
2706                         r = amdgpu_device_ip_suspend(adev);
2707                         need_full_reset = true;
2708                         goto retry;
2709                 }
2710         }
2711
2712         if (reset_flags) {
2713                 if (vram_lost)
2714                         (*reset_flags) |= AMDGPU_RESET_INFO_VRAM_LOST;
2715
2716                 if (need_full_reset)
2717                         (*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET;
2718         }
2719
2720         return r;
2721 }
2722
2723 /*
2724  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
2725  *
2726  * @adev: amdgpu device pointer
2727  * @reset_flags: output param tells caller the reset result
2728  *
2729  * do VF FLR and reinitialize Asic
2730  * return 0 means successed otherwise failed
2731 */
2732 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
2733                                      uint64_t *reset_flags,
2734                                      bool from_hypervisor)
2735 {
2736         int r;
2737
2738         if (from_hypervisor)
2739                 r = amdgpu_virt_request_full_gpu(adev, true);
2740         else
2741                 r = amdgpu_virt_reset_gpu(adev);
2742         if (r)
2743                 return r;
2744
2745         /* Resume IP prior to SMC */
2746         r = amdgpu_device_ip_reinit_early_sriov(adev);
2747         if (r)
2748                 goto error;
2749
2750         /* we need recover gart prior to run SMC/CP/SDMA resume */
2751         amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]);
2752
2753         /* now we are okay to resume SMC/CP/SDMA */
2754         r = amdgpu_device_ip_reinit_late_sriov(adev);
2755         if (r)
2756                 goto error;
2757
2758         amdgpu_irq_gpu_reset_resume_helper(adev);
2759         r = amdgpu_ib_ring_tests(adev);
2760         if (r)
2761                 dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r);
2762
2763 error:
2764         /* release full control of GPU after ib test */
2765         amdgpu_virt_release_full_gpu(adev, true);
2766
2767         if (reset_flags) {
2768                 if (adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
2769                         (*reset_flags) |= AMDGPU_RESET_INFO_VRAM_LOST;
2770                         atomic_inc(&adev->vram_lost_counter);
2771                 }
2772
2773                 /* VF FLR or hotlink reset is always full-reset */
2774                 (*reset_flags) |= AMDGPU_RESET_INFO_FULLRESET;
2775         }
2776
2777         return r;
2778 }
2779
2780 /**
2781  * amdgpu_gpu_recover - reset the asic and recover scheduler
2782  *
2783  * @adev: amdgpu device pointer
2784  * @job: which job trigger hang
2785  * @force forces reset regardless of amdgpu_gpu_recovery
2786  *
2787  * Attempt to reset the GPU if it has hung (all asics).
2788  * Returns 0 for success or an error on failure.
2789  */
2790 int amdgpu_gpu_recover(struct amdgpu_device *adev, struct amdgpu_job *job, bool force)
2791 {
2792         struct drm_atomic_state *state = NULL;
2793         uint64_t reset_flags = 0;
2794         int i, r, resched;
2795
2796         if (!amdgpu_device_ip_check_soft_reset(adev)) {
2797                 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
2798                 return 0;
2799         }
2800
2801         if (!force && (amdgpu_gpu_recovery == 0 ||
2802                         (amdgpu_gpu_recovery == -1  && !amdgpu_sriov_vf(adev)))) {
2803                 DRM_INFO("GPU recovery disabled.\n");
2804                 return 0;
2805         }
2806
2807         dev_info(adev->dev, "GPU reset begin!\n");
2808
2809         mutex_lock(&adev->lock_reset);
2810         atomic_inc(&adev->gpu_reset_counter);
2811         adev->in_gpu_reset = 1;
2812
2813         /* block TTM */
2814         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
2815         /* store modesetting */
2816         if (amdgpu_device_has_dc_support(adev))
2817                 state = drm_atomic_helper_suspend(adev->ddev);
2818
2819         /* block scheduler */
2820         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2821                 struct amdgpu_ring *ring = adev->rings[i];
2822
2823                 if (!ring || !ring->sched.thread)
2824                         continue;
2825
2826                 /* only focus on the ring hit timeout if &job not NULL */
2827                 if (job && job->ring->idx != i)
2828                         continue;
2829
2830                 kthread_park(ring->sched.thread);
2831                 drm_sched_hw_job_reset(&ring->sched, &job->base);
2832
2833                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2834                 amdgpu_fence_driver_force_completion(ring);
2835         }
2836
2837         if (amdgpu_sriov_vf(adev))
2838                 r = amdgpu_device_reset_sriov(adev, &reset_flags, job ? false : true);
2839         else
2840                 r = amdgpu_device_reset(adev, &reset_flags);
2841
2842         if (!r) {
2843                 if (((reset_flags & AMDGPU_RESET_INFO_FULLRESET) && !(adev->flags & AMD_IS_APU)) ||
2844                         (reset_flags & AMDGPU_RESET_INFO_VRAM_LOST)) {
2845                         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2846                         struct amdgpu_bo *bo, *tmp;
2847                         struct dma_fence *fence = NULL, *next = NULL;
2848
2849                         DRM_INFO("recover vram bo from shadow\n");
2850                         mutex_lock(&adev->shadow_list_lock);
2851                         list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2852                                 next = NULL;
2853                                 amdgpu_device_recover_vram_from_shadow(adev, ring, bo, &next);
2854                                 if (fence) {
2855                                         r = dma_fence_wait(fence, false);
2856                                         if (r) {
2857                                                 WARN(r, "recovery from shadow isn't completed\n");
2858                                                 break;
2859                                         }
2860                                 }
2861
2862                                 dma_fence_put(fence);
2863                                 fence = next;
2864                         }
2865                         mutex_unlock(&adev->shadow_list_lock);
2866                         if (fence) {
2867                                 r = dma_fence_wait(fence, false);
2868                                 if (r)
2869                                         WARN(r, "recovery from shadow isn't completed\n");
2870                         }
2871                         dma_fence_put(fence);
2872                 }
2873
2874                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2875                         struct amdgpu_ring *ring = adev->rings[i];
2876
2877                         if (!ring || !ring->sched.thread)
2878                                 continue;
2879
2880                         /* only focus on the ring hit timeout if &job not NULL */
2881                         if (job && job->ring->idx != i)
2882                                 continue;
2883
2884                         drm_sched_job_recovery(&ring->sched);
2885                         kthread_unpark(ring->sched.thread);
2886                 }
2887         } else {
2888                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2889                         struct amdgpu_ring *ring = adev->rings[i];
2890
2891                         if (!ring || !ring->sched.thread)
2892                                 continue;
2893
2894                         /* only focus on the ring hit timeout if &job not NULL */
2895                         if (job && job->ring->idx != i)
2896                                 continue;
2897
2898                         kthread_unpark(adev->rings[i]->sched.thread);
2899                 }
2900         }
2901
2902         if (amdgpu_device_has_dc_support(adev)) {
2903                 if (drm_atomic_helper_resume(adev->ddev, state))
2904                         dev_info(adev->dev, "drm resume failed:%d\n", r);
2905                 amdgpu_dm_display_resume(adev);
2906         } else {
2907                 drm_helper_resume_force_mode(adev->ddev);
2908         }
2909
2910         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
2911
2912         if (r) {
2913                 /* bad news, how to tell it to userspace ? */
2914                 dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
2915                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
2916         } else {
2917                 dev_info(adev->dev, "GPU reset(%d) successed!\n",atomic_read(&adev->gpu_reset_counter));
2918         }
2919
2920         amdgpu_vf_error_trans_all(adev);
2921         adev->in_gpu_reset = 0;
2922         mutex_unlock(&adev->lock_reset);
2923         return r;
2924 }
2925
2926 void amdgpu_get_pcie_info(struct amdgpu_device *adev)
2927 {
2928         u32 mask;
2929         int ret;
2930
2931         if (amdgpu_pcie_gen_cap)
2932                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
2933
2934         if (amdgpu_pcie_lane_cap)
2935                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
2936
2937         /* covers APUs as well */
2938         if (pci_is_root_bus(adev->pdev->bus)) {
2939                 if (adev->pm.pcie_gen_mask == 0)
2940                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
2941                 if (adev->pm.pcie_mlw_mask == 0)
2942                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
2943                 return;
2944         }
2945
2946         if (adev->pm.pcie_gen_mask == 0) {
2947                 ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
2948                 if (!ret) {
2949                         adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
2950                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
2951                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
2952
2953                         if (mask & DRM_PCIE_SPEED_25)
2954                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
2955                         if (mask & DRM_PCIE_SPEED_50)
2956                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
2957                         if (mask & DRM_PCIE_SPEED_80)
2958                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
2959                 } else {
2960                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
2961                 }
2962         }
2963         if (adev->pm.pcie_mlw_mask == 0) {
2964                 ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
2965                 if (!ret) {
2966                         switch (mask) {
2967                         case 32:
2968                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
2969                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
2970                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2971                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2972                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2973                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2974                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2975                                 break;
2976                         case 16:
2977                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
2978                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2979                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2980                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2981                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2982                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2983                                 break;
2984                         case 12:
2985                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2986                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2987                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2988                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2989                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2990                                 break;
2991                         case 8:
2992                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2993                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2994                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2995                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2996                                 break;
2997                         case 4:
2998                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2999                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3000                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3001                                 break;
3002                         case 2:
3003                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
3004                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
3005                                 break;
3006                         case 1:
3007                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
3008                                 break;
3009                         default:
3010                                 break;
3011                         }
3012                 } else {
3013                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
3014                 }
3015         }
3016 }
3017