drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/kthread.h>
  29 #include <linux/console.h>
  30 #include <linux/slab.h>
  31 #include <drm/drmP.h>
  32 #include <drm/drm_crtc_helper.h>
  33 #include <drm/drm_atomic_helper.h>
  34 #include <drm/amdgpu_drm.h>
  35 #include <linux/vgaarb.h>
  36 #include <linux/vga_switcheroo.h>
  37 #include <linux/efi.h>
  38 #include "amdgpu.h"
  39 #include "amdgpu_trace.h"
  40 #include "amdgpu_i2c.h"
  41 #include "atom.h"
  42 #include "amdgpu_atombios.h"
  43 #include "amdgpu_atomfirmware.h"
  44 #include "amd_pcie.h"
  45 #ifdef CONFIG_DRM_AMDGPU_SI
  46 #include "si.h"
  47 #endif
  48 #ifdef CONFIG_DRM_AMDGPU_CIK
  49 #include "cik.h"
  50 #endif
  51 #include "vi.h"
  52 #include "soc15.h"
  53 #include "bif/bif_4_1_d.h"
  54 #include <linux/pci.h>
  55 #include <linux/firmware.h>
  56 #include "amdgpu_vf_error.h"
  57
  58 #include "amdgpu_amdkfd.h"
  59 #include "amdgpu_pm.h"
  60
  61 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  62 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  63
  64 #define AMDGPU_RESUME_MS                2000
  65
  66 static const char *amdgpu_asic_name[] = {
  67         "TAHITI",
  68         "PITCAIRN",
  69         "VERDE",
  70         "OLAND",
  71         "HAINAN",
  72         "BONAIRE",
  73         "KAVERI",
  74         "KABINI",
  75         "HAWAII",
  76         "MULLINS",
  77         "TOPAZ",
  78         "TONGA",
  79         "FIJI",
  80         "CARRIZO",
  81         "STONEY",
  82         "POLARIS10",
  83         "POLARIS11",
  84         "POLARIS12",
  85         "VEGA10",
  86         "RAVEN",
  87         "LAST",
  88 };
  89
  90 bool amdgpu_device_is_px(struct drm_device *dev)
  91 {
  92         struct amdgpu_device *adev = dev->dev_private;
  93
  94         if (adev->flags & AMD_IS_PX)
  95                 return true;
  96         return false;
  97 }
  98
  99 /*
 100  * MMIO register access helper functions.
 101  */
 102 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
 103                         uint32_t acc_flags)
 104 {
 105         uint32_t ret;
 106
 107         if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
 108                 return amdgpu_virt_kiq_rreg(adev, reg);
 109
 110         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
 111                 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 112         else {
 113                 unsigned long flags;
 114
 115                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 116                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 117                 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 118                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 119         }
 120         trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
 121         return ret;
 122 }
 123
 124 /*
 125  * MMIO register read with bytes helper functions
 126  * @offset:bytes offset from MMIO start
 127  *
 128 */
 129
 130 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) {
 131         if (offset < adev->rmmio_size)
 132                 return (readb(adev->rmmio + offset));
 133         BUG();
 134 }
 135
 136 /*
 137  * MMIO register write with bytes helper functions
 138  * @offset:bytes offset from MMIO start
 139  * @value: the value want to be written to the register
 140  *
 141 */
 142 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) {
 143         if (offset < adev->rmmio_size)
 144                 writeb(value, adev->rmmio + offset);
 145         else
 146                 BUG();
 147 }
 148
 149
 150 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 151                     uint32_t acc_flags)
 152 {
 153         trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 154
 155         if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
 156                 adev->last_mm_index = v;
 157         }
 158
 159         if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev))
 160                 return amdgpu_virt_kiq_wreg(adev, reg, v);
 161
 162         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
 163                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 164         else {
 165                 unsigned long flags;
 166
 167                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 168                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 169                 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 170                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 171         }
 172
 173         if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
 174                 udelay(500);
 175         }
 176 }
 177
 178 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
 179 {
 180         if ((reg * 4) < adev->rio_mem_size)
 181                 return ioread32(adev->rio_mem + (reg * 4));
 182         else {
 183                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 184                 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
 185         }
 186 }
 187
 188 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 189 {
 190         if (adev->asic_type >= CHIP_VEGA10 && reg == 0) {
 191                 adev->last_mm_index = v;
 192         }
 193
 194         if ((reg * 4) < adev->rio_mem_size)
 195                 iowrite32(v, adev->rio_mem + (reg * 4));
 196         else {
 197                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 198                 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
 199         }
 200
 201         if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) {
 202                 udelay(500);
 203         }
 204 }
 205
 206 /**
 207  * amdgpu_mm_rdoorbell - read a doorbell dword
 208  *
 209  * @adev: amdgpu_device pointer
 210  * @index: doorbell index
 211  *
 212  * Returns the value in the doorbell aperture at the
 213  * requested doorbell index (CIK).
 214  */
 215 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 216 {
 217         if (index < adev->doorbell.num_doorbells) {
 218                 return readl(adev->doorbell.ptr + index);
 219         } else {
 220                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 221                 return 0;
 222         }
 223 }
 224
 225 /**
 226  * amdgpu_mm_wdoorbell - write a doorbell dword
 227  *
 228  * @adev: amdgpu_device pointer
 229  * @index: doorbell index
 230  * @v: value to write
 231  *
 232  * Writes @v to the doorbell aperture at the
 233  * requested doorbell index (CIK).
 234  */
 235 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 236 {
 237         if (index < adev->doorbell.num_doorbells) {
 238                 writel(v, adev->doorbell.ptr + index);
 239         } else {
 240                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 241         }
 242 }
 243
 244 /**
 245  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 246  *
 247  * @adev: amdgpu_device pointer
 248  * @index: doorbell index
 249  *
 250  * Returns the value in the doorbell aperture at the
 251  * requested doorbell index (VEGA10+).
 252  */
 253 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 254 {
 255         if (index < adev->doorbell.num_doorbells) {
 256                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 257         } else {
 258                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 259                 return 0;
 260         }
 261 }
 262
 263 /**
 264  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 265  *
 266  * @adev: amdgpu_device pointer
 267  * @index: doorbell index
 268  * @v: value to write
 269  *
 270  * Writes @v to the doorbell aperture at the
 271  * requested doorbell index (VEGA10+).
 272  */
 273 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 274 {
 275         if (index < adev->doorbell.num_doorbells) {
 276                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 277         } else {
 278                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 279         }
 280 }
 281
 282 /**
 283  * amdgpu_invalid_rreg - dummy reg read function
 284  *
 285  * @adev: amdgpu device pointer
 286  * @reg: offset of register
 287  *
 288  * Dummy register read function.  Used for register blocks
 289  * that certain asics don't have (all asics).
 290  * Returns the value in the register.
 291  */
 292 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 293 {
 294         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 295         BUG();
 296         return 0;
 297 }
 298
 299 /**
 300  * amdgpu_invalid_wreg - dummy reg write function
 301  *
 302  * @adev: amdgpu device pointer
 303  * @reg: offset of register
 304  * @v: value to write to the register
 305  *
 306  * Dummy register read function.  Used for register blocks
 307  * that certain asics don't have (all asics).
 308  */
 309 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 310 {
 311         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 312                   reg, v);
 313         BUG();
 314 }
 315
 316 /**
 317  * amdgpu_block_invalid_rreg - dummy reg read function
 318  *
 319  * @adev: amdgpu device pointer
 320  * @block: offset of instance
 321  * @reg: offset of register
 322  *
 323  * Dummy register read function.  Used for register blocks
 324  * that certain asics don't have (all asics).
 325  * Returns the value in the register.
 326  */
 327 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 328                                           uint32_t block, uint32_t reg)
 329 {
 330         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 331                   reg, block);
 332         BUG();
 333         return 0;
 334 }
 335
 336 /**
 337  * amdgpu_block_invalid_wreg - dummy reg write function
 338  *
 339  * @adev: amdgpu device pointer
 340  * @block: offset of instance
 341  * @reg: offset of register
 342  * @v: value to write to the register
 343  *
 344  * Dummy register read function.  Used for register blocks
 345  * that certain asics don't have (all asics).
 346  */
 347 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 348                                       uint32_t block,
 349                                       uint32_t reg, uint32_t v)
 350 {
 351         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 352                   reg, block, v);
 353         BUG();
 354 }
 355
 356 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
 357 {
 358         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
 359                                        PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 360                                        &adev->vram_scratch.robj,
 361                                        &adev->vram_scratch.gpu_addr,
 362                                        (void **)&adev->vram_scratch.ptr);
 363 }
 364
 365 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
 366 {
 367         amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
 368 }
 369
 370 /**
 371  * amdgpu_device_program_register_sequence - program an array of registers.
 372  *
 373  * @adev: amdgpu_device pointer
 374  * @registers: pointer to the register array
 375  * @array_size: size of the register array
 376  *
 377  * Programs an array or registers with and and or masks.
 378  * This is a helper for setting golden registers.
 379  */
 380 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
 381                                              const u32 *registers,
 382                                              const u32 array_size)
 383 {
 384         u32 tmp, reg, and_mask, or_mask;
 385         int i;
 386
 387         if (array_size % 3)
 388                 return;
 389
 390         for (i = 0; i < array_size; i +=3) {
 391                 reg = registers[i + 0];
 392                 and_mask = registers[i + 1];
 393                 or_mask = registers[i + 2];
 394
 395                 if (and_mask == 0xffffffff) {
 396                         tmp = or_mask;
 397                 } else {
 398                         tmp = RREG32(reg);
 399                         tmp &= ~and_mask;
 400                         tmp |= or_mask;
 401                 }
 402                 WREG32(reg, tmp);
 403         }
 404 }
 405
 406 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
 407 {
 408         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
 409 }
 410
 411 /*
 412  * GPU doorbell aperture helpers function.
 413  */
 414 /**
 415  * amdgpu_device_doorbell_init - Init doorbell driver information.
 416  *
 417  * @adev: amdgpu_device pointer
 418  *
 419  * Init doorbell driver information (CIK)
 420  * Returns 0 on success, error on failure.
 421  */
 422 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
 423 {
 424         /* No doorbell on SI hardware generation */
 425         if (adev->asic_type < CHIP_BONAIRE) {
 426                 adev->doorbell.base = 0;
 427                 adev->doorbell.size = 0;
 428                 adev->doorbell.num_doorbells = 0;
 429                 adev->doorbell.ptr = NULL;
 430                 return 0;
 431         }
 432
 433         if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
 434                 return -EINVAL;
 435
 436         /* doorbell bar mapping */
 437         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
 438         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
 439
 440         adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
 441                                              AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
 442         if (adev->doorbell.num_doorbells == 0)
 443                 return -EINVAL;
 444
 445         adev->doorbell.ptr = ioremap(adev->doorbell.base,
 446                                      adev->doorbell.num_doorbells *
 447                                      sizeof(u32));
 448         if (adev->doorbell.ptr == NULL)
 449                 return -ENOMEM;
 450
 451         return 0;
 452 }
 453
 454 /**
 455  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
 456  *
 457  * @adev: amdgpu_device pointer
 458  *
 459  * Tear down doorbell driver information (CIK)
 460  */
 461 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
 462 {
 463         iounmap(adev->doorbell.ptr);
 464         adev->doorbell.ptr = NULL;
 465 }
 466
 467
 468
 469 /*
 470  * amdgpu_device_wb_*()
 471  * Writeback is the method by which the GPU updates special pages in memory
 472  * with the status of certain GPU events (fences, ring pointers,etc.).
 473  */
 474
 475 /**
 476  * amdgpu_device_wb_fini - Disable Writeback and free memory
 477  *
 478  * @adev: amdgpu_device pointer
 479  *
 480  * Disables Writeback and frees the Writeback memory (all asics).
 481  * Used at driver shutdown.
 482  */
 483 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
 484 {
 485         if (adev->wb.wb_obj) {
 486                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
 487                                       &adev->wb.gpu_addr,
 488                                       (void **)&adev->wb.wb);
 489                 adev->wb.wb_obj = NULL;
 490         }
 491 }
 492
 493 /**
 494  * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
 495  *
 496  * @adev: amdgpu_device pointer
 497  *
 498  * Initializes writeback and allocates writeback memory (all asics).
 499  * Used at driver startup.
 500  * Returns 0 on success or an -error on failure.
 501  */
 502 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
 503 {
 504         int r;
 505
 506         if (adev->wb.wb_obj == NULL) {
 507                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
 508                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
 509                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 510                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
 511                                             (void **)&adev->wb.wb);
 512                 if (r) {
 513                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
 514                         return r;
 515                 }
 516
 517                 adev->wb.num_wb = AMDGPU_MAX_WB;
 518                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
 519
 520                 /* clear wb memory */
 521                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
 522         }
 523
 524         return 0;
 525 }
 526
 527 /**
 528  * amdgpu_device_wb_get - Allocate a wb entry
 529  *
 530  * @adev: amdgpu_device pointer
 531  * @wb: wb index
 532  *
 533  * Allocate a wb slot for use by the driver (all asics).
 534  * Returns 0 on success or -EINVAL on failure.
 535  */
 536 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
 537 {
 538         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
 539
 540         if (offset < adev->wb.num_wb) {
 541                 __set_bit(offset, adev->wb.used);
 542                 *wb = offset << 3; /* convert to dw offset */
 543                 return 0;
 544         } else {
 545                 return -EINVAL;
 546         }
 547 }
 548
 549 /**
 550  * amdgpu_device_wb_free - Free a wb entry
 551  *
 552  * @adev: amdgpu_device pointer
 553  * @wb: wb index
 554  *
 555  * Free a wb slot allocated for use by the driver (all asics)
 556  */
 557 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
 558 {
 559         wb >>= 3;
 560         if (wb < adev->wb.num_wb)
 561                 __clear_bit(wb, adev->wb.used);
 562 }
 563
 564 /**
 565  * amdgpu_device_vram_location - try to find VRAM location
 566  * @adev: amdgpu device structure holding all necessary informations
 567  * @mc: memory controller structure holding memory informations
 568  * @base: base address at which to put VRAM
 569  *
 570  * Function will try to place VRAM at base address provided
 571  * as parameter.
 572  */
 573 void amdgpu_device_vram_location(struct amdgpu_device *adev,
 574                                  struct amdgpu_gmc *mc, u64 base)
 575 {
 576         uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
 577
 578         mc->vram_start = base;
 579         mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 580         if (limit && limit < mc->real_vram_size)
 581                 mc->real_vram_size = limit;
 582         dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
 583                         mc->mc_vram_size >> 20, mc->vram_start,
 584                         mc->vram_end, mc->real_vram_size >> 20);
 585 }
 586
 587 /**
 588  * amdgpu_device_gart_location - try to find GTT location
 589  * @adev: amdgpu device structure holding all necessary informations
 590  * @mc: memory controller structure holding memory informations
 591  *
 592  * Function will place try to place GTT before or after VRAM.
 593  *
 594  * If GTT size is bigger than space left then we ajust GTT size.
 595  * Thus function will never fails.
 596  *
 597  * FIXME: when reducing GTT size align new size on power of 2.
 598  */
 599 void amdgpu_device_gart_location(struct amdgpu_device *adev,
 600                                  struct amdgpu_gmc *mc)
 601 {
 602         u64 size_af, size_bf;
 603
 604         size_af = adev->gmc.mc_mask - mc->vram_end;
 605         size_bf = mc->vram_start;
 606         if (size_bf > size_af) {
 607                 if (mc->gart_size > size_bf) {
 608                         dev_warn(adev->dev, "limiting GTT\n");
 609                         mc->gart_size = size_bf;
 610                 }
 611                 mc->gart_start = 0;
 612         } else {
 613                 if (mc->gart_size > size_af) {
 614                         dev_warn(adev->dev, "limiting GTT\n");
 615                         mc->gart_size = size_af;
 616                 }
 617                 /* VCE doesn't like it when BOs cross a 4GB segment, so align
 618                  * the GART base on a 4GB boundary as well.
 619                  */
 620                 mc->gart_start = ALIGN(mc->vram_end + 1, 0x100000000ULL);
 621         }
 622         mc->gart_end = mc->gart_start + mc->gart_size - 1;
 623         dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
 624                         mc->gart_size >> 20, mc->gart_start, mc->gart_end);
 625 }
 626
 627 /**
 628  * amdgpu_device_resize_fb_bar - try to resize FB BAR
 629  *
 630  * @adev: amdgpu_device pointer
 631  *
 632  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
 633  * to fail, but if any of the BARs is not accessible after the size we abort
 634  * driver loading by returning -ENODEV.
 635  */
 636 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
 637 {
 638         u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size);
 639         u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1;
 640         struct pci_bus *root;
 641         struct resource *res;
 642         unsigned i;
 643         u16 cmd;
 644         int r;
 645
 646         /* Bypass for VF */
 647         if (amdgpu_sriov_vf(adev))
 648                 return 0;
 649
 650         /* Check if the root BUS has 64bit memory resources */
 651         root = adev->pdev->bus;
 652         while (root->parent)
 653                 root = root->parent;
 654
 655         pci_bus_for_each_resource(root, res, i) {
 656                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
 657                     res->start > 0x100000000ull)
 658                         break;
 659         }
 660
 661         /* Trying to resize is pointless without a root hub window above 4GB */
 662         if (!res)
 663                 return 0;
 664
 665         /* Disable memory decoding while we change the BAR addresses and size */
 666         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
 667         pci_write_config_word(adev->pdev, PCI_COMMAND,
 668                               cmd & ~PCI_COMMAND_MEMORY);
 669
 670         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
 671         amdgpu_device_doorbell_fini(adev);
 672         if (adev->asic_type >= CHIP_BONAIRE)
 673                 pci_release_resource(adev->pdev, 2);
 674
 675         pci_release_resource(adev->pdev, 0);
 676
 677         r = pci_resize_resource(adev->pdev, 0, rbar_size);
 678         if (r == -ENOSPC)
 679                 DRM_INFO("Not enough PCI address space for a large BAR.");
 680         else if (r && r != -ENOTSUPP)
 681                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
 682
 683         pci_assign_unassigned_bus_resources(adev->pdev->bus);
 684
 685         /* When the doorbell or fb BAR isn't available we have no chance of
 686          * using the device.
 687          */
 688         r = amdgpu_device_doorbell_init(adev);
 689         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
 690                 return -ENODEV;
 691
 692         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
 693
 694         return 0;
 695 }
 696
 697 /*
 698  * GPU helpers function.
 699  */
 700 /**
 701  * amdgpu_device_need_post - check if the hw need post or not
 702  *
 703  * @adev: amdgpu_device pointer
 704  *
 705  * Check if the asic has been initialized (all asics) at driver startup
 706  * or post is needed if  hw reset is performed.
 707  * Returns true if need or false if not.
 708  */
 709 bool amdgpu_device_need_post(struct amdgpu_device *adev)
 710 {
 711         uint32_t reg;
 712
 713         if (amdgpu_sriov_vf(adev))
 714                 return false;
 715
 716         if (amdgpu_passthrough(adev)) {
 717                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
 718                  * some old smc fw still need driver do vPost otherwise gpu hang, while
 719                  * those smc fw version above 22.15 doesn't have this flaw, so we force
 720                  * vpost executed for smc version below 22.15
 721                  */
 722                 if (adev->asic_type == CHIP_FIJI) {
 723                         int err;
 724                         uint32_t fw_ver;
 725                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
 726                         /* force vPost if error occured */
 727                         if (err)
 728                                 return true;
 729
 730                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
 731                         if (fw_ver < 0x00160e00)
 732                                 return true;
 733                 }
 734         }
 735
 736         if (adev->has_hw_reset) {
 737                 adev->has_hw_reset = false;
 738                 return true;
 739         }
 740
 741         /* bios scratch used on CIK+ */
 742         if (adev->asic_type >= CHIP_BONAIRE)
 743                 return amdgpu_atombios_scratch_need_asic_init(adev);
 744
 745         /* check MEM_SIZE for older asics */
 746         reg = amdgpu_asic_get_config_memsize(adev);
 747
 748         if ((reg != 0) && (reg != 0xffffffff))
 749                 return false;
 750
 751         return true;
 752 }
 753
 754 /* if we get transitioned to only one device, take VGA back */
 755 /**
 756  * amdgpu_device_vga_set_decode - enable/disable vga decode
 757  *
 758  * @cookie: amdgpu_device pointer
 759  * @state: enable/disable vga decode
 760  *
 761  * Enable/disable vga decode (all asics).
 762  * Returns VGA resource flags.
 763  */
 764 static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
 765 {
 766         struct amdgpu_device *adev = cookie;
 767         amdgpu_asic_set_vga_state(adev, state);
 768         if (state)
 769                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
 770                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 771         else
 772                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 773 }
 774
 775 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
 776 {
 777         /* defines number of bits in page table versus page directory,
 778          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
 779          * page table and the remaining bits are in the page directory */
 780         if (amdgpu_vm_block_size == -1)
 781                 return;
 782
 783         if (amdgpu_vm_block_size < 9) {
 784                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
 785                          amdgpu_vm_block_size);
 786                 amdgpu_vm_block_size = -1;
 787         }
 788 }
 789
 790 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
 791 {
 792         /* no need to check the default value */
 793         if (amdgpu_vm_size == -1)
 794                 return;
 795
 796         if (amdgpu_vm_size < 1) {
 797                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
 798                          amdgpu_vm_size);
 799                 amdgpu_vm_size = -1;
 800         }
 801 }
 802
 803 /**
 804  * amdgpu_device_check_arguments - validate module params
 805  *
 806  * @adev: amdgpu_device pointer
 807  *
 808  * Validates certain module parameters and updates
 809  * the associated values used by the driver (all asics).
 810  */
 811 static void amdgpu_device_check_arguments(struct amdgpu_device *adev)
 812 {
 813         if (amdgpu_sched_jobs < 4) {
 814                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
 815                          amdgpu_sched_jobs);
 816                 amdgpu_sched_jobs = 4;
 817         } else if (!is_power_of_2(amdgpu_sched_jobs)){
 818                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
 819                          amdgpu_sched_jobs);
 820                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
 821         }
 822
 823         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
 824                 /* gart size must be greater or equal to 32M */
 825                 dev_warn(adev->dev, "gart size (%d) too small\n",
 826                          amdgpu_gart_size);
 827                 amdgpu_gart_size = -1;
 828         }
 829
 830         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
 831                 /* gtt size must be greater or equal to 32M */
 832                 dev_warn(adev->dev, "gtt size (%d) too small\n",
 833                                  amdgpu_gtt_size);
 834                 amdgpu_gtt_size = -1;
 835         }
 836
 837         /* valid range is between 4 and 9 inclusive */
 838         if (amdgpu_vm_fragment_size != -1 &&
 839             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
 840                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
 841                 amdgpu_vm_fragment_size = -1;
 842         }
 843
 844         amdgpu_device_check_vm_size(adev);
 845
 846         amdgpu_device_check_block_size(adev);
 847
 848         if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
 849             !is_power_of_2(amdgpu_vram_page_split))) {
 850                 dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
 851                          amdgpu_vram_page_split);
 852                 amdgpu_vram_page_split = 1024;
 853         }
 854
 855         if (amdgpu_lockup_timeout == 0) {
 856                 dev_warn(adev->dev, "lockup_timeout msut be > 0, adjusting to 10000\n");
 857                 amdgpu_lockup_timeout = 10000;
 858         }
 859 }
 860
 861 /**
 862  * amdgpu_switcheroo_set_state - set switcheroo state
 863  *
 864  * @pdev: pci dev pointer
 865  * @state: vga_switcheroo state
 866  *
 867  * Callback for the switcheroo driver.  Suspends or resumes the
 868  * the asics before or after it is powered up using ACPI methods.
 869  */
 870 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
 871 {
 872         struct drm_device *dev = pci_get_drvdata(pdev);
 873
 874         if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
 875                 return;
 876
 877         if (state == VGA_SWITCHEROO_ON) {
 878                 pr_info("amdgpu: switched on\n");
 879                 /* don't suspend or resume card normally */
 880                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
 881
 882                 amdgpu_device_resume(dev, true, true);
 883
 884                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
 885                 drm_kms_helper_poll_enable(dev);
 886         } else {
 887                 pr_info("amdgpu: switched off\n");
 888                 drm_kms_helper_poll_disable(dev);
 889                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
 890                 amdgpu_device_suspend(dev, true, true);
 891                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
 892         }
 893 }
 894
 895 /**
 896  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
 897  *
 898  * @pdev: pci dev pointer
 899  *
 900  * Callback for the switcheroo driver.  Check of the switcheroo
 901  * state can be changed.
 902  * Returns true if the state can be changed, false if not.
 903  */
 904 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
 905 {
 906         struct drm_device *dev = pci_get_drvdata(pdev);
 907
 908         /*
 909         * FIXME: open_count is protected by drm_global_mutex but that would lead to
 910         * locking inversion with the driver load path. And the access here is
 911         * completely racy anyway. So don't bother with locking for now.
 912         */
 913         return dev->open_count == 0;
 914 }
 915
 916 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
 917         .set_gpu_state = amdgpu_switcheroo_set_state,
 918         .reprobe = NULL,
 919         .can_switch = amdgpu_switcheroo_can_switch,
 920 };
 921
 922 int amdgpu_device_ip_set_clockgating_state(struct amdgpu_device *adev,
 923                                            enum amd_ip_block_type block_type,
 924                                            enum amd_clockgating_state state)
 925 {
 926         int i, r = 0;
 927
 928         for (i = 0; i < adev->num_ip_blocks; i++) {
 929                 if (!adev->ip_blocks[i].status.valid)
 930                         continue;
 931                 if (adev->ip_blocks[i].version->type != block_type)
 932                         continue;
 933                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
 934                         continue;
 935                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
 936                         (void *)adev, state);
 937                 if (r)
 938                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
 939                                   adev->ip_blocks[i].version->funcs->name, r);
 940         }
 941         return r;
 942 }
 943
 944 int amdgpu_device_ip_set_powergating_state(struct amdgpu_device *adev,
 945                                            enum amd_ip_block_type block_type,
 946                                            enum amd_powergating_state state)
 947 {
 948         int i, r = 0;
 949
 950         for (i = 0; i < adev->num_ip_blocks; i++) {
 951                 if (!adev->ip_blocks[i].status.valid)
 952                         continue;
 953                 if (adev->ip_blocks[i].version->type != block_type)
 954                         continue;
 955                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
 956                         continue;
 957                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
 958                         (void *)adev, state);
 959                 if (r)
 960                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
 961                                   adev->ip_blocks[i].version->funcs->name, r);
 962         }
 963         return r;
 964 }
 965
 966 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
 967                                             u32 *flags)
 968 {
 969         int i;
 970
 971         for (i = 0; i < adev->num_ip_blocks; i++) {
 972                 if (!adev->ip_blocks[i].status.valid)
 973                         continue;
 974                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
 975                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
 976         }
 977 }
 978
 979 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
 980                                    enum amd_ip_block_type block_type)
 981 {
 982         int i, r;
 983
 984         for (i = 0; i < adev->num_ip_blocks; i++) {
 985                 if (!adev->ip_blocks[i].status.valid)
 986                         continue;
 987                 if (adev->ip_blocks[i].version->type == block_type) {
 988                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
 989                         if (r)
 990                                 return r;
 991                         break;
 992                 }
 993         }
 994         return 0;
 995
 996 }
 997
 998 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
 999                               enum amd_ip_block_type block_type)
1000 {
1001         int i;
1002
1003         for (i = 0; i < adev->num_ip_blocks; i++) {
1004                 if (!adev->ip_blocks[i].status.valid)
1005                         continue;
1006                 if (adev->ip_blocks[i].version->type == block_type)
1007                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1008         }
1009         return true;
1010
1011 }
1012
1013 struct amdgpu_ip_block *
1014 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1015                               enum amd_ip_block_type type)
1016 {
1017         int i;
1018
1019         for (i = 0; i < adev->num_ip_blocks; i++)
1020                 if (adev->ip_blocks[i].version->type == type)
1021                         return &adev->ip_blocks[i];
1022
1023         return NULL;
1024 }
1025
1026 /**
1027  * amdgpu_device_ip_block_version_cmp
1028  *
1029  * @adev: amdgpu_device pointer
1030  * @type: enum amd_ip_block_type
1031  * @major: major version
1032  * @minor: minor version
1033  *
1034  * return 0 if equal or greater
1035  * return 1 if smaller or the ip_block doesn't exist
1036  */
1037 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1038                                        enum amd_ip_block_type type,
1039                                        u32 major, u32 minor)
1040 {
1041         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1042
1043         if (ip_block && ((ip_block->version->major > major) ||
1044                         ((ip_block->version->major == major) &&
1045                         (ip_block->version->minor >= minor))))
1046                 return 0;
1047
1048         return 1;
1049 }
1050
1051 /**
1052  * amdgpu_device_ip_block_add
1053  *
1054  * @adev: amdgpu_device pointer
1055  * @ip_block_version: pointer to the IP to add
1056  *
1057  * Adds the IP block driver information to the collection of IPs
1058  * on the asic.
1059  */
1060 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1061                                const struct amdgpu_ip_block_version *ip_block_version)
1062 {
1063         if (!ip_block_version)
1064                 return -EINVAL;
1065
1066         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1067                   ip_block_version->funcs->name);
1068
1069         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1070
1071         return 0;
1072 }
1073
1074 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1075 {
1076         adev->enable_virtual_display = false;
1077
1078         if (amdgpu_virtual_display) {
1079                 struct drm_device *ddev = adev->ddev;
1080                 const char *pci_address_name = pci_name(ddev->pdev);
1081                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1082
1083                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1084                 pciaddstr_tmp = pciaddstr;
1085                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1086                         pciaddname = strsep(&pciaddname_tmp, ",");
1087                         if (!strcmp("all", pciaddname)
1088                             || !strcmp(pci_address_name, pciaddname)) {
1089                                 long num_crtc;
1090                                 int res = -1;
1091
1092                                 adev->enable_virtual_display = true;
1093
1094                                 if (pciaddname_tmp)
1095                                         res = kstrtol(pciaddname_tmp, 10,
1096                                                       &num_crtc);
1097
1098                                 if (!res) {
1099                                         if (num_crtc < 1)
1100                                                 num_crtc = 1;
1101                                         if (num_crtc > 6)
1102                                                 num_crtc = 6;
1103                                         adev->mode_info.num_crtc = num_crtc;
1104                                 } else {
1105                                         adev->mode_info.num_crtc = 1;
1106                                 }
1107                                 break;
1108                         }
1109                 }
1110
1111                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1112                          amdgpu_virtual_display, pci_address_name,
1113                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1114
1115                 kfree(pciaddstr);
1116         }
1117 }
1118
1119 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1120 {
1121         const char *chip_name;
1122         char fw_name[30];
1123         int err;
1124         const struct gpu_info_firmware_header_v1_0 *hdr;
1125
1126         adev->firmware.gpu_info_fw = NULL;
1127
1128         switch (adev->asic_type) {
1129         case CHIP_TOPAZ:
1130         case CHIP_TONGA:
1131         case CHIP_FIJI:
1132         case CHIP_POLARIS11:
1133         case CHIP_POLARIS10:
1134         case CHIP_POLARIS12:
1135         case CHIP_CARRIZO:
1136         case CHIP_STONEY:
1137 #ifdef CONFIG_DRM_AMDGPU_SI
1138         case CHIP_VERDE:
1139         case CHIP_TAHITI:
1140         case CHIP_PITCAIRN:
1141         case CHIP_OLAND:
1142         case CHIP_HAINAN:
1143 #endif
1144 #ifdef CONFIG_DRM_AMDGPU_CIK
1145         case CHIP_BONAIRE:
1146         case CHIP_HAWAII:
1147         case CHIP_KAVERI:
1148         case CHIP_KABINI:
1149         case CHIP_MULLINS:
1150 #endif
1151         default:
1152                 return 0;
1153         case CHIP_VEGA10:
1154                 chip_name = "vega10";
1155                 break;
1156         case CHIP_RAVEN:
1157                 chip_name = "raven";
1158                 break;
1159         }
1160
1161         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1162         err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1163         if (err) {
1164                 dev_err(adev->dev,
1165                         "Failed to load gpu_info firmware \"%s\"\n",
1166                         fw_name);
1167                 goto out;
1168         }
1169         err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1170         if (err) {
1171                 dev_err(adev->dev,
1172                         "Failed to validate gpu_info firmware \"%s\"\n",
1173                         fw_name);
1174                 goto out;
1175         }
1176
1177         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1178         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1179
1180         switch (hdr->version_major) {
1181         case 1:
1182         {
1183                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1184                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1185                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1186
1187                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1188                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1189                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1190                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1191                 adev->gfx.config.max_texture_channel_caches =
1192                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
1193                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1194                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1195                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1196                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1197                 adev->gfx.config.double_offchip_lds_buf =
1198                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1199                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1200                 adev->gfx.cu_info.max_waves_per_simd =
1201                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1202                 adev->gfx.cu_info.max_scratch_slots_per_cu =
1203                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1204                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1205                 break;
1206         }
1207         default:
1208                 dev_err(adev->dev,
1209                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1210                 err = -EINVAL;
1211                 goto out;
1212         }
1213 out:
1214         return err;
1215 }
1216
1217 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
1218 {
1219         int i, r;
1220
1221         amdgpu_device_enable_virtual_display(adev);
1222
1223         switch (adev->asic_type) {
1224         case CHIP_TOPAZ:
1225         case CHIP_TONGA:
1226         case CHIP_FIJI:
1227         case CHIP_POLARIS11:
1228         case CHIP_POLARIS10:
1229         case CHIP_POLARIS12:
1230         case CHIP_CARRIZO:
1231         case CHIP_STONEY:
1232                 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1233                         adev->family = AMDGPU_FAMILY_CZ;
1234                 else
1235                         adev->family = AMDGPU_FAMILY_VI;
1236
1237                 r = vi_set_ip_blocks(adev);
1238                 if (r)
1239                         return r;
1240                 break;
1241 #ifdef CONFIG_DRM_AMDGPU_SI
1242         case CHIP_VERDE:
1243         case CHIP_TAHITI:
1244         case CHIP_PITCAIRN:
1245         case CHIP_OLAND:
1246         case CHIP_HAINAN:
1247                 adev->family = AMDGPU_FAMILY_SI;
1248                 r = si_set_ip_blocks(adev);
1249                 if (r)
1250                         return r;
1251                 break;
1252 #endif
1253 #ifdef CONFIG_DRM_AMDGPU_CIK
1254         case CHIP_BONAIRE:
1255         case CHIP_HAWAII:
1256         case CHIP_KAVERI:
1257         case CHIP_KABINI:
1258         case CHIP_MULLINS:
1259                 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1260                         adev->family = AMDGPU_FAMILY_CI;
1261                 else
1262                         adev->family = AMDGPU_FAMILY_KV;
1263
1264                 r = cik_set_ip_blocks(adev);
1265                 if (r)
1266                         return r;
1267                 break;
1268 #endif
1269         case  CHIP_VEGA10:
1270         case  CHIP_RAVEN:
1271                 if (adev->asic_type == CHIP_RAVEN)
1272                         adev->family = AMDGPU_FAMILY_RV;
1273                 else
1274                         adev->family = AMDGPU_FAMILY_AI;
1275
1276                 r = soc15_set_ip_blocks(adev);
1277                 if (r)
1278                         return r;
1279                 break;
1280         default:
1281                 /* FIXME: not supported yet */
1282                 return -EINVAL;
1283         }
1284
1285         r = amdgpu_device_parse_gpu_info_fw(adev);
1286         if (r)
1287                 return r;
1288
1289         amdgpu_amdkfd_device_probe(adev);
1290
1291         if (amdgpu_sriov_vf(adev)) {
1292                 r = amdgpu_virt_request_full_gpu(adev, true);
1293                 if (r)
1294                         return -EAGAIN;
1295         }
1296
1297         for (i = 0; i < adev->num_ip_blocks; i++) {
1298                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1299                         DRM_ERROR("disabled ip block: %d <%s>\n",
1300                                   i, adev->ip_blocks[i].version->funcs->name);
1301                         adev->ip_blocks[i].status.valid = false;
1302                 } else {
1303                         if (adev->ip_blocks[i].version->funcs->early_init) {
1304                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1305                                 if (r == -ENOENT) {
1306                                         adev->ip_blocks[i].status.valid = false;
1307                                 } else if (r) {
1308                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
1309                                                   adev->ip_blocks[i].version->funcs->name, r);
1310                                         return r;
1311                                 } else {
1312                                         adev->ip_blocks[i].status.valid = true;
1313                                 }
1314                         } else {
1315                                 adev->ip_blocks[i].status.valid = true;
1316                         }
1317                 }
1318         }
1319
1320         adev->cg_flags &= amdgpu_cg_mask;
1321         adev->pg_flags &= amdgpu_pg_mask;
1322
1323         return 0;
1324 }
1325
1326 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
1327 {
1328         int i, r;
1329
1330         for (i = 0; i < adev->num_ip_blocks; i++) {
1331                 if (!adev->ip_blocks[i].status.valid)
1332                         continue;
1333                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1334                 if (r) {
1335                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1336                                   adev->ip_blocks[i].version->funcs->name, r);
1337                         return r;
1338                 }
1339                 adev->ip_blocks[i].status.sw = true;
1340
1341                 /* need to do gmc hw init early so we can allocate gpu mem */
1342                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1343                         r = amdgpu_device_vram_scratch_init(adev);
1344                         if (r) {
1345                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1346                                 return r;
1347                         }
1348                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1349                         if (r) {
1350                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
1351                                 return r;
1352                         }
1353                         r = amdgpu_device_wb_init(adev);
1354                         if (r) {
1355                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
1356                                 return r;
1357                         }
1358                         adev->ip_blocks[i].status.hw = true;
1359
1360                         /* right after GMC hw init, we create CSA */
1361                         if (amdgpu_sriov_vf(adev)) {
1362                                 r = amdgpu_allocate_static_csa(adev);
1363                                 if (r) {
1364                                         DRM_ERROR("allocate CSA failed %d\n", r);
1365                                         return r;
1366                                 }
1367                         }
1368                 }
1369         }
1370
1371         for (i = 0; i < adev->num_ip_blocks; i++) {
1372                 if (!adev->ip_blocks[i].status.sw)
1373                         continue;
1374                 if (adev->ip_blocks[i].status.hw)
1375                         continue;
1376                 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1377                 if (r) {
1378                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1379                                   adev->ip_blocks[i].version->funcs->name, r);
1380                         return r;
1381                 }
1382                 adev->ip_blocks[i].status.hw = true;
1383         }
1384
1385         amdgpu_amdkfd_device_init(adev);
1386
1387         if (amdgpu_sriov_vf(adev))
1388                 amdgpu_virt_release_full_gpu(adev, true);
1389
1390         return 0;
1391 }
1392
1393 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
1394 {
1395         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
1396 }
1397
1398 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
1399 {
1400         return !!memcmp(adev->gart.ptr, adev->reset_magic,
1401                         AMDGPU_RESET_MAGIC_NUM);
1402 }
1403
1404 static int amdgpu_device_ip_late_set_cg_state(struct amdgpu_device *adev)
1405 {
1406         int i = 0, r;
1407
1408         if (amdgpu_emu_mode == 1)
1409                 return 0;
1410
1411         for (i = 0; i < adev->num_ip_blocks; i++) {
1412                 if (!adev->ip_blocks[i].status.valid)
1413                         continue;
1414                 /* skip CG for VCE/UVD, it's handled specially */
1415                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1416                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1417                         /* enable clockgating to save power */
1418                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1419                                                                                      AMD_CG_STATE_GATE);
1420                         if (r) {
1421                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1422                                           adev->ip_blocks[i].version->funcs->name, r);
1423                                 return r;
1424                         }
1425                 }
1426         }
1427         return 0;
1428 }
1429
1430 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
1431 {
1432         int i = 0, r;
1433
1434         for (i = 0; i < adev->num_ip_blocks; i++) {
1435                 if (!adev->ip_blocks[i].status.valid)
1436                         continue;
1437                 if (adev->ip_blocks[i].version->funcs->late_init) {
1438                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
1439                         if (r) {
1440                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
1441                                           adev->ip_blocks[i].version->funcs->name, r);
1442                                 return r;
1443                         }
1444                         adev->ip_blocks[i].status.late_initialized = true;
1445                 }
1446         }
1447
1448         mod_delayed_work(system_wq, &adev->late_init_work,
1449                         msecs_to_jiffies(AMDGPU_RESUME_MS));
1450
1451         amdgpu_device_fill_reset_magic(adev);
1452
1453         return 0;
1454 }
1455
1456 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
1457 {
1458         int i, r;
1459
1460         amdgpu_amdkfd_device_fini(adev);
1461         /* need to disable SMC first */
1462         for (i = 0; i < adev->num_ip_blocks; i++) {
1463                 if (!adev->ip_blocks[i].status.hw)
1464                         continue;
1465                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
1466                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1467                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1468                                                                                      AMD_CG_STATE_UNGATE);
1469                         if (r) {
1470                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1471                                           adev->ip_blocks[i].version->funcs->name, r);
1472                                 return r;
1473                         }
1474                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1475                         /* XXX handle errors */
1476                         if (r) {
1477                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1478                                           adev->ip_blocks[i].version->funcs->name, r);
1479                         }
1480                         adev->ip_blocks[i].status.hw = false;
1481                         break;
1482                 }
1483         }
1484
1485         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1486                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC &&
1487                         adev->firmware.load_type == AMDGPU_FW_LOAD_SMU)
1488                         amdgpu_ucode_fini_bo(adev);
1489                 if (!adev->ip_blocks[i].status.hw)
1490                         continue;
1491
1492                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1493                         adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1494                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1495                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1496                                                                                      AMD_CG_STATE_UNGATE);
1497                         if (r) {
1498                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1499                                           adev->ip_blocks[i].version->funcs->name, r);
1500                                 return r;
1501                         }
1502                 }
1503
1504                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1505                 /* XXX handle errors */
1506                 if (r) {
1507                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1508                                   adev->ip_blocks[i].version->funcs->name, r);
1509                 }
1510
1511                 adev->ip_blocks[i].status.hw = false;
1512         }
1513
1514         /* disable all interrupts */
1515         amdgpu_irq_disable_all(adev);
1516
1517         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1518                 if (!adev->ip_blocks[i].status.sw)
1519                         continue;
1520
1521                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1522                         amdgpu_free_static_csa(adev);
1523                         amdgpu_device_wb_fini(adev);
1524                         amdgpu_device_vram_scratch_fini(adev);
1525                 }
1526
1527                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
1528                 /* XXX handle errors */
1529                 if (r) {
1530                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
1531                                   adev->ip_blocks[i].version->funcs->name, r);
1532                 }
1533                 adev->ip_blocks[i].status.sw = false;
1534                 adev->ip_blocks[i].status.valid = false;
1535         }
1536
1537         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1538                 if (!adev->ip_blocks[i].status.late_initialized)
1539                         continue;
1540                 if (adev->ip_blocks[i].version->funcs->late_fini)
1541                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
1542                 adev->ip_blocks[i].status.late_initialized = false;
1543         }
1544
1545         if (amdgpu_sriov_vf(adev))
1546                 if (amdgpu_virt_release_full_gpu(adev, false))
1547                         DRM_ERROR("failed to release exclusive mode on fini\n");
1548
1549         return 0;
1550 }
1551
1552 static void amdgpu_device_ip_late_init_func_handler(struct work_struct *work)
1553 {
1554         struct amdgpu_device *adev =
1555                 container_of(work, struct amdgpu_device, late_init_work.work);
1556         amdgpu_device_ip_late_set_cg_state(adev);
1557 }
1558
1559 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
1560 {
1561         int i, r;
1562
1563         if (amdgpu_sriov_vf(adev))
1564                 amdgpu_virt_request_full_gpu(adev, false);
1565
1566         /* ungate SMC block first */
1567         r = amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
1568                                                    AMD_CG_STATE_UNGATE);
1569         if (r) {
1570                 DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n", r);
1571         }
1572
1573         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1574                 if (!adev->ip_blocks[i].status.valid)
1575                         continue;
1576                 /* ungate blocks so that suspend can properly shut them down */
1577                 if (i != AMD_IP_BLOCK_TYPE_SMC) {
1578                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1579                                                                                      AMD_CG_STATE_UNGATE);
1580                         if (r) {
1581                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1582                                           adev->ip_blocks[i].version->funcs->name, r);
1583                         }
1584                 }
1585                 /* XXX handle errors */
1586                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
1587                 /* XXX handle errors */
1588                 if (r) {
1589                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
1590                                   adev->ip_blocks[i].version->funcs->name, r);
1591                 }
1592         }
1593
1594         if (amdgpu_sriov_vf(adev))
1595                 amdgpu_virt_release_full_gpu(adev, false);
1596
1597         return 0;
1598 }
1599
1600 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
1601 {
1602         int i, r;
1603
1604         static enum amd_ip_block_type ip_order[] = {
1605                 AMD_IP_BLOCK_TYPE_GMC,
1606                 AMD_IP_BLOCK_TYPE_COMMON,
1607                 AMD_IP_BLOCK_TYPE_IH,
1608         };
1609
1610         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1611                 int j;
1612                 struct amdgpu_ip_block *block;
1613
1614                 for (j = 0; j < adev->num_ip_blocks; j++) {
1615                         block = &adev->ip_blocks[j];
1616
1617                         if (block->version->type != ip_order[i] ||
1618                                 !block->status.valid)
1619                                 continue;
1620
1621                         r = block->version->funcs->hw_init(adev);
1622                         DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
1623                         if (r)
1624                                 return r;
1625                 }
1626         }
1627
1628         return 0;
1629 }
1630
1631 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
1632 {
1633         int i, r;
1634
1635         static enum amd_ip_block_type ip_order[] = {
1636                 AMD_IP_BLOCK_TYPE_SMC,
1637                 AMD_IP_BLOCK_TYPE_PSP,
1638                 AMD_IP_BLOCK_TYPE_DCE,
1639                 AMD_IP_BLOCK_TYPE_GFX,
1640                 AMD_IP_BLOCK_TYPE_SDMA,
1641                 AMD_IP_BLOCK_TYPE_UVD,
1642                 AMD_IP_BLOCK_TYPE_VCE
1643         };
1644
1645         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
1646                 int j;
1647                 struct amdgpu_ip_block *block;
1648
1649                 for (j = 0; j < adev->num_ip_blocks; j++) {
1650                         block = &adev->ip_blocks[j];
1651
1652                         if (block->version->type != ip_order[i] ||
1653                                 !block->status.valid)
1654                                 continue;
1655
1656                         r = block->version->funcs->hw_init(adev);
1657                         DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed");
1658                         if (r)
1659                                 return r;
1660                 }
1661         }
1662
1663         return 0;
1664 }
1665
1666 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
1667 {
1668         int i, r;
1669
1670         for (i = 0; i < adev->num_ip_blocks; i++) {
1671                 if (!adev->ip_blocks[i].status.valid)
1672                         continue;
1673                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1674                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
1675                                 adev->ip_blocks[i].version->type ==
1676                                 AMD_IP_BLOCK_TYPE_IH) {
1677                         r = adev->ip_blocks[i].version->funcs->resume(adev);
1678                         if (r) {
1679                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
1680                                           adev->ip_blocks[i].version->funcs->name, r);
1681                                 return r;
1682                         }
1683                 }
1684         }
1685
1686         return 0;
1687 }
1688
1689 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
1690 {
1691         int i, r;
1692
1693         for (i = 0; i < adev->num_ip_blocks; i++) {
1694                 if (!adev->ip_blocks[i].status.valid)
1695                         continue;
1696                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1697                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
1698                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH )
1699                         continue;
1700                 r = adev->ip_blocks[i].version->funcs->resume(adev);
1701                 if (r) {
1702                         DRM_ERROR("resume of IP block <%s> failed %d\n",
1703                                   adev->ip_blocks[i].version->funcs->name, r);
1704                         return r;
1705                 }
1706         }
1707
1708         return 0;
1709 }
1710
1711 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
1712 {
1713         int r;
1714
1715         r = amdgpu_device_ip_resume_phase1(adev);
1716         if (r)
1717                 return r;
1718         r = amdgpu_device_ip_resume_phase2(adev);
1719
1720         return r;
1721 }
1722
1723 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
1724 {
1725         if (amdgpu_sriov_vf(adev)) {
1726                 if (adev->is_atom_fw) {
1727                         if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
1728                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
1729                 } else {
1730                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
1731                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
1732                 }
1733
1734                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
1735                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
1736         }
1737 }
1738
1739 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
1740 {
1741         switch (asic_type) {
1742 #if defined(CONFIG_DRM_AMD_DC)
1743         case CHIP_BONAIRE:
1744         case CHIP_HAWAII:
1745         case CHIP_KAVERI:
1746         case CHIP_KABINI:
1747         case CHIP_MULLINS:
1748         case CHIP_CARRIZO:
1749         case CHIP_STONEY:
1750         case CHIP_POLARIS11:
1751         case CHIP_POLARIS10:
1752         case CHIP_POLARIS12:
1753         case CHIP_TONGA:
1754         case CHIP_FIJI:
1755 #if defined(CONFIG_DRM_AMD_DC_PRE_VEGA)
1756                 return amdgpu_dc != 0;
1757 #endif
1758         case CHIP_VEGA10:
1759 #if defined(CONFIG_DRM_AMD_DC_DCN1_0)
1760         case CHIP_RAVEN:
1761 #endif
1762                 return amdgpu_dc != 0;
1763 #endif
1764         default:
1765                 return false;
1766         }
1767 }
1768
1769 /**
1770  * amdgpu_device_has_dc_support - check if dc is supported
1771  *
1772  * @adev: amdgpu_device_pointer
1773  *
1774  * Returns true for supported, false for not supported
1775  */
1776 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
1777 {
1778         if (amdgpu_sriov_vf(adev))
1779                 return false;
1780
1781         return amdgpu_device_asic_has_dc_support(adev->asic_type);
1782 }
1783
1784 /**
1785  * amdgpu_device_init - initialize the driver
1786  *
1787  * @adev: amdgpu_device pointer
1788  * @pdev: drm dev pointer
1789  * @pdev: pci dev pointer
1790  * @flags: driver flags
1791  *
1792  * Initializes the driver info and hw (all asics).
1793  * Returns 0 for success or an error on failure.
1794  * Called at driver startup.
1795  */
1796 int amdgpu_device_init(struct amdgpu_device *adev,
1797                        struct drm_device *ddev,
1798                        struct pci_dev *pdev,
1799                        uint32_t flags)
1800 {
1801         int r, i;
1802         bool runtime = false;
1803         u32 max_MBps;
1804
1805         adev->shutdown = false;
1806         adev->dev = &pdev->dev;
1807         adev->ddev = ddev;
1808         adev->pdev = pdev;
1809         adev->flags = flags;
1810         adev->asic_type = flags & AMD_ASIC_MASK;
1811         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
1812         if (amdgpu_emu_mode == 1)
1813                 adev->usec_timeout *= 2;
1814         adev->gmc.gart_size = 512 * 1024 * 1024;
1815         adev->accel_working = false;
1816         adev->num_rings = 0;
1817         adev->mman.buffer_funcs = NULL;
1818         adev->mman.buffer_funcs_ring = NULL;
1819         adev->vm_manager.vm_pte_funcs = NULL;
1820         adev->vm_manager.vm_pte_num_rings = 0;
1821         adev->gmc.gmc_funcs = NULL;
1822         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
1823         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1824
1825         adev->smc_rreg = &amdgpu_invalid_rreg;
1826         adev->smc_wreg = &amdgpu_invalid_wreg;
1827         adev->pcie_rreg = &amdgpu_invalid_rreg;
1828         adev->pcie_wreg = &amdgpu_invalid_wreg;
1829         adev->pciep_rreg = &amdgpu_invalid_rreg;
1830         adev->pciep_wreg = &amdgpu_invalid_wreg;
1831         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
1832         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
1833         adev->didt_rreg = &amdgpu_invalid_rreg;
1834         adev->didt_wreg = &amdgpu_invalid_wreg;
1835         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
1836         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
1837         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
1838         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
1839
1840         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
1841                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
1842                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
1843
1844         /* mutex initialization are all done here so we
1845          * can recall function without having locking issues */
1846         atomic_set(&adev->irq.ih.lock, 0);
1847         mutex_init(&adev->firmware.mutex);
1848         mutex_init(&adev->pm.mutex);
1849         mutex_init(&adev->gfx.gpu_clock_mutex);
1850         mutex_init(&adev->srbm_mutex);
1851         mutex_init(&adev->gfx.pipe_reserve_mutex);
1852         mutex_init(&adev->grbm_idx_mutex);
1853         mutex_init(&adev->mn_lock);
1854         mutex_init(&adev->virt.vf_errors.lock);
1855         hash_init(adev->mn_hash);
1856         mutex_init(&adev->lock_reset);
1857
1858         amdgpu_device_check_arguments(adev);
1859
1860         spin_lock_init(&adev->mmio_idx_lock);
1861         spin_lock_init(&adev->smc_idx_lock);
1862         spin_lock_init(&adev->pcie_idx_lock);
1863         spin_lock_init(&adev->uvd_ctx_idx_lock);
1864         spin_lock_init(&adev->didt_idx_lock);
1865         spin_lock_init(&adev->gc_cac_idx_lock);
1866         spin_lock_init(&adev->se_cac_idx_lock);
1867         spin_lock_init(&adev->audio_endpt_idx_lock);
1868         spin_lock_init(&adev->mm_stats.lock);
1869
1870         INIT_LIST_HEAD(&adev->shadow_list);
1871         mutex_init(&adev->shadow_list_lock);
1872
1873         INIT_LIST_HEAD(&adev->ring_lru_list);
1874         spin_lock_init(&adev->ring_lru_list_lock);
1875
1876         INIT_DELAYED_WORK(&adev->late_init_work,
1877                           amdgpu_device_ip_late_init_func_handler);
1878
1879         /* Registers mapping */
1880         /* TODO: block userspace mapping of io register */
1881         if (adev->asic_type >= CHIP_BONAIRE) {
1882                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
1883                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
1884         } else {
1885                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
1886                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
1887         }
1888
1889         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
1890         if (adev->rmmio == NULL) {
1891                 return -ENOMEM;
1892         }
1893         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
1894         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
1895
1896         /* doorbell bar mapping */
1897         amdgpu_device_doorbell_init(adev);
1898
1899         /* io port mapping */
1900         for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
1901                 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
1902                         adev->rio_mem_size = pci_resource_len(adev->pdev, i);
1903                         adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
1904                         break;
1905                 }
1906         }
1907         if (adev->rio_mem == NULL)
1908                 DRM_INFO("PCI I/O BAR is not found.\n");
1909
1910         /* early init functions */
1911         r = amdgpu_device_ip_early_init(adev);
1912         if (r)
1913                 return r;
1914
1915         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
1916         /* this will fail for cards that aren't VGA class devices, just
1917          * ignore it */
1918         vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
1919
1920         if (amdgpu_device_is_px(ddev))
1921                 runtime = true;
1922         if (!pci_is_thunderbolt_attached(adev->pdev))
1923                 vga_switcheroo_register_client(adev->pdev,
1924                                                &amdgpu_switcheroo_ops, runtime);
1925         if (runtime)
1926                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
1927
1928         if (amdgpu_emu_mode == 1) {
1929                 /* post the asic on emulation mode */
1930                 emu_soc_asic_init(adev);
1931                 goto fence_driver_init;
1932         }
1933
1934         /* Read BIOS */
1935         if (!amdgpu_get_bios(adev)) {
1936                 r = -EINVAL;
1937                 goto failed;
1938         }
1939
1940         r = amdgpu_atombios_init(adev);
1941         if (r) {
1942                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
1943                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
1944                 goto failed;
1945         }
1946
1947         /* detect if we are with an SRIOV vbios */
1948         amdgpu_device_detect_sriov_bios(adev);
1949
1950         /* Post card if necessary */
1951         if (amdgpu_device_need_post(adev)) {
1952                 if (!adev->bios) {
1953                         dev_err(adev->dev, "no vBIOS found\n");
1954                         r = -EINVAL;
1955                         goto failed;
1956                 }
1957                 DRM_INFO("GPU posting now...\n");
1958                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
1959                 if (r) {
1960                         dev_err(adev->dev, "gpu post error!\n");
1961                         goto failed;
1962                 }
1963         }
1964
1965         if (adev->is_atom_fw) {
1966                 /* Initialize clocks */
1967                 r = amdgpu_atomfirmware_get_clock_info(adev);
1968                 if (r) {
1969                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
1970                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
1971                         goto failed;
1972                 }
1973         } else {
1974                 /* Initialize clocks */
1975                 r = amdgpu_atombios_get_clock_info(adev);
1976                 if (r) {
1977                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
1978                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
1979                         goto failed;
1980                 }
1981                 /* init i2c buses */
1982                 if (!amdgpu_device_has_dc_support(adev))
1983                         amdgpu_atombios_i2c_init(adev);
1984         }
1985
1986 fence_driver_init:
1987         /* Fence driver */
1988         r = amdgpu_fence_driver_init(adev);
1989         if (r) {
1990                 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
1991                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
1992                 goto failed;
1993         }
1994
1995         /* init the mode config */
1996         drm_mode_config_init(adev->ddev);
1997
1998         r = amdgpu_device_ip_init(adev);
1999         if (r) {
2000                 /* failed in exclusive mode due to timeout */
2001                 if (amdgpu_sriov_vf(adev) &&
2002                     !amdgpu_sriov_runtime(adev) &&
2003                     amdgpu_virt_mmio_blocked(adev) &&
2004                     !amdgpu_virt_wait_reset(adev)) {
2005                         dev_err(adev->dev, "VF exclusive mode timeout\n");
2006                         /* Don't send request since VF is inactive. */
2007                         adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
2008                         adev->virt.ops = NULL;
2009                         r = -EAGAIN;
2010                         goto failed;
2011                 }
2012                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
2013                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
2014                 amdgpu_device_ip_fini(adev);
2015                 goto failed;
2016         }
2017
2018         adev->accel_working = true;
2019
2020         amdgpu_vm_check_compute_bug(adev);
2021
2022         /* Initialize the buffer migration limit. */
2023         if (amdgpu_moverate >= 0)
2024                 max_MBps = amdgpu_moverate;
2025         else
2026                 max_MBps = 8; /* Allow 8 MB/s. */
2027         /* Get a log2 for easy divisions. */
2028         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
2029
2030         r = amdgpu_ib_pool_init(adev);
2031         if (r) {
2032                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2033                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2034                 goto failed;
2035         }
2036
2037         r = amdgpu_ib_ring_tests(adev);
2038         if (r)
2039                 DRM_ERROR("ib ring test failed (%d).\n", r);
2040
2041         if (amdgpu_sriov_vf(adev))
2042                 amdgpu_virt_init_data_exchange(adev);
2043
2044         amdgpu_fbdev_init(adev);
2045
2046         r = amdgpu_pm_sysfs_init(adev);
2047         if (r)
2048                 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
2049
2050         r = amdgpu_debugfs_gem_init(adev);
2051         if (r)
2052                 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
2053
2054         r = amdgpu_debugfs_regs_init(adev);
2055         if (r)
2056                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2057
2058         r = amdgpu_debugfs_firmware_init(adev);
2059         if (r)
2060                 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
2061
2062         r = amdgpu_debugfs_init(adev);
2063         if (r)
2064                 DRM_ERROR("Creating debugfs files failed (%d).\n", r);
2065
2066         if ((amdgpu_testing & 1)) {
2067                 if (adev->accel_working)
2068                         amdgpu_test_moves(adev);
2069                 else
2070                         DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
2071         }
2072         if (amdgpu_benchmarking) {
2073                 if (adev->accel_working)
2074                         amdgpu_benchmark(adev, amdgpu_benchmarking);
2075                 else
2076                         DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
2077         }
2078
2079         /* enable clockgating, etc. after ib tests, etc. since some blocks require
2080          * explicit gating rather than handling it automatically.
2081          */
2082         r = amdgpu_device_ip_late_init(adev);
2083         if (r) {
2084                 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
2085                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
2086                 goto failed;
2087         }
2088
2089         return 0;
2090
2091 failed:
2092         amdgpu_vf_error_trans_all(adev);
2093         if (runtime)
2094                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2095
2096         return r;
2097 }
2098
2099 /**
2100  * amdgpu_device_fini - tear down the driver
2101  *
2102  * @adev: amdgpu_device pointer
2103  *
2104  * Tear down the driver info (all asics).
2105  * Called at driver shutdown.
2106  */
2107 void amdgpu_device_fini(struct amdgpu_device *adev)
2108 {
2109         int r;
2110
2111         DRM_INFO("amdgpu: finishing device.\n");
2112         adev->shutdown = true;
2113         if (adev->mode_info.mode_config_initialized)
2114                 drm_crtc_force_disable_all(adev->ddev);
2115
2116         amdgpu_ib_pool_fini(adev);
2117         amdgpu_fence_driver_fini(adev);
2118         amdgpu_pm_sysfs_fini(adev);
2119         amdgpu_fbdev_fini(adev);
2120         r = amdgpu_device_ip_fini(adev);
2121         if (adev->firmware.gpu_info_fw) {
2122                 release_firmware(adev->firmware.gpu_info_fw);
2123                 adev->firmware.gpu_info_fw = NULL;
2124         }
2125         adev->accel_working = false;
2126         cancel_delayed_work_sync(&adev->late_init_work);
2127         /* free i2c buses */
2128         if (!amdgpu_device_has_dc_support(adev))
2129                 amdgpu_i2c_fini(adev);
2130
2131         if (amdgpu_emu_mode != 1)
2132                 amdgpu_atombios_fini(adev);
2133
2134         kfree(adev->bios);
2135         adev->bios = NULL;
2136         if (!pci_is_thunderbolt_attached(adev->pdev))
2137                 vga_switcheroo_unregister_client(adev->pdev);
2138         if (adev->flags & AMD_IS_PX)
2139                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
2140         vga_client_register(adev->pdev, NULL, NULL, NULL);
2141         if (adev->rio_mem)
2142                 pci_iounmap(adev->pdev, adev->rio_mem);
2143         adev->rio_mem = NULL;
2144         iounmap(adev->rmmio);
2145         adev->rmmio = NULL;
2146         amdgpu_device_doorbell_fini(adev);
2147         amdgpu_debugfs_regs_cleanup(adev);
2148 }
2149
2150
2151 /*
2152  * Suspend & resume.
2153  */
2154 /**
2155  * amdgpu_device_suspend - initiate device suspend
2156  *
2157  * @pdev: drm dev pointer
2158  * @state: suspend state
2159  *
2160  * Puts the hw in the suspend state (all asics).
2161  * Returns 0 for success or an error on failure.
2162  * Called at driver suspend.
2163  */
2164 int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
2165 {
2166         struct amdgpu_device *adev;
2167         struct drm_crtc *crtc;
2168         struct drm_connector *connector;
2169         int r;
2170
2171         if (dev == NULL || dev->dev_private == NULL) {
2172                 return -ENODEV;
2173         }
2174
2175         adev = dev->dev_private;
2176
2177         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2178                 return 0;
2179
2180         drm_kms_helper_poll_disable(dev);
2181
2182         if (!amdgpu_device_has_dc_support(adev)) {
2183                 /* turn off display hw */
2184                 drm_modeset_lock_all(dev);
2185                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2186                         drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
2187                 }
2188                 drm_modeset_unlock_all(dev);
2189         }
2190
2191         amdgpu_amdkfd_suspend(adev);
2192
2193         /* unpin the front buffers and cursors */
2194         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2195                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2196                 struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
2197                 struct amdgpu_bo *robj;
2198
2199                 if (amdgpu_crtc->cursor_bo) {
2200                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2201                         r = amdgpu_bo_reserve(aobj, true);
2202                         if (r == 0) {
2203                                 amdgpu_bo_unpin(aobj);
2204                                 amdgpu_bo_unreserve(aobj);
2205                         }
2206                 }
2207
2208                 if (rfb == NULL || rfb->obj == NULL) {
2209                         continue;
2210                 }
2211                 robj = gem_to_amdgpu_bo(rfb->obj);
2212                 /* don't unpin kernel fb objects */
2213                 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
2214                         r = amdgpu_bo_reserve(robj, true);
2215                         if (r == 0) {
2216                                 amdgpu_bo_unpin(robj);
2217                                 amdgpu_bo_unreserve(robj);
2218                         }
2219                 }
2220         }
2221         /* evict vram memory */
2222         amdgpu_bo_evict_vram(adev);
2223
2224         amdgpu_fence_driver_suspend(adev);
2225
2226         r = amdgpu_device_ip_suspend(adev);
2227
2228         /* evict remaining vram memory
2229          * This second call to evict vram is to evict the gart page table
2230          * using the CPU.
2231          */
2232         amdgpu_bo_evict_vram(adev);
2233
2234         pci_save_state(dev->pdev);
2235         if (suspend) {
2236                 /* Shut down the device */
2237                 pci_disable_device(dev->pdev);
2238                 pci_set_power_state(dev->pdev, PCI_D3hot);
2239         } else {
2240                 r = amdgpu_asic_reset(adev);
2241                 if (r)
2242                         DRM_ERROR("amdgpu asic reset failed\n");
2243         }
2244
2245         if (fbcon) {
2246                 console_lock();
2247                 amdgpu_fbdev_set_suspend(adev, 1);
2248                 console_unlock();
2249         }
2250         return 0;
2251 }
2252
2253 /**
2254  * amdgpu_device_resume - initiate device resume
2255  *
2256  * @pdev: drm dev pointer
2257  *
2258  * Bring the hw back to operating state (all asics).
2259  * Returns 0 for success or an error on failure.
2260  * Called at driver resume.
2261  */
2262 int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2263 {
2264         struct drm_connector *connector;
2265         struct amdgpu_device *adev = dev->dev_private;
2266         struct drm_crtc *crtc;
2267         int r = 0;
2268
2269         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2270                 return 0;
2271
2272         if (fbcon)
2273                 console_lock();
2274
2275         if (resume) {
2276                 pci_set_power_state(dev->pdev, PCI_D0);
2277                 pci_restore_state(dev->pdev);
2278                 r = pci_enable_device(dev->pdev);
2279                 if (r)
2280                         goto unlock;
2281         }
2282
2283         /* post card */
2284         if (amdgpu_device_need_post(adev)) {
2285                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2286                 if (r)
2287                         DRM_ERROR("amdgpu asic init failed\n");
2288         }
2289
2290         r = amdgpu_device_ip_resume(adev);
2291         if (r) {
2292                 DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r);
2293                 goto unlock;
2294         }
2295         amdgpu_fence_driver_resume(adev);
2296
2297         if (resume) {
2298                 r = amdgpu_ib_ring_tests(adev);
2299                 if (r)
2300                         DRM_ERROR("ib ring test failed (%d).\n", r);
2301         }
2302
2303         r = amdgpu_device_ip_late_init(adev);
2304         if (r)
2305                 goto unlock;
2306
2307         /* pin cursors */
2308         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2309                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2310
2311                 if (amdgpu_crtc->cursor_bo) {
2312                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2313                         r = amdgpu_bo_reserve(aobj, true);
2314                         if (r == 0) {
2315                                 r = amdgpu_bo_pin(aobj,
2316                                                   AMDGPU_GEM_DOMAIN_VRAM,
2317                                                   &amdgpu_crtc->cursor_addr);
2318                                 if (r != 0)
2319                                         DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
2320                                 amdgpu_bo_unreserve(aobj);
2321                         }
2322                 }
2323         }
2324         r = amdgpu_amdkfd_resume(adev);
2325         if (r)
2326                 return r;
2327
2328         /* blat the mode back in */
2329         if (fbcon) {
2330                 if (!amdgpu_device_has_dc_support(adev)) {
2331                         /* pre DCE11 */
2332                         drm_helper_resume_force_mode(dev);
2333
2334                         /* turn on display hw */
2335                         drm_modeset_lock_all(dev);
2336                         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2337                                 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
2338                         }
2339                         drm_modeset_unlock_all(dev);
2340                 }
2341         }
2342
2343         drm_kms_helper_poll_enable(dev);
2344
2345         /*
2346          * Most of the connector probing functions try to acquire runtime pm
2347          * refs to ensure that the GPU is powered on when connector polling is
2348          * performed. Since we're calling this from a runtime PM callback,
2349          * trying to acquire rpm refs will cause us to deadlock.
2350          *
2351          * Since we're guaranteed to be holding the rpm lock, it's safe to
2352          * temporarily disable the rpm helpers so this doesn't deadlock us.
2353          */
2354 #ifdef CONFIG_PM
2355         dev->dev->power.disable_depth++;
2356 #endif
2357         if (!amdgpu_device_has_dc_support(adev))
2358                 drm_helper_hpd_irq_event(dev);
2359         else
2360                 drm_kms_helper_hotplug_event(dev);
2361 #ifdef CONFIG_PM
2362         dev->dev->power.disable_depth--;
2363 #endif
2364
2365         if (fbcon)
2366                 amdgpu_fbdev_set_suspend(adev, 0);
2367
2368 unlock:
2369         if (fbcon)
2370                 console_unlock();
2371
2372         return r;
2373 }
2374
2375 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
2376 {
2377         int i;
2378         bool asic_hang = false;
2379
2380         if (amdgpu_sriov_vf(adev))
2381                 return true;
2382
2383         for (i = 0; i < adev->num_ip_blocks; i++) {
2384                 if (!adev->ip_blocks[i].status.valid)
2385                         continue;
2386                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
2387                         adev->ip_blocks[i].status.hang =
2388                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
2389                 if (adev->ip_blocks[i].status.hang) {
2390                         DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
2391                         asic_hang = true;
2392                 }
2393         }
2394         return asic_hang;
2395 }
2396
2397 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
2398 {
2399         int i, r = 0;
2400
2401         for (i = 0; i < adev->num_ip_blocks; i++) {
2402                 if (!adev->ip_blocks[i].status.valid)
2403                         continue;
2404                 if (adev->ip_blocks[i].status.hang &&
2405                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
2406                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
2407                         if (r)
2408                                 return r;
2409                 }
2410         }
2411
2412         return 0;
2413 }
2414
2415 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
2416 {
2417         int i;
2418
2419         for (i = 0; i < adev->num_ip_blocks; i++) {
2420                 if (!adev->ip_blocks[i].status.valid)
2421                         continue;
2422                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
2423                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
2424                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
2425                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
2426                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2427                         if (adev->ip_blocks[i].status.hang) {
2428                                 DRM_INFO("Some block need full reset!\n");
2429                                 return true;
2430                         }
2431                 }
2432         }
2433         return false;
2434 }
2435
2436 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
2437 {
2438         int i, r = 0;
2439
2440         for (i = 0; i < adev->num_ip_blocks; i++) {
2441                 if (!adev->ip_blocks[i].status.valid)
2442                         continue;
2443                 if (adev->ip_blocks[i].status.hang &&
2444                     adev->ip_blocks[i].version->funcs->soft_reset) {
2445                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
2446                         if (r)
2447                                 return r;
2448                 }
2449         }
2450
2451         return 0;
2452 }
2453
2454 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
2455 {
2456         int i, r = 0;
2457
2458         for (i = 0; i < adev->num_ip_blocks; i++) {
2459                 if (!adev->ip_blocks[i].status.valid)
2460                         continue;
2461                 if (adev->ip_blocks[i].status.hang &&
2462                     adev->ip_blocks[i].version->funcs->post_soft_reset)
2463                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
2464                 if (r)
2465                         return r;
2466         }
2467
2468         return 0;
2469 }
2470
2471 static int amdgpu_device_recover_vram_from_shadow(struct amdgpu_device *adev,
2472                                                   struct amdgpu_ring *ring,
2473                                                   struct amdgpu_bo *bo,
2474                                                   struct dma_fence **fence)
2475 {
2476         uint32_t domain;
2477         int r;
2478
2479         if (!bo->shadow)
2480                 return 0;
2481
2482         r = amdgpu_bo_reserve(bo, true);
2483         if (r)
2484                 return r;
2485         domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
2486         /* if bo has been evicted, then no need to recover */
2487         if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
2488                 r = amdgpu_bo_validate(bo->shadow);
2489                 if (r) {
2490                         DRM_ERROR("bo validate failed!\n");
2491                         goto err;
2492                 }
2493
2494                 r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
2495                                                  NULL, fence, true);
2496                 if (r) {
2497                         DRM_ERROR("recover page table failed!\n");
2498                         goto err;
2499                 }
2500         }
2501 err:
2502         amdgpu_bo_unreserve(bo);
2503         return r;
2504 }
2505
2506 static int amdgpu_device_handle_vram_lost(struct amdgpu_device *adev)
2507 {
2508         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2509         struct amdgpu_bo *bo, *tmp;
2510         struct dma_fence *fence = NULL, *next = NULL;
2511         long r = 1;
2512         int i = 0;
2513         long tmo;
2514
2515         if (amdgpu_sriov_runtime(adev))
2516                 tmo = msecs_to_jiffies(amdgpu_lockup_timeout);
2517         else
2518                 tmo = msecs_to_jiffies(100);
2519
2520         DRM_INFO("recover vram bo from shadow start\n");
2521         mutex_lock(&adev->shadow_list_lock);
2522         list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2523                 next = NULL;
2524                 amdgpu_device_recover_vram_from_shadow(adev, ring, bo, &next);
2525                 if (fence) {
2526                         r = dma_fence_wait_timeout(fence, false, tmo);
2527                         if (r == 0)
2528                                 pr_err("wait fence %p[%d] timeout\n", fence, i);
2529                         else if (r < 0)
2530                                 pr_err("wait fence %p[%d] interrupted\n", fence, i);
2531                         if (r < 1) {
2532                                 dma_fence_put(fence);
2533                                 fence = next;
2534                                 break;
2535                         }
2536                         i++;
2537                 }
2538
2539                 dma_fence_put(fence);
2540                 fence = next;
2541         }
2542         mutex_unlock(&adev->shadow_list_lock);
2543
2544         if (fence) {
2545                 r = dma_fence_wait_timeout(fence, false, tmo);
2546                 if (r == 0)
2547                         pr_err("wait fence %p[%d] timeout\n", fence, i);
2548                 else if (r < 0)
2549                         pr_err("wait fence %p[%d] interrupted\n", fence, i);
2550
2551         }
2552         dma_fence_put(fence);
2553
2554         if (r > 0)
2555                 DRM_INFO("recover vram bo from shadow done\n");
2556         else
2557                 DRM_ERROR("recover vram bo from shadow failed\n");
2558
2559         return (r > 0?0:1);
2560 }
2561
2562 /*
2563  * amdgpu_device_reset - reset ASIC/GPU for bare-metal or passthrough
2564  *
2565  * @adev: amdgpu device pointer
2566  *
2567  * attempt to do soft-reset or full-reset and reinitialize Asic
2568  * return 0 means successed otherwise failed
2569 */
2570 static int amdgpu_device_reset(struct amdgpu_device *adev)
2571 {
2572         bool need_full_reset, vram_lost = 0;
2573         int r;
2574
2575         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
2576
2577         if (!need_full_reset) {
2578                 amdgpu_device_ip_pre_soft_reset(adev);
2579                 r = amdgpu_device_ip_soft_reset(adev);
2580                 amdgpu_device_ip_post_soft_reset(adev);
2581                 if (r || amdgpu_device_ip_check_soft_reset(adev)) {
2582                         DRM_INFO("soft reset failed, will fallback to full reset!\n");
2583                         need_full_reset = true;
2584                 }
2585         }
2586
2587         if (need_full_reset) {
2588                 r = amdgpu_device_ip_suspend(adev);
2589
2590 retry:
2591                 r = amdgpu_asic_reset(adev);
2592                 /* post card */
2593                 amdgpu_atom_asic_init(adev->mode_info.atom_context);
2594
2595                 if (!r) {
2596                         dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
2597                         r = amdgpu_device_ip_resume_phase1(adev);
2598                         if (r)
2599                                 goto out;
2600
2601                         vram_lost = amdgpu_device_check_vram_lost(adev);
2602                         if (vram_lost) {
2603                                 DRM_ERROR("VRAM is lost!\n");
2604                                 atomic_inc(&adev->vram_lost_counter);
2605                         }
2606
2607                         r = amdgpu_gtt_mgr_recover(
2608                                 &adev->mman.bdev.man[TTM_PL_TT]);
2609                         if (r)
2610                                 goto out;
2611
2612                         r = amdgpu_device_ip_resume_phase2(adev);
2613                         if (r)
2614                                 goto out;
2615
2616                         if (vram_lost)
2617                                 amdgpu_device_fill_reset_magic(adev);
2618                 }
2619         }
2620
2621 out:
2622         if (!r) {
2623                 amdgpu_irq_gpu_reset_resume_helper(adev);
2624                 r = amdgpu_ib_ring_tests(adev);
2625                 if (r) {
2626                         dev_err(adev->dev, "ib ring test failed (%d).\n", r);
2627                         r = amdgpu_device_ip_suspend(adev);
2628                         need_full_reset = true;
2629                         goto retry;
2630                 }
2631         }
2632
2633         if (!r && ((need_full_reset && !(adev->flags & AMD_IS_APU)) || vram_lost))
2634                 r = amdgpu_device_handle_vram_lost(adev);
2635
2636         return r;
2637 }
2638
2639 /*
2640  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
2641  *
2642  * @adev: amdgpu device pointer
2643  *
2644  * do VF FLR and reinitialize Asic
2645  * return 0 means successed otherwise failed
2646 */
2647 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, bool from_hypervisor)
2648 {
2649         int r;
2650
2651         if (from_hypervisor)
2652                 r = amdgpu_virt_request_full_gpu(adev, true);
2653         else
2654                 r = amdgpu_virt_reset_gpu(adev);
2655         if (r)
2656                 return r;
2657
2658         /* Resume IP prior to SMC */
2659         r = amdgpu_device_ip_reinit_early_sriov(adev);
2660         if (r)
2661                 goto error;
2662
2663         /* we need recover gart prior to run SMC/CP/SDMA resume */
2664         amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]);
2665
2666         /* now we are okay to resume SMC/CP/SDMA */
2667         r = amdgpu_device_ip_reinit_late_sriov(adev);
2668         amdgpu_virt_release_full_gpu(adev, true);
2669         if (r)
2670                 goto error;
2671
2672         amdgpu_irq_gpu_reset_resume_helper(adev);
2673         r = amdgpu_ib_ring_tests(adev);
2674
2675         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
2676                 atomic_inc(&adev->vram_lost_counter);
2677                 r = amdgpu_device_handle_vram_lost(adev);
2678         }
2679
2680 error:
2681
2682         return r;
2683 }
2684
2685 /**
2686  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
2687  *
2688  * @adev: amdgpu device pointer
2689  * @job: which job trigger hang
2690  * @force forces reset regardless of amdgpu_gpu_recovery
2691  *
2692  * Attempt to reset the GPU if it has hung (all asics).
2693  * Returns 0 for success or an error on failure.
2694  */
2695 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
2696                               struct amdgpu_job *job, bool force)
2697 {
2698         struct drm_atomic_state *state = NULL;
2699         int i, r, resched;
2700
2701         if (!force && !amdgpu_device_ip_check_soft_reset(adev)) {
2702                 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
2703                 return 0;
2704         }
2705
2706         if (!force && (amdgpu_gpu_recovery == 0 ||
2707                         (amdgpu_gpu_recovery == -1  && !amdgpu_sriov_vf(adev)))) {
2708                 DRM_INFO("GPU recovery disabled.\n");
2709                 return 0;
2710         }
2711
2712         dev_info(adev->dev, "GPU reset begin!\n");
2713
2714         mutex_lock(&adev->lock_reset);
2715         atomic_inc(&adev->gpu_reset_counter);
2716         adev->in_gpu_reset = 1;
2717
2718         /* block TTM */
2719         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
2720
2721         /* store modesetting */
2722         if (amdgpu_device_has_dc_support(adev))
2723                 state = drm_atomic_helper_suspend(adev->ddev);
2724
2725         /* block all schedulers and reset given job's ring */
2726         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2727                 struct amdgpu_ring *ring = adev->rings[i];
2728
2729                 if (!ring || !ring->sched.thread)
2730                         continue;
2731
2732                 kthread_park(ring->sched.thread);
2733
2734                 if (job && job->ring->idx != i)
2735                         continue;
2736
2737                 drm_sched_hw_job_reset(&ring->sched, &job->base);
2738
2739                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2740                 amdgpu_fence_driver_force_completion(ring);
2741         }
2742
2743         if (amdgpu_sriov_vf(adev))
2744                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
2745         else
2746                 r = amdgpu_device_reset(adev);
2747
2748         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2749                 struct amdgpu_ring *ring = adev->rings[i];
2750
2751                 if (!ring || !ring->sched.thread)
2752                         continue;
2753
2754                 /* only need recovery sched of the given job's ring
2755                  * or all rings (in the case @job is NULL)
2756                  * after above amdgpu_reset accomplished
2757                  */
2758                 if ((!job || job->ring->idx == i) && !r)
2759                         drm_sched_job_recovery(&ring->sched);
2760
2761                 kthread_unpark(ring->sched.thread);
2762         }
2763
2764         if (amdgpu_device_has_dc_support(adev)) {
2765                 if (drm_atomic_helper_resume(adev->ddev, state))
2766                         dev_info(adev->dev, "drm resume failed:%d\n", r);
2767         } else {
2768                 drm_helper_resume_force_mode(adev->ddev);
2769         }
2770
2771         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
2772
2773         if (r) {
2774                 /* bad news, how to tell it to userspace ? */
2775                 dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter));
2776                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
2777         } else {
2778                 dev_info(adev->dev, "GPU reset(%d) successed!\n",atomic_read(&adev->gpu_reset_counter));
2779         }
2780
2781         amdgpu_vf_error_trans_all(adev);
2782         adev->in_gpu_reset = 0;
2783         mutex_unlock(&adev->lock_reset);
2784         return r;
2785 }
2786
2787 void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
2788 {
2789         u32 mask;
2790         int ret;
2791
2792         if (amdgpu_pcie_gen_cap)
2793                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
2794
2795         if (amdgpu_pcie_lane_cap)
2796                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
2797
2798         /* covers APUs as well */
2799         if (pci_is_root_bus(adev->pdev->bus)) {
2800                 if (adev->pm.pcie_gen_mask == 0)
2801                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
2802                 if (adev->pm.pcie_mlw_mask == 0)
2803                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
2804                 return;
2805         }
2806
2807         if (adev->pm.pcie_gen_mask == 0) {
2808                 ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
2809                 if (!ret) {
2810                         adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
2811                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
2812                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
2813
2814                         if (mask & DRM_PCIE_SPEED_25)
2815                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
2816                         if (mask & DRM_PCIE_SPEED_50)
2817                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
2818                         if (mask & DRM_PCIE_SPEED_80)
2819                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
2820                 } else {
2821                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
2822                 }
2823         }
2824         if (adev->pm.pcie_mlw_mask == 0) {
2825                 ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
2826                 if (!ret) {
2827                         switch (mask) {
2828                         case 32:
2829                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
2830                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
2831                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2832                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2833                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2834                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2835                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2836                                 break;
2837                         case 16:
2838                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
2839                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2840                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2841                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2842                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2843                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2844                                 break;
2845                         case 12:
2846                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2847                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2848                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2849                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2850                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2851                                 break;
2852                         case 8:
2853                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2854                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2855                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2856                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2857                                 break;
2858                         case 4:
2859                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2860                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2861                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2862                                 break;
2863                         case 2:
2864                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2865                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2866                                 break;
2867                         case 1:
2868                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
2869                                 break;
2870                         default:
2871                                 break;
2872                         }
2873                 } else {
2874                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
2875                 }
2876         }
2877 }
2878