drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/kthread.h>
  29 #include <linux/console.h>
  30 #include <linux/slab.h>
  31 #include <linux/debugfs.h>
  32 #include <drm/drmP.h>
  33 #include <drm/drm_crtc_helper.h>
  34 #include <drm/amdgpu_drm.h>
  35 #include <linux/vgaarb.h>
  36 #include <linux/vga_switcheroo.h>
  37 #include <linux/efi.h>
  38 #include "amdgpu.h"
  39 #include "amdgpu_trace.h"
  40 #include "amdgpu_i2c.h"
  41 #include "atom.h"
  42 #include "amdgpu_atombios.h"
  43 #include "amd_pcie.h"
  44 #ifdef CONFIG_DRM_AMDGPU_SI
  45 #include "si.h"
  46 #endif
  47 #ifdef CONFIG_DRM_AMDGPU_CIK
  48 #include "cik.h"
  49 #endif
  50 #include "vi.h"
  51 #include "bif/bif_4_1_d.h"
  52 #include <linux/pci.h>
  53 #include <linux/firmware.h>
  54
  55 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
  56 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
  57
  58 static const char *amdgpu_asic_name[] = {
  59         "TAHITI",
  60         "PITCAIRN",
  61         "VERDE",
  62         "OLAND",
  63         "HAINAN",
  64         "BONAIRE",
  65         "KAVERI",
  66         "KABINI",
  67         "HAWAII",
  68         "MULLINS",
  69         "TOPAZ",
  70         "TONGA",
  71         "FIJI",
  72         "CARRIZO",
  73         "STONEY",
  74         "POLARIS10",
  75         "POLARIS11",
  76         "POLARIS12",
  77         "LAST",
  78 };
  79
  80 bool amdgpu_device_is_px(struct drm_device *dev)
  81 {
  82         struct amdgpu_device *adev = dev->dev_private;
  83
  84         if (adev->flags & AMD_IS_PX)
  85                 return true;
  86         return false;
  87 }
  88
  89 /*
  90  * MMIO register access helper functions.
  91  */
  92 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
  93                         uint32_t acc_flags)
  94 {
  95         uint32_t ret;
  96
  97         if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
  98                 BUG_ON(in_interrupt());
  99                 return amdgpu_virt_kiq_rreg(adev, reg);
 100         }
 101
 102         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
 103                 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 104         else {
 105                 unsigned long flags;
 106
 107                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 108                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 109                 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 110                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 111         }
 112         trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
 113         return ret;
 114 }
 115
 116 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 117                     uint32_t acc_flags)
 118 {
 119         trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 120
 121         if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) {
 122                 BUG_ON(in_interrupt());
 123                 return amdgpu_virt_kiq_wreg(adev, reg, v);
 124         }
 125
 126         if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX))
 127                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 128         else {
 129                 unsigned long flags;
 130
 131                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 132                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 133                 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 134                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 135         }
 136 }
 137
 138 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
 139 {
 140         if ((reg * 4) < adev->rio_mem_size)
 141                 return ioread32(adev->rio_mem + (reg * 4));
 142         else {
 143                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 144                 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
 145         }
 146 }
 147
 148 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 149 {
 150
 151         if ((reg * 4) < adev->rio_mem_size)
 152                 iowrite32(v, adev->rio_mem + (reg * 4));
 153         else {
 154                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 155                 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
 156         }
 157 }
 158
 159 /**
 160  * amdgpu_mm_rdoorbell - read a doorbell dword
 161  *
 162  * @adev: amdgpu_device pointer
 163  * @index: doorbell index
 164  *
 165  * Returns the value in the doorbell aperture at the
 166  * requested doorbell index (CIK).
 167  */
 168 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 169 {
 170         if (index < adev->doorbell.num_doorbells) {
 171                 return readl(adev->doorbell.ptr + index);
 172         } else {
 173                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 174                 return 0;
 175         }
 176 }
 177
 178 /**
 179  * amdgpu_mm_wdoorbell - write a doorbell dword
 180  *
 181  * @adev: amdgpu_device pointer
 182  * @index: doorbell index
 183  * @v: value to write
 184  *
 185  * Writes @v to the doorbell aperture at the
 186  * requested doorbell index (CIK).
 187  */
 188 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 189 {
 190         if (index < adev->doorbell.num_doorbells) {
 191                 writel(v, adev->doorbell.ptr + index);
 192         } else {
 193                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 194         }
 195 }
 196
 197 /**
 198  * amdgpu_invalid_rreg - dummy reg read function
 199  *
 200  * @adev: amdgpu device pointer
 201  * @reg: offset of register
 202  *
 203  * Dummy register read function.  Used for register blocks
 204  * that certain asics don't have (all asics).
 205  * Returns the value in the register.
 206  */
 207 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 208 {
 209         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 210         BUG();
 211         return 0;
 212 }
 213
 214 /**
 215  * amdgpu_invalid_wreg - dummy reg write function
 216  *
 217  * @adev: amdgpu device pointer
 218  * @reg: offset of register
 219  * @v: value to write to the register
 220  *
 221  * Dummy register read function.  Used for register blocks
 222  * that certain asics don't have (all asics).
 223  */
 224 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 225 {
 226         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 227                   reg, v);
 228         BUG();
 229 }
 230
 231 /**
 232  * amdgpu_block_invalid_rreg - dummy reg read function
 233  *
 234  * @adev: amdgpu device pointer
 235  * @block: offset of instance
 236  * @reg: offset of register
 237  *
 238  * Dummy register read function.  Used for register blocks
 239  * that certain asics don't have (all asics).
 240  * Returns the value in the register.
 241  */
 242 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 243                                           uint32_t block, uint32_t reg)
 244 {
 245         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 246                   reg, block);
 247         BUG();
 248         return 0;
 249 }
 250
 251 /**
 252  * amdgpu_block_invalid_wreg - dummy reg write function
 253  *
 254  * @adev: amdgpu device pointer
 255  * @block: offset of instance
 256  * @reg: offset of register
 257  * @v: value to write to the register
 258  *
 259  * Dummy register read function.  Used for register blocks
 260  * that certain asics don't have (all asics).
 261  */
 262 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 263                                       uint32_t block,
 264                                       uint32_t reg, uint32_t v)
 265 {
 266         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 267                   reg, block, v);
 268         BUG();
 269 }
 270
 271 static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
 272 {
 273         int r;
 274
 275         if (adev->vram_scratch.robj == NULL) {
 276                 r = amdgpu_bo_create(adev, AMDGPU_GPU_PAGE_SIZE,
 277                                      PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
 278                                      AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
 279                                      AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 280                                      NULL, NULL, &adev->vram_scratch.robj);
 281                 if (r) {
 282                         return r;
 283                 }
 284         }
 285
 286         r = amdgpu_bo_reserve(adev->vram_scratch.robj, false);
 287         if (unlikely(r != 0))
 288                 return r;
 289         r = amdgpu_bo_pin(adev->vram_scratch.robj,
 290                           AMDGPU_GEM_DOMAIN_VRAM, &adev->vram_scratch.gpu_addr);
 291         if (r) {
 292                 amdgpu_bo_unreserve(adev->vram_scratch.robj);
 293                 return r;
 294         }
 295         r = amdgpu_bo_kmap(adev->vram_scratch.robj,
 296                                 (void **)&adev->vram_scratch.ptr);
 297         if (r)
 298                 amdgpu_bo_unpin(adev->vram_scratch.robj);
 299         amdgpu_bo_unreserve(adev->vram_scratch.robj);
 300
 301         return r;
 302 }
 303
 304 static void amdgpu_vram_scratch_fini(struct amdgpu_device *adev)
 305 {
 306         int r;
 307
 308         if (adev->vram_scratch.robj == NULL) {
 309                 return;
 310         }
 311         r = amdgpu_bo_reserve(adev->vram_scratch.robj, false);
 312         if (likely(r == 0)) {
 313                 amdgpu_bo_kunmap(adev->vram_scratch.robj);
 314                 amdgpu_bo_unpin(adev->vram_scratch.robj);
 315                 amdgpu_bo_unreserve(adev->vram_scratch.robj);
 316         }
 317         amdgpu_bo_unref(&adev->vram_scratch.robj);
 318 }
 319
 320 /**
 321  * amdgpu_program_register_sequence - program an array of registers.
 322  *
 323  * @adev: amdgpu_device pointer
 324  * @registers: pointer to the register array
 325  * @array_size: size of the register array
 326  *
 327  * Programs an array or registers with and and or masks.
 328  * This is a helper for setting golden registers.
 329  */
 330 void amdgpu_program_register_sequence(struct amdgpu_device *adev,
 331                                       const u32 *registers,
 332                                       const u32 array_size)
 333 {
 334         u32 tmp, reg, and_mask, or_mask;
 335         int i;
 336
 337         if (array_size % 3)
 338                 return;
 339
 340         for (i = 0; i < array_size; i +=3) {
 341                 reg = registers[i + 0];
 342                 and_mask = registers[i + 1];
 343                 or_mask = registers[i + 2];
 344
 345                 if (and_mask == 0xffffffff) {
 346                         tmp = or_mask;
 347                 } else {
 348                         tmp = RREG32(reg);
 349                         tmp &= ~and_mask;
 350                         tmp |= or_mask;
 351                 }
 352                 WREG32(reg, tmp);
 353         }
 354 }
 355
 356 void amdgpu_pci_config_reset(struct amdgpu_device *adev)
 357 {
 358         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
 359 }
 360
 361 /*
 362  * GPU doorbell aperture helpers function.
 363  */
 364 /**
 365  * amdgpu_doorbell_init - Init doorbell driver information.
 366  *
 367  * @adev: amdgpu_device pointer
 368  *
 369  * Init doorbell driver information (CIK)
 370  * Returns 0 on success, error on failure.
 371  */
 372 static int amdgpu_doorbell_init(struct amdgpu_device *adev)
 373 {
 374         /* doorbell bar mapping */
 375         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
 376         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
 377
 378         adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
 379                                              AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
 380         if (adev->doorbell.num_doorbells == 0)
 381                 return -EINVAL;
 382
 383         adev->doorbell.ptr = ioremap(adev->doorbell.base, adev->doorbell.num_doorbells * sizeof(u32));
 384         if (adev->doorbell.ptr == NULL) {
 385                 return -ENOMEM;
 386         }
 387         DRM_INFO("doorbell mmio base: 0x%08X\n", (uint32_t)adev->doorbell.base);
 388         DRM_INFO("doorbell mmio size: %u\n", (unsigned)adev->doorbell.size);
 389
 390         return 0;
 391 }
 392
 393 /**
 394  * amdgpu_doorbell_fini - Tear down doorbell driver information.
 395  *
 396  * @adev: amdgpu_device pointer
 397  *
 398  * Tear down doorbell driver information (CIK)
 399  */
 400 static void amdgpu_doorbell_fini(struct amdgpu_device *adev)
 401 {
 402         iounmap(adev->doorbell.ptr);
 403         adev->doorbell.ptr = NULL;
 404 }
 405
 406 /**
 407  * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to
 408  *                                setup amdkfd
 409  *
 410  * @adev: amdgpu_device pointer
 411  * @aperture_base: output returning doorbell aperture base physical address
 412  * @aperture_size: output returning doorbell aperture size in bytes
 413  * @start_offset: output returning # of doorbell bytes reserved for amdgpu.
 414  *
 415  * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up,
 416  * takes doorbells required for its own rings and reports the setup to amdkfd.
 417  * amdgpu reserved doorbells are at the start of the doorbell aperture.
 418  */
 419 void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
 420                                 phys_addr_t *aperture_base,
 421                                 size_t *aperture_size,
 422                                 size_t *start_offset)
 423 {
 424         /*
 425          * The first num_doorbells are used by amdgpu.
 426          * amdkfd takes whatever's left in the aperture.
 427          */
 428         if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) {
 429                 *aperture_base = adev->doorbell.base;
 430                 *aperture_size = adev->doorbell.size;
 431                 *start_offset = adev->doorbell.num_doorbells * sizeof(u32);
 432         } else {
 433                 *aperture_base = 0;
 434                 *aperture_size = 0;
 435                 *start_offset = 0;
 436         }
 437 }
 438
 439 /*
 440  * amdgpu_wb_*()
 441  * Writeback is the the method by which the the GPU updates special pages
 442  * in memory with the status of certain GPU events (fences, ring pointers,
 443  * etc.).
 444  */
 445
 446 /**
 447  * amdgpu_wb_fini - Disable Writeback and free memory
 448  *
 449  * @adev: amdgpu_device pointer
 450  *
 451  * Disables Writeback and frees the Writeback memory (all asics).
 452  * Used at driver shutdown.
 453  */
 454 static void amdgpu_wb_fini(struct amdgpu_device *adev)
 455 {
 456         if (adev->wb.wb_obj) {
 457                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
 458                                       &adev->wb.gpu_addr,
 459                                       (void **)&adev->wb.wb);
 460                 adev->wb.wb_obj = NULL;
 461         }
 462 }
 463
 464 /**
 465  * amdgpu_wb_init- Init Writeback driver info and allocate memory
 466  *
 467  * @adev: amdgpu_device pointer
 468  *
 469  * Disables Writeback and frees the Writeback memory (all asics).
 470  * Used at driver startup.
 471  * Returns 0 on success or an -error on failure.
 472  */
 473 static int amdgpu_wb_init(struct amdgpu_device *adev)
 474 {
 475         int r;
 476
 477         if (adev->wb.wb_obj == NULL) {
 478                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t),
 479                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 480                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
 481                                             (void **)&adev->wb.wb);
 482                 if (r) {
 483                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
 484                         return r;
 485                 }
 486
 487                 adev->wb.num_wb = AMDGPU_MAX_WB;
 488                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
 489
 490                 /* clear wb memory */
 491                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t));
 492         }
 493
 494         return 0;
 495 }
 496
 497 /**
 498  * amdgpu_wb_get - Allocate a wb entry
 499  *
 500  * @adev: amdgpu_device pointer
 501  * @wb: wb index
 502  *
 503  * Allocate a wb slot for use by the driver (all asics).
 504  * Returns 0 on success or -EINVAL on failure.
 505  */
 506 int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
 507 {
 508         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
 509         if (offset < adev->wb.num_wb) {
 510                 __set_bit(offset, adev->wb.used);
 511                 *wb = offset;
 512                 return 0;
 513         } else {
 514                 return -EINVAL;
 515         }
 516 }
 517
 518 /**
 519  * amdgpu_wb_free - Free a wb entry
 520  *
 521  * @adev: amdgpu_device pointer
 522  * @wb: wb index
 523  *
 524  * Free a wb slot allocated for use by the driver (all asics)
 525  */
 526 void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
 527 {
 528         if (wb < adev->wb.num_wb)
 529                 __clear_bit(wb, adev->wb.used);
 530 }
 531
 532 /**
 533  * amdgpu_vram_location - try to find VRAM location
 534  * @adev: amdgpu device structure holding all necessary informations
 535  * @mc: memory controller structure holding memory informations
 536  * @base: base address at which to put VRAM
 537  *
 538  * Function will place try to place VRAM at base address provided
 539  * as parameter (which is so far either PCI aperture address or
 540  * for IGP TOM base address).
 541  *
 542  * If there is not enough space to fit the unvisible VRAM in the 32bits
 543  * address space then we limit the VRAM size to the aperture.
 544  *
 545  * Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,
 546  * this shouldn't be a problem as we are using the PCI aperture as a reference.
 547  * Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
 548  * not IGP.
 549  *
 550  * Note: we use mc_vram_size as on some board we need to program the mc to
 551  * cover the whole aperture even if VRAM size is inferior to aperture size
 552  * Novell bug 204882 + along with lots of ubuntu ones
 553  *
 554  * Note: when limiting vram it's safe to overwritte real_vram_size because
 555  * we are not in case where real_vram_size is inferior to mc_vram_size (ie
 556  * note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu
 557  * ones)
 558  *
 559  * Note: IGP TOM addr should be the same as the aperture addr, we don't
 560  * explicitly check for that thought.
 561  *
 562  * FIXME: when reducing VRAM size align new size on power of 2.
 563  */
 564 void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base)
 565 {
 566         uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
 567
 568         mc->vram_start = base;
 569         if (mc->mc_vram_size > (adev->mc.mc_mask - base + 1)) {
 570                 dev_warn(adev->dev, "limiting VRAM to PCI aperture size\n");
 571                 mc->real_vram_size = mc->aper_size;
 572                 mc->mc_vram_size = mc->aper_size;
 573         }
 574         mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 575         if (limit && limit < mc->real_vram_size)
 576                 mc->real_vram_size = limit;
 577         dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
 578                         mc->mc_vram_size >> 20, mc->vram_start,
 579                         mc->vram_end, mc->real_vram_size >> 20);
 580 }
 581
 582 /**
 583  * amdgpu_gtt_location - try to find GTT location
 584  * @adev: amdgpu device structure holding all necessary informations
 585  * @mc: memory controller structure holding memory informations
 586  *
 587  * Function will place try to place GTT before or after VRAM.
 588  *
 589  * If GTT size is bigger than space left then we ajust GTT size.
 590  * Thus function will never fails.
 591  *
 592  * FIXME: when reducing GTT size align new size on power of 2.
 593  */
 594 void amdgpu_gtt_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
 595 {
 596         u64 size_af, size_bf;
 597
 598         size_af = ((adev->mc.mc_mask - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align;
 599         size_bf = mc->vram_start & ~mc->gtt_base_align;
 600         if (size_bf > size_af) {
 601                 if (mc->gtt_size > size_bf) {
 602                         dev_warn(adev->dev, "limiting GTT\n");
 603                         mc->gtt_size = size_bf;
 604                 }
 605                 mc->gtt_start = (mc->vram_start & ~mc->gtt_base_align) - mc->gtt_size;
 606         } else {
 607                 if (mc->gtt_size > size_af) {
 608                         dev_warn(adev->dev, "limiting GTT\n");
 609                         mc->gtt_size = size_af;
 610                 }
 611                 mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align;
 612         }
 613         mc->gtt_end = mc->gtt_start + mc->gtt_size - 1;
 614         dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
 615                         mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end);
 616 }
 617
 618 /*
 619  * GPU helpers function.
 620  */
 621 /**
 622  * amdgpu_need_post - check if the hw need post or not
 623  *
 624  * @adev: amdgpu_device pointer
 625  *
 626  * Check if the asic has been initialized (all asics) at driver startup
 627  * or post is needed if  hw reset is performed.
 628  * Returns true if need or false if not.
 629  */
 630 bool amdgpu_need_post(struct amdgpu_device *adev)
 631 {
 632         uint32_t reg;
 633
 634         if (adev->has_hw_reset) {
 635                 adev->has_hw_reset = false;
 636                 return true;
 637         }
 638         /* then check MEM_SIZE, in case the crtcs are off */
 639         reg = RREG32(mmCONFIG_MEMSIZE);
 640
 641         if (reg)
 642                 return false;
 643
 644         return true;
 645
 646 }
 647
 648 static bool amdgpu_vpost_needed(struct amdgpu_device *adev)
 649 {
 650         if (amdgpu_sriov_vf(adev))
 651                 return false;
 652
 653         if (amdgpu_passthrough(adev)) {
 654                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
 655                  * some old smc fw still need driver do vPost otherwise gpu hang, while
 656                  * those smc fw version above 22.15 doesn't have this flaw, so we force
 657                  * vpost executed for smc version below 22.15
 658                  */
 659                 if (adev->asic_type == CHIP_FIJI) {
 660                         int err;
 661                         uint32_t fw_ver;
 662                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
 663                         /* force vPost if error occured */
 664                         if (err)
 665                                 return true;
 666
 667                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
 668                         if (fw_ver < 0x00160e00)
 669                                 return true;
 670                 }
 671         }
 672         return amdgpu_need_post(adev);
 673 }
 674
 675 /**
 676  * amdgpu_dummy_page_init - init dummy page used by the driver
 677  *
 678  * @adev: amdgpu_device pointer
 679  *
 680  * Allocate the dummy page used by the driver (all asics).
 681  * This dummy page is used by the driver as a filler for gart entries
 682  * when pages are taken out of the GART
 683  * Returns 0 on sucess, -ENOMEM on failure.
 684  */
 685 int amdgpu_dummy_page_init(struct amdgpu_device *adev)
 686 {
 687         if (adev->dummy_page.page)
 688                 return 0;
 689         adev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
 690         if (adev->dummy_page.page == NULL)
 691                 return -ENOMEM;
 692         adev->dummy_page.addr = pci_map_page(adev->pdev, adev->dummy_page.page,
 693                                         0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 694         if (pci_dma_mapping_error(adev->pdev, adev->dummy_page.addr)) {
 695                 dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n");
 696                 __free_page(adev->dummy_page.page);
 697                 adev->dummy_page.page = NULL;
 698                 return -ENOMEM;
 699         }
 700         return 0;
 701 }
 702
 703 /**
 704  * amdgpu_dummy_page_fini - free dummy page used by the driver
 705  *
 706  * @adev: amdgpu_device pointer
 707  *
 708  * Frees the dummy page used by the driver (all asics).
 709  */
 710 void amdgpu_dummy_page_fini(struct amdgpu_device *adev)
 711 {
 712         if (adev->dummy_page.page == NULL)
 713                 return;
 714         pci_unmap_page(adev->pdev, adev->dummy_page.addr,
 715                         PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 716         __free_page(adev->dummy_page.page);
 717         adev->dummy_page.page = NULL;
 718 }
 719
 720
 721 /* ATOM accessor methods */
 722 /*
 723  * ATOM is an interpreted byte code stored in tables in the vbios.  The
 724  * driver registers callbacks to access registers and the interpreter
 725  * in the driver parses the tables and executes then to program specific
 726  * actions (set display modes, asic init, etc.).  See amdgpu_atombios.c,
 727  * atombios.h, and atom.c
 728  */
 729
 730 /**
 731  * cail_pll_read - read PLL register
 732  *
 733  * @info: atom card_info pointer
 734  * @reg: PLL register offset
 735  *
 736  * Provides a PLL register accessor for the atom interpreter (r4xx+).
 737  * Returns the value of the PLL register.
 738  */
 739 static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
 740 {
 741         return 0;
 742 }
 743
 744 /**
 745  * cail_pll_write - write PLL register
 746  *
 747  * @info: atom card_info pointer
 748  * @reg: PLL register offset
 749  * @val: value to write to the pll register
 750  *
 751  * Provides a PLL register accessor for the atom interpreter (r4xx+).
 752  */
 753 static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
 754 {
 755
 756 }
 757
 758 /**
 759  * cail_mc_read - read MC (Memory Controller) register
 760  *
 761  * @info: atom card_info pointer
 762  * @reg: MC register offset
 763  *
 764  * Provides an MC register accessor for the atom interpreter (r4xx+).
 765  * Returns the value of the MC register.
 766  */
 767 static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
 768 {
 769         return 0;
 770 }
 771
 772 /**
 773  * cail_mc_write - write MC (Memory Controller) register
 774  *
 775  * @info: atom card_info pointer
 776  * @reg: MC register offset
 777  * @val: value to write to the pll register
 778  *
 779  * Provides a MC register accessor for the atom interpreter (r4xx+).
 780  */
 781 static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
 782 {
 783
 784 }
 785
 786 /**
 787  * cail_reg_write - write MMIO register
 788  *
 789  * @info: atom card_info pointer
 790  * @reg: MMIO register offset
 791  * @val: value to write to the pll register
 792  *
 793  * Provides a MMIO register accessor for the atom interpreter (r4xx+).
 794  */
 795 static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
 796 {
 797         struct amdgpu_device *adev = info->dev->dev_private;
 798
 799         WREG32(reg, val);
 800 }
 801
 802 /**
 803  * cail_reg_read - read MMIO register
 804  *
 805  * @info: atom card_info pointer
 806  * @reg: MMIO register offset
 807  *
 808  * Provides an MMIO register accessor for the atom interpreter (r4xx+).
 809  * Returns the value of the MMIO register.
 810  */
 811 static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
 812 {
 813         struct amdgpu_device *adev = info->dev->dev_private;
 814         uint32_t r;
 815
 816         r = RREG32(reg);
 817         return r;
 818 }
 819
 820 /**
 821  * cail_ioreg_write - write IO register
 822  *
 823  * @info: atom card_info pointer
 824  * @reg: IO register offset
 825  * @val: value to write to the pll register
 826  *
 827  * Provides a IO register accessor for the atom interpreter (r4xx+).
 828  */
 829 static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
 830 {
 831         struct amdgpu_device *adev = info->dev->dev_private;
 832
 833         WREG32_IO(reg, val);
 834 }
 835
 836 /**
 837  * cail_ioreg_read - read IO register
 838  *
 839  * @info: atom card_info pointer
 840  * @reg: IO register offset
 841  *
 842  * Provides an IO register accessor for the atom interpreter (r4xx+).
 843  * Returns the value of the IO register.
 844  */
 845 static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
 846 {
 847         struct amdgpu_device *adev = info->dev->dev_private;
 848         uint32_t r;
 849
 850         r = RREG32_IO(reg);
 851         return r;
 852 }
 853
 854 /**
 855  * amdgpu_atombios_fini - free the driver info and callbacks for atombios
 856  *
 857  * @adev: amdgpu_device pointer
 858  *
 859  * Frees the driver info and register access callbacks for the ATOM
 860  * interpreter (r4xx+).
 861  * Called at driver shutdown.
 862  */
 863 static void amdgpu_atombios_fini(struct amdgpu_device *adev)
 864 {
 865         if (adev->mode_info.atom_context) {
 866                 kfree(adev->mode_info.atom_context->scratch);
 867                 kfree(adev->mode_info.atom_context->iio);
 868         }
 869         kfree(adev->mode_info.atom_context);
 870         adev->mode_info.atom_context = NULL;
 871         kfree(adev->mode_info.atom_card_info);
 872         adev->mode_info.atom_card_info = NULL;
 873 }
 874
 875 /**
 876  * amdgpu_atombios_init - init the driver info and callbacks for atombios
 877  *
 878  * @adev: amdgpu_device pointer
 879  *
 880  * Initializes the driver info and register access callbacks for the
 881  * ATOM interpreter (r4xx+).
 882  * Returns 0 on sucess, -ENOMEM on failure.
 883  * Called at driver startup.
 884  */
 885 static int amdgpu_atombios_init(struct amdgpu_device *adev)
 886 {
 887         struct card_info *atom_card_info =
 888             kzalloc(sizeof(struct card_info), GFP_KERNEL);
 889
 890         if (!atom_card_info)
 891                 return -ENOMEM;
 892
 893         adev->mode_info.atom_card_info = atom_card_info;
 894         atom_card_info->dev = adev->ddev;
 895         atom_card_info->reg_read = cail_reg_read;
 896         atom_card_info->reg_write = cail_reg_write;
 897         /* needed for iio ops */
 898         if (adev->rio_mem) {
 899                 atom_card_info->ioreg_read = cail_ioreg_read;
 900                 atom_card_info->ioreg_write = cail_ioreg_write;
 901         } else {
 902                 DRM_INFO("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n");
 903                 atom_card_info->ioreg_read = cail_reg_read;
 904                 atom_card_info->ioreg_write = cail_reg_write;
 905         }
 906         atom_card_info->mc_read = cail_mc_read;
 907         atom_card_info->mc_write = cail_mc_write;
 908         atom_card_info->pll_read = cail_pll_read;
 909         atom_card_info->pll_write = cail_pll_write;
 910
 911         adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios);
 912         if (!adev->mode_info.atom_context) {
 913                 amdgpu_atombios_fini(adev);
 914                 return -ENOMEM;
 915         }
 916
 917         mutex_init(&adev->mode_info.atom_context->mutex);
 918         amdgpu_atombios_scratch_regs_init(adev);
 919         amdgpu_atom_allocate_fb_scratch(adev->mode_info.atom_context);
 920         return 0;
 921 }
 922
 923 /* if we get transitioned to only one device, take VGA back */
 924 /**
 925  * amdgpu_vga_set_decode - enable/disable vga decode
 926  *
 927  * @cookie: amdgpu_device pointer
 928  * @state: enable/disable vga decode
 929  *
 930  * Enable/disable vga decode (all asics).
 931  * Returns VGA resource flags.
 932  */
 933 static unsigned int amdgpu_vga_set_decode(void *cookie, bool state)
 934 {
 935         struct amdgpu_device *adev = cookie;
 936         amdgpu_asic_set_vga_state(adev, state);
 937         if (state)
 938                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
 939                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 940         else
 941                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 942 }
 943
 944 /**
 945  * amdgpu_check_pot_argument - check that argument is a power of two
 946  *
 947  * @arg: value to check
 948  *
 949  * Validates that a certain argument is a power of two (all asics).
 950  * Returns true if argument is valid.
 951  */
 952 static bool amdgpu_check_pot_argument(int arg)
 953 {
 954         return (arg & (arg - 1)) == 0;
 955 }
 956
 957 /**
 958  * amdgpu_check_arguments - validate module params
 959  *
 960  * @adev: amdgpu_device pointer
 961  *
 962  * Validates certain module parameters and updates
 963  * the associated values used by the driver (all asics).
 964  */
 965 static void amdgpu_check_arguments(struct amdgpu_device *adev)
 966 {
 967         if (amdgpu_sched_jobs < 4) {
 968                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
 969                          amdgpu_sched_jobs);
 970                 amdgpu_sched_jobs = 4;
 971         } else if (!amdgpu_check_pot_argument(amdgpu_sched_jobs)){
 972                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
 973                          amdgpu_sched_jobs);
 974                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
 975         }
 976
 977         if (amdgpu_gart_size != -1) {
 978                 /* gtt size must be greater or equal to 32M */
 979                 if (amdgpu_gart_size < 32) {
 980                         dev_warn(adev->dev, "gart size (%d) too small\n",
 981                                  amdgpu_gart_size);
 982                         amdgpu_gart_size = -1;
 983                 }
 984         }
 985
 986         if (!amdgpu_check_pot_argument(amdgpu_vm_size)) {
 987                 dev_warn(adev->dev, "VM size (%d) must be a power of 2\n",
 988                          amdgpu_vm_size);
 989                 amdgpu_vm_size = 8;
 990         }
 991
 992         if (amdgpu_vm_size < 1) {
 993                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
 994                          amdgpu_vm_size);
 995                 amdgpu_vm_size = 8;
 996         }
 997
 998         /*
 999          * Max GPUVM size for Cayman, SI and CI are 40 bits.
1000          */
1001         if (amdgpu_vm_size > 1024) {
1002                 dev_warn(adev->dev, "VM size (%d) too large, max is 1TB\n",
1003                          amdgpu_vm_size);
1004                 amdgpu_vm_size = 8;
1005         }
1006
1007         /* defines number of bits in page table versus page directory,
1008          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1009          * page table and the remaining bits are in the page directory */
1010         if (amdgpu_vm_block_size == -1) {
1011
1012                 /* Total bits covered by PD + PTs */
1013                 unsigned bits = ilog2(amdgpu_vm_size) + 18;
1014
1015                 /* Make sure the PD is 4K in size up to 8GB address space.
1016                    Above that split equal between PD and PTs */
1017                 if (amdgpu_vm_size <= 8)
1018                         amdgpu_vm_block_size = bits - 9;
1019                 else
1020                         amdgpu_vm_block_size = (bits + 3) / 2;
1021
1022         } else if (amdgpu_vm_block_size < 9) {
1023                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1024                          amdgpu_vm_block_size);
1025                 amdgpu_vm_block_size = 9;
1026         }
1027
1028         if (amdgpu_vm_block_size > 24 ||
1029             (amdgpu_vm_size * 1024) < (1ull << amdgpu_vm_block_size)) {
1030                 dev_warn(adev->dev, "VM page table size (%d) too large\n",
1031                          amdgpu_vm_block_size);
1032                 amdgpu_vm_block_size = 9;
1033         }
1034
1035         if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
1036             !amdgpu_check_pot_argument(amdgpu_vram_page_split))) {
1037                 dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
1038                          amdgpu_vram_page_split);
1039                 amdgpu_vram_page_split = 1024;
1040         }
1041 }
1042
1043 /**
1044  * amdgpu_switcheroo_set_state - set switcheroo state
1045  *
1046  * @pdev: pci dev pointer
1047  * @state: vga_switcheroo state
1048  *
1049  * Callback for the switcheroo driver.  Suspends or resumes the
1050  * the asics before or after it is powered up using ACPI methods.
1051  */
1052 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1053 {
1054         struct drm_device *dev = pci_get_drvdata(pdev);
1055
1056         if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1057                 return;
1058
1059         if (state == VGA_SWITCHEROO_ON) {
1060                 unsigned d3_delay = dev->pdev->d3_delay;
1061
1062                 printk(KERN_INFO "amdgpu: switched on\n");
1063                 /* don't suspend or resume card normally */
1064                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1065
1066                 amdgpu_device_resume(dev, true, true);
1067
1068                 dev->pdev->d3_delay = d3_delay;
1069
1070                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1071                 drm_kms_helper_poll_enable(dev);
1072         } else {
1073                 printk(KERN_INFO "amdgpu: switched off\n");
1074                 drm_kms_helper_poll_disable(dev);
1075                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1076                 amdgpu_device_suspend(dev, true, true);
1077                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1078         }
1079 }
1080
1081 /**
1082  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1083  *
1084  * @pdev: pci dev pointer
1085  *
1086  * Callback for the switcheroo driver.  Check of the switcheroo
1087  * state can be changed.
1088  * Returns true if the state can be changed, false if not.
1089  */
1090 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1091 {
1092         struct drm_device *dev = pci_get_drvdata(pdev);
1093
1094         /*
1095         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1096         * locking inversion with the driver load path. And the access here is
1097         * completely racy anyway. So don't bother with locking for now.
1098         */
1099         return dev->open_count == 0;
1100 }
1101
1102 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1103         .set_gpu_state = amdgpu_switcheroo_set_state,
1104         .reprobe = NULL,
1105         .can_switch = amdgpu_switcheroo_can_switch,
1106 };
1107
1108 int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
1109                                   enum amd_ip_block_type block_type,
1110                                   enum amd_clockgating_state state)
1111 {
1112         int i, r = 0;
1113
1114         for (i = 0; i < adev->num_ip_blocks; i++) {
1115                 if (!adev->ip_blocks[i].status.valid)
1116                         continue;
1117                 if (adev->ip_blocks[i].version->type == block_type) {
1118                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1119                                                                                      state);
1120                         if (r)
1121                                 return r;
1122                         break;
1123                 }
1124         }
1125         return r;
1126 }
1127
1128 int amdgpu_set_powergating_state(struct amdgpu_device *adev,
1129                                   enum amd_ip_block_type block_type,
1130                                   enum amd_powergating_state state)
1131 {
1132         int i, r = 0;
1133
1134         for (i = 0; i < adev->num_ip_blocks; i++) {
1135                 if (!adev->ip_blocks[i].status.valid)
1136                         continue;
1137                 if (adev->ip_blocks[i].version->type == block_type) {
1138                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
1139                                                                                      state);
1140                         if (r)
1141                                 return r;
1142                         break;
1143                 }
1144         }
1145         return r;
1146 }
1147
1148 void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
1149 {
1150         int i;
1151
1152         for (i = 0; i < adev->num_ip_blocks; i++) {
1153                 if (!adev->ip_blocks[i].status.valid)
1154                         continue;
1155                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1156                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1157         }
1158 }
1159
1160 int amdgpu_wait_for_idle(struct amdgpu_device *adev,
1161                          enum amd_ip_block_type block_type)
1162 {
1163         int i, r;
1164
1165         for (i = 0; i < adev->num_ip_blocks; i++) {
1166                 if (!adev->ip_blocks[i].status.valid)
1167                         continue;
1168                 if (adev->ip_blocks[i].version->type == block_type) {
1169                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1170                         if (r)
1171                                 return r;
1172                         break;
1173                 }
1174         }
1175         return 0;
1176
1177 }
1178
1179 bool amdgpu_is_idle(struct amdgpu_device *adev,
1180                     enum amd_ip_block_type block_type)
1181 {
1182         int i;
1183
1184         for (i = 0; i < adev->num_ip_blocks; i++) {
1185                 if (!adev->ip_blocks[i].status.valid)
1186                         continue;
1187                 if (adev->ip_blocks[i].version->type == block_type)
1188                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1189         }
1190         return true;
1191
1192 }
1193
1194 struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
1195                                              enum amd_ip_block_type type)
1196 {
1197         int i;
1198
1199         for (i = 0; i < adev->num_ip_blocks; i++)
1200                 if (adev->ip_blocks[i].version->type == type)
1201                         return &adev->ip_blocks[i];
1202
1203         return NULL;
1204 }
1205
1206 /**
1207  * amdgpu_ip_block_version_cmp
1208  *
1209  * @adev: amdgpu_device pointer
1210  * @type: enum amd_ip_block_type
1211  * @major: major version
1212  * @minor: minor version
1213  *
1214  * return 0 if equal or greater
1215  * return 1 if smaller or the ip_block doesn't exist
1216  */
1217 int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
1218                                 enum amd_ip_block_type type,
1219                                 u32 major, u32 minor)
1220 {
1221         struct amdgpu_ip_block *ip_block = amdgpu_get_ip_block(adev, type);
1222
1223         if (ip_block && ((ip_block->version->major > major) ||
1224                         ((ip_block->version->major == major) &&
1225                         (ip_block->version->minor >= minor))))
1226                 return 0;
1227
1228         return 1;
1229 }
1230
1231 /**
1232  * amdgpu_ip_block_add
1233  *
1234  * @adev: amdgpu_device pointer
1235  * @ip_block_version: pointer to the IP to add
1236  *
1237  * Adds the IP block driver information to the collection of IPs
1238  * on the asic.
1239  */
1240 int amdgpu_ip_block_add(struct amdgpu_device *adev,
1241                         const struct amdgpu_ip_block_version *ip_block_version)
1242 {
1243         if (!ip_block_version)
1244                 return -EINVAL;
1245
1246         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1247
1248         return 0;
1249 }
1250
1251 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1252 {
1253         adev->enable_virtual_display = false;
1254
1255         if (amdgpu_virtual_display) {
1256                 struct drm_device *ddev = adev->ddev;
1257                 const char *pci_address_name = pci_name(ddev->pdev);
1258                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1259
1260                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1261                 pciaddstr_tmp = pciaddstr;
1262                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1263                         pciaddname = strsep(&pciaddname_tmp, ",");
1264                         if (!strcmp("all", pciaddname)
1265                             || !strcmp(pci_address_name, pciaddname)) {
1266                                 long num_crtc;
1267                                 int res = -1;
1268
1269                                 adev->enable_virtual_display = true;
1270
1271                                 if (pciaddname_tmp)
1272                                         res = kstrtol(pciaddname_tmp, 10,
1273                                                       &num_crtc);
1274
1275                                 if (!res) {
1276                                         if (num_crtc < 1)
1277                                                 num_crtc = 1;
1278                                         if (num_crtc > 6)
1279                                                 num_crtc = 6;
1280                                         adev->mode_info.num_crtc = num_crtc;
1281                                 } else {
1282                                         adev->mode_info.num_crtc = 1;
1283                                 }
1284                                 break;
1285                         }
1286                 }
1287
1288                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1289                          amdgpu_virtual_display, pci_address_name,
1290                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1291
1292                 kfree(pciaddstr);
1293         }
1294 }
1295
1296 static int amdgpu_early_init(struct amdgpu_device *adev)
1297 {
1298         int i, r;
1299
1300         amdgpu_device_enable_virtual_display(adev);
1301
1302         switch (adev->asic_type) {
1303         case CHIP_TOPAZ:
1304         case CHIP_TONGA:
1305         case CHIP_FIJI:
1306         case CHIP_POLARIS11:
1307         case CHIP_POLARIS10:
1308         case CHIP_POLARIS12:
1309         case CHIP_CARRIZO:
1310         case CHIP_STONEY:
1311                 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1312                         adev->family = AMDGPU_FAMILY_CZ;
1313                 else
1314                         adev->family = AMDGPU_FAMILY_VI;
1315
1316                 r = vi_set_ip_blocks(adev);
1317                 if (r)
1318                         return r;
1319                 break;
1320 #ifdef CONFIG_DRM_AMDGPU_SI
1321         case CHIP_VERDE:
1322         case CHIP_TAHITI:
1323         case CHIP_PITCAIRN:
1324         case CHIP_OLAND:
1325         case CHIP_HAINAN:
1326                 adev->family = AMDGPU_FAMILY_SI;
1327                 r = si_set_ip_blocks(adev);
1328                 if (r)
1329                         return r;
1330                 break;
1331 #endif
1332 #ifdef CONFIG_DRM_AMDGPU_CIK
1333         case CHIP_BONAIRE:
1334         case CHIP_HAWAII:
1335         case CHIP_KAVERI:
1336         case CHIP_KABINI:
1337         case CHIP_MULLINS:
1338                 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1339                         adev->family = AMDGPU_FAMILY_CI;
1340                 else
1341                         adev->family = AMDGPU_FAMILY_KV;
1342
1343                 r = cik_set_ip_blocks(adev);
1344                 if (r)
1345                         return r;
1346                 break;
1347 #endif
1348         default:
1349                 /* FIXME: not supported yet */
1350                 return -EINVAL;
1351         }
1352
1353         if (amdgpu_sriov_vf(adev)) {
1354                 r = amdgpu_virt_request_full_gpu(adev, true);
1355                 if (r)
1356                         return r;
1357         }
1358
1359         for (i = 0; i < adev->num_ip_blocks; i++) {
1360                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1361                         DRM_ERROR("disabled ip block: %d\n", i);
1362                         adev->ip_blocks[i].status.valid = false;
1363                 } else {
1364                         if (adev->ip_blocks[i].version->funcs->early_init) {
1365                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1366                                 if (r == -ENOENT) {
1367                                         adev->ip_blocks[i].status.valid = false;
1368                                 } else if (r) {
1369                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
1370                                                   adev->ip_blocks[i].version->funcs->name, r);
1371                                         return r;
1372                                 } else {
1373                                         adev->ip_blocks[i].status.valid = true;
1374                                 }
1375                         } else {
1376                                 adev->ip_blocks[i].status.valid = true;
1377                         }
1378                 }
1379         }
1380
1381         adev->cg_flags &= amdgpu_cg_mask;
1382         adev->pg_flags &= amdgpu_pg_mask;
1383
1384         return 0;
1385 }
1386
1387 static int amdgpu_init(struct amdgpu_device *adev)
1388 {
1389         int i, r;
1390
1391         for (i = 0; i < adev->num_ip_blocks; i++) {
1392                 if (!adev->ip_blocks[i].status.valid)
1393                         continue;
1394                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1395                 if (r) {
1396                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1397                                   adev->ip_blocks[i].version->funcs->name, r);
1398                         return r;
1399                 }
1400                 adev->ip_blocks[i].status.sw = true;
1401                 /* need to do gmc hw init early so we can allocate gpu mem */
1402                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1403                         r = amdgpu_vram_scratch_init(adev);
1404                         if (r) {
1405                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1406                                 return r;
1407                         }
1408                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1409                         if (r) {
1410                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
1411                                 return r;
1412                         }
1413                         r = amdgpu_wb_init(adev);
1414                         if (r) {
1415                                 DRM_ERROR("amdgpu_wb_init failed %d\n", r);
1416                                 return r;
1417                         }
1418                         adev->ip_blocks[i].status.hw = true;
1419
1420                         /* right after GMC hw init, we create CSA */
1421                         if (amdgpu_sriov_vf(adev)) {
1422                                 r = amdgpu_allocate_static_csa(adev);
1423                                 if (r) {
1424                                         DRM_ERROR("allocate CSA failed %d\n", r);
1425                                         return r;
1426                                 }
1427                         }
1428                 }
1429         }
1430
1431         for (i = 0; i < adev->num_ip_blocks; i++) {
1432                 if (!adev->ip_blocks[i].status.sw)
1433                         continue;
1434                 /* gmc hw init is done early */
1435                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC)
1436                         continue;
1437                 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1438                 if (r) {
1439                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1440                                   adev->ip_blocks[i].version->funcs->name, r);
1441                         return r;
1442                 }
1443                 adev->ip_blocks[i].status.hw = true;
1444         }
1445
1446         return 0;
1447 }
1448
1449 static int amdgpu_late_init(struct amdgpu_device *adev)
1450 {
1451         int i = 0, r;
1452
1453         for (i = 0; i < adev->num_ip_blocks; i++) {
1454                 if (!adev->ip_blocks[i].status.valid)
1455                         continue;
1456                 if (adev->ip_blocks[i].version->funcs->late_init) {
1457                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
1458                         if (r) {
1459                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
1460                                           adev->ip_blocks[i].version->funcs->name, r);
1461                                 return r;
1462                         }
1463                         adev->ip_blocks[i].status.late_initialized = true;
1464                 }
1465                 /* skip CG for VCE/UVD, it's handled specially */
1466                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1467                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1468                         /* enable clockgating to save power */
1469                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1470                                                                                      AMD_CG_STATE_GATE);
1471                         if (r) {
1472                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1473                                           adev->ip_blocks[i].version->funcs->name, r);
1474                                 return r;
1475                         }
1476                 }
1477         }
1478
1479         return 0;
1480 }
1481
1482 static int amdgpu_fini(struct amdgpu_device *adev)
1483 {
1484         int i, r;
1485
1486         /* need to disable SMC first */
1487         for (i = 0; i < adev->num_ip_blocks; i++) {
1488                 if (!adev->ip_blocks[i].status.hw)
1489                         continue;
1490                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
1491                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1492                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1493                                                                                      AMD_CG_STATE_UNGATE);
1494                         if (r) {
1495                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1496                                           adev->ip_blocks[i].version->funcs->name, r);
1497                                 return r;
1498                         }
1499                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1500                         /* XXX handle errors */
1501                         if (r) {
1502                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1503                                           adev->ip_blocks[i].version->funcs->name, r);
1504                         }
1505                         adev->ip_blocks[i].status.hw = false;
1506                         break;
1507                 }
1508         }
1509
1510         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1511                 if (!adev->ip_blocks[i].status.hw)
1512                         continue;
1513                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1514                         amdgpu_wb_fini(adev);
1515                         amdgpu_vram_scratch_fini(adev);
1516                 }
1517
1518                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1519                         adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1520                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1521                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1522                                                                                      AMD_CG_STATE_UNGATE);
1523                         if (r) {
1524                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1525                                           adev->ip_blocks[i].version->funcs->name, r);
1526                                 return r;
1527                         }
1528                 }
1529
1530                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1531                 /* XXX handle errors */
1532                 if (r) {
1533                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1534                                   adev->ip_blocks[i].version->funcs->name, r);
1535                 }
1536
1537                 adev->ip_blocks[i].status.hw = false;
1538         }
1539
1540         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1541                 if (!adev->ip_blocks[i].status.sw)
1542                         continue;
1543                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
1544                 /* XXX handle errors */
1545                 if (r) {
1546                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
1547                                   adev->ip_blocks[i].version->funcs->name, r);
1548                 }
1549                 adev->ip_blocks[i].status.sw = false;
1550                 adev->ip_blocks[i].status.valid = false;
1551         }
1552
1553         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1554                 if (!adev->ip_blocks[i].status.late_initialized)
1555                         continue;
1556                 if (adev->ip_blocks[i].version->funcs->late_fini)
1557                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
1558                 adev->ip_blocks[i].status.late_initialized = false;
1559         }
1560
1561         if (amdgpu_sriov_vf(adev)) {
1562                 amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
1563                 amdgpu_virt_release_full_gpu(adev, false);
1564         }
1565
1566         return 0;
1567 }
1568
1569 int amdgpu_suspend(struct amdgpu_device *adev)
1570 {
1571         int i, r;
1572
1573         if (amdgpu_sriov_vf(adev))
1574                 amdgpu_virt_request_full_gpu(adev, false);
1575
1576         /* ungate SMC block first */
1577         r = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
1578                                          AMD_CG_STATE_UNGATE);
1579         if (r) {
1580                 DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n",r);
1581         }
1582
1583         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1584                 if (!adev->ip_blocks[i].status.valid)
1585                         continue;
1586                 /* ungate blocks so that suspend can properly shut them down */
1587                 if (i != AMD_IP_BLOCK_TYPE_SMC) {
1588                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1589                                                                                      AMD_CG_STATE_UNGATE);
1590                         if (r) {
1591                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1592                                           adev->ip_blocks[i].version->funcs->name, r);
1593                         }
1594                 }
1595                 /* XXX handle errors */
1596                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
1597                 /* XXX handle errors */
1598                 if (r) {
1599                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
1600                                   adev->ip_blocks[i].version->funcs->name, r);
1601                 }
1602         }
1603
1604         if (amdgpu_sriov_vf(adev))
1605                 amdgpu_virt_release_full_gpu(adev, false);
1606
1607         return 0;
1608 }
1609
1610 static int amdgpu_sriov_reinit_early(struct amdgpu_device *adev)
1611 {
1612         int i, r;
1613
1614         for (i = 0; i < adev->num_ip_blocks; i++) {
1615                 if (!adev->ip_blocks[i].status.valid)
1616                         continue;
1617
1618                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1619                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
1620                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)
1621                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
1622
1623                 if (r) {
1624                         DRM_ERROR("resume of IP block <%s> failed %d\n",
1625                                   adev->ip_blocks[i].version->funcs->name, r);
1626                         return r;
1627                 }
1628         }
1629
1630         return 0;
1631 }
1632
1633 static int amdgpu_sriov_reinit_late(struct amdgpu_device *adev)
1634 {
1635         int i, r;
1636
1637         for (i = 0; i < adev->num_ip_blocks; i++) {
1638                 if (!adev->ip_blocks[i].status.valid)
1639                         continue;
1640
1641                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
1642                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
1643                                 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH )
1644                         continue;
1645
1646                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
1647                 if (r) {
1648                         DRM_ERROR("resume of IP block <%s> failed %d\n",
1649                                   adev->ip_blocks[i].version->funcs->name, r);
1650                         return r;
1651                 }
1652         }
1653
1654         return 0;
1655 }
1656
1657 static int amdgpu_resume(struct amdgpu_device *adev)
1658 {
1659         int i, r;
1660
1661         for (i = 0; i < adev->num_ip_blocks; i++) {
1662                 if (!adev->ip_blocks[i].status.valid)
1663                         continue;
1664                 r = adev->ip_blocks[i].version->funcs->resume(adev);
1665                 if (r) {
1666                         DRM_ERROR("resume of IP block <%s> failed %d\n",
1667                                   adev->ip_blocks[i].version->funcs->name, r);
1668                         return r;
1669                 }
1670         }
1671
1672         return 0;
1673 }
1674
1675 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
1676 {
1677         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
1678                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
1679 }
1680
1681 /**
1682  * amdgpu_device_init - initialize the driver
1683  *
1684  * @adev: amdgpu_device pointer
1685  * @pdev: drm dev pointer
1686  * @pdev: pci dev pointer
1687  * @flags: driver flags
1688  *
1689  * Initializes the driver info and hw (all asics).
1690  * Returns 0 for success or an error on failure.
1691  * Called at driver startup.
1692  */
1693 int amdgpu_device_init(struct amdgpu_device *adev,
1694                        struct drm_device *ddev,
1695                        struct pci_dev *pdev,
1696                        uint32_t flags)
1697 {
1698         int r, i;
1699         bool runtime = false;
1700         u32 max_MBps;
1701
1702         adev->shutdown = false;
1703         adev->dev = &pdev->dev;
1704         adev->ddev = ddev;
1705         adev->pdev = pdev;
1706         adev->flags = flags;
1707         adev->asic_type = flags & AMD_ASIC_MASK;
1708         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
1709         adev->mc.gtt_size = 512 * 1024 * 1024;
1710         adev->accel_working = false;
1711         adev->num_rings = 0;
1712         adev->mman.buffer_funcs = NULL;
1713         adev->mman.buffer_funcs_ring = NULL;
1714         adev->vm_manager.vm_pte_funcs = NULL;
1715         adev->vm_manager.vm_pte_num_rings = 0;
1716         adev->gart.gart_funcs = NULL;
1717         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
1718
1719         adev->smc_rreg = &amdgpu_invalid_rreg;
1720         adev->smc_wreg = &amdgpu_invalid_wreg;
1721         adev->pcie_rreg = &amdgpu_invalid_rreg;
1722         adev->pcie_wreg = &amdgpu_invalid_wreg;
1723         adev->pciep_rreg = &amdgpu_invalid_rreg;
1724         adev->pciep_wreg = &amdgpu_invalid_wreg;
1725         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
1726         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
1727         adev->didt_rreg = &amdgpu_invalid_rreg;
1728         adev->didt_wreg = &amdgpu_invalid_wreg;
1729         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
1730         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
1731         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
1732         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
1733
1734
1735         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
1736                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
1737                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
1738
1739         /* mutex initialization are all done here so we
1740          * can recall function without having locking issues */
1741         mutex_init(&adev->vm_manager.lock);
1742         atomic_set(&adev->irq.ih.lock, 0);
1743         mutex_init(&adev->pm.mutex);
1744         mutex_init(&adev->gfx.gpu_clock_mutex);
1745         mutex_init(&adev->srbm_mutex);
1746         mutex_init(&adev->grbm_idx_mutex);
1747         mutex_init(&adev->mn_lock);
1748         hash_init(adev->mn_hash);
1749
1750         amdgpu_check_arguments(adev);
1751
1752         /* Registers mapping */
1753         /* TODO: block userspace mapping of io register */
1754         spin_lock_init(&adev->mmio_idx_lock);
1755         spin_lock_init(&adev->smc_idx_lock);
1756         spin_lock_init(&adev->pcie_idx_lock);
1757         spin_lock_init(&adev->uvd_ctx_idx_lock);
1758         spin_lock_init(&adev->didt_idx_lock);
1759         spin_lock_init(&adev->gc_cac_idx_lock);
1760         spin_lock_init(&adev->audio_endpt_idx_lock);
1761         spin_lock_init(&adev->mm_stats.lock);
1762
1763         INIT_LIST_HEAD(&adev->shadow_list);
1764         mutex_init(&adev->shadow_list_lock);
1765
1766         INIT_LIST_HEAD(&adev->gtt_list);
1767         spin_lock_init(&adev->gtt_list_lock);
1768
1769         if (adev->asic_type >= CHIP_BONAIRE) {
1770                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
1771                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
1772         } else {
1773                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
1774                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
1775         }
1776
1777         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
1778         if (adev->rmmio == NULL) {
1779                 return -ENOMEM;
1780         }
1781         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
1782         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
1783
1784         if (adev->asic_type >= CHIP_BONAIRE)
1785                 /* doorbell bar mapping */
1786                 amdgpu_doorbell_init(adev);
1787
1788         /* io port mapping */
1789         for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
1790                 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
1791                         adev->rio_mem_size = pci_resource_len(adev->pdev, i);
1792                         adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
1793                         break;
1794                 }
1795         }
1796         if (adev->rio_mem == NULL)
1797                 DRM_INFO("PCI I/O BAR is not found.\n");
1798
1799         /* early init functions */
1800         r = amdgpu_early_init(adev);
1801         if (r)
1802                 return r;
1803
1804         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
1805         /* this will fail for cards that aren't VGA class devices, just
1806          * ignore it */
1807         vga_client_register(adev->pdev, adev, NULL, amdgpu_vga_set_decode);
1808
1809         if (amdgpu_runtime_pm == 1)
1810                 runtime = true;
1811         if (amdgpu_device_is_px(ddev))
1812                 runtime = true;
1813         vga_switcheroo_register_client(adev->pdev, &amdgpu_switcheroo_ops, runtime);
1814         if (runtime)
1815                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
1816
1817         /* Read BIOS */
1818         if (!amdgpu_get_bios(adev)) {
1819                 r = -EINVAL;
1820                 goto failed;
1821         }
1822
1823         r = amdgpu_atombios_init(adev);
1824         if (r) {
1825                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
1826                 goto failed;
1827         }
1828
1829         /* detect if we are with an SRIOV vbios */
1830         amdgpu_device_detect_sriov_bios(adev);
1831
1832         /* Post card if necessary */
1833         if (amdgpu_vpost_needed(adev)) {
1834                 if (!adev->bios) {
1835                         dev_err(adev->dev, "no vBIOS found\n");
1836                         r = -EINVAL;
1837                         goto failed;
1838                 }
1839                 DRM_INFO("GPU posting now...\n");
1840                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
1841                 if (r) {
1842                         dev_err(adev->dev, "gpu post error!\n");
1843                         goto failed;
1844                 }
1845         } else {
1846                 DRM_INFO("GPU post is not needed\n");
1847         }
1848
1849         /* Initialize clocks */
1850         r = amdgpu_atombios_get_clock_info(adev);
1851         if (r) {
1852                 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
1853                 goto failed;
1854         }
1855         /* init i2c buses */
1856         amdgpu_atombios_i2c_init(adev);
1857
1858         /* Fence driver */
1859         r = amdgpu_fence_driver_init(adev);
1860         if (r) {
1861                 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
1862                 goto failed;
1863         }
1864
1865         /* init the mode config */
1866         drm_mode_config_init(adev->ddev);
1867
1868         r = amdgpu_init(adev);
1869         if (r) {
1870                 dev_err(adev->dev, "amdgpu_init failed\n");
1871                 amdgpu_fini(adev);
1872                 goto failed;
1873         }
1874
1875         adev->accel_working = true;
1876
1877         /* Initialize the buffer migration limit. */
1878         if (amdgpu_moverate >= 0)
1879                 max_MBps = amdgpu_moverate;
1880         else
1881                 max_MBps = 8; /* Allow 8 MB/s. */
1882         /* Get a log2 for easy divisions. */
1883         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
1884
1885         r = amdgpu_ib_pool_init(adev);
1886         if (r) {
1887                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
1888                 goto failed;
1889         }
1890
1891         r = amdgpu_ib_ring_tests(adev);
1892         if (r)
1893                 DRM_ERROR("ib ring test failed (%d).\n", r);
1894
1895         amdgpu_fbdev_init(adev);
1896
1897         r = amdgpu_gem_debugfs_init(adev);
1898         if (r)
1899                 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
1900
1901         r = amdgpu_debugfs_regs_init(adev);
1902         if (r)
1903                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1904
1905         r = amdgpu_debugfs_firmware_init(adev);
1906         if (r)
1907                 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
1908
1909         if ((amdgpu_testing & 1)) {
1910                 if (adev->accel_working)
1911                         amdgpu_test_moves(adev);
1912                 else
1913                         DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
1914         }
1915         if ((amdgpu_testing & 2)) {
1916                 if (adev->accel_working)
1917                         amdgpu_test_syncing(adev);
1918                 else
1919                         DRM_INFO("amdgpu: acceleration disabled, skipping sync tests\n");
1920         }
1921         if (amdgpu_benchmarking) {
1922                 if (adev->accel_working)
1923                         amdgpu_benchmark(adev, amdgpu_benchmarking);
1924                 else
1925                         DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
1926         }
1927
1928         /* enable clockgating, etc. after ib tests, etc. since some blocks require
1929          * explicit gating rather than handling it automatically.
1930          */
1931         r = amdgpu_late_init(adev);
1932         if (r) {
1933                 dev_err(adev->dev, "amdgpu_late_init failed\n");
1934                 goto failed;
1935         }
1936
1937         return 0;
1938
1939 failed:
1940         if (runtime)
1941                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
1942         return r;
1943 }
1944
1945 /**
1946  * amdgpu_device_fini - tear down the driver
1947  *
1948  * @adev: amdgpu_device pointer
1949  *
1950  * Tear down the driver info (all asics).
1951  * Called at driver shutdown.
1952  */
1953 void amdgpu_device_fini(struct amdgpu_device *adev)
1954 {
1955         int r;
1956
1957         DRM_INFO("amdgpu: finishing device.\n");
1958         adev->shutdown = true;
1959         drm_crtc_force_disable_all(adev->ddev);
1960         /* evict vram memory */
1961         amdgpu_bo_evict_vram(adev);
1962         amdgpu_ib_pool_fini(adev);
1963         amdgpu_fence_driver_fini(adev);
1964         amdgpu_fbdev_fini(adev);
1965         r = amdgpu_fini(adev);
1966         adev->accel_working = false;
1967         /* free i2c buses */
1968         amdgpu_i2c_fini(adev);
1969         amdgpu_atombios_fini(adev);
1970         kfree(adev->bios);
1971         adev->bios = NULL;
1972         vga_switcheroo_unregister_client(adev->pdev);
1973         if (adev->flags & AMD_IS_PX)
1974                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
1975         vga_client_register(adev->pdev, NULL, NULL, NULL);
1976         if (adev->rio_mem)
1977                 pci_iounmap(adev->pdev, adev->rio_mem);
1978         adev->rio_mem = NULL;
1979         iounmap(adev->rmmio);
1980         adev->rmmio = NULL;
1981         if (adev->asic_type >= CHIP_BONAIRE)
1982                 amdgpu_doorbell_fini(adev);
1983         amdgpu_debugfs_regs_cleanup(adev);
1984 }
1985
1986
1987 /*
1988  * Suspend & resume.
1989  */
1990 /**
1991  * amdgpu_device_suspend - initiate device suspend
1992  *
1993  * @pdev: drm dev pointer
1994  * @state: suspend state
1995  *
1996  * Puts the hw in the suspend state (all asics).
1997  * Returns 0 for success or an error on failure.
1998  * Called at driver suspend.
1999  */
2000 int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
2001 {
2002         struct amdgpu_device *adev;
2003         struct drm_crtc *crtc;
2004         struct drm_connector *connector;
2005         int r;
2006
2007         if (dev == NULL || dev->dev_private == NULL) {
2008                 return -ENODEV;
2009         }
2010
2011         adev = dev->dev_private;
2012
2013         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2014                 return 0;
2015
2016         drm_kms_helper_poll_disable(dev);
2017
2018         /* turn off display hw */
2019         drm_modeset_lock_all(dev);
2020         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2021                 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
2022         }
2023         drm_modeset_unlock_all(dev);
2024
2025         /* unpin the front buffers and cursors */
2026         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2027                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2028                 struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
2029                 struct amdgpu_bo *robj;
2030
2031                 if (amdgpu_crtc->cursor_bo) {
2032                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2033                         r = amdgpu_bo_reserve(aobj, false);
2034                         if (r == 0) {
2035                                 amdgpu_bo_unpin(aobj);
2036                                 amdgpu_bo_unreserve(aobj);
2037                         }
2038                 }
2039
2040                 if (rfb == NULL || rfb->obj == NULL) {
2041                         continue;
2042                 }
2043                 robj = gem_to_amdgpu_bo(rfb->obj);
2044                 /* don't unpin kernel fb objects */
2045                 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
2046                         r = amdgpu_bo_reserve(robj, false);
2047                         if (r == 0) {
2048                                 amdgpu_bo_unpin(robj);
2049                                 amdgpu_bo_unreserve(robj);
2050                         }
2051                 }
2052         }
2053         /* evict vram memory */
2054         amdgpu_bo_evict_vram(adev);
2055
2056         amdgpu_fence_driver_suspend(adev);
2057
2058         r = amdgpu_suspend(adev);
2059
2060         /* evict remaining vram memory
2061          * This second call to evict vram is to evict the gart page table
2062          * using the CPU.
2063          */
2064         amdgpu_bo_evict_vram(adev);
2065
2066         amdgpu_atombios_scratch_regs_save(adev);
2067         pci_save_state(dev->pdev);
2068         if (suspend) {
2069                 /* Shut down the device */
2070                 pci_disable_device(dev->pdev);
2071                 pci_set_power_state(dev->pdev, PCI_D3hot);
2072         } else {
2073                 r = amdgpu_asic_reset(adev);
2074                 if (r)
2075                         DRM_ERROR("amdgpu asic reset failed\n");
2076         }
2077
2078         if (fbcon) {
2079                 console_lock();
2080                 amdgpu_fbdev_set_suspend(adev, 1);
2081                 console_unlock();
2082         }
2083         return 0;
2084 }
2085
2086 /**
2087  * amdgpu_device_resume - initiate device resume
2088  *
2089  * @pdev: drm dev pointer
2090  *
2091  * Bring the hw back to operating state (all asics).
2092  * Returns 0 for success or an error on failure.
2093  * Called at driver resume.
2094  */
2095 int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2096 {
2097         struct drm_connector *connector;
2098         struct amdgpu_device *adev = dev->dev_private;
2099         struct drm_crtc *crtc;
2100         int r;
2101
2102         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2103                 return 0;
2104
2105         if (fbcon)
2106                 console_lock();
2107
2108         if (resume) {
2109                 pci_set_power_state(dev->pdev, PCI_D0);
2110                 pci_restore_state(dev->pdev);
2111                 r = pci_enable_device(dev->pdev);
2112                 if (r) {
2113                         if (fbcon)
2114                                 console_unlock();
2115                         return r;
2116                 }
2117         }
2118         amdgpu_atombios_scratch_regs_restore(adev);
2119
2120         /* post card */
2121         if (amdgpu_need_post(adev)) {
2122                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2123                 if (r)
2124                         DRM_ERROR("amdgpu asic init failed\n");
2125         }
2126
2127         r = amdgpu_resume(adev);
2128         if (r)
2129                 DRM_ERROR("amdgpu_resume failed (%d).\n", r);
2130
2131         amdgpu_fence_driver_resume(adev);
2132
2133         if (resume) {
2134                 r = amdgpu_ib_ring_tests(adev);
2135                 if (r)
2136                         DRM_ERROR("ib ring test failed (%d).\n", r);
2137         }
2138
2139         r = amdgpu_late_init(adev);
2140         if (r) {
2141                 if (fbcon)
2142                         console_unlock();
2143                 return r;
2144         }
2145
2146         /* pin cursors */
2147         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2148                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2149
2150                 if (amdgpu_crtc->cursor_bo) {
2151                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2152                         r = amdgpu_bo_reserve(aobj, false);
2153                         if (r == 0) {
2154                                 r = amdgpu_bo_pin(aobj,
2155                                                   AMDGPU_GEM_DOMAIN_VRAM,
2156                                                   &amdgpu_crtc->cursor_addr);
2157                                 if (r != 0)
2158                                         DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
2159                                 amdgpu_bo_unreserve(aobj);
2160                         }
2161                 }
2162         }
2163
2164         /* blat the mode back in */
2165         if (fbcon) {
2166                 drm_helper_resume_force_mode(dev);
2167                 /* turn on display hw */
2168                 drm_modeset_lock_all(dev);
2169                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2170                         drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
2171                 }
2172                 drm_modeset_unlock_all(dev);
2173         }
2174
2175         drm_kms_helper_poll_enable(dev);
2176
2177         /*
2178          * Most of the connector probing functions try to acquire runtime pm
2179          * refs to ensure that the GPU is powered on when connector polling is
2180          * performed. Since we're calling this from a runtime PM callback,
2181          * trying to acquire rpm refs will cause us to deadlock.
2182          *
2183          * Since we're guaranteed to be holding the rpm lock, it's safe to
2184          * temporarily disable the rpm helpers so this doesn't deadlock us.
2185          */
2186 #ifdef CONFIG_PM
2187         dev->dev->power.disable_depth++;
2188 #endif
2189         drm_helper_hpd_irq_event(dev);
2190 #ifdef CONFIG_PM
2191         dev->dev->power.disable_depth--;
2192 #endif
2193
2194         if (fbcon) {
2195                 amdgpu_fbdev_set_suspend(adev, 0);
2196                 console_unlock();
2197         }
2198
2199         return 0;
2200 }
2201
2202 static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
2203 {
2204         int i;
2205         bool asic_hang = false;
2206
2207         for (i = 0; i < adev->num_ip_blocks; i++) {
2208                 if (!adev->ip_blocks[i].status.valid)
2209                         continue;
2210                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
2211                         adev->ip_blocks[i].status.hang =
2212                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
2213                 if (adev->ip_blocks[i].status.hang) {
2214                         DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
2215                         asic_hang = true;
2216                 }
2217         }
2218         return asic_hang;
2219 }
2220
2221 static int amdgpu_pre_soft_reset(struct amdgpu_device *adev)
2222 {
2223         int i, r = 0;
2224
2225         for (i = 0; i < adev->num_ip_blocks; i++) {
2226                 if (!adev->ip_blocks[i].status.valid)
2227                         continue;
2228                 if (adev->ip_blocks[i].status.hang &&
2229                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
2230                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
2231                         if (r)
2232                                 return r;
2233                 }
2234         }
2235
2236         return 0;
2237 }
2238
2239 static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
2240 {
2241         int i;
2242
2243         for (i = 0; i < adev->num_ip_blocks; i++) {
2244                 if (!adev->ip_blocks[i].status.valid)
2245                         continue;
2246                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
2247                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
2248                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
2249                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)) {
2250                         if (adev->ip_blocks[i].status.hang) {
2251                                 DRM_INFO("Some block need full reset!\n");
2252                                 return true;
2253                         }
2254                 }
2255         }
2256         return false;
2257 }
2258
2259 static int amdgpu_soft_reset(struct amdgpu_device *adev)
2260 {
2261         int i, r = 0;
2262
2263         for (i = 0; i < adev->num_ip_blocks; i++) {
2264                 if (!adev->ip_blocks[i].status.valid)
2265                         continue;
2266                 if (adev->ip_blocks[i].status.hang &&
2267                     adev->ip_blocks[i].version->funcs->soft_reset) {
2268                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
2269                         if (r)
2270                                 return r;
2271                 }
2272         }
2273
2274         return 0;
2275 }
2276
2277 static int amdgpu_post_soft_reset(struct amdgpu_device *adev)
2278 {
2279         int i, r = 0;
2280
2281         for (i = 0; i < adev->num_ip_blocks; i++) {
2282                 if (!adev->ip_blocks[i].status.valid)
2283                         continue;
2284                 if (adev->ip_blocks[i].status.hang &&
2285                     adev->ip_blocks[i].version->funcs->post_soft_reset)
2286                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
2287                 if (r)
2288                         return r;
2289         }
2290
2291         return 0;
2292 }
2293
2294 bool amdgpu_need_backup(struct amdgpu_device *adev)
2295 {
2296         if (adev->flags & AMD_IS_APU)
2297                 return false;
2298
2299         return amdgpu_lockup_timeout > 0 ? true : false;
2300 }
2301
2302 static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
2303                                            struct amdgpu_ring *ring,
2304                                            struct amdgpu_bo *bo,
2305                                            struct dma_fence **fence)
2306 {
2307         uint32_t domain;
2308         int r;
2309
2310        if (!bo->shadow)
2311                return 0;
2312
2313        r = amdgpu_bo_reserve(bo, false);
2314        if (r)
2315                return r;
2316        domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
2317        /* if bo has been evicted, then no need to recover */
2318        if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
2319                r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
2320                                                  NULL, fence, true);
2321                if (r) {
2322                        DRM_ERROR("recover page table failed!\n");
2323                        goto err;
2324                }
2325        }
2326 err:
2327        amdgpu_bo_unreserve(bo);
2328        return r;
2329 }
2330
2331 /**
2332  * amdgpu_sriov_gpu_reset - reset the asic
2333  *
2334  * @adev: amdgpu device pointer
2335  * @voluntary: if this reset is requested by guest.
2336  *             (true means by guest and false means by HYPERVISOR )
2337  *
2338  * Attempt the reset the GPU if it has hung (all asics).
2339  * for SRIOV case.
2340  * Returns 0 for success or an error on failure.
2341  */
2342 int amdgpu_sriov_gpu_reset(struct amdgpu_device *adev, bool voluntary)
2343 {
2344         int i, r = 0;
2345         int resched;
2346         struct amdgpu_bo *bo, *tmp;
2347         struct amdgpu_ring *ring;
2348         struct dma_fence *fence = NULL, *next = NULL;
2349
2350         mutex_lock(&adev->virt.lock_reset);
2351         atomic_inc(&adev->gpu_reset_counter);
2352         adev->gfx.in_reset = true;
2353
2354         /* block TTM */
2355         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
2356
2357         /* block scheduler */
2358         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2359                 ring = adev->rings[i];
2360
2361                 if (!ring || !ring->sched.thread)
2362                         continue;
2363
2364                 kthread_park(ring->sched.thread);
2365                 amd_sched_hw_job_reset(&ring->sched);
2366         }
2367
2368         /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2369         amdgpu_fence_driver_force_completion(adev);
2370
2371         /* request to take full control of GPU before re-initialization  */
2372         if (voluntary)
2373                 amdgpu_virt_reset_gpu(adev);
2374         else
2375                 amdgpu_virt_request_full_gpu(adev, true);
2376
2377
2378         /* Resume IP prior to SMC */
2379         amdgpu_sriov_reinit_early(adev);
2380
2381         /* we need recover gart prior to run SMC/CP/SDMA resume */
2382         amdgpu_ttm_recover_gart(adev);
2383
2384         /* now we are okay to resume SMC/CP/SDMA */
2385         amdgpu_sriov_reinit_late(adev);
2386
2387         amdgpu_irq_gpu_reset_resume_helper(adev);
2388
2389         if (amdgpu_ib_ring_tests(adev))
2390                 dev_err(adev->dev, "[GPU_RESET] ib ring test failed (%d).\n", r);
2391
2392         /* release full control of GPU after ib test */
2393         amdgpu_virt_release_full_gpu(adev, true);
2394
2395         DRM_INFO("recover vram bo from shadow\n");
2396
2397         ring = adev->mman.buffer_funcs_ring;
2398         mutex_lock(&adev->shadow_list_lock);
2399         list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2400                 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2401                 if (fence) {
2402                         r = dma_fence_wait(fence, false);
2403                         if (r) {
2404                                 WARN(r, "recovery from shadow isn't completed\n");
2405                                 break;
2406                         }
2407                 }
2408
2409                 dma_fence_put(fence);
2410                 fence = next;
2411         }
2412         mutex_unlock(&adev->shadow_list_lock);
2413
2414         if (fence) {
2415                 r = dma_fence_wait(fence, false);
2416                 if (r)
2417                         WARN(r, "recovery from shadow isn't completed\n");
2418         }
2419         dma_fence_put(fence);
2420
2421         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2422                 struct amdgpu_ring *ring = adev->rings[i];
2423                 if (!ring || !ring->sched.thread)
2424                         continue;
2425
2426                 amd_sched_job_recovery(&ring->sched);
2427                 kthread_unpark(ring->sched.thread);
2428         }
2429
2430         drm_helper_resume_force_mode(adev->ddev);
2431         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
2432         if (r) {
2433                 /* bad news, how to tell it to userspace ? */
2434                 dev_info(adev->dev, "GPU reset failed\n");
2435         }
2436
2437         adev->gfx.in_reset = false;
2438         mutex_unlock(&adev->virt.lock_reset);
2439         return r;
2440 }
2441
2442 /**
2443  * amdgpu_gpu_reset - reset the asic
2444  *
2445  * @adev: amdgpu device pointer
2446  *
2447  * Attempt the reset the GPU if it has hung (all asics).
2448  * Returns 0 for success or an error on failure.
2449  */
2450 int amdgpu_gpu_reset(struct amdgpu_device *adev)
2451 {
2452         int i, r;
2453         int resched;
2454         bool need_full_reset;
2455
2456         if (amdgpu_sriov_vf(adev))
2457                 return amdgpu_sriov_gpu_reset(adev, true);
2458
2459         if (!amdgpu_check_soft_reset(adev)) {
2460                 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
2461                 return 0;
2462         }
2463
2464         atomic_inc(&adev->gpu_reset_counter);
2465
2466         /* block TTM */
2467         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
2468
2469         /* block scheduler */
2470         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2471                 struct amdgpu_ring *ring = adev->rings[i];
2472
2473                 if (!ring)
2474                         continue;
2475                 kthread_park(ring->sched.thread);
2476                 amd_sched_hw_job_reset(&ring->sched);
2477         }
2478         /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2479         amdgpu_fence_driver_force_completion(adev);
2480
2481         need_full_reset = amdgpu_need_full_reset(adev);
2482
2483         if (!need_full_reset) {
2484                 amdgpu_pre_soft_reset(adev);
2485                 r = amdgpu_soft_reset(adev);
2486                 amdgpu_post_soft_reset(adev);
2487                 if (r || amdgpu_check_soft_reset(adev)) {
2488                         DRM_INFO("soft reset failed, will fallback to full reset!\n");
2489                         need_full_reset = true;
2490                 }
2491         }
2492
2493         if (need_full_reset) {
2494                 r = amdgpu_suspend(adev);
2495
2496 retry:
2497                 /* Disable fb access */
2498                 if (adev->mode_info.num_crtc) {
2499                         struct amdgpu_mode_mc_save save;
2500                         amdgpu_display_stop_mc_access(adev, &save);
2501                         amdgpu_wait_for_idle(adev, AMD_IP_BLOCK_TYPE_GMC);
2502                 }
2503                 amdgpu_atombios_scratch_regs_save(adev);
2504                 r = amdgpu_asic_reset(adev);
2505                 amdgpu_atombios_scratch_regs_restore(adev);
2506                 /* post card */
2507                 amdgpu_atom_asic_init(adev->mode_info.atom_context);
2508
2509                 if (!r) {
2510                         dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
2511                         r = amdgpu_resume(adev);
2512                 }
2513         }
2514         if (!r) {
2515                 amdgpu_irq_gpu_reset_resume_helper(adev);
2516                 if (need_full_reset && amdgpu_need_backup(adev)) {
2517                         r = amdgpu_ttm_recover_gart(adev);
2518                         if (r)
2519                                 DRM_ERROR("gart recovery failed!!!\n");
2520                 }
2521                 r = amdgpu_ib_ring_tests(adev);
2522                 if (r) {
2523                         dev_err(adev->dev, "ib ring test failed (%d).\n", r);
2524                         r = amdgpu_suspend(adev);
2525                         need_full_reset = true;
2526                         goto retry;
2527                 }
2528                 /**
2529                  * recovery vm page tables, since we cannot depend on VRAM is
2530                  * consistent after gpu full reset.
2531                  */
2532                 if (need_full_reset && amdgpu_need_backup(adev)) {
2533                         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2534                         struct amdgpu_bo *bo, *tmp;
2535                         struct dma_fence *fence = NULL, *next = NULL;
2536
2537                         DRM_INFO("recover vram bo from shadow\n");
2538                         mutex_lock(&adev->shadow_list_lock);
2539                         list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2540                                 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2541                                 if (fence) {
2542                                         r = dma_fence_wait(fence, false);
2543                                         if (r) {
2544                                                 WARN(r, "recovery from shadow isn't completed\n");
2545                                                 break;
2546                                         }
2547                                 }
2548
2549                                 dma_fence_put(fence);
2550                                 fence = next;
2551                         }
2552                         mutex_unlock(&adev->shadow_list_lock);
2553                         if (fence) {
2554                                 r = dma_fence_wait(fence, false);
2555                                 if (r)
2556                                         WARN(r, "recovery from shadow isn't completed\n");
2557                         }
2558                         dma_fence_put(fence);
2559                 }
2560                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2561                         struct amdgpu_ring *ring = adev->rings[i];
2562                         if (!ring)
2563                                 continue;
2564
2565                         amd_sched_job_recovery(&ring->sched);
2566                         kthread_unpark(ring->sched.thread);
2567                 }
2568         } else {
2569                 dev_err(adev->dev, "asic resume failed (%d).\n", r);
2570                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2571                         if (adev->rings[i]) {
2572                                 kthread_unpark(adev->rings[i]->sched.thread);
2573                         }
2574                 }
2575         }
2576
2577         drm_helper_resume_force_mode(adev->ddev);
2578
2579         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
2580         if (r) {
2581                 /* bad news, how to tell it to userspace ? */
2582                 dev_info(adev->dev, "GPU reset failed\n");
2583         }
2584
2585         return r;
2586 }
2587
2588 void amdgpu_get_pcie_info(struct amdgpu_device *adev)
2589 {
2590         u32 mask;
2591         int ret;
2592
2593         if (amdgpu_pcie_gen_cap)
2594                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
2595
2596         if (amdgpu_pcie_lane_cap)
2597                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
2598
2599         /* covers APUs as well */
2600         if (pci_is_root_bus(adev->pdev->bus)) {
2601                 if (adev->pm.pcie_gen_mask == 0)
2602                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
2603                 if (adev->pm.pcie_mlw_mask == 0)
2604                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
2605                 return;
2606         }
2607
2608         if (adev->pm.pcie_gen_mask == 0) {
2609                 ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
2610                 if (!ret) {
2611                         adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
2612                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
2613                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
2614
2615                         if (mask & DRM_PCIE_SPEED_25)
2616                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
2617                         if (mask & DRM_PCIE_SPEED_50)
2618                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
2619                         if (mask & DRM_PCIE_SPEED_80)
2620                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
2621                 } else {
2622                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
2623                 }
2624         }
2625         if (adev->pm.pcie_mlw_mask == 0) {
2626                 ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
2627                 if (!ret) {
2628                         switch (mask) {
2629                         case 32:
2630                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
2631                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
2632                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2633                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2634                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2635                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2636                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2637                                 break;
2638                         case 16:
2639                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
2640                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2641                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2642                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2643                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2644                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2645                                 break;
2646                         case 12:
2647                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2648                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2649                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2650                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2651                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2652                                 break;
2653                         case 8:
2654                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2655                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2656                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2657                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2658                                 break;
2659                         case 4:
2660                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2661                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2662                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2663                                 break;
2664                         case 2:
2665                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2666                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2667                                 break;
2668                         case 1:
2669                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
2670                                 break;
2671                         default:
2672                                 break;
2673                         }
2674                 } else {
2675                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
2676                 }
2677         }
2678 }
2679
2680 /*
2681  * Debugfs
2682  */
2683 int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
2684                              const struct drm_info_list *files,
2685                              unsigned nfiles)
2686 {
2687         unsigned i;
2688
2689         for (i = 0; i < adev->debugfs_count; i++) {
2690                 if (adev->debugfs[i].files == files) {
2691                         /* Already registered */
2692                         return 0;
2693                 }
2694         }
2695
2696         i = adev->debugfs_count + 1;
2697         if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
2698                 DRM_ERROR("Reached maximum number of debugfs components.\n");
2699                 DRM_ERROR("Report so we increase "
2700                           "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
2701                 return -EINVAL;
2702         }
2703         adev->debugfs[adev->debugfs_count].files = files;
2704         adev->debugfs[adev->debugfs_count].num_files = nfiles;
2705         adev->debugfs_count = i;
2706 #if defined(CONFIG_DEBUG_FS)
2707         drm_debugfs_create_files(files, nfiles,
2708                                  adev->ddev->primary->debugfs_root,
2709                                  adev->ddev->primary);
2710 #endif
2711         return 0;
2712 }
2713
2714 #if defined(CONFIG_DEBUG_FS)
2715
2716 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
2717                                         size_t size, loff_t *pos)
2718 {
2719         struct amdgpu_device *adev = file_inode(f)->i_private;
2720         ssize_t result = 0;
2721         int r;
2722         bool pm_pg_lock, use_bank;
2723         unsigned instance_bank, sh_bank, se_bank;
2724
2725         if (size & 0x3 || *pos & 0x3)
2726                 return -EINVAL;
2727
2728         /* are we reading registers for which a PG lock is necessary? */
2729         pm_pg_lock = (*pos >> 23) & 1;
2730
2731         if (*pos & (1ULL << 62)) {
2732                 se_bank = (*pos >> 24) & 0x3FF;
2733                 sh_bank = (*pos >> 34) & 0x3FF;
2734                 instance_bank = (*pos >> 44) & 0x3FF;
2735
2736                 if (se_bank == 0x3FF)
2737                         se_bank = 0xFFFFFFFF;
2738                 if (sh_bank == 0x3FF)
2739                         sh_bank = 0xFFFFFFFF;
2740                 if (instance_bank == 0x3FF)
2741                         instance_bank = 0xFFFFFFFF;
2742                 use_bank = 1;
2743         } else {
2744                 use_bank = 0;
2745         }
2746
2747         *pos &= (1UL << 22) - 1;
2748
2749         if (use_bank) {
2750                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
2751                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
2752                         return -EINVAL;
2753                 mutex_lock(&adev->grbm_idx_mutex);
2754                 amdgpu_gfx_select_se_sh(adev, se_bank,
2755                                         sh_bank, instance_bank);
2756         }
2757
2758         if (pm_pg_lock)
2759                 mutex_lock(&adev->pm.mutex);
2760
2761         while (size) {
2762                 uint32_t value;
2763
2764                 if (*pos > adev->rmmio_size)
2765                         goto end;
2766
2767                 value = RREG32(*pos >> 2);
2768                 r = put_user(value, (uint32_t *)buf);
2769                 if (r) {
2770                         result = r;
2771                         goto end;
2772                 }
2773
2774                 result += 4;
2775                 buf += 4;
2776                 *pos += 4;
2777                 size -= 4;
2778         }
2779
2780 end:
2781         if (use_bank) {
2782                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2783                 mutex_unlock(&adev->grbm_idx_mutex);
2784         }
2785
2786         if (pm_pg_lock)
2787                 mutex_unlock(&adev->pm.mutex);
2788
2789         return result;
2790 }
2791
2792 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
2793                                          size_t size, loff_t *pos)
2794 {
2795         struct amdgpu_device *adev = file_inode(f)->i_private;
2796         ssize_t result = 0;
2797         int r;
2798         bool pm_pg_lock, use_bank;
2799         unsigned instance_bank, sh_bank, se_bank;
2800
2801         if (size & 0x3 || *pos & 0x3)
2802                 return -EINVAL;
2803
2804         /* are we reading registers for which a PG lock is necessary? */
2805         pm_pg_lock = (*pos >> 23) & 1;
2806
2807         if (*pos & (1ULL << 62)) {
2808                 se_bank = (*pos >> 24) & 0x3FF;
2809                 sh_bank = (*pos >> 34) & 0x3FF;
2810                 instance_bank = (*pos >> 44) & 0x3FF;
2811
2812                 if (se_bank == 0x3FF)
2813                         se_bank = 0xFFFFFFFF;
2814                 if (sh_bank == 0x3FF)
2815                         sh_bank = 0xFFFFFFFF;
2816                 if (instance_bank == 0x3FF)
2817                         instance_bank = 0xFFFFFFFF;
2818                 use_bank = 1;
2819         } else {
2820                 use_bank = 0;
2821         }
2822
2823         *pos &= (1UL << 22) - 1;
2824
2825         if (use_bank) {
2826                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
2827                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
2828                         return -EINVAL;
2829                 mutex_lock(&adev->grbm_idx_mutex);
2830                 amdgpu_gfx_select_se_sh(adev, se_bank,
2831                                         sh_bank, instance_bank);
2832         }
2833
2834         if (pm_pg_lock)
2835                 mutex_lock(&adev->pm.mutex);
2836
2837         while (size) {
2838                 uint32_t value;
2839
2840                 if (*pos > adev->rmmio_size)
2841                         return result;
2842
2843                 r = get_user(value, (uint32_t *)buf);
2844                 if (r)
2845                         return r;
2846
2847                 WREG32(*pos >> 2, value);
2848
2849                 result += 4;
2850                 buf += 4;
2851                 *pos += 4;
2852                 size -= 4;
2853         }
2854
2855         if (use_bank) {
2856                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2857                 mutex_unlock(&adev->grbm_idx_mutex);
2858         }
2859
2860         if (pm_pg_lock)
2861                 mutex_unlock(&adev->pm.mutex);
2862
2863         return result;
2864 }
2865
2866 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
2867                                         size_t size, loff_t *pos)
2868 {
2869         struct amdgpu_device *adev = file_inode(f)->i_private;
2870         ssize_t result = 0;
2871         int r;
2872
2873         if (size & 0x3 || *pos & 0x3)
2874                 return -EINVAL;
2875
2876         while (size) {
2877                 uint32_t value;
2878
2879                 value = RREG32_PCIE(*pos >> 2);
2880                 r = put_user(value, (uint32_t *)buf);
2881                 if (r)
2882                         return r;
2883
2884                 result += 4;
2885                 buf += 4;
2886                 *pos += 4;
2887                 size -= 4;
2888         }
2889
2890         return result;
2891 }
2892
2893 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
2894                                          size_t size, loff_t *pos)
2895 {
2896         struct amdgpu_device *adev = file_inode(f)->i_private;
2897         ssize_t result = 0;
2898         int r;
2899
2900         if (size & 0x3 || *pos & 0x3)
2901                 return -EINVAL;
2902
2903         while (size) {
2904                 uint32_t value;
2905
2906                 r = get_user(value, (uint32_t *)buf);
2907                 if (r)
2908                         return r;
2909
2910                 WREG32_PCIE(*pos >> 2, value);
2911
2912                 result += 4;
2913                 buf += 4;
2914                 *pos += 4;
2915                 size -= 4;
2916         }
2917
2918         return result;
2919 }
2920
2921 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
2922                                         size_t size, loff_t *pos)
2923 {
2924         struct amdgpu_device *adev = file_inode(f)->i_private;
2925         ssize_t result = 0;
2926         int r;
2927
2928         if (size & 0x3 || *pos & 0x3)
2929                 return -EINVAL;
2930
2931         while (size) {
2932                 uint32_t value;
2933
2934                 value = RREG32_DIDT(*pos >> 2);
2935                 r = put_user(value, (uint32_t *)buf);
2936                 if (r)
2937                         return r;
2938
2939                 result += 4;
2940                 buf += 4;
2941                 *pos += 4;
2942                 size -= 4;
2943         }
2944
2945         return result;
2946 }
2947
2948 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
2949                                          size_t size, loff_t *pos)
2950 {
2951         struct amdgpu_device *adev = file_inode(f)->i_private;
2952         ssize_t result = 0;
2953         int r;
2954
2955         if (size & 0x3 || *pos & 0x3)
2956                 return -EINVAL;
2957
2958         while (size) {
2959                 uint32_t value;
2960
2961                 r = get_user(value, (uint32_t *)buf);
2962                 if (r)
2963                         return r;
2964
2965                 WREG32_DIDT(*pos >> 2, value);
2966
2967                 result += 4;
2968                 buf += 4;
2969                 *pos += 4;
2970                 size -= 4;
2971         }
2972
2973         return result;
2974 }
2975
2976 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
2977                                         size_t size, loff_t *pos)
2978 {
2979         struct amdgpu_device *adev = file_inode(f)->i_private;
2980         ssize_t result = 0;
2981         int r;
2982
2983         if (size & 0x3 || *pos & 0x3)
2984                 return -EINVAL;
2985
2986         while (size) {
2987                 uint32_t value;
2988
2989                 value = RREG32_SMC(*pos);
2990                 r = put_user(value, (uint32_t *)buf);
2991                 if (r)
2992                         return r;
2993
2994                 result += 4;
2995                 buf += 4;
2996                 *pos += 4;
2997                 size -= 4;
2998         }
2999
3000         return result;
3001 }
3002
3003 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
3004                                          size_t size, loff_t *pos)
3005 {
3006         struct amdgpu_device *adev = file_inode(f)->i_private;
3007         ssize_t result = 0;
3008         int r;
3009
3010         if (size & 0x3 || *pos & 0x3)
3011                 return -EINVAL;
3012
3013         while (size) {
3014                 uint32_t value;
3015
3016                 r = get_user(value, (uint32_t *)buf);
3017                 if (r)
3018                         return r;
3019
3020                 WREG32_SMC(*pos, value);
3021
3022                 result += 4;
3023                 buf += 4;
3024                 *pos += 4;
3025                 size -= 4;
3026         }
3027
3028         return result;
3029 }
3030
3031 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
3032                                         size_t size, loff_t *pos)
3033 {
3034         struct amdgpu_device *adev = file_inode(f)->i_private;
3035         ssize_t result = 0;
3036         int r;
3037         uint32_t *config, no_regs = 0;
3038
3039         if (size & 0x3 || *pos & 0x3)
3040                 return -EINVAL;
3041
3042         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
3043         if (!config)
3044                 return -ENOMEM;
3045
3046         /* version, increment each time something is added */
3047         config[no_regs++] = 3;
3048         config[no_regs++] = adev->gfx.config.max_shader_engines;
3049         config[no_regs++] = adev->gfx.config.max_tile_pipes;
3050         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
3051         config[no_regs++] = adev->gfx.config.max_sh_per_se;
3052         config[no_regs++] = adev->gfx.config.max_backends_per_se;
3053         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
3054         config[no_regs++] = adev->gfx.config.max_gprs;
3055         config[no_regs++] = adev->gfx.config.max_gs_threads;
3056         config[no_regs++] = adev->gfx.config.max_hw_contexts;
3057         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
3058         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
3059         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
3060         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
3061         config[no_regs++] = adev->gfx.config.num_tile_pipes;
3062         config[no_regs++] = adev->gfx.config.backend_enable_mask;
3063         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
3064         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
3065         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
3066         config[no_regs++] = adev->gfx.config.num_gpus;
3067         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
3068         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
3069         config[no_regs++] = adev->gfx.config.gb_addr_config;
3070         config[no_regs++] = adev->gfx.config.num_rbs;
3071
3072         /* rev==1 */
3073         config[no_regs++] = adev->rev_id;
3074         config[no_regs++] = adev->pg_flags;
3075         config[no_regs++] = adev->cg_flags;
3076
3077         /* rev==2 */
3078         config[no_regs++] = adev->family;
3079         config[no_regs++] = adev->external_rev_id;
3080
3081         /* rev==3 */
3082         config[no_regs++] = adev->pdev->device;
3083         config[no_regs++] = adev->pdev->revision;
3084         config[no_regs++] = adev->pdev->subsystem_device;
3085         config[no_regs++] = adev->pdev->subsystem_vendor;
3086
3087         while (size && (*pos < no_regs * 4)) {
3088                 uint32_t value;
3089
3090                 value = config[*pos >> 2];
3091                 r = put_user(value, (uint32_t *)buf);
3092                 if (r) {
3093                         kfree(config);
3094                         return r;
3095                 }
3096
3097                 result += 4;
3098                 buf += 4;
3099                 *pos += 4;
3100                 size -= 4;
3101         }
3102
3103         kfree(config);
3104         return result;
3105 }
3106
3107 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
3108                                         size_t size, loff_t *pos)
3109 {
3110         struct amdgpu_device *adev = file_inode(f)->i_private;
3111         int idx, x, outsize, r, valuesize;
3112         uint32_t values[16];
3113
3114         if (size & 3 || *pos & 0x3)
3115                 return -EINVAL;
3116
3117         if (amdgpu_dpm == 0)
3118                 return -EINVAL;
3119
3120         /* convert offset to sensor number */
3121         idx = *pos >> 2;
3122
3123         valuesize = sizeof(values);
3124         if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
3125                 r = adev->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, idx, &values[0], &valuesize);
3126         else if (adev->pm.funcs && adev->pm.funcs->read_sensor)
3127                 r = adev->pm.funcs->read_sensor(adev, idx, &values[0],
3128                                                 &valuesize);
3129         else
3130                 return -EINVAL;
3131
3132         if (size > valuesize)
3133                 return -EINVAL;
3134
3135         outsize = 0;
3136         x = 0;
3137         if (!r) {
3138                 while (size) {
3139                         r = put_user(values[x++], (int32_t *)buf);
3140                         buf += 4;
3141                         size -= 4;
3142                         outsize += 4;
3143                 }
3144         }
3145
3146         return !r ? outsize : r;
3147 }
3148
3149 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
3150                                         size_t size, loff_t *pos)
3151 {
3152         struct amdgpu_device *adev = f->f_inode->i_private;
3153         int r, x;
3154         ssize_t result=0;
3155         uint32_t offset, se, sh, cu, wave, simd, data[32];
3156
3157         if (size & 3 || *pos & 3)
3158                 return -EINVAL;
3159
3160         /* decode offset */
3161         offset = (*pos & 0x7F);
3162         se = ((*pos >> 7) & 0xFF);
3163         sh = ((*pos >> 15) & 0xFF);
3164         cu = ((*pos >> 23) & 0xFF);
3165         wave = ((*pos >> 31) & 0xFF);
3166         simd = ((*pos >> 37) & 0xFF);
3167
3168         /* switch to the specific se/sh/cu */
3169         mutex_lock(&adev->grbm_idx_mutex);
3170         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
3171
3172         x = 0;
3173         if (adev->gfx.funcs->read_wave_data)
3174                 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
3175
3176         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3177         mutex_unlock(&adev->grbm_idx_mutex);
3178
3179         if (!x)
3180                 return -EINVAL;
3181
3182         while (size && (offset < x * 4)) {
3183                 uint32_t value;
3184
3185                 value = data[offset >> 2];
3186                 r = put_user(value, (uint32_t *)buf);
3187                 if (r)
3188                         return r;
3189
3190                 result += 4;
3191                 buf += 4;
3192                 offset += 4;
3193                 size -= 4;
3194         }
3195
3196         return result;
3197 }
3198
3199 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
3200                                         size_t size, loff_t *pos)
3201 {
3202         struct amdgpu_device *adev = f->f_inode->i_private;
3203         int r;
3204         ssize_t result = 0;
3205         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
3206
3207         if (size & 3 || *pos & 3)
3208                 return -EINVAL;
3209
3210         /* decode offset */
3211         offset = (*pos & 0xFFF);       /* in dwords */
3212         se = ((*pos >> 12) & 0xFF);
3213         sh = ((*pos >> 20) & 0xFF);
3214         cu = ((*pos >> 28) & 0xFF);
3215         wave = ((*pos >> 36) & 0xFF);
3216         simd = ((*pos >> 44) & 0xFF);
3217         thread = ((*pos >> 52) & 0xFF);
3218         bank = ((*pos >> 60) & 1);
3219
3220         data = kmalloc_array(1024, sizeof(*data), GFP_KERNEL);
3221         if (!data)
3222                 return -ENOMEM;
3223
3224         /* switch to the specific se/sh/cu */
3225         mutex_lock(&adev->grbm_idx_mutex);
3226         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
3227
3228         if (bank == 0) {
3229                 if (adev->gfx.funcs->read_wave_vgprs)
3230                         adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
3231         } else {
3232                 if (adev->gfx.funcs->read_wave_sgprs)
3233                         adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
3234         }
3235
3236         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3237         mutex_unlock(&adev->grbm_idx_mutex);
3238
3239         while (size) {
3240                 uint32_t value;
3241
3242                 value = data[offset++];
3243                 r = put_user(value, (uint32_t *)buf);
3244                 if (r) {
3245                         result = r;
3246                         goto err;
3247                 }
3248
3249                 result += 4;
3250                 buf += 4;
3251                 size -= 4;
3252         }
3253
3254 err:
3255         kfree(data);
3256         return result;
3257 }
3258
3259 static const struct file_operations amdgpu_debugfs_regs_fops = {
3260         .owner = THIS_MODULE,
3261         .read = amdgpu_debugfs_regs_read,
3262         .write = amdgpu_debugfs_regs_write,
3263         .llseek = default_llseek
3264 };
3265 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
3266         .owner = THIS_MODULE,
3267         .read = amdgpu_debugfs_regs_didt_read,
3268         .write = amdgpu_debugfs_regs_didt_write,
3269         .llseek = default_llseek
3270 };
3271 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
3272         .owner = THIS_MODULE,
3273         .read = amdgpu_debugfs_regs_pcie_read,
3274         .write = amdgpu_debugfs_regs_pcie_write,
3275         .llseek = default_llseek
3276 };
3277 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
3278         .owner = THIS_MODULE,
3279         .read = amdgpu_debugfs_regs_smc_read,
3280         .write = amdgpu_debugfs_regs_smc_write,
3281         .llseek = default_llseek
3282 };
3283
3284 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
3285         .owner = THIS_MODULE,
3286         .read = amdgpu_debugfs_gca_config_read,
3287         .llseek = default_llseek
3288 };
3289
3290 static const struct file_operations amdgpu_debugfs_sensors_fops = {
3291         .owner = THIS_MODULE,
3292         .read = amdgpu_debugfs_sensor_read,
3293         .llseek = default_llseek
3294 };
3295
3296 static const struct file_operations amdgpu_debugfs_wave_fops = {
3297         .owner = THIS_MODULE,
3298         .read = amdgpu_debugfs_wave_read,
3299         .llseek = default_llseek
3300 };
3301 static const struct file_operations amdgpu_debugfs_gpr_fops = {
3302         .owner = THIS_MODULE,
3303         .read = amdgpu_debugfs_gpr_read,
3304         .llseek = default_llseek
3305 };
3306
3307 static const struct file_operations *debugfs_regs[] = {
3308         &amdgpu_debugfs_regs_fops,
3309         &amdgpu_debugfs_regs_didt_fops,
3310         &amdgpu_debugfs_regs_pcie_fops,
3311         &amdgpu_debugfs_regs_smc_fops,
3312         &amdgpu_debugfs_gca_config_fops,
3313         &amdgpu_debugfs_sensors_fops,
3314         &amdgpu_debugfs_wave_fops,
3315         &amdgpu_debugfs_gpr_fops,
3316 };
3317
3318 static const char *debugfs_regs_names[] = {
3319         "amdgpu_regs",
3320         "amdgpu_regs_didt",
3321         "amdgpu_regs_pcie",
3322         "amdgpu_regs_smc",
3323         "amdgpu_gca_config",
3324         "amdgpu_sensors",
3325         "amdgpu_wave",
3326         "amdgpu_gpr",
3327 };
3328
3329 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3330 {
3331         struct drm_minor *minor = adev->ddev->primary;
3332         struct dentry *ent, *root = minor->debugfs_root;
3333         unsigned i, j;
3334
3335         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3336                 ent = debugfs_create_file(debugfs_regs_names[i],
3337                                           S_IFREG | S_IRUGO, root,
3338                                           adev, debugfs_regs[i]);
3339                 if (IS_ERR(ent)) {
3340                         for (j = 0; j < i; j++) {
3341                                 debugfs_remove(adev->debugfs_regs[i]);
3342                                 adev->debugfs_regs[i] = NULL;
3343                         }
3344                         return PTR_ERR(ent);
3345                 }
3346
3347                 if (!i)
3348                         i_size_write(ent->d_inode, adev->rmmio_size);
3349                 adev->debugfs_regs[i] = ent;
3350         }
3351
3352         return 0;
3353 }
3354
3355 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
3356 {
3357         unsigned i;
3358
3359         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3360                 if (adev->debugfs_regs[i]) {
3361                         debugfs_remove(adev->debugfs_regs[i]);
3362                         adev->debugfs_regs[i] = NULL;
3363                 }
3364         }
3365 }
3366
3367 int amdgpu_debugfs_init(struct drm_minor *minor)
3368 {
3369         return 0;
3370 }
3371 #else
3372 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3373 {
3374         return 0;
3375 }
3376 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
3377 #endif