drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/kthread.h>
  29 #include <linux/console.h>
  30 #include <linux/slab.h>
  31 #include <linux/debugfs.h>
  32 #include <drm/drmP.h>
  33 #include <drm/drm_crtc_helper.h>
  34 #include <drm/amdgpu_drm.h>
  35 #include <linux/vgaarb.h>
  36 #include <linux/vga_switcheroo.h>
  37 #include <linux/efi.h>
  38 #include "amdgpu.h"
  39 #include "amdgpu_trace.h"
  40 #include "amdgpu_i2c.h"
  41 #include "atom.h"
  42 #include "amdgpu_atombios.h"
  43 #include "amd_pcie.h"
  44 #ifdef CONFIG_DRM_AMDGPU_SI
  45 #include "si.h"
  46 #endif
  47 #ifdef CONFIG_DRM_AMDGPU_CIK
  48 #include "cik.h"
  49 #endif
  50 #include "vi.h"
  51 #include "bif/bif_4_1_d.h"
  52 #include <linux/pci.h>
  53 #include <linux/firmware.h>
  54
  55 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev);
  56 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev);
  57
  58 static const char *amdgpu_asic_name[] = {
  59         "TAHITI",
  60         "PITCAIRN",
  61         "VERDE",
  62         "OLAND",
  63         "HAINAN",
  64         "BONAIRE",
  65         "KAVERI",
  66         "KABINI",
  67         "HAWAII",
  68         "MULLINS",
  69         "TOPAZ",
  70         "TONGA",
  71         "FIJI",
  72         "CARRIZO",
  73         "STONEY",
  74         "POLARIS10",
  75         "POLARIS11",
  76         "POLARIS12",
  77         "LAST",
  78 };
  79
  80 bool amdgpu_device_is_px(struct drm_device *dev)
  81 {
  82         struct amdgpu_device *adev = dev->dev_private;
  83
  84         if (adev->flags & AMD_IS_PX)
  85                 return true;
  86         return false;
  87 }
  88
  89 /*
  90  * MMIO register access helper functions.
  91  */
  92 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
  93                         bool always_indirect)
  94 {
  95         uint32_t ret;
  96
  97         if (amdgpu_sriov_runtime(adev)) {
  98                 BUG_ON(in_interrupt());
  99                 return amdgpu_virt_kiq_rreg(adev, reg);
 100         }
 101
 102         if ((reg * 4) < adev->rmmio_size && !always_indirect)
 103                 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 104         else {
 105                 unsigned long flags;
 106
 107                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 108                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 109                 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 110                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 111         }
 112         trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
 113         return ret;
 114 }
 115
 116 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
 117                     bool always_indirect)
 118 {
 119         trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
 120
 121         if (amdgpu_sriov_runtime(adev)) {
 122                 BUG_ON(in_interrupt());
 123                 return amdgpu_virt_kiq_wreg(adev, reg, v);
 124         }
 125
 126         if ((reg * 4) < adev->rmmio_size && !always_indirect)
 127                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 128         else {
 129                 unsigned long flags;
 130
 131                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 132                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
 133                 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
 134                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 135         }
 136 }
 137
 138 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg)
 139 {
 140         if ((reg * 4) < adev->rio_mem_size)
 141                 return ioread32(adev->rio_mem + (reg * 4));
 142         else {
 143                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 144                 return ioread32(adev->rio_mem + (mmMM_DATA * 4));
 145         }
 146 }
 147
 148 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
 149 {
 150
 151         if ((reg * 4) < adev->rio_mem_size)
 152                 iowrite32(v, adev->rio_mem + (reg * 4));
 153         else {
 154                 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4));
 155                 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4));
 156         }
 157 }
 158
 159 /**
 160  * amdgpu_mm_rdoorbell - read a doorbell dword
 161  *
 162  * @adev: amdgpu_device pointer
 163  * @index: doorbell index
 164  *
 165  * Returns the value in the doorbell aperture at the
 166  * requested doorbell index (CIK).
 167  */
 168 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 169 {
 170         if (index < adev->doorbell.num_doorbells) {
 171                 return readl(adev->doorbell.ptr + index);
 172         } else {
 173                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 174                 return 0;
 175         }
 176 }
 177
 178 /**
 179  * amdgpu_mm_wdoorbell - write a doorbell dword
 180  *
 181  * @adev: amdgpu_device pointer
 182  * @index: doorbell index
 183  * @v: value to write
 184  *
 185  * Writes @v to the doorbell aperture at the
 186  * requested doorbell index (CIK).
 187  */
 188 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 189 {
 190         if (index < adev->doorbell.num_doorbells) {
 191                 writel(v, adev->doorbell.ptr + index);
 192         } else {
 193                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 194         }
 195 }
 196
 197 /**
 198  * amdgpu_invalid_rreg - dummy reg read function
 199  *
 200  * @adev: amdgpu device pointer
 201  * @reg: offset of register
 202  *
 203  * Dummy register read function.  Used for register blocks
 204  * that certain asics don't have (all asics).
 205  * Returns the value in the register.
 206  */
 207 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 208 {
 209         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 210         BUG();
 211         return 0;
 212 }
 213
 214 /**
 215  * amdgpu_invalid_wreg - dummy reg write function
 216  *
 217  * @adev: amdgpu device pointer
 218  * @reg: offset of register
 219  * @v: value to write to the register
 220  *
 221  * Dummy register read function.  Used for register blocks
 222  * that certain asics don't have (all asics).
 223  */
 224 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 225 {
 226         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 227                   reg, v);
 228         BUG();
 229 }
 230
 231 /**
 232  * amdgpu_block_invalid_rreg - dummy reg read function
 233  *
 234  * @adev: amdgpu device pointer
 235  * @block: offset of instance
 236  * @reg: offset of register
 237  *
 238  * Dummy register read function.  Used for register blocks
 239  * that certain asics don't have (all asics).
 240  * Returns the value in the register.
 241  */
 242 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 243                                           uint32_t block, uint32_t reg)
 244 {
 245         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 246                   reg, block);
 247         BUG();
 248         return 0;
 249 }
 250
 251 /**
 252  * amdgpu_block_invalid_wreg - dummy reg write function
 253  *
 254  * @adev: amdgpu device pointer
 255  * @block: offset of instance
 256  * @reg: offset of register
 257  * @v: value to write to the register
 258  *
 259  * Dummy register read function.  Used for register blocks
 260  * that certain asics don't have (all asics).
 261  */
 262 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 263                                       uint32_t block,
 264                                       uint32_t reg, uint32_t v)
 265 {
 266         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 267                   reg, block, v);
 268         BUG();
 269 }
 270
 271 static int amdgpu_vram_scratch_init(struct amdgpu_device *adev)
 272 {
 273         int r;
 274
 275         if (adev->vram_scratch.robj == NULL) {
 276                 r = amdgpu_bo_create(adev, AMDGPU_GPU_PAGE_SIZE,
 277                                      PAGE_SIZE, true, AMDGPU_GEM_DOMAIN_VRAM,
 278                                      AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
 279                                      AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
 280                                      NULL, NULL, &adev->vram_scratch.robj);
 281                 if (r) {
 282                         return r;
 283                 }
 284         }
 285
 286         r = amdgpu_bo_reserve(adev->vram_scratch.robj, false);
 287         if (unlikely(r != 0))
 288                 return r;
 289         r = amdgpu_bo_pin(adev->vram_scratch.robj,
 290                           AMDGPU_GEM_DOMAIN_VRAM, &adev->vram_scratch.gpu_addr);
 291         if (r) {
 292                 amdgpu_bo_unreserve(adev->vram_scratch.robj);
 293                 return r;
 294         }
 295         r = amdgpu_bo_kmap(adev->vram_scratch.robj,
 296                                 (void **)&adev->vram_scratch.ptr);
 297         if (r)
 298                 amdgpu_bo_unpin(adev->vram_scratch.robj);
 299         amdgpu_bo_unreserve(adev->vram_scratch.robj);
 300
 301         return r;
 302 }
 303
 304 static void amdgpu_vram_scratch_fini(struct amdgpu_device *adev)
 305 {
 306         int r;
 307
 308         if (adev->vram_scratch.robj == NULL) {
 309                 return;
 310         }
 311         r = amdgpu_bo_reserve(adev->vram_scratch.robj, false);
 312         if (likely(r == 0)) {
 313                 amdgpu_bo_kunmap(adev->vram_scratch.robj);
 314                 amdgpu_bo_unpin(adev->vram_scratch.robj);
 315                 amdgpu_bo_unreserve(adev->vram_scratch.robj);
 316         }
 317         amdgpu_bo_unref(&adev->vram_scratch.robj);
 318 }
 319
 320 /**
 321  * amdgpu_program_register_sequence - program an array of registers.
 322  *
 323  * @adev: amdgpu_device pointer
 324  * @registers: pointer to the register array
 325  * @array_size: size of the register array
 326  *
 327  * Programs an array or registers with and and or masks.
 328  * This is a helper for setting golden registers.
 329  */
 330 void amdgpu_program_register_sequence(struct amdgpu_device *adev,
 331                                       const u32 *registers,
 332                                       const u32 array_size)
 333 {
 334         u32 tmp, reg, and_mask, or_mask;
 335         int i;
 336
 337         if (array_size % 3)
 338                 return;
 339
 340         for (i = 0; i < array_size; i +=3) {
 341                 reg = registers[i + 0];
 342                 and_mask = registers[i + 1];
 343                 or_mask = registers[i + 2];
 344
 345                 if (and_mask == 0xffffffff) {
 346                         tmp = or_mask;
 347                 } else {
 348                         tmp = RREG32(reg);
 349                         tmp &= ~and_mask;
 350                         tmp |= or_mask;
 351                 }
 352                 WREG32(reg, tmp);
 353         }
 354 }
 355
 356 void amdgpu_pci_config_reset(struct amdgpu_device *adev)
 357 {
 358         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
 359 }
 360
 361 /*
 362  * GPU doorbell aperture helpers function.
 363  */
 364 /**
 365  * amdgpu_doorbell_init - Init doorbell driver information.
 366  *
 367  * @adev: amdgpu_device pointer
 368  *
 369  * Init doorbell driver information (CIK)
 370  * Returns 0 on success, error on failure.
 371  */
 372 static int amdgpu_doorbell_init(struct amdgpu_device *adev)
 373 {
 374         /* doorbell bar mapping */
 375         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
 376         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
 377
 378         adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
 379                                              AMDGPU_DOORBELL_MAX_ASSIGNMENT+1);
 380         if (adev->doorbell.num_doorbells == 0)
 381                 return -EINVAL;
 382
 383         adev->doorbell.ptr = ioremap(adev->doorbell.base, adev->doorbell.num_doorbells * sizeof(u32));
 384         if (adev->doorbell.ptr == NULL) {
 385                 return -ENOMEM;
 386         }
 387         DRM_INFO("doorbell mmio base: 0x%08X\n", (uint32_t)adev->doorbell.base);
 388         DRM_INFO("doorbell mmio size: %u\n", (unsigned)adev->doorbell.size);
 389
 390         return 0;
 391 }
 392
 393 /**
 394  * amdgpu_doorbell_fini - Tear down doorbell driver information.
 395  *
 396  * @adev: amdgpu_device pointer
 397  *
 398  * Tear down doorbell driver information (CIK)
 399  */
 400 static void amdgpu_doorbell_fini(struct amdgpu_device *adev)
 401 {
 402         iounmap(adev->doorbell.ptr);
 403         adev->doorbell.ptr = NULL;
 404 }
 405
 406 /**
 407  * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to
 408  *                                setup amdkfd
 409  *
 410  * @adev: amdgpu_device pointer
 411  * @aperture_base: output returning doorbell aperture base physical address
 412  * @aperture_size: output returning doorbell aperture size in bytes
 413  * @start_offset: output returning # of doorbell bytes reserved for amdgpu.
 414  *
 415  * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up,
 416  * takes doorbells required for its own rings and reports the setup to amdkfd.
 417  * amdgpu reserved doorbells are at the start of the doorbell aperture.
 418  */
 419 void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev,
 420                                 phys_addr_t *aperture_base,
 421                                 size_t *aperture_size,
 422                                 size_t *start_offset)
 423 {
 424         /*
 425          * The first num_doorbells are used by amdgpu.
 426          * amdkfd takes whatever's left in the aperture.
 427          */
 428         if (adev->doorbell.size > adev->doorbell.num_doorbells * sizeof(u32)) {
 429                 *aperture_base = adev->doorbell.base;
 430                 *aperture_size = adev->doorbell.size;
 431                 *start_offset = adev->doorbell.num_doorbells * sizeof(u32);
 432         } else {
 433                 *aperture_base = 0;
 434                 *aperture_size = 0;
 435                 *start_offset = 0;
 436         }
 437 }
 438
 439 /*
 440  * amdgpu_wb_*()
 441  * Writeback is the the method by which the the GPU updates special pages
 442  * in memory with the status of certain GPU events (fences, ring pointers,
 443  * etc.).
 444  */
 445
 446 /**
 447  * amdgpu_wb_fini - Disable Writeback and free memory
 448  *
 449  * @adev: amdgpu_device pointer
 450  *
 451  * Disables Writeback and frees the Writeback memory (all asics).
 452  * Used at driver shutdown.
 453  */
 454 static void amdgpu_wb_fini(struct amdgpu_device *adev)
 455 {
 456         if (adev->wb.wb_obj) {
 457                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
 458                                       &adev->wb.gpu_addr,
 459                                       (void **)&adev->wb.wb);
 460                 adev->wb.wb_obj = NULL;
 461         }
 462 }
 463
 464 /**
 465  * amdgpu_wb_init- Init Writeback driver info and allocate memory
 466  *
 467  * @adev: amdgpu_device pointer
 468  *
 469  * Disables Writeback and frees the Writeback memory (all asics).
 470  * Used at driver startup.
 471  * Returns 0 on success or an -error on failure.
 472  */
 473 static int amdgpu_wb_init(struct amdgpu_device *adev)
 474 {
 475         int r;
 476
 477         if (adev->wb.wb_obj == NULL) {
 478                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * 4,
 479                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
 480                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
 481                                             (void **)&adev->wb.wb);
 482                 if (r) {
 483                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
 484                         return r;
 485                 }
 486
 487                 adev->wb.num_wb = AMDGPU_MAX_WB;
 488                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
 489
 490                 /* clear wb memory */
 491                 memset((char *)adev->wb.wb, 0, AMDGPU_GPU_PAGE_SIZE);
 492         }
 493
 494         return 0;
 495 }
 496
 497 /**
 498  * amdgpu_wb_get - Allocate a wb entry
 499  *
 500  * @adev: amdgpu_device pointer
 501  * @wb: wb index
 502  *
 503  * Allocate a wb slot for use by the driver (all asics).
 504  * Returns 0 on success or -EINVAL on failure.
 505  */
 506 int amdgpu_wb_get(struct amdgpu_device *adev, u32 *wb)
 507 {
 508         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
 509         if (offset < adev->wb.num_wb) {
 510                 __set_bit(offset, adev->wb.used);
 511                 *wb = offset;
 512                 return 0;
 513         } else {
 514                 return -EINVAL;
 515         }
 516 }
 517
 518 /**
 519  * amdgpu_wb_free - Free a wb entry
 520  *
 521  * @adev: amdgpu_device pointer
 522  * @wb: wb index
 523  *
 524  * Free a wb slot allocated for use by the driver (all asics)
 525  */
 526 void amdgpu_wb_free(struct amdgpu_device *adev, u32 wb)
 527 {
 528         if (wb < adev->wb.num_wb)
 529                 __clear_bit(wb, adev->wb.used);
 530 }
 531
 532 /**
 533  * amdgpu_vram_location - try to find VRAM location
 534  * @adev: amdgpu device structure holding all necessary informations
 535  * @mc: memory controller structure holding memory informations
 536  * @base: base address at which to put VRAM
 537  *
 538  * Function will place try to place VRAM at base address provided
 539  * as parameter (which is so far either PCI aperture address or
 540  * for IGP TOM base address).
 541  *
 542  * If there is not enough space to fit the unvisible VRAM in the 32bits
 543  * address space then we limit the VRAM size to the aperture.
 544  *
 545  * Note: We don't explicitly enforce VRAM start to be aligned on VRAM size,
 546  * this shouldn't be a problem as we are using the PCI aperture as a reference.
 547  * Otherwise this would be needed for rv280, all r3xx, and all r4xx, but
 548  * not IGP.
 549  *
 550  * Note: we use mc_vram_size as on some board we need to program the mc to
 551  * cover the whole aperture even if VRAM size is inferior to aperture size
 552  * Novell bug 204882 + along with lots of ubuntu ones
 553  *
 554  * Note: when limiting vram it's safe to overwritte real_vram_size because
 555  * we are not in case where real_vram_size is inferior to mc_vram_size (ie
 556  * note afected by bogus hw of Novell bug 204882 + along with lots of ubuntu
 557  * ones)
 558  *
 559  * Note: IGP TOM addr should be the same as the aperture addr, we don't
 560  * explicitly check for that thought.
 561  *
 562  * FIXME: when reducing VRAM size align new size on power of 2.
 563  */
 564 void amdgpu_vram_location(struct amdgpu_device *adev, struct amdgpu_mc *mc, u64 base)
 565 {
 566         uint64_t limit = (uint64_t)amdgpu_vram_limit << 20;
 567
 568         mc->vram_start = base;
 569         if (mc->mc_vram_size > (adev->mc.mc_mask - base + 1)) {
 570                 dev_warn(adev->dev, "limiting VRAM to PCI aperture size\n");
 571                 mc->real_vram_size = mc->aper_size;
 572                 mc->mc_vram_size = mc->aper_size;
 573         }
 574         mc->vram_end = mc->vram_start + mc->mc_vram_size - 1;
 575         if (limit && limit < mc->real_vram_size)
 576                 mc->real_vram_size = limit;
 577         dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n",
 578                         mc->mc_vram_size >> 20, mc->vram_start,
 579                         mc->vram_end, mc->real_vram_size >> 20);
 580 }
 581
 582 /**
 583  * amdgpu_gtt_location - try to find GTT location
 584  * @adev: amdgpu device structure holding all necessary informations
 585  * @mc: memory controller structure holding memory informations
 586  *
 587  * Function will place try to place GTT before or after VRAM.
 588  *
 589  * If GTT size is bigger than space left then we ajust GTT size.
 590  * Thus function will never fails.
 591  *
 592  * FIXME: when reducing GTT size align new size on power of 2.
 593  */
 594 void amdgpu_gtt_location(struct amdgpu_device *adev, struct amdgpu_mc *mc)
 595 {
 596         u64 size_af, size_bf;
 597
 598         size_af = ((adev->mc.mc_mask - mc->vram_end) + mc->gtt_base_align) & ~mc->gtt_base_align;
 599         size_bf = mc->vram_start & ~mc->gtt_base_align;
 600         if (size_bf > size_af) {
 601                 if (mc->gtt_size > size_bf) {
 602                         dev_warn(adev->dev, "limiting GTT\n");
 603                         mc->gtt_size = size_bf;
 604                 }
 605                 mc->gtt_start = (mc->vram_start & ~mc->gtt_base_align) - mc->gtt_size;
 606         } else {
 607                 if (mc->gtt_size > size_af) {
 608                         dev_warn(adev->dev, "limiting GTT\n");
 609                         mc->gtt_size = size_af;
 610                 }
 611                 mc->gtt_start = (mc->vram_end + 1 + mc->gtt_base_align) & ~mc->gtt_base_align;
 612         }
 613         mc->gtt_end = mc->gtt_start + mc->gtt_size - 1;
 614         dev_info(adev->dev, "GTT: %lluM 0x%016llX - 0x%016llX\n",
 615                         mc->gtt_size >> 20, mc->gtt_start, mc->gtt_end);
 616 }
 617
 618 /*
 619  * GPU helpers function.
 620  */
 621 /**
 622  * amdgpu_need_post - check if the hw need post or not
 623  *
 624  * @adev: amdgpu_device pointer
 625  *
 626  * Check if the asic has been initialized (all asics) at driver startup
 627  * or post is needed if  hw reset is performed.
 628  * Returns true if need or false if not.
 629  */
 630 bool amdgpu_need_post(struct amdgpu_device *adev)
 631 {
 632         uint32_t reg;
 633
 634         if (adev->has_hw_reset) {
 635                 adev->has_hw_reset = false;
 636                 return true;
 637         }
 638         /* then check MEM_SIZE, in case the crtcs are off */
 639         reg = RREG32(mmCONFIG_MEMSIZE);
 640
 641         if (reg)
 642                 return false;
 643
 644         return true;
 645
 646 }
 647
 648 static bool amdgpu_vpost_needed(struct amdgpu_device *adev)
 649 {
 650         if (amdgpu_sriov_vf(adev))
 651                 return false;
 652
 653         if (amdgpu_passthrough(adev)) {
 654                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
 655                  * some old smc fw still need driver do vPost otherwise gpu hang, while
 656                  * those smc fw version above 22.15 doesn't have this flaw, so we force
 657                  * vpost executed for smc version below 22.15
 658                  */
 659                 if (adev->asic_type == CHIP_FIJI) {
 660                         int err;
 661                         uint32_t fw_ver;
 662                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
 663                         /* force vPost if error occured */
 664                         if (err)
 665                                 return true;
 666
 667                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
 668                         if (fw_ver < 0x00160e00)
 669                                 return true;
 670                 }
 671         }
 672         return amdgpu_need_post(adev);
 673 }
 674
 675 /**
 676  * amdgpu_dummy_page_init - init dummy page used by the driver
 677  *
 678  * @adev: amdgpu_device pointer
 679  *
 680  * Allocate the dummy page used by the driver (all asics).
 681  * This dummy page is used by the driver as a filler for gart entries
 682  * when pages are taken out of the GART
 683  * Returns 0 on sucess, -ENOMEM on failure.
 684  */
 685 int amdgpu_dummy_page_init(struct amdgpu_device *adev)
 686 {
 687         if (adev->dummy_page.page)
 688                 return 0;
 689         adev->dummy_page.page = alloc_page(GFP_DMA32 | GFP_KERNEL | __GFP_ZERO);
 690         if (adev->dummy_page.page == NULL)
 691                 return -ENOMEM;
 692         adev->dummy_page.addr = pci_map_page(adev->pdev, adev->dummy_page.page,
 693                                         0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 694         if (pci_dma_mapping_error(adev->pdev, adev->dummy_page.addr)) {
 695                 dev_err(&adev->pdev->dev, "Failed to DMA MAP the dummy page\n");
 696                 __free_page(adev->dummy_page.page);
 697                 adev->dummy_page.page = NULL;
 698                 return -ENOMEM;
 699         }
 700         return 0;
 701 }
 702
 703 /**
 704  * amdgpu_dummy_page_fini - free dummy page used by the driver
 705  *
 706  * @adev: amdgpu_device pointer
 707  *
 708  * Frees the dummy page used by the driver (all asics).
 709  */
 710 void amdgpu_dummy_page_fini(struct amdgpu_device *adev)
 711 {
 712         if (adev->dummy_page.page == NULL)
 713                 return;
 714         pci_unmap_page(adev->pdev, adev->dummy_page.addr,
 715                         PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 716         __free_page(adev->dummy_page.page);
 717         adev->dummy_page.page = NULL;
 718 }
 719
 720
 721 /* ATOM accessor methods */
 722 /*
 723  * ATOM is an interpreted byte code stored in tables in the vbios.  The
 724  * driver registers callbacks to access registers and the interpreter
 725  * in the driver parses the tables and executes then to program specific
 726  * actions (set display modes, asic init, etc.).  See amdgpu_atombios.c,
 727  * atombios.h, and atom.c
 728  */
 729
 730 /**
 731  * cail_pll_read - read PLL register
 732  *
 733  * @info: atom card_info pointer
 734  * @reg: PLL register offset
 735  *
 736  * Provides a PLL register accessor for the atom interpreter (r4xx+).
 737  * Returns the value of the PLL register.
 738  */
 739 static uint32_t cail_pll_read(struct card_info *info, uint32_t reg)
 740 {
 741         return 0;
 742 }
 743
 744 /**
 745  * cail_pll_write - write PLL register
 746  *
 747  * @info: atom card_info pointer
 748  * @reg: PLL register offset
 749  * @val: value to write to the pll register
 750  *
 751  * Provides a PLL register accessor for the atom interpreter (r4xx+).
 752  */
 753 static void cail_pll_write(struct card_info *info, uint32_t reg, uint32_t val)
 754 {
 755
 756 }
 757
 758 /**
 759  * cail_mc_read - read MC (Memory Controller) register
 760  *
 761  * @info: atom card_info pointer
 762  * @reg: MC register offset
 763  *
 764  * Provides an MC register accessor for the atom interpreter (r4xx+).
 765  * Returns the value of the MC register.
 766  */
 767 static uint32_t cail_mc_read(struct card_info *info, uint32_t reg)
 768 {
 769         return 0;
 770 }
 771
 772 /**
 773  * cail_mc_write - write MC (Memory Controller) register
 774  *
 775  * @info: atom card_info pointer
 776  * @reg: MC register offset
 777  * @val: value to write to the pll register
 778  *
 779  * Provides a MC register accessor for the atom interpreter (r4xx+).
 780  */
 781 static void cail_mc_write(struct card_info *info, uint32_t reg, uint32_t val)
 782 {
 783
 784 }
 785
 786 /**
 787  * cail_reg_write - write MMIO register
 788  *
 789  * @info: atom card_info pointer
 790  * @reg: MMIO register offset
 791  * @val: value to write to the pll register
 792  *
 793  * Provides a MMIO register accessor for the atom interpreter (r4xx+).
 794  */
 795 static void cail_reg_write(struct card_info *info, uint32_t reg, uint32_t val)
 796 {
 797         struct amdgpu_device *adev = info->dev->dev_private;
 798
 799         WREG32(reg, val);
 800 }
 801
 802 /**
 803  * cail_reg_read - read MMIO register
 804  *
 805  * @info: atom card_info pointer
 806  * @reg: MMIO register offset
 807  *
 808  * Provides an MMIO register accessor for the atom interpreter (r4xx+).
 809  * Returns the value of the MMIO register.
 810  */
 811 static uint32_t cail_reg_read(struct card_info *info, uint32_t reg)
 812 {
 813         struct amdgpu_device *adev = info->dev->dev_private;
 814         uint32_t r;
 815
 816         r = RREG32(reg);
 817         return r;
 818 }
 819
 820 /**
 821  * cail_ioreg_write - write IO register
 822  *
 823  * @info: atom card_info pointer
 824  * @reg: IO register offset
 825  * @val: value to write to the pll register
 826  *
 827  * Provides a IO register accessor for the atom interpreter (r4xx+).
 828  */
 829 static void cail_ioreg_write(struct card_info *info, uint32_t reg, uint32_t val)
 830 {
 831         struct amdgpu_device *adev = info->dev->dev_private;
 832
 833         WREG32_IO(reg, val);
 834 }
 835
 836 /**
 837  * cail_ioreg_read - read IO register
 838  *
 839  * @info: atom card_info pointer
 840  * @reg: IO register offset
 841  *
 842  * Provides an IO register accessor for the atom interpreter (r4xx+).
 843  * Returns the value of the IO register.
 844  */
 845 static uint32_t cail_ioreg_read(struct card_info *info, uint32_t reg)
 846 {
 847         struct amdgpu_device *adev = info->dev->dev_private;
 848         uint32_t r;
 849
 850         r = RREG32_IO(reg);
 851         return r;
 852 }
 853
 854 /**
 855  * amdgpu_atombios_fini - free the driver info and callbacks for atombios
 856  *
 857  * @adev: amdgpu_device pointer
 858  *
 859  * Frees the driver info and register access callbacks for the ATOM
 860  * interpreter (r4xx+).
 861  * Called at driver shutdown.
 862  */
 863 static void amdgpu_atombios_fini(struct amdgpu_device *adev)
 864 {
 865         if (adev->mode_info.atom_context) {
 866                 kfree(adev->mode_info.atom_context->scratch);
 867                 kfree(adev->mode_info.atom_context->iio);
 868         }
 869         kfree(adev->mode_info.atom_context);
 870         adev->mode_info.atom_context = NULL;
 871         kfree(adev->mode_info.atom_card_info);
 872         adev->mode_info.atom_card_info = NULL;
 873 }
 874
 875 /**
 876  * amdgpu_atombios_init - init the driver info and callbacks for atombios
 877  *
 878  * @adev: amdgpu_device pointer
 879  *
 880  * Initializes the driver info and register access callbacks for the
 881  * ATOM interpreter (r4xx+).
 882  * Returns 0 on sucess, -ENOMEM on failure.
 883  * Called at driver startup.
 884  */
 885 static int amdgpu_atombios_init(struct amdgpu_device *adev)
 886 {
 887         struct card_info *atom_card_info =
 888             kzalloc(sizeof(struct card_info), GFP_KERNEL);
 889
 890         if (!atom_card_info)
 891                 return -ENOMEM;
 892
 893         adev->mode_info.atom_card_info = atom_card_info;
 894         atom_card_info->dev = adev->ddev;
 895         atom_card_info->reg_read = cail_reg_read;
 896         atom_card_info->reg_write = cail_reg_write;
 897         /* needed for iio ops */
 898         if (adev->rio_mem) {
 899                 atom_card_info->ioreg_read = cail_ioreg_read;
 900                 atom_card_info->ioreg_write = cail_ioreg_write;
 901         } else {
 902                 DRM_INFO("PCI I/O BAR is not found. Using MMIO to access ATOM BIOS\n");
 903                 atom_card_info->ioreg_read = cail_reg_read;
 904                 atom_card_info->ioreg_write = cail_reg_write;
 905         }
 906         atom_card_info->mc_read = cail_mc_read;
 907         atom_card_info->mc_write = cail_mc_write;
 908         atom_card_info->pll_read = cail_pll_read;
 909         atom_card_info->pll_write = cail_pll_write;
 910
 911         adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios);
 912         if (!adev->mode_info.atom_context) {
 913                 amdgpu_atombios_fini(adev);
 914                 return -ENOMEM;
 915         }
 916
 917         mutex_init(&adev->mode_info.atom_context->mutex);
 918         amdgpu_atombios_scratch_regs_init(adev);
 919         amdgpu_atom_allocate_fb_scratch(adev->mode_info.atom_context);
 920         return 0;
 921 }
 922
 923 /* if we get transitioned to only one device, take VGA back */
 924 /**
 925  * amdgpu_vga_set_decode - enable/disable vga decode
 926  *
 927  * @cookie: amdgpu_device pointer
 928  * @state: enable/disable vga decode
 929  *
 930  * Enable/disable vga decode (all asics).
 931  * Returns VGA resource flags.
 932  */
 933 static unsigned int amdgpu_vga_set_decode(void *cookie, bool state)
 934 {
 935         struct amdgpu_device *adev = cookie;
 936         amdgpu_asic_set_vga_state(adev, state);
 937         if (state)
 938                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
 939                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 940         else
 941                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
 942 }
 943
 944 /**
 945  * amdgpu_check_pot_argument - check that argument is a power of two
 946  *
 947  * @arg: value to check
 948  *
 949  * Validates that a certain argument is a power of two (all asics).
 950  * Returns true if argument is valid.
 951  */
 952 static bool amdgpu_check_pot_argument(int arg)
 953 {
 954         return (arg & (arg - 1)) == 0;
 955 }
 956
 957 /**
 958  * amdgpu_check_arguments - validate module params
 959  *
 960  * @adev: amdgpu_device pointer
 961  *
 962  * Validates certain module parameters and updates
 963  * the associated values used by the driver (all asics).
 964  */
 965 static void amdgpu_check_arguments(struct amdgpu_device *adev)
 966 {
 967         if (amdgpu_sched_jobs < 4) {
 968                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
 969                          amdgpu_sched_jobs);
 970                 amdgpu_sched_jobs = 4;
 971         } else if (!amdgpu_check_pot_argument(amdgpu_sched_jobs)){
 972                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
 973                          amdgpu_sched_jobs);
 974                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
 975         }
 976
 977         if (amdgpu_gart_size != -1) {
 978                 /* gtt size must be greater or equal to 32M */
 979                 if (amdgpu_gart_size < 32) {
 980                         dev_warn(adev->dev, "gart size (%d) too small\n",
 981                                  amdgpu_gart_size);
 982                         amdgpu_gart_size = -1;
 983                 }
 984         }
 985
 986         if (!amdgpu_check_pot_argument(amdgpu_vm_size)) {
 987                 dev_warn(adev->dev, "VM size (%d) must be a power of 2\n",
 988                          amdgpu_vm_size);
 989                 amdgpu_vm_size = 8;
 990         }
 991
 992         if (amdgpu_vm_size < 1) {
 993                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
 994                          amdgpu_vm_size);
 995                 amdgpu_vm_size = 8;
 996         }
 997
 998         /*
 999          * Max GPUVM size for Cayman, SI and CI are 40 bits.
1000          */
1001         if (amdgpu_vm_size > 1024) {
1002                 dev_warn(adev->dev, "VM size (%d) too large, max is 1TB\n",
1003                          amdgpu_vm_size);
1004                 amdgpu_vm_size = 8;
1005         }
1006
1007         /* defines number of bits in page table versus page directory,
1008          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1009          * page table and the remaining bits are in the page directory */
1010         if (amdgpu_vm_block_size == -1) {
1011
1012                 /* Total bits covered by PD + PTs */
1013                 unsigned bits = ilog2(amdgpu_vm_size) + 18;
1014
1015                 /* Make sure the PD is 4K in size up to 8GB address space.
1016                    Above that split equal between PD and PTs */
1017                 if (amdgpu_vm_size <= 8)
1018                         amdgpu_vm_block_size = bits - 9;
1019                 else
1020                         amdgpu_vm_block_size = (bits + 3) / 2;
1021
1022         } else if (amdgpu_vm_block_size < 9) {
1023                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1024                          amdgpu_vm_block_size);
1025                 amdgpu_vm_block_size = 9;
1026         }
1027
1028         if (amdgpu_vm_block_size > 24 ||
1029             (amdgpu_vm_size * 1024) < (1ull << amdgpu_vm_block_size)) {
1030                 dev_warn(adev->dev, "VM page table size (%d) too large\n",
1031                          amdgpu_vm_block_size);
1032                 amdgpu_vm_block_size = 9;
1033         }
1034
1035         if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 ||
1036             !amdgpu_check_pot_argument(amdgpu_vram_page_split))) {
1037                 dev_warn(adev->dev, "invalid VRAM page split (%d)\n",
1038                          amdgpu_vram_page_split);
1039                 amdgpu_vram_page_split = 1024;
1040         }
1041 }
1042
1043 /**
1044  * amdgpu_switcheroo_set_state - set switcheroo state
1045  *
1046  * @pdev: pci dev pointer
1047  * @state: vga_switcheroo state
1048  *
1049  * Callback for the switcheroo driver.  Suspends or resumes the
1050  * the asics before or after it is powered up using ACPI methods.
1051  */
1052 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
1053 {
1054         struct drm_device *dev = pci_get_drvdata(pdev);
1055
1056         if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF)
1057                 return;
1058
1059         if (state == VGA_SWITCHEROO_ON) {
1060                 unsigned d3_delay = dev->pdev->d3_delay;
1061
1062                 printk(KERN_INFO "amdgpu: switched on\n");
1063                 /* don't suspend or resume card normally */
1064                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1065
1066                 amdgpu_device_resume(dev, true, true);
1067
1068                 dev->pdev->d3_delay = d3_delay;
1069
1070                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1071                 drm_kms_helper_poll_enable(dev);
1072         } else {
1073                 printk(KERN_INFO "amdgpu: switched off\n");
1074                 drm_kms_helper_poll_disable(dev);
1075                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1076                 amdgpu_device_suspend(dev, true, true);
1077                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1078         }
1079 }
1080
1081 /**
1082  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1083  *
1084  * @pdev: pci dev pointer
1085  *
1086  * Callback for the switcheroo driver.  Check of the switcheroo
1087  * state can be changed.
1088  * Returns true if the state can be changed, false if not.
1089  */
1090 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1091 {
1092         struct drm_device *dev = pci_get_drvdata(pdev);
1093
1094         /*
1095         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1096         * locking inversion with the driver load path. And the access here is
1097         * completely racy anyway. So don't bother with locking for now.
1098         */
1099         return dev->open_count == 0;
1100 }
1101
1102 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1103         .set_gpu_state = amdgpu_switcheroo_set_state,
1104         .reprobe = NULL,
1105         .can_switch = amdgpu_switcheroo_can_switch,
1106 };
1107
1108 int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
1109                                   enum amd_ip_block_type block_type,
1110                                   enum amd_clockgating_state state)
1111 {
1112         int i, r = 0;
1113
1114         for (i = 0; i < adev->num_ip_blocks; i++) {
1115                 if (!adev->ip_blocks[i].status.valid)
1116                         continue;
1117                 if (adev->ip_blocks[i].version->type == block_type) {
1118                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1119                                                                                      state);
1120                         if (r)
1121                                 return r;
1122                         break;
1123                 }
1124         }
1125         return r;
1126 }
1127
1128 int amdgpu_set_powergating_state(struct amdgpu_device *adev,
1129                                   enum amd_ip_block_type block_type,
1130                                   enum amd_powergating_state state)
1131 {
1132         int i, r = 0;
1133
1134         for (i = 0; i < adev->num_ip_blocks; i++) {
1135                 if (!adev->ip_blocks[i].status.valid)
1136                         continue;
1137                 if (adev->ip_blocks[i].version->type == block_type) {
1138                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
1139                                                                                      state);
1140                         if (r)
1141                                 return r;
1142                         break;
1143                 }
1144         }
1145         return r;
1146 }
1147
1148 void amdgpu_get_clockgating_state(struct amdgpu_device *adev, u32 *flags)
1149 {
1150         int i;
1151
1152         for (i = 0; i < adev->num_ip_blocks; i++) {
1153                 if (!adev->ip_blocks[i].status.valid)
1154                         continue;
1155                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1156                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1157         }
1158 }
1159
1160 int amdgpu_wait_for_idle(struct amdgpu_device *adev,
1161                          enum amd_ip_block_type block_type)
1162 {
1163         int i, r;
1164
1165         for (i = 0; i < adev->num_ip_blocks; i++) {
1166                 if (!adev->ip_blocks[i].status.valid)
1167                         continue;
1168                 if (adev->ip_blocks[i].version->type == block_type) {
1169                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1170                         if (r)
1171                                 return r;
1172                         break;
1173                 }
1174         }
1175         return 0;
1176
1177 }
1178
1179 bool amdgpu_is_idle(struct amdgpu_device *adev,
1180                     enum amd_ip_block_type block_type)
1181 {
1182         int i;
1183
1184         for (i = 0; i < adev->num_ip_blocks; i++) {
1185                 if (!adev->ip_blocks[i].status.valid)
1186                         continue;
1187                 if (adev->ip_blocks[i].version->type == block_type)
1188                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1189         }
1190         return true;
1191
1192 }
1193
1194 struct amdgpu_ip_block * amdgpu_get_ip_block(struct amdgpu_device *adev,
1195                                              enum amd_ip_block_type type)
1196 {
1197         int i;
1198
1199         for (i = 0; i < adev->num_ip_blocks; i++)
1200                 if (adev->ip_blocks[i].version->type == type)
1201                         return &adev->ip_blocks[i];
1202
1203         return NULL;
1204 }
1205
1206 /**
1207  * amdgpu_ip_block_version_cmp
1208  *
1209  * @adev: amdgpu_device pointer
1210  * @type: enum amd_ip_block_type
1211  * @major: major version
1212  * @minor: minor version
1213  *
1214  * return 0 if equal or greater
1215  * return 1 if smaller or the ip_block doesn't exist
1216  */
1217 int amdgpu_ip_block_version_cmp(struct amdgpu_device *adev,
1218                                 enum amd_ip_block_type type,
1219                                 u32 major, u32 minor)
1220 {
1221         struct amdgpu_ip_block *ip_block = amdgpu_get_ip_block(adev, type);
1222
1223         if (ip_block && ((ip_block->version->major > major) ||
1224                         ((ip_block->version->major == major) &&
1225                         (ip_block->version->minor >= minor))))
1226                 return 0;
1227
1228         return 1;
1229 }
1230
1231 /**
1232  * amdgpu_ip_block_add
1233  *
1234  * @adev: amdgpu_device pointer
1235  * @ip_block_version: pointer to the IP to add
1236  *
1237  * Adds the IP block driver information to the collection of IPs
1238  * on the asic.
1239  */
1240 int amdgpu_ip_block_add(struct amdgpu_device *adev,
1241                         const struct amdgpu_ip_block_version *ip_block_version)
1242 {
1243         if (!ip_block_version)
1244                 return -EINVAL;
1245
1246         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1247
1248         return 0;
1249 }
1250
1251 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1252 {
1253         adev->enable_virtual_display = false;
1254
1255         if (amdgpu_virtual_display) {
1256                 struct drm_device *ddev = adev->ddev;
1257                 const char *pci_address_name = pci_name(ddev->pdev);
1258                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1259
1260                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1261                 pciaddstr_tmp = pciaddstr;
1262                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1263                         pciaddname = strsep(&pciaddname_tmp, ",");
1264                         if (!strcmp("all", pciaddname)
1265                             || !strcmp(pci_address_name, pciaddname)) {
1266                                 long num_crtc;
1267                                 int res = -1;
1268
1269                                 adev->enable_virtual_display = true;
1270
1271                                 if (pciaddname_tmp)
1272                                         res = kstrtol(pciaddname_tmp, 10,
1273                                                       &num_crtc);
1274
1275                                 if (!res) {
1276                                         if (num_crtc < 1)
1277                                                 num_crtc = 1;
1278                                         if (num_crtc > 6)
1279                                                 num_crtc = 6;
1280                                         adev->mode_info.num_crtc = num_crtc;
1281                                 } else {
1282                                         adev->mode_info.num_crtc = 1;
1283                                 }
1284                                 break;
1285                         }
1286                 }
1287
1288                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1289                          amdgpu_virtual_display, pci_address_name,
1290                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1291
1292                 kfree(pciaddstr);
1293         }
1294 }
1295
1296 static int amdgpu_early_init(struct amdgpu_device *adev)
1297 {
1298         int i, r;
1299
1300         amdgpu_device_enable_virtual_display(adev);
1301
1302         switch (adev->asic_type) {
1303         case CHIP_TOPAZ:
1304         case CHIP_TONGA:
1305         case CHIP_FIJI:
1306         case CHIP_POLARIS11:
1307         case CHIP_POLARIS10:
1308         case CHIP_POLARIS12:
1309         case CHIP_CARRIZO:
1310         case CHIP_STONEY:
1311                 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY)
1312                         adev->family = AMDGPU_FAMILY_CZ;
1313                 else
1314                         adev->family = AMDGPU_FAMILY_VI;
1315
1316                 r = vi_set_ip_blocks(adev);
1317                 if (r)
1318                         return r;
1319                 break;
1320 #ifdef CONFIG_DRM_AMDGPU_SI
1321         case CHIP_VERDE:
1322         case CHIP_TAHITI:
1323         case CHIP_PITCAIRN:
1324         case CHIP_OLAND:
1325         case CHIP_HAINAN:
1326                 adev->family = AMDGPU_FAMILY_SI;
1327                 r = si_set_ip_blocks(adev);
1328                 if (r)
1329                         return r;
1330                 break;
1331 #endif
1332 #ifdef CONFIG_DRM_AMDGPU_CIK
1333         case CHIP_BONAIRE:
1334         case CHIP_HAWAII:
1335         case CHIP_KAVERI:
1336         case CHIP_KABINI:
1337         case CHIP_MULLINS:
1338                 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII))
1339                         adev->family = AMDGPU_FAMILY_CI;
1340                 else
1341                         adev->family = AMDGPU_FAMILY_KV;
1342
1343                 r = cik_set_ip_blocks(adev);
1344                 if (r)
1345                         return r;
1346                 break;
1347 #endif
1348         default:
1349                 /* FIXME: not supported yet */
1350                 return -EINVAL;
1351         }
1352
1353         if (amdgpu_sriov_vf(adev)) {
1354                 r = amdgpu_virt_request_full_gpu(adev, true);
1355                 if (r)
1356                         return r;
1357         }
1358
1359         for (i = 0; i < adev->num_ip_blocks; i++) {
1360                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
1361                         DRM_ERROR("disabled ip block: %d\n", i);
1362                         adev->ip_blocks[i].status.valid = false;
1363                 } else {
1364                         if (adev->ip_blocks[i].version->funcs->early_init) {
1365                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
1366                                 if (r == -ENOENT) {
1367                                         adev->ip_blocks[i].status.valid = false;
1368                                 } else if (r) {
1369                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
1370                                                   adev->ip_blocks[i].version->funcs->name, r);
1371                                         return r;
1372                                 } else {
1373                                         adev->ip_blocks[i].status.valid = true;
1374                                 }
1375                         } else {
1376                                 adev->ip_blocks[i].status.valid = true;
1377                         }
1378                 }
1379         }
1380
1381         adev->cg_flags &= amdgpu_cg_mask;
1382         adev->pg_flags &= amdgpu_pg_mask;
1383
1384         return 0;
1385 }
1386
1387 static int amdgpu_init(struct amdgpu_device *adev)
1388 {
1389         int i, r;
1390
1391         for (i = 0; i < adev->num_ip_blocks; i++) {
1392                 if (!adev->ip_blocks[i].status.valid)
1393                         continue;
1394                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
1395                 if (r) {
1396                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
1397                                   adev->ip_blocks[i].version->funcs->name, r);
1398                         return r;
1399                 }
1400                 adev->ip_blocks[i].status.sw = true;
1401                 /* need to do gmc hw init early so we can allocate gpu mem */
1402                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1403                         r = amdgpu_vram_scratch_init(adev);
1404                         if (r) {
1405                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
1406                                 return r;
1407                         }
1408                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1409                         if (r) {
1410                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
1411                                 return r;
1412                         }
1413                         r = amdgpu_wb_init(adev);
1414                         if (r) {
1415                                 DRM_ERROR("amdgpu_wb_init failed %d\n", r);
1416                                 return r;
1417                         }
1418                         adev->ip_blocks[i].status.hw = true;
1419
1420                         /* right after GMC hw init, we create CSA */
1421                         if (amdgpu_sriov_vf(adev)) {
1422                                 r = amdgpu_allocate_static_csa(adev);
1423                                 if (r) {
1424                                         DRM_ERROR("allocate CSA failed %d\n", r);
1425                                         return r;
1426                                 }
1427                         }
1428                 }
1429         }
1430
1431         for (i = 0; i < adev->num_ip_blocks; i++) {
1432                 if (!adev->ip_blocks[i].status.sw)
1433                         continue;
1434                 /* gmc hw init is done early */
1435                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC)
1436                         continue;
1437                 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
1438                 if (r) {
1439                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
1440                                   adev->ip_blocks[i].version->funcs->name, r);
1441                         return r;
1442                 }
1443                 adev->ip_blocks[i].status.hw = true;
1444         }
1445
1446         return 0;
1447 }
1448
1449 static int amdgpu_late_init(struct amdgpu_device *adev)
1450 {
1451         int i = 0, r;
1452
1453         for (i = 0; i < adev->num_ip_blocks; i++) {
1454                 if (!adev->ip_blocks[i].status.valid)
1455                         continue;
1456                 if (adev->ip_blocks[i].version->funcs->late_init) {
1457                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
1458                         if (r) {
1459                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
1460                                           adev->ip_blocks[i].version->funcs->name, r);
1461                                 return r;
1462                         }
1463                         adev->ip_blocks[i].status.late_initialized = true;
1464                 }
1465                 /* skip CG for VCE/UVD, it's handled specially */
1466                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1467                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1468                         /* enable clockgating to save power */
1469                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1470                                                                                      AMD_CG_STATE_GATE);
1471                         if (r) {
1472                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
1473                                           adev->ip_blocks[i].version->funcs->name, r);
1474                                 return r;
1475                         }
1476                 }
1477         }
1478
1479         return 0;
1480 }
1481
1482 static int amdgpu_fini(struct amdgpu_device *adev)
1483 {
1484         int i, r;
1485
1486         /* need to disable SMC first */
1487         for (i = 0; i < adev->num_ip_blocks; i++) {
1488                 if (!adev->ip_blocks[i].status.hw)
1489                         continue;
1490                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
1491                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1492                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1493                                                                                      AMD_CG_STATE_UNGATE);
1494                         if (r) {
1495                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1496                                           adev->ip_blocks[i].version->funcs->name, r);
1497                                 return r;
1498                         }
1499                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1500                         /* XXX handle errors */
1501                         if (r) {
1502                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1503                                           adev->ip_blocks[i].version->funcs->name, r);
1504                         }
1505                         adev->ip_blocks[i].status.hw = false;
1506                         break;
1507                 }
1508         }
1509
1510         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1511                 if (!adev->ip_blocks[i].status.hw)
1512                         continue;
1513                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
1514                         amdgpu_wb_fini(adev);
1515                         amdgpu_vram_scratch_fini(adev);
1516                 }
1517
1518                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
1519                         adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE) {
1520                         /* ungate blocks before hw fini so that we can shutdown the blocks safely */
1521                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1522                                                                                      AMD_CG_STATE_UNGATE);
1523                         if (r) {
1524                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1525                                           adev->ip_blocks[i].version->funcs->name, r);
1526                                 return r;
1527                         }
1528                 }
1529
1530                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
1531                 /* XXX handle errors */
1532                 if (r) {
1533                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
1534                                   adev->ip_blocks[i].version->funcs->name, r);
1535                 }
1536
1537                 adev->ip_blocks[i].status.hw = false;
1538         }
1539
1540         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1541                 if (!adev->ip_blocks[i].status.sw)
1542                         continue;
1543                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
1544                 /* XXX handle errors */
1545                 if (r) {
1546                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
1547                                   adev->ip_blocks[i].version->funcs->name, r);
1548                 }
1549                 adev->ip_blocks[i].status.sw = false;
1550                 adev->ip_blocks[i].status.valid = false;
1551         }
1552
1553         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1554                 if (!adev->ip_blocks[i].status.late_initialized)
1555                         continue;
1556                 if (adev->ip_blocks[i].version->funcs->late_fini)
1557                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
1558                 adev->ip_blocks[i].status.late_initialized = false;
1559         }
1560
1561         if (amdgpu_sriov_vf(adev)) {
1562                 amdgpu_bo_free_kernel(&adev->virt.csa_obj, &adev->virt.csa_vmid0_addr, NULL);
1563                 amdgpu_virt_release_full_gpu(adev, false);
1564         }
1565
1566         return 0;
1567 }
1568
1569 int amdgpu_suspend(struct amdgpu_device *adev)
1570 {
1571         int i, r;
1572
1573         if (amdgpu_sriov_vf(adev))
1574                 amdgpu_virt_request_full_gpu(adev, false);
1575
1576         /* ungate SMC block first */
1577         r = amdgpu_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC,
1578                                          AMD_CG_STATE_UNGATE);
1579         if (r) {
1580                 DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n",r);
1581         }
1582
1583         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
1584                 if (!adev->ip_blocks[i].status.valid)
1585                         continue;
1586                 /* ungate blocks so that suspend can properly shut them down */
1587                 if (i != AMD_IP_BLOCK_TYPE_SMC) {
1588                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
1589                                                                                      AMD_CG_STATE_UNGATE);
1590                         if (r) {
1591                                 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n",
1592                                           adev->ip_blocks[i].version->funcs->name, r);
1593                         }
1594                 }
1595                 /* XXX handle errors */
1596                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
1597                 /* XXX handle errors */
1598                 if (r) {
1599                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
1600                                   adev->ip_blocks[i].version->funcs->name, r);
1601                 }
1602         }
1603
1604         if (amdgpu_sriov_vf(adev))
1605                 amdgpu_virt_release_full_gpu(adev, false);
1606
1607         return 0;
1608 }
1609
1610 static int amdgpu_resume(struct amdgpu_device *adev)
1611 {
1612         int i, r;
1613
1614         for (i = 0; i < adev->num_ip_blocks; i++) {
1615                 if (!adev->ip_blocks[i].status.valid)
1616                         continue;
1617                 r = adev->ip_blocks[i].version->funcs->resume(adev);
1618                 if (r) {
1619                         DRM_ERROR("resume of IP block <%s> failed %d\n",
1620                                   adev->ip_blocks[i].version->funcs->name, r);
1621                         return r;
1622                 }
1623         }
1624
1625         return 0;
1626 }
1627
1628 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
1629 {
1630         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
1631                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
1632 }
1633
1634 /**
1635  * amdgpu_device_init - initialize the driver
1636  *
1637  * @adev: amdgpu_device pointer
1638  * @pdev: drm dev pointer
1639  * @pdev: pci dev pointer
1640  * @flags: driver flags
1641  *
1642  * Initializes the driver info and hw (all asics).
1643  * Returns 0 for success or an error on failure.
1644  * Called at driver startup.
1645  */
1646 int amdgpu_device_init(struct amdgpu_device *adev,
1647                        struct drm_device *ddev,
1648                        struct pci_dev *pdev,
1649                        uint32_t flags)
1650 {
1651         int r, i;
1652         bool runtime = false;
1653         u32 max_MBps;
1654
1655         adev->shutdown = false;
1656         adev->dev = &pdev->dev;
1657         adev->ddev = ddev;
1658         adev->pdev = pdev;
1659         adev->flags = flags;
1660         adev->asic_type = flags & AMD_ASIC_MASK;
1661         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
1662         adev->mc.gtt_size = 512 * 1024 * 1024;
1663         adev->accel_working = false;
1664         adev->num_rings = 0;
1665         adev->mman.buffer_funcs = NULL;
1666         adev->mman.buffer_funcs_ring = NULL;
1667         adev->vm_manager.vm_pte_funcs = NULL;
1668         adev->vm_manager.vm_pte_num_rings = 0;
1669         adev->gart.gart_funcs = NULL;
1670         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
1671
1672         adev->smc_rreg = &amdgpu_invalid_rreg;
1673         adev->smc_wreg = &amdgpu_invalid_wreg;
1674         adev->pcie_rreg = &amdgpu_invalid_rreg;
1675         adev->pcie_wreg = &amdgpu_invalid_wreg;
1676         adev->pciep_rreg = &amdgpu_invalid_rreg;
1677         adev->pciep_wreg = &amdgpu_invalid_wreg;
1678         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
1679         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
1680         adev->didt_rreg = &amdgpu_invalid_rreg;
1681         adev->didt_wreg = &amdgpu_invalid_wreg;
1682         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
1683         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
1684         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
1685         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
1686
1687
1688         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
1689                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
1690                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
1691
1692         /* mutex initialization are all done here so we
1693          * can recall function without having locking issues */
1694         mutex_init(&adev->vm_manager.lock);
1695         atomic_set(&adev->irq.ih.lock, 0);
1696         mutex_init(&adev->pm.mutex);
1697         mutex_init(&adev->gfx.gpu_clock_mutex);
1698         mutex_init(&adev->srbm_mutex);
1699         mutex_init(&adev->grbm_idx_mutex);
1700         mutex_init(&adev->mn_lock);
1701         hash_init(adev->mn_hash);
1702
1703         amdgpu_check_arguments(adev);
1704
1705         /* Registers mapping */
1706         /* TODO: block userspace mapping of io register */
1707         spin_lock_init(&adev->mmio_idx_lock);
1708         spin_lock_init(&adev->smc_idx_lock);
1709         spin_lock_init(&adev->pcie_idx_lock);
1710         spin_lock_init(&adev->uvd_ctx_idx_lock);
1711         spin_lock_init(&adev->didt_idx_lock);
1712         spin_lock_init(&adev->gc_cac_idx_lock);
1713         spin_lock_init(&adev->audio_endpt_idx_lock);
1714         spin_lock_init(&adev->mm_stats.lock);
1715
1716         INIT_LIST_HEAD(&adev->shadow_list);
1717         mutex_init(&adev->shadow_list_lock);
1718
1719         INIT_LIST_HEAD(&adev->gtt_list);
1720         spin_lock_init(&adev->gtt_list_lock);
1721
1722         if (adev->asic_type >= CHIP_BONAIRE) {
1723                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
1724                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
1725         } else {
1726                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
1727                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
1728         }
1729
1730         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
1731         if (adev->rmmio == NULL) {
1732                 return -ENOMEM;
1733         }
1734         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
1735         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
1736
1737         if (adev->asic_type >= CHIP_BONAIRE)
1738                 /* doorbell bar mapping */
1739                 amdgpu_doorbell_init(adev);
1740
1741         /* io port mapping */
1742         for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) {
1743                 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) {
1744                         adev->rio_mem_size = pci_resource_len(adev->pdev, i);
1745                         adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size);
1746                         break;
1747                 }
1748         }
1749         if (adev->rio_mem == NULL)
1750                 DRM_INFO("PCI I/O BAR is not found.\n");
1751
1752         /* early init functions */
1753         r = amdgpu_early_init(adev);
1754         if (r)
1755                 return r;
1756
1757         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
1758         /* this will fail for cards that aren't VGA class devices, just
1759          * ignore it */
1760         vga_client_register(adev->pdev, adev, NULL, amdgpu_vga_set_decode);
1761
1762         if (amdgpu_runtime_pm == 1)
1763                 runtime = true;
1764         if (amdgpu_device_is_px(ddev))
1765                 runtime = true;
1766         vga_switcheroo_register_client(adev->pdev, &amdgpu_switcheroo_ops, runtime);
1767         if (runtime)
1768                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
1769
1770         /* Read BIOS */
1771         if (!amdgpu_get_bios(adev)) {
1772                 r = -EINVAL;
1773                 goto failed;
1774         }
1775
1776         r = amdgpu_atombios_init(adev);
1777         if (r) {
1778                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
1779                 goto failed;
1780         }
1781
1782         /* detect if we are with an SRIOV vbios */
1783         amdgpu_device_detect_sriov_bios(adev);
1784
1785         /* Post card if necessary */
1786         if (amdgpu_vpost_needed(adev)) {
1787                 if (!adev->bios) {
1788                         dev_err(adev->dev, "no vBIOS found\n");
1789                         r = -EINVAL;
1790                         goto failed;
1791                 }
1792                 DRM_INFO("GPU posting now...\n");
1793                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
1794                 if (r) {
1795                         dev_err(adev->dev, "gpu post error!\n");
1796                         goto failed;
1797                 }
1798         } else {
1799                 DRM_INFO("GPU post is not needed\n");
1800         }
1801
1802         /* Initialize clocks */
1803         r = amdgpu_atombios_get_clock_info(adev);
1804         if (r) {
1805                 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
1806                 goto failed;
1807         }
1808         /* init i2c buses */
1809         amdgpu_atombios_i2c_init(adev);
1810
1811         /* Fence driver */
1812         r = amdgpu_fence_driver_init(adev);
1813         if (r) {
1814                 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
1815                 goto failed;
1816         }
1817
1818         /* init the mode config */
1819         drm_mode_config_init(adev->ddev);
1820
1821         r = amdgpu_init(adev);
1822         if (r) {
1823                 dev_err(adev->dev, "amdgpu_init failed\n");
1824                 amdgpu_fini(adev);
1825                 goto failed;
1826         }
1827
1828         adev->accel_working = true;
1829
1830         /* Initialize the buffer migration limit. */
1831         if (amdgpu_moverate >= 0)
1832                 max_MBps = amdgpu_moverate;
1833         else
1834                 max_MBps = 8; /* Allow 8 MB/s. */
1835         /* Get a log2 for easy divisions. */
1836         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
1837
1838         amdgpu_fbdev_init(adev);
1839
1840         r = amdgpu_ib_pool_init(adev);
1841         if (r) {
1842                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
1843                 goto failed;
1844         }
1845
1846         r = amdgpu_ib_ring_tests(adev);
1847         if (r)
1848                 DRM_ERROR("ib ring test failed (%d).\n", r);
1849
1850         r = amdgpu_gem_debugfs_init(adev);
1851         if (r) {
1852                 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
1853         }
1854
1855         r = amdgpu_debugfs_regs_init(adev);
1856         if (r) {
1857                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1858         }
1859
1860         r = amdgpu_debugfs_firmware_init(adev);
1861         if (r) {
1862                 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
1863                 return r;
1864         }
1865
1866         if ((amdgpu_testing & 1)) {
1867                 if (adev->accel_working)
1868                         amdgpu_test_moves(adev);
1869                 else
1870                         DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
1871         }
1872         if ((amdgpu_testing & 2)) {
1873                 if (adev->accel_working)
1874                         amdgpu_test_syncing(adev);
1875                 else
1876                         DRM_INFO("amdgpu: acceleration disabled, skipping sync tests\n");
1877         }
1878         if (amdgpu_benchmarking) {
1879                 if (adev->accel_working)
1880                         amdgpu_benchmark(adev, amdgpu_benchmarking);
1881                 else
1882                         DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
1883         }
1884
1885         /* enable clockgating, etc. after ib tests, etc. since some blocks require
1886          * explicit gating rather than handling it automatically.
1887          */
1888         r = amdgpu_late_init(adev);
1889         if (r) {
1890                 dev_err(adev->dev, "amdgpu_late_init failed\n");
1891                 goto failed;
1892         }
1893
1894         return 0;
1895
1896 failed:
1897         if (runtime)
1898                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
1899         return r;
1900 }
1901
1902 /**
1903  * amdgpu_device_fini - tear down the driver
1904  *
1905  * @adev: amdgpu_device pointer
1906  *
1907  * Tear down the driver info (all asics).
1908  * Called at driver shutdown.
1909  */
1910 void amdgpu_device_fini(struct amdgpu_device *adev)
1911 {
1912         int r;
1913
1914         DRM_INFO("amdgpu: finishing device.\n");
1915         adev->shutdown = true;
1916         drm_crtc_force_disable_all(adev->ddev);
1917         /* evict vram memory */
1918         amdgpu_bo_evict_vram(adev);
1919         amdgpu_ib_pool_fini(adev);
1920         amdgpu_fence_driver_fini(adev);
1921         amdgpu_fbdev_fini(adev);
1922         r = amdgpu_fini(adev);
1923         adev->accel_working = false;
1924         /* free i2c buses */
1925         amdgpu_i2c_fini(adev);
1926         amdgpu_atombios_fini(adev);
1927         kfree(adev->bios);
1928         adev->bios = NULL;
1929         vga_switcheroo_unregister_client(adev->pdev);
1930         if (adev->flags & AMD_IS_PX)
1931                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
1932         vga_client_register(adev->pdev, NULL, NULL, NULL);
1933         if (adev->rio_mem)
1934                 pci_iounmap(adev->pdev, adev->rio_mem);
1935         adev->rio_mem = NULL;
1936         iounmap(adev->rmmio);
1937         adev->rmmio = NULL;
1938         if (adev->asic_type >= CHIP_BONAIRE)
1939                 amdgpu_doorbell_fini(adev);
1940         amdgpu_debugfs_regs_cleanup(adev);
1941 }
1942
1943
1944 /*
1945  * Suspend & resume.
1946  */
1947 /**
1948  * amdgpu_device_suspend - initiate device suspend
1949  *
1950  * @pdev: drm dev pointer
1951  * @state: suspend state
1952  *
1953  * Puts the hw in the suspend state (all asics).
1954  * Returns 0 for success or an error on failure.
1955  * Called at driver suspend.
1956  */
1957 int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon)
1958 {
1959         struct amdgpu_device *adev;
1960         struct drm_crtc *crtc;
1961         struct drm_connector *connector;
1962         int r;
1963
1964         if (dev == NULL || dev->dev_private == NULL) {
1965                 return -ENODEV;
1966         }
1967
1968         adev = dev->dev_private;
1969
1970         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
1971                 return 0;
1972
1973         drm_kms_helper_poll_disable(dev);
1974
1975         /* turn off display hw */
1976         drm_modeset_lock_all(dev);
1977         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1978                 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF);
1979         }
1980         drm_modeset_unlock_all(dev);
1981
1982         /* unpin the front buffers and cursors */
1983         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
1984                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
1985                 struct amdgpu_framebuffer *rfb = to_amdgpu_framebuffer(crtc->primary->fb);
1986                 struct amdgpu_bo *robj;
1987
1988                 if (amdgpu_crtc->cursor_bo) {
1989                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
1990                         r = amdgpu_bo_reserve(aobj, false);
1991                         if (r == 0) {
1992                                 amdgpu_bo_unpin(aobj);
1993                                 amdgpu_bo_unreserve(aobj);
1994                         }
1995                 }
1996
1997                 if (rfb == NULL || rfb->obj == NULL) {
1998                         continue;
1999                 }
2000                 robj = gem_to_amdgpu_bo(rfb->obj);
2001                 /* don't unpin kernel fb objects */
2002                 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) {
2003                         r = amdgpu_bo_reserve(robj, false);
2004                         if (r == 0) {
2005                                 amdgpu_bo_unpin(robj);
2006                                 amdgpu_bo_unreserve(robj);
2007                         }
2008                 }
2009         }
2010         /* evict vram memory */
2011         amdgpu_bo_evict_vram(adev);
2012
2013         amdgpu_fence_driver_suspend(adev);
2014
2015         r = amdgpu_suspend(adev);
2016
2017         /* evict remaining vram memory
2018          * This second call to evict vram is to evict the gart page table
2019          * using the CPU.
2020          */
2021         amdgpu_bo_evict_vram(adev);
2022
2023         amdgpu_atombios_scratch_regs_save(adev);
2024         pci_save_state(dev->pdev);
2025         if (suspend) {
2026                 /* Shut down the device */
2027                 pci_disable_device(dev->pdev);
2028                 pci_set_power_state(dev->pdev, PCI_D3hot);
2029         } else {
2030                 r = amdgpu_asic_reset(adev);
2031                 if (r)
2032                         DRM_ERROR("amdgpu asic reset failed\n");
2033         }
2034
2035         if (fbcon) {
2036                 console_lock();
2037                 amdgpu_fbdev_set_suspend(adev, 1);
2038                 console_unlock();
2039         }
2040         return 0;
2041 }
2042
2043 /**
2044  * amdgpu_device_resume - initiate device resume
2045  *
2046  * @pdev: drm dev pointer
2047  *
2048  * Bring the hw back to operating state (all asics).
2049  * Returns 0 for success or an error on failure.
2050  * Called at driver resume.
2051  */
2052 int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon)
2053 {
2054         struct drm_connector *connector;
2055         struct amdgpu_device *adev = dev->dev_private;
2056         struct drm_crtc *crtc;
2057         int r;
2058
2059         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
2060                 return 0;
2061
2062         if (fbcon)
2063                 console_lock();
2064
2065         if (resume) {
2066                 pci_set_power_state(dev->pdev, PCI_D0);
2067                 pci_restore_state(dev->pdev);
2068                 r = pci_enable_device(dev->pdev);
2069                 if (r) {
2070                         if (fbcon)
2071                                 console_unlock();
2072                         return r;
2073                 }
2074         }
2075         amdgpu_atombios_scratch_regs_restore(adev);
2076
2077         /* post card */
2078         if (amdgpu_need_post(adev)) {
2079                 r = amdgpu_atom_asic_init(adev->mode_info.atom_context);
2080                 if (r)
2081                         DRM_ERROR("amdgpu asic init failed\n");
2082         }
2083
2084         r = amdgpu_resume(adev);
2085         if (r)
2086                 DRM_ERROR("amdgpu_resume failed (%d).\n", r);
2087
2088         amdgpu_fence_driver_resume(adev);
2089
2090         if (resume) {
2091                 r = amdgpu_ib_ring_tests(adev);
2092                 if (r)
2093                         DRM_ERROR("ib ring test failed (%d).\n", r);
2094         }
2095
2096         r = amdgpu_late_init(adev);
2097         if (r) {
2098                 if (fbcon)
2099                         console_unlock();
2100                 return r;
2101         }
2102
2103         /* pin cursors */
2104         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
2105                 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
2106
2107                 if (amdgpu_crtc->cursor_bo) {
2108                         struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo);
2109                         r = amdgpu_bo_reserve(aobj, false);
2110                         if (r == 0) {
2111                                 r = amdgpu_bo_pin(aobj,
2112                                                   AMDGPU_GEM_DOMAIN_VRAM,
2113                                                   &amdgpu_crtc->cursor_addr);
2114                                 if (r != 0)
2115                                         DRM_ERROR("Failed to pin cursor BO (%d)\n", r);
2116                                 amdgpu_bo_unreserve(aobj);
2117                         }
2118                 }
2119         }
2120
2121         /* blat the mode back in */
2122         if (fbcon) {
2123                 drm_helper_resume_force_mode(dev);
2124                 /* turn on display hw */
2125                 drm_modeset_lock_all(dev);
2126                 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
2127                         drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
2128                 }
2129                 drm_modeset_unlock_all(dev);
2130         }
2131
2132         drm_kms_helper_poll_enable(dev);
2133
2134         /*
2135          * Most of the connector probing functions try to acquire runtime pm
2136          * refs to ensure that the GPU is powered on when connector polling is
2137          * performed. Since we're calling this from a runtime PM callback,
2138          * trying to acquire rpm refs will cause us to deadlock.
2139          *
2140          * Since we're guaranteed to be holding the rpm lock, it's safe to
2141          * temporarily disable the rpm helpers so this doesn't deadlock us.
2142          */
2143 #ifdef CONFIG_PM
2144         dev->dev->power.disable_depth++;
2145 #endif
2146         drm_helper_hpd_irq_event(dev);
2147 #ifdef CONFIG_PM
2148         dev->dev->power.disable_depth--;
2149 #endif
2150
2151         if (fbcon) {
2152                 amdgpu_fbdev_set_suspend(adev, 0);
2153                 console_unlock();
2154         }
2155
2156         return 0;
2157 }
2158
2159 static bool amdgpu_check_soft_reset(struct amdgpu_device *adev)
2160 {
2161         int i;
2162         bool asic_hang = false;
2163
2164         for (i = 0; i < adev->num_ip_blocks; i++) {
2165                 if (!adev->ip_blocks[i].status.valid)
2166                         continue;
2167                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
2168                         adev->ip_blocks[i].status.hang =
2169                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
2170                 if (adev->ip_blocks[i].status.hang) {
2171                         DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
2172                         asic_hang = true;
2173                 }
2174         }
2175         return asic_hang;
2176 }
2177
2178 static int amdgpu_pre_soft_reset(struct amdgpu_device *adev)
2179 {
2180         int i, r = 0;
2181
2182         for (i = 0; i < adev->num_ip_blocks; i++) {
2183                 if (!adev->ip_blocks[i].status.valid)
2184                         continue;
2185                 if (adev->ip_blocks[i].status.hang &&
2186                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
2187                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
2188                         if (r)
2189                                 return r;
2190                 }
2191         }
2192
2193         return 0;
2194 }
2195
2196 static bool amdgpu_need_full_reset(struct amdgpu_device *adev)
2197 {
2198         int i;
2199
2200         for (i = 0; i < adev->num_ip_blocks; i++) {
2201                 if (!adev->ip_blocks[i].status.valid)
2202                         continue;
2203                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
2204                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
2205                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
2206                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)) {
2207                         if (adev->ip_blocks[i].status.hang) {
2208                                 DRM_INFO("Some block need full reset!\n");
2209                                 return true;
2210                         }
2211                 }
2212         }
2213         return false;
2214 }
2215
2216 static int amdgpu_soft_reset(struct amdgpu_device *adev)
2217 {
2218         int i, r = 0;
2219
2220         for (i = 0; i < adev->num_ip_blocks; i++) {
2221                 if (!adev->ip_blocks[i].status.valid)
2222                         continue;
2223                 if (adev->ip_blocks[i].status.hang &&
2224                     adev->ip_blocks[i].version->funcs->soft_reset) {
2225                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
2226                         if (r)
2227                                 return r;
2228                 }
2229         }
2230
2231         return 0;
2232 }
2233
2234 static int amdgpu_post_soft_reset(struct amdgpu_device *adev)
2235 {
2236         int i, r = 0;
2237
2238         for (i = 0; i < adev->num_ip_blocks; i++) {
2239                 if (!adev->ip_blocks[i].status.valid)
2240                         continue;
2241                 if (adev->ip_blocks[i].status.hang &&
2242                     adev->ip_blocks[i].version->funcs->post_soft_reset)
2243                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
2244                 if (r)
2245                         return r;
2246         }
2247
2248         return 0;
2249 }
2250
2251 bool amdgpu_need_backup(struct amdgpu_device *adev)
2252 {
2253         if (adev->flags & AMD_IS_APU)
2254                 return false;
2255
2256         return amdgpu_lockup_timeout > 0 ? true : false;
2257 }
2258
2259 static int amdgpu_recover_vram_from_shadow(struct amdgpu_device *adev,
2260                                            struct amdgpu_ring *ring,
2261                                            struct amdgpu_bo *bo,
2262                                            struct dma_fence **fence)
2263 {
2264         uint32_t domain;
2265         int r;
2266
2267        if (!bo->shadow)
2268                return 0;
2269
2270        r = amdgpu_bo_reserve(bo, false);
2271        if (r)
2272                return r;
2273        domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type);
2274        /* if bo has been evicted, then no need to recover */
2275        if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
2276                r = amdgpu_bo_restore_from_shadow(adev, ring, bo,
2277                                                  NULL, fence, true);
2278                if (r) {
2279                        DRM_ERROR("recover page table failed!\n");
2280                        goto err;
2281                }
2282        }
2283 err:
2284        amdgpu_bo_unreserve(bo);
2285        return r;
2286 }
2287
2288 /**
2289  * amdgpu_gpu_reset - reset the asic
2290  *
2291  * @adev: amdgpu device pointer
2292  *
2293  * Attempt the reset the GPU if it has hung (all asics).
2294  * Returns 0 for success or an error on failure.
2295  */
2296 int amdgpu_gpu_reset(struct amdgpu_device *adev)
2297 {
2298         int i, r;
2299         int resched;
2300         bool need_full_reset;
2301
2302         if (amdgpu_sriov_vf(adev))
2303                 return 0;
2304
2305         if (!amdgpu_check_soft_reset(adev)) {
2306                 DRM_INFO("No hardware hang detected. Did some blocks stall?\n");
2307                 return 0;
2308         }
2309
2310         atomic_inc(&adev->gpu_reset_counter);
2311
2312         /* block TTM */
2313         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
2314
2315         /* block scheduler */
2316         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2317                 struct amdgpu_ring *ring = adev->rings[i];
2318
2319                 if (!ring)
2320                         continue;
2321                 kthread_park(ring->sched.thread);
2322                 amd_sched_hw_job_reset(&ring->sched);
2323         }
2324         /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
2325         amdgpu_fence_driver_force_completion(adev);
2326
2327         need_full_reset = amdgpu_need_full_reset(adev);
2328
2329         if (!need_full_reset) {
2330                 amdgpu_pre_soft_reset(adev);
2331                 r = amdgpu_soft_reset(adev);
2332                 amdgpu_post_soft_reset(adev);
2333                 if (r || amdgpu_check_soft_reset(adev)) {
2334                         DRM_INFO("soft reset failed, will fallback to full reset!\n");
2335                         need_full_reset = true;
2336                 }
2337         }
2338
2339         if (need_full_reset) {
2340                 r = amdgpu_suspend(adev);
2341
2342 retry:
2343                 /* Disable fb access */
2344                 if (adev->mode_info.num_crtc) {
2345                         struct amdgpu_mode_mc_save save;
2346                         amdgpu_display_stop_mc_access(adev, &save);
2347                         amdgpu_wait_for_idle(adev, AMD_IP_BLOCK_TYPE_GMC);
2348                 }
2349                 amdgpu_atombios_scratch_regs_save(adev);
2350                 r = amdgpu_asic_reset(adev);
2351                 amdgpu_atombios_scratch_regs_restore(adev);
2352                 /* post card */
2353                 amdgpu_atom_asic_init(adev->mode_info.atom_context);
2354
2355                 if (!r) {
2356                         dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
2357                         r = amdgpu_resume(adev);
2358                 }
2359         }
2360         if (!r) {
2361                 amdgpu_irq_gpu_reset_resume_helper(adev);
2362                 if (need_full_reset && amdgpu_need_backup(adev)) {
2363                         r = amdgpu_ttm_recover_gart(adev);
2364                         if (r)
2365                                 DRM_ERROR("gart recovery failed!!!\n");
2366                 }
2367                 r = amdgpu_ib_ring_tests(adev);
2368                 if (r) {
2369                         dev_err(adev->dev, "ib ring test failed (%d).\n", r);
2370                         r = amdgpu_suspend(adev);
2371                         need_full_reset = true;
2372                         goto retry;
2373                 }
2374                 /**
2375                  * recovery vm page tables, since we cannot depend on VRAM is
2376                  * consistent after gpu full reset.
2377                  */
2378                 if (need_full_reset && amdgpu_need_backup(adev)) {
2379                         struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
2380                         struct amdgpu_bo *bo, *tmp;
2381                         struct dma_fence *fence = NULL, *next = NULL;
2382
2383                         DRM_INFO("recover vram bo from shadow\n");
2384                         mutex_lock(&adev->shadow_list_lock);
2385                         list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) {
2386                                 amdgpu_recover_vram_from_shadow(adev, ring, bo, &next);
2387                                 if (fence) {
2388                                         r = dma_fence_wait(fence, false);
2389                                         if (r) {
2390                                                 WARN(r, "recovery from shadow isn't comleted\n");
2391                                                 break;
2392                                         }
2393                                 }
2394
2395                                 dma_fence_put(fence);
2396                                 fence = next;
2397                         }
2398                         mutex_unlock(&adev->shadow_list_lock);
2399                         if (fence) {
2400                                 r = dma_fence_wait(fence, false);
2401                                 if (r)
2402                                         WARN(r, "recovery from shadow isn't comleted\n");
2403                         }
2404                         dma_fence_put(fence);
2405                 }
2406                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2407                         struct amdgpu_ring *ring = adev->rings[i];
2408                         if (!ring)
2409                                 continue;
2410
2411                         amd_sched_job_recovery(&ring->sched);
2412                         kthread_unpark(ring->sched.thread);
2413                 }
2414         } else {
2415                 dev_err(adev->dev, "asic resume failed (%d).\n", r);
2416                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2417                         if (adev->rings[i]) {
2418                                 kthread_unpark(adev->rings[i]->sched.thread);
2419                         }
2420                 }
2421         }
2422
2423         drm_helper_resume_force_mode(adev->ddev);
2424
2425         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
2426         if (r) {
2427                 /* bad news, how to tell it to userspace ? */
2428                 dev_info(adev->dev, "GPU reset failed\n");
2429         }
2430
2431         return r;
2432 }
2433
2434 void amdgpu_get_pcie_info(struct amdgpu_device *adev)
2435 {
2436         u32 mask;
2437         int ret;
2438
2439         if (amdgpu_pcie_gen_cap)
2440                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
2441
2442         if (amdgpu_pcie_lane_cap)
2443                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
2444
2445         /* covers APUs as well */
2446         if (pci_is_root_bus(adev->pdev->bus)) {
2447                 if (adev->pm.pcie_gen_mask == 0)
2448                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
2449                 if (adev->pm.pcie_mlw_mask == 0)
2450                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
2451                 return;
2452         }
2453
2454         if (adev->pm.pcie_gen_mask == 0) {
2455                 ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
2456                 if (!ret) {
2457                         adev->pm.pcie_gen_mask = (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
2458                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
2459                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
2460
2461                         if (mask & DRM_PCIE_SPEED_25)
2462                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
2463                         if (mask & DRM_PCIE_SPEED_50)
2464                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2;
2465                         if (mask & DRM_PCIE_SPEED_80)
2466                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3;
2467                 } else {
2468                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
2469                 }
2470         }
2471         if (adev->pm.pcie_mlw_mask == 0) {
2472                 ret = drm_pcie_get_max_link_width(adev->ddev, &mask);
2473                 if (!ret) {
2474                         switch (mask) {
2475                         case 32:
2476                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
2477                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
2478                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2479                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2480                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2481                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2482                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2483                                 break;
2484                         case 16:
2485                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
2486                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2487                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2488                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2489                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2490                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2491                                 break;
2492                         case 12:
2493                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
2494                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2495                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2496                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2497                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2498                                 break;
2499                         case 8:
2500                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
2501                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2502                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2503                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2504                                 break;
2505                         case 4:
2506                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
2507                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2508                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2509                                 break;
2510                         case 2:
2511                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
2512                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
2513                                 break;
2514                         case 1:
2515                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
2516                                 break;
2517                         default:
2518                                 break;
2519                         }
2520                 } else {
2521                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
2522                 }
2523         }
2524 }
2525
2526 /*
2527  * Debugfs
2528  */
2529 int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
2530                              const struct drm_info_list *files,
2531                              unsigned nfiles)
2532 {
2533         unsigned i;
2534
2535         for (i = 0; i < adev->debugfs_count; i++) {
2536                 if (adev->debugfs[i].files == files) {
2537                         /* Already registered */
2538                         return 0;
2539                 }
2540         }
2541
2542         i = adev->debugfs_count + 1;
2543         if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
2544                 DRM_ERROR("Reached maximum number of debugfs components.\n");
2545                 DRM_ERROR("Report so we increase "
2546                           "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
2547                 return -EINVAL;
2548         }
2549         adev->debugfs[adev->debugfs_count].files = files;
2550         adev->debugfs[adev->debugfs_count].num_files = nfiles;
2551         adev->debugfs_count = i;
2552 #if defined(CONFIG_DEBUG_FS)
2553         drm_debugfs_create_files(files, nfiles,
2554                                  adev->ddev->primary->debugfs_root,
2555                                  adev->ddev->primary);
2556 #endif
2557         return 0;
2558 }
2559
2560 #if defined(CONFIG_DEBUG_FS)
2561
2562 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
2563                                         size_t size, loff_t *pos)
2564 {
2565         struct amdgpu_device *adev = file_inode(f)->i_private;
2566         ssize_t result = 0;
2567         int r;
2568         bool pm_pg_lock, use_bank;
2569         unsigned instance_bank, sh_bank, se_bank;
2570
2571         if (size & 0x3 || *pos & 0x3)
2572                 return -EINVAL;
2573
2574         /* are we reading registers for which a PG lock is necessary? */
2575         pm_pg_lock = (*pos >> 23) & 1;
2576
2577         if (*pos & (1ULL << 62)) {
2578                 se_bank = (*pos >> 24) & 0x3FF;
2579                 sh_bank = (*pos >> 34) & 0x3FF;
2580                 instance_bank = (*pos >> 44) & 0x3FF;
2581
2582                 if (se_bank == 0x3FF)
2583                         se_bank = 0xFFFFFFFF;
2584                 if (sh_bank == 0x3FF)
2585                         sh_bank = 0xFFFFFFFF;
2586                 if (instance_bank == 0x3FF)
2587                         instance_bank = 0xFFFFFFFF;
2588                 use_bank = 1;
2589         } else {
2590                 use_bank = 0;
2591         }
2592
2593         *pos &= (1UL << 22) - 1;
2594
2595         if (use_bank) {
2596                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
2597                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
2598                         return -EINVAL;
2599                 mutex_lock(&adev->grbm_idx_mutex);
2600                 amdgpu_gfx_select_se_sh(adev, se_bank,
2601                                         sh_bank, instance_bank);
2602         }
2603
2604         if (pm_pg_lock)
2605                 mutex_lock(&adev->pm.mutex);
2606
2607         while (size) {
2608                 uint32_t value;
2609
2610                 if (*pos > adev->rmmio_size)
2611                         goto end;
2612
2613                 value = RREG32(*pos >> 2);
2614                 r = put_user(value, (uint32_t *)buf);
2615                 if (r) {
2616                         result = r;
2617                         goto end;
2618                 }
2619
2620                 result += 4;
2621                 buf += 4;
2622                 *pos += 4;
2623                 size -= 4;
2624         }
2625
2626 end:
2627         if (use_bank) {
2628                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2629                 mutex_unlock(&adev->grbm_idx_mutex);
2630         }
2631
2632         if (pm_pg_lock)
2633                 mutex_unlock(&adev->pm.mutex);
2634
2635         return result;
2636 }
2637
2638 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
2639                                          size_t size, loff_t *pos)
2640 {
2641         struct amdgpu_device *adev = file_inode(f)->i_private;
2642         ssize_t result = 0;
2643         int r;
2644         bool pm_pg_lock, use_bank;
2645         unsigned instance_bank, sh_bank, se_bank;
2646
2647         if (size & 0x3 || *pos & 0x3)
2648                 return -EINVAL;
2649
2650         /* are we reading registers for which a PG lock is necessary? */
2651         pm_pg_lock = (*pos >> 23) & 1;
2652
2653         if (*pos & (1ULL << 62)) {
2654                 se_bank = (*pos >> 24) & 0x3FF;
2655                 sh_bank = (*pos >> 34) & 0x3FF;
2656                 instance_bank = (*pos >> 44) & 0x3FF;
2657
2658                 if (se_bank == 0x3FF)
2659                         se_bank = 0xFFFFFFFF;
2660                 if (sh_bank == 0x3FF)
2661                         sh_bank = 0xFFFFFFFF;
2662                 if (instance_bank == 0x3FF)
2663                         instance_bank = 0xFFFFFFFF;
2664                 use_bank = 1;
2665         } else {
2666                 use_bank = 0;
2667         }
2668
2669         *pos &= (1UL << 22) - 1;
2670
2671         if (use_bank) {
2672                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
2673                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
2674                         return -EINVAL;
2675                 mutex_lock(&adev->grbm_idx_mutex);
2676                 amdgpu_gfx_select_se_sh(adev, se_bank,
2677                                         sh_bank, instance_bank);
2678         }
2679
2680         if (pm_pg_lock)
2681                 mutex_lock(&adev->pm.mutex);
2682
2683         while (size) {
2684                 uint32_t value;
2685
2686                 if (*pos > adev->rmmio_size)
2687                         return result;
2688
2689                 r = get_user(value, (uint32_t *)buf);
2690                 if (r)
2691                         return r;
2692
2693                 WREG32(*pos >> 2, value);
2694
2695                 result += 4;
2696                 buf += 4;
2697                 *pos += 4;
2698                 size -= 4;
2699         }
2700
2701         if (use_bank) {
2702                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2703                 mutex_unlock(&adev->grbm_idx_mutex);
2704         }
2705
2706         if (pm_pg_lock)
2707                 mutex_unlock(&adev->pm.mutex);
2708
2709         return result;
2710 }
2711
2712 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
2713                                         size_t size, loff_t *pos)
2714 {
2715         struct amdgpu_device *adev = file_inode(f)->i_private;
2716         ssize_t result = 0;
2717         int r;
2718
2719         if (size & 0x3 || *pos & 0x3)
2720                 return -EINVAL;
2721
2722         while (size) {
2723                 uint32_t value;
2724
2725                 value = RREG32_PCIE(*pos >> 2);
2726                 r = put_user(value, (uint32_t *)buf);
2727                 if (r)
2728                         return r;
2729
2730                 result += 4;
2731                 buf += 4;
2732                 *pos += 4;
2733                 size -= 4;
2734         }
2735
2736         return result;
2737 }
2738
2739 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
2740                                          size_t size, loff_t *pos)
2741 {
2742         struct amdgpu_device *adev = file_inode(f)->i_private;
2743         ssize_t result = 0;
2744         int r;
2745
2746         if (size & 0x3 || *pos & 0x3)
2747                 return -EINVAL;
2748
2749         while (size) {
2750                 uint32_t value;
2751
2752                 r = get_user(value, (uint32_t *)buf);
2753                 if (r)
2754                         return r;
2755
2756                 WREG32_PCIE(*pos >> 2, value);
2757
2758                 result += 4;
2759                 buf += 4;
2760                 *pos += 4;
2761                 size -= 4;
2762         }
2763
2764         return result;
2765 }
2766
2767 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
2768                                         size_t size, loff_t *pos)
2769 {
2770         struct amdgpu_device *adev = file_inode(f)->i_private;
2771         ssize_t result = 0;
2772         int r;
2773
2774         if (size & 0x3 || *pos & 0x3)
2775                 return -EINVAL;
2776
2777         while (size) {
2778                 uint32_t value;
2779
2780                 value = RREG32_DIDT(*pos >> 2);
2781                 r = put_user(value, (uint32_t *)buf);
2782                 if (r)
2783                         return r;
2784
2785                 result += 4;
2786                 buf += 4;
2787                 *pos += 4;
2788                 size -= 4;
2789         }
2790
2791         return result;
2792 }
2793
2794 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
2795                                          size_t size, loff_t *pos)
2796 {
2797         struct amdgpu_device *adev = file_inode(f)->i_private;
2798         ssize_t result = 0;
2799         int r;
2800
2801         if (size & 0x3 || *pos & 0x3)
2802                 return -EINVAL;
2803
2804         while (size) {
2805                 uint32_t value;
2806
2807                 r = get_user(value, (uint32_t *)buf);
2808                 if (r)
2809                         return r;
2810
2811                 WREG32_DIDT(*pos >> 2, value);
2812
2813                 result += 4;
2814                 buf += 4;
2815                 *pos += 4;
2816                 size -= 4;
2817         }
2818
2819         return result;
2820 }
2821
2822 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
2823                                         size_t size, loff_t *pos)
2824 {
2825         struct amdgpu_device *adev = file_inode(f)->i_private;
2826         ssize_t result = 0;
2827         int r;
2828
2829         if (size & 0x3 || *pos & 0x3)
2830                 return -EINVAL;
2831
2832         while (size) {
2833                 uint32_t value;
2834
2835                 value = RREG32_SMC(*pos);
2836                 r = put_user(value, (uint32_t *)buf);
2837                 if (r)
2838                         return r;
2839
2840                 result += 4;
2841                 buf += 4;
2842                 *pos += 4;
2843                 size -= 4;
2844         }
2845
2846         return result;
2847 }
2848
2849 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
2850                                          size_t size, loff_t *pos)
2851 {
2852         struct amdgpu_device *adev = file_inode(f)->i_private;
2853         ssize_t result = 0;
2854         int r;
2855
2856         if (size & 0x3 || *pos & 0x3)
2857                 return -EINVAL;
2858
2859         while (size) {
2860                 uint32_t value;
2861
2862                 r = get_user(value, (uint32_t *)buf);
2863                 if (r)
2864                         return r;
2865
2866                 WREG32_SMC(*pos, value);
2867
2868                 result += 4;
2869                 buf += 4;
2870                 *pos += 4;
2871                 size -= 4;
2872         }
2873
2874         return result;
2875 }
2876
2877 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
2878                                         size_t size, loff_t *pos)
2879 {
2880         struct amdgpu_device *adev = file_inode(f)->i_private;
2881         ssize_t result = 0;
2882         int r;
2883         uint32_t *config, no_regs = 0;
2884
2885         if (size & 0x3 || *pos & 0x3)
2886                 return -EINVAL;
2887
2888         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
2889         if (!config)
2890                 return -ENOMEM;
2891
2892         /* version, increment each time something is added */
2893         config[no_regs++] = 3;
2894         config[no_regs++] = adev->gfx.config.max_shader_engines;
2895         config[no_regs++] = adev->gfx.config.max_tile_pipes;
2896         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
2897         config[no_regs++] = adev->gfx.config.max_sh_per_se;
2898         config[no_regs++] = adev->gfx.config.max_backends_per_se;
2899         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
2900         config[no_regs++] = adev->gfx.config.max_gprs;
2901         config[no_regs++] = adev->gfx.config.max_gs_threads;
2902         config[no_regs++] = adev->gfx.config.max_hw_contexts;
2903         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
2904         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
2905         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
2906         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
2907         config[no_regs++] = adev->gfx.config.num_tile_pipes;
2908         config[no_regs++] = adev->gfx.config.backend_enable_mask;
2909         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
2910         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
2911         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
2912         config[no_regs++] = adev->gfx.config.num_gpus;
2913         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
2914         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
2915         config[no_regs++] = adev->gfx.config.gb_addr_config;
2916         config[no_regs++] = adev->gfx.config.num_rbs;
2917
2918         /* rev==1 */
2919         config[no_regs++] = adev->rev_id;
2920         config[no_regs++] = adev->pg_flags;
2921         config[no_regs++] = adev->cg_flags;
2922
2923         /* rev==2 */
2924         config[no_regs++] = adev->family;
2925         config[no_regs++] = adev->external_rev_id;
2926
2927         /* rev==3 */
2928         config[no_regs++] = adev->pdev->device;
2929         config[no_regs++] = adev->pdev->revision;
2930         config[no_regs++] = adev->pdev->subsystem_device;
2931         config[no_regs++] = adev->pdev->subsystem_vendor;
2932
2933         while (size && (*pos < no_regs * 4)) {
2934                 uint32_t value;
2935
2936                 value = config[*pos >> 2];
2937                 r = put_user(value, (uint32_t *)buf);
2938                 if (r) {
2939                         kfree(config);
2940                         return r;
2941                 }
2942
2943                 result += 4;
2944                 buf += 4;
2945                 *pos += 4;
2946                 size -= 4;
2947         }
2948
2949         kfree(config);
2950         return result;
2951 }
2952
2953 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
2954                                         size_t size, loff_t *pos)
2955 {
2956         struct amdgpu_device *adev = file_inode(f)->i_private;
2957         int idx, r;
2958         int32_t value;
2959
2960         if (size != 4 || *pos & 0x3)
2961                 return -EINVAL;
2962
2963         /* convert offset to sensor number */
2964         idx = *pos >> 2;
2965
2966         if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
2967                 r = adev->powerplay.pp_funcs->read_sensor(adev->powerplay.pp_handle, idx, &value);
2968         else
2969                 return -EINVAL;
2970
2971         if (!r)
2972                 r = put_user(value, (int32_t *)buf);
2973
2974         return !r ? 4 : r;
2975 }
2976
2977 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
2978                                         size_t size, loff_t *pos)
2979 {
2980         struct amdgpu_device *adev = f->f_inode->i_private;
2981         int r, x;
2982         ssize_t result=0;
2983         uint32_t offset, se, sh, cu, wave, simd, data[32];
2984
2985         if (size & 3 || *pos & 3)
2986                 return -EINVAL;
2987
2988         /* decode offset */
2989         offset = (*pos & 0x7F);
2990         se = ((*pos >> 7) & 0xFF);
2991         sh = ((*pos >> 15) & 0xFF);
2992         cu = ((*pos >> 23) & 0xFF);
2993         wave = ((*pos >> 31) & 0xFF);
2994         simd = ((*pos >> 37) & 0xFF);
2995
2996         /* switch to the specific se/sh/cu */
2997         mutex_lock(&adev->grbm_idx_mutex);
2998         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
2999
3000         x = 0;
3001         if (adev->gfx.funcs->read_wave_data)
3002                 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
3003
3004         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3005         mutex_unlock(&adev->grbm_idx_mutex);
3006
3007         if (!x)
3008                 return -EINVAL;
3009
3010         while (size && (offset < x * 4)) {
3011                 uint32_t value;
3012
3013                 value = data[offset >> 2];
3014                 r = put_user(value, (uint32_t *)buf);
3015                 if (r)
3016                         return r;
3017
3018                 result += 4;
3019                 buf += 4;
3020                 offset += 4;
3021                 size -= 4;
3022         }
3023
3024         return result;
3025 }
3026
3027 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
3028                                         size_t size, loff_t *pos)
3029 {
3030         struct amdgpu_device *adev = f->f_inode->i_private;
3031         int r;
3032         ssize_t result = 0;
3033         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
3034
3035         if (size & 3 || *pos & 3)
3036                 return -EINVAL;
3037
3038         /* decode offset */
3039         offset = (*pos & 0xFFF);       /* in dwords */
3040         se = ((*pos >> 12) & 0xFF);
3041         sh = ((*pos >> 20) & 0xFF);
3042         cu = ((*pos >> 28) & 0xFF);
3043         wave = ((*pos >> 36) & 0xFF);
3044         simd = ((*pos >> 44) & 0xFF);
3045         thread = ((*pos >> 52) & 0xFF);
3046         bank = ((*pos >> 60) & 1);
3047
3048         data = kmalloc_array(1024, sizeof(*data), GFP_KERNEL);
3049         if (!data)
3050                 return -ENOMEM;
3051
3052         /* switch to the specific se/sh/cu */
3053         mutex_lock(&adev->grbm_idx_mutex);
3054         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
3055
3056         if (bank == 0) {
3057                 if (adev->gfx.funcs->read_wave_vgprs)
3058                         adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
3059         } else {
3060                 if (adev->gfx.funcs->read_wave_sgprs)
3061                         adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
3062         }
3063
3064         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
3065         mutex_unlock(&adev->grbm_idx_mutex);
3066
3067         while (size) {
3068                 uint32_t value;
3069
3070                 value = data[offset++];
3071                 r = put_user(value, (uint32_t *)buf);
3072                 if (r) {
3073                         result = r;
3074                         goto err;
3075                 }
3076
3077                 result += 4;
3078                 buf += 4;
3079                 size -= 4;
3080         }
3081
3082 err:
3083         kfree(data);
3084         return result;
3085 }
3086
3087 static const struct file_operations amdgpu_debugfs_regs_fops = {
3088         .owner = THIS_MODULE,
3089         .read = amdgpu_debugfs_regs_read,
3090         .write = amdgpu_debugfs_regs_write,
3091         .llseek = default_llseek
3092 };
3093 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
3094         .owner = THIS_MODULE,
3095         .read = amdgpu_debugfs_regs_didt_read,
3096         .write = amdgpu_debugfs_regs_didt_write,
3097         .llseek = default_llseek
3098 };
3099 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
3100         .owner = THIS_MODULE,
3101         .read = amdgpu_debugfs_regs_pcie_read,
3102         .write = amdgpu_debugfs_regs_pcie_write,
3103         .llseek = default_llseek
3104 };
3105 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
3106         .owner = THIS_MODULE,
3107         .read = amdgpu_debugfs_regs_smc_read,
3108         .write = amdgpu_debugfs_regs_smc_write,
3109         .llseek = default_llseek
3110 };
3111
3112 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
3113         .owner = THIS_MODULE,
3114         .read = amdgpu_debugfs_gca_config_read,
3115         .llseek = default_llseek
3116 };
3117
3118 static const struct file_operations amdgpu_debugfs_sensors_fops = {
3119         .owner = THIS_MODULE,
3120         .read = amdgpu_debugfs_sensor_read,
3121         .llseek = default_llseek
3122 };
3123
3124 static const struct file_operations amdgpu_debugfs_wave_fops = {
3125         .owner = THIS_MODULE,
3126         .read = amdgpu_debugfs_wave_read,
3127         .llseek = default_llseek
3128 };
3129 static const struct file_operations amdgpu_debugfs_gpr_fops = {
3130         .owner = THIS_MODULE,
3131         .read = amdgpu_debugfs_gpr_read,
3132         .llseek = default_llseek
3133 };
3134
3135 static const struct file_operations *debugfs_regs[] = {
3136         &amdgpu_debugfs_regs_fops,
3137         &amdgpu_debugfs_regs_didt_fops,
3138         &amdgpu_debugfs_regs_pcie_fops,
3139         &amdgpu_debugfs_regs_smc_fops,
3140         &amdgpu_debugfs_gca_config_fops,
3141         &amdgpu_debugfs_sensors_fops,
3142         &amdgpu_debugfs_wave_fops,
3143         &amdgpu_debugfs_gpr_fops,
3144 };
3145
3146 static const char *debugfs_regs_names[] = {
3147         "amdgpu_regs",
3148         "amdgpu_regs_didt",
3149         "amdgpu_regs_pcie",
3150         "amdgpu_regs_smc",
3151         "amdgpu_gca_config",
3152         "amdgpu_sensors",
3153         "amdgpu_wave",
3154         "amdgpu_gpr",
3155 };
3156
3157 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3158 {
3159         struct drm_minor *minor = adev->ddev->primary;
3160         struct dentry *ent, *root = minor->debugfs_root;
3161         unsigned i, j;
3162
3163         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3164                 ent = debugfs_create_file(debugfs_regs_names[i],
3165                                           S_IFREG | S_IRUGO, root,
3166                                           adev, debugfs_regs[i]);
3167                 if (IS_ERR(ent)) {
3168                         for (j = 0; j < i; j++) {
3169                                 debugfs_remove(adev->debugfs_regs[i]);
3170                                 adev->debugfs_regs[i] = NULL;
3171                         }
3172                         return PTR_ERR(ent);
3173                 }
3174
3175                 if (!i)
3176                         i_size_write(ent->d_inode, adev->rmmio_size);
3177                 adev->debugfs_regs[i] = ent;
3178         }
3179
3180         return 0;
3181 }
3182
3183 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
3184 {
3185         unsigned i;
3186
3187         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
3188                 if (adev->debugfs_regs[i]) {
3189                         debugfs_remove(adev->debugfs_regs[i]);
3190                         adev->debugfs_regs[i] = NULL;
3191                 }
3192         }
3193 }
3194
3195 int amdgpu_debugfs_init(struct drm_minor *minor)
3196 {
3197         return 0;
3198 }
3199 #else
3200 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
3201 {
3202         return 0;
3203 }
3204 static void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
3205 #endif