drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include "amdgpu.h"
  24 #include "mmhub_v9_4.h"
  25
  26 #include "mmhub/mmhub_9_4_1_offset.h"
  27 #include "mmhub/mmhub_9_4_1_sh_mask.h"
  28 #include "mmhub/mmhub_9_4_1_default.h"
  29 #include "athub/athub_1_0_offset.h"
  30 #include "athub/athub_1_0_sh_mask.h"
  31 #include "vega10_enum.h"
  32
  33 #include "soc15_common.h"
  34
  35 #define MMHUB_NUM_INSTANCES                     2
  36 #define MMHUB_INSTANCE_REGISTER_OFFSET          0x3000
  37
  38 u64 mmhub_v9_4_get_fb_location(struct amdgpu_device *adev)
  39 {
  40         /* The base should be same b/t 2 mmhubs on Acrturus. Read one here. */
  41         u64 base = RREG32_SOC15(MMHUB, 0, mmVMSHAREDVC0_MC_VM_FB_LOCATION_BASE);
  42         u64 top = RREG32_SOC15(MMHUB, 0, mmVMSHAREDVC0_MC_VM_FB_LOCATION_TOP);
  43
  44         base &= VMSHAREDVC0_MC_VM_FB_LOCATION_BASE__FB_BASE_MASK;
  45         base <<= 24;
  46
  47         top &= VMSHAREDVC0_MC_VM_FB_LOCATION_TOP__FB_TOP_MASK;
  48         top <<= 24;
  49
  50         adev->gmc.fb_start = base;
  51         adev->gmc.fb_end = top;
  52
  53         return base;
  54 }
  55
  56 void mmhub_v9_4_setup_vm_pt_regs(struct amdgpu_device *adev, int hubid,
  57                                 uint32_t vmid, uint64_t value)
  58 {
  59         /* two registers distance between mmVML2VC0_VM_CONTEXT0_* to
  60          * mmVML2VC0_VM_CONTEXT1_*
  61          */
  62         int dist = mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32
  63                         - mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
  64
  65         WREG32_SOC15_OFFSET(MMHUB, 0,
  66                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
  67                             dist * vmid + hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  68                             lower_32_bits(value));
  69
  70         WREG32_SOC15_OFFSET(MMHUB, 0,
  71                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
  72                             dist * vmid + hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  73                             upper_32_bits(value));
  74
  75 }
  76
  77 static void mmhub_v9_4_init_gart_aperture_regs(struct amdgpu_device *adev,
  78                                                int hubid)
  79 {
  80         uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
  81
  82         mmhub_v9_4_setup_vm_pt_regs(adev, hubid, 0, pt_base);
  83
  84         WREG32_SOC15_OFFSET(MMHUB, 0,
  85                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
  86                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  87                             (u32)(adev->gmc.gart_start >> 12));
  88         WREG32_SOC15_OFFSET(MMHUB, 0,
  89                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
  90                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  91                             (u32)(adev->gmc.gart_start >> 44));
  92
  93         WREG32_SOC15_OFFSET(MMHUB, 0,
  94                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
  95                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  96                             (u32)(adev->gmc.gart_end >> 12));
  97         WREG32_SOC15_OFFSET(MMHUB, 0,
  98                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
  99                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 100                             (u32)(adev->gmc.gart_end >> 44));
 101 }
 102
 103 static void mmhub_v9_4_init_system_aperture_regs(struct amdgpu_device *adev,
 104                                                  int hubid)
 105 {
 106         uint64_t value;
 107         uint32_t tmp;
 108
 109         /* Program the AGP BAR */
 110         WREG32_SOC15_OFFSET(MMHUB, 0, mmVMSHAREDVC0_MC_VM_AGP_BASE,
 111                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 112                             0);
 113         WREG32_SOC15_OFFSET(MMHUB, 0, mmVMSHAREDVC0_MC_VM_AGP_TOP,
 114                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 115                             adev->gmc.agp_end >> 24);
 116         WREG32_SOC15_OFFSET(MMHUB, 0, mmVMSHAREDVC0_MC_VM_AGP_BOT,
 117                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 118                             adev->gmc.agp_start >> 24);
 119
 120         /* Program the system aperture low logical page number. */
 121         WREG32_SOC15_OFFSET(MMHUB, 0,
 122                             mmVMSHAREDVC0_MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 123                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 124                             min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18);
 125         WREG32_SOC15_OFFSET(MMHUB, 0,
 126                             mmVMSHAREDVC0_MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 127                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 128                             max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18);
 129
 130         /* Set default page address. */
 131         value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start +
 132                 adev->vm_manager.vram_base_offset;
 133         WREG32_SOC15_OFFSET(MMHUB, 0,
 134                         mmVMSHAREDPF0_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
 135                         hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 136                         (u32)(value >> 12));
 137         WREG32_SOC15_OFFSET(MMHUB, 0,
 138                         mmVMSHAREDPF0_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
 139                         hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 140                         (u32)(value >> 44));
 141
 142         /* Program "protection fault". */
 143         WREG32_SOC15_OFFSET(MMHUB, 0,
 144                             mmVML2PF0_VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
 145                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 146                             (u32)(adev->dummy_page_addr >> 12));
 147         WREG32_SOC15_OFFSET(MMHUB, 0,
 148                             mmVML2PF0_VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
 149                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 150                             (u32)((u64)adev->dummy_page_addr >> 44));
 151
 152         tmp = RREG32_SOC15_OFFSET(MMHUB, 0,
 153                                   mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL2,
 154                                   hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 155         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL2,
 156                             ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
 157         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL2,
 158                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 159 }
 160
 161 static void mmhub_v9_4_init_tlb_regs(struct amdgpu_device *adev, int hubid)
 162 {
 163         uint32_t tmp;
 164
 165         /* Setup TLB control */
 166         tmp = RREG32_SOC15_OFFSET(MMHUB, 0,
 167                            mmVMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 168                            hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 169
 170         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 171                             ENABLE_L1_TLB, 1);
 172         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 173                             SYSTEM_ACCESS_MODE, 3);
 174         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 175                             ENABLE_ADVANCED_DRIVER_MODEL, 1);
 176         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 177                             SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
 178         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 179                             ECO_BITS, 0);
 180         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 181                             MTYPE, MTYPE_UC);/* XXX for emulation. */
 182         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 183                             ATC_EN, 1);
 184
 185         WREG32_SOC15_OFFSET(MMHUB, 0, mmVMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 186                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 187 }
 188
 189 static void mmhub_v9_4_init_cache_regs(struct amdgpu_device *adev, int hubid)
 190 {
 191         uint32_t tmp;
 192
 193         /* Setup L2 cache */
 194         tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL,
 195                                   hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 196         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 197                             ENABLE_L2_CACHE, 1);
 198         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 199                             ENABLE_L2_FRAGMENT_PROCESSING, 1);
 200         /* XXX for emulation, Refer to closed source code.*/
 201         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 202                             L2_PDE0_CACHE_TAG_GENERATION_MODE, 0);
 203         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 204                             PDE_FAULT_CLASSIFICATION, 0);
 205         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 206                             CONTEXT1_IDENTITY_ACCESS_MODE, 1);
 207         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 208                             IDENTITY_MODE_FRAGMENT_SIZE, 0);
 209         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL,
 210                      hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 211
 212         tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL2,
 213                                   hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 214         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL2,
 215                             INVALIDATE_ALL_L1_TLBS, 1);
 216         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL2,
 217                             INVALIDATE_L2_CACHE, 1);
 218         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL2,
 219                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 220
 221         tmp = mmVML2PF0_VM_L2_CNTL3_DEFAULT;
 222         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL3,
 223                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 224
 225         tmp = mmVML2PF0_VM_L2_CNTL4_DEFAULT;
 226         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL4,
 227                             VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
 228         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL4,
 229                             VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
 230         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL4,
 231                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 232 }
 233
 234 static void mmhub_v9_4_enable_system_domain(struct amdgpu_device *adev,
 235                                             int hubid)
 236 {
 237         uint32_t tmp;
 238
 239         tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2VC0_VM_CONTEXT0_CNTL,
 240                                   hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 241         tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
 242         tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
 243         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2VC0_VM_CONTEXT0_CNTL,
 244                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 245 }
 246
 247 static void mmhub_v9_4_disable_identity_aperture(struct amdgpu_device *adev,
 248                                                  int hubid)
 249 {
 250         WREG32_SOC15_OFFSET(MMHUB, 0,
 251                     mmVML2PF0_VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
 252                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0XFFFFFFFF);
 253         WREG32_SOC15_OFFSET(MMHUB, 0,
 254                     mmVML2PF0_VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
 255                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0x0000000F);
 256
 257         WREG32_SOC15_OFFSET(MMHUB, 0,
 258                     mmVML2PF0_VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
 259                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 260         WREG32_SOC15_OFFSET(MMHUB, 0,
 261                     mmVML2PF0_VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
 262                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 263
 264         WREG32_SOC15_OFFSET(MMHUB, 0,
 265                     mmVML2PF0_VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32,
 266                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 267         WREG32_SOC15_OFFSET(MMHUB, 0,
 268                     mmVML2PF0_VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32,
 269                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 270 }
 271
 272 static void mmhub_v9_4_setup_vmid_config(struct amdgpu_device *adev, int hubid)
 273 {
 274         uint32_t tmp;
 275         int i;
 276
 277         for (i = 0; i <= 14; i++) {
 278                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2VC0_VM_CONTEXT1_CNTL,
 279                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i);
 280                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 281                                     ENABLE_CONTEXT, 1);
 282                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 283                                     PAGE_TABLE_DEPTH,
 284                                     adev->vm_manager.num_level);
 285                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 286                                     RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 287                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 288                                     DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
 289                                     1);
 290                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 291                                     PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 292                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 293                                     VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 294                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 295                                     READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 296                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 297                                     WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 298                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 299                                     EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 300                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 301                                     PAGE_TABLE_BLOCK_SIZE,
 302                                     adev->vm_manager.block_size - 9);
 303                 /* Send no-retry XNACK on fault to suppress VM fault storm. */
 304                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 305                                     RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
 306                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2VC0_VM_CONTEXT1_CNTL,
 307                                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i,
 308                                     tmp);
 309                 WREG32_SOC15_OFFSET(MMHUB, 0,
 310                             mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
 311                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i*2, 0);
 312                 WREG32_SOC15_OFFSET(MMHUB, 0,
 313                             mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
 314                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i*2, 0);
 315                 WREG32_SOC15_OFFSET(MMHUB, 0,
 316                                 mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
 317                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i*2,
 318                                 lower_32_bits(adev->vm_manager.max_pfn - 1));
 319                 WREG32_SOC15_OFFSET(MMHUB, 0,
 320                                 mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
 321                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i*2,
 322                                 upper_32_bits(adev->vm_manager.max_pfn - 1));
 323         }
 324 }
 325
 326 static void mmhub_v9_4_program_invalidation(struct amdgpu_device *adev,
 327                                             int hubid)
 328 {
 329         unsigned i;
 330
 331         for (i = 0; i < 18; ++i) {
 332                 WREG32_SOC15_OFFSET(MMHUB, 0,
 333                                 mmVML2VC0_VM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
 334                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + 2 * i,
 335                                 0xffffffff);
 336                 WREG32_SOC15_OFFSET(MMHUB, 0,
 337                                 mmVML2VC0_VM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
 338                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + 2 * i,
 339                                 0x1f);
 340         }
 341 }
 342
 343 int mmhub_v9_4_gart_enable(struct amdgpu_device *adev)
 344 {
 345         int i;
 346
 347         for (i = 0; i < MMHUB_NUM_INSTANCES; i++) {
 348                 if (amdgpu_sriov_vf(adev)) {
 349                         /*
 350                          * MC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase
 351                          * they are VF copy registers so vbios post doesn't
 352                          * program them, for SRIOV driver need to program them
 353                          */
 354                         WREG32_SOC15_OFFSET(MMHUB, 0,
 355                                      mmVMSHAREDVC0_MC_VM_FB_LOCATION_BASE,
 356                                      i * MMHUB_INSTANCE_REGISTER_OFFSET,
 357                                      adev->gmc.vram_start >> 24);
 358                         WREG32_SOC15_OFFSET(MMHUB, 0,
 359                                      mmVMSHAREDVC0_MC_VM_FB_LOCATION_TOP,
 360                                      i * MMHUB_INSTANCE_REGISTER_OFFSET,
 361                                      adev->gmc.vram_end >> 24);
 362                 }
 363
 364                 /* GART Enable. */
 365                 mmhub_v9_4_init_gart_aperture_regs(adev, i);
 366                 mmhub_v9_4_init_system_aperture_regs(adev, i);
 367                 mmhub_v9_4_init_tlb_regs(adev, i);
 368                 mmhub_v9_4_init_cache_regs(adev, i);
 369
 370                 mmhub_v9_4_enable_system_domain(adev, i);
 371                 mmhub_v9_4_disable_identity_aperture(adev, i);
 372                 mmhub_v9_4_setup_vmid_config(adev, i);
 373                 mmhub_v9_4_program_invalidation(adev, i);
 374         }
 375
 376         return 0;
 377 }
 378
 379 void mmhub_v9_4_gart_disable(struct amdgpu_device *adev)
 380 {
 381         u32 tmp;
 382         u32 i, j;
 383
 384         for (j = 0; j < MMHUB_NUM_INSTANCES; j++) {
 385                 /* Disable all tables */
 386                 for (i = 0; i < 16; i++)
 387                         WREG32_SOC15_OFFSET(MMHUB, 0,
 388                                             mmVML2VC0_VM_CONTEXT0_CNTL,
 389                                             j * MMHUB_INSTANCE_REGISTER_OFFSET +
 390                                             i, 0);
 391
 392                 /* Setup TLB control */
 393                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0,
 394                                    mmVMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 395                                    j * MMHUB_INSTANCE_REGISTER_OFFSET);
 396                 tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 397                                     ENABLE_L1_TLB, 0);
 398                 tmp = REG_SET_FIELD(tmp,
 399                                     VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 400                                     ENABLE_ADVANCED_DRIVER_MODEL, 0);
 401                 WREG32_SOC15_OFFSET(MMHUB, 0,
 402                                     mmVMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 403                                     j * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 404
 405                 /* Setup L2 cache */
 406                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL,
 407                                           j * MMHUB_INSTANCE_REGISTER_OFFSET);
 408                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 409                                     ENABLE_L2_CACHE, 0);
 410                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL,
 411                                     j * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 412                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL3,
 413                                     j * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 414         }
 415 }
 416
 417 /**
 418  * mmhub_v1_0_set_fault_enable_default - update GART/VM fault handling
 419  *
 420  * @adev: amdgpu_device pointer
 421  * @value: true redirects VM faults to the default page
 422  */
 423 void mmhub_v9_4_set_fault_enable_default(struct amdgpu_device *adev, bool value)
 424 {
 425         u32 tmp;
 426         int i;
 427
 428         for (i = 0; i < MMHUB_NUM_INSTANCES; i++) {
 429                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0,
 430                                           mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 431                                           i * MMHUB_INSTANCE_REGISTER_OFFSET);
 432                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 433                                     RANGE_PROTECTION_FAULT_ENABLE_DEFAULT,
 434                                     value);
 435                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 436                                     PDE0_PROTECTION_FAULT_ENABLE_DEFAULT,
 437                                     value);
 438                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 439                                     PDE1_PROTECTION_FAULT_ENABLE_DEFAULT,
 440                                     value);
 441                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 442                                     PDE2_PROTECTION_FAULT_ENABLE_DEFAULT,
 443                                     value);
 444                 tmp = REG_SET_FIELD(tmp,
 445                             VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 446                             TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
 447                             value);
 448                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 449                                     NACK_PROTECTION_FAULT_ENABLE_DEFAULT,
 450                                     value);
 451                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 452                                     DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
 453                                     value);
 454                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 455                                     VALID_PROTECTION_FAULT_ENABLE_DEFAULT,
 456                                     value);
 457                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 458                                     READ_PROTECTION_FAULT_ENABLE_DEFAULT,
 459                                     value);
 460                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 461                                     WRITE_PROTECTION_FAULT_ENABLE_DEFAULT,
 462                                     value);
 463                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 464                                     EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT,
 465                                     value);
 466                 if (!value) {
 467                         tmp = REG_SET_FIELD(tmp,
 468                                             VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 469                                             CRASH_ON_NO_RETRY_FAULT, 1);
 470                         tmp = REG_SET_FIELD(tmp,
 471                                             VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 472                                             CRASH_ON_RETRY_FAULT, 1);
 473                 }
 474
 475                 WREG32_SOC15_OFFSET(MMHUB, 0,
 476                                     mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 477                                     i * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 478         }
 479 }
 480
 481 void mmhub_v9_4_init(struct amdgpu_device *adev)
 482 {
 483         struct amdgpu_vmhub *hub[MMHUB_NUM_INSTANCES] =
 484                 {&adev->vmhub[AMDGPU_MMHUB_0], &adev->vmhub[AMDGPU_MMHUB_1]};
 485         int i;
 486
 487         for (i = 0; i < MMHUB_NUM_INSTANCES; i++) {
 488                 hub[i]->ctx0_ptb_addr_lo32 =
 489                         SOC15_REG_OFFSET(MMHUB, 0,
 490                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32) +
 491                             i * MMHUB_INSTANCE_REGISTER_OFFSET;
 492                 hub[i]->ctx0_ptb_addr_hi32 =
 493                         SOC15_REG_OFFSET(MMHUB, 0,
 494                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32) +
 495                             i * MMHUB_INSTANCE_REGISTER_OFFSET;
 496                 hub[i]->vm_inv_eng0_req =
 497                         SOC15_REG_OFFSET(MMHUB, 0,
 498                                          mmVML2VC0_VM_INVALIDATE_ENG0_REQ) +
 499                                          i * MMHUB_INSTANCE_REGISTER_OFFSET;
 500                 hub[i]->vm_inv_eng0_ack =
 501                         SOC15_REG_OFFSET(MMHUB, 0,
 502                                          mmVML2VC0_VM_INVALIDATE_ENG0_ACK) +
 503                                          i * MMHUB_INSTANCE_REGISTER_OFFSET;
 504                 hub[i]->vm_context0_cntl =
 505                         SOC15_REG_OFFSET(MMHUB, 0,
 506                                          mmVML2VC0_VM_CONTEXT0_CNTL) +
 507                                          i * MMHUB_INSTANCE_REGISTER_OFFSET;
 508                 hub[i]->vm_l2_pro_fault_status =
 509                         SOC15_REG_OFFSET(MMHUB, 0,
 510                                     mmVML2PF0_VM_L2_PROTECTION_FAULT_STATUS) +
 511                                     i * MMHUB_INSTANCE_REGISTER_OFFSET;
 512                 hub[i]->vm_l2_pro_fault_cntl =
 513                         SOC15_REG_OFFSET(MMHUB, 0,
 514                                     mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL) +
 515                                     i * MMHUB_INSTANCE_REGISTER_OFFSET;
 516         }
 517 }
 518
 519 static void mmhub_v9_4_update_medium_grain_clock_gating(struct amdgpu_device *adev,
 520                                                         bool enable)
 521 {
 522         uint32_t def, data, def1, data1;
 523         int i, j;
 524         int dist = mmDAGB1_CNTL_MISC2 - mmDAGB0_CNTL_MISC2;
 525
 526         for (i = 0; i < MMHUB_NUM_INSTANCES; i++) {
 527                 def = data = RREG32_SOC15_OFFSET(MMHUB, 0,
 528                                         mmATCL2_0_ATC_L2_MISC_CG,
 529                                         i * MMHUB_INSTANCE_REGISTER_OFFSET);
 530
 531                 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG))
 532                         data |= ATCL2_0_ATC_L2_MISC_CG__ENABLE_MASK;
 533                 else
 534                         data &= ~ATCL2_0_ATC_L2_MISC_CG__ENABLE_MASK;
 535
 536                 if (def != data)
 537                         WREG32_SOC15_OFFSET(MMHUB, 0, mmATCL2_0_ATC_L2_MISC_CG,
 538                                 i * MMHUB_INSTANCE_REGISTER_OFFSET, data);
 539
 540                 for (j = 0; j < 5; j++) {
 541                         def1 = data1 = RREG32_SOC15_OFFSET(MMHUB, 0,
 542                                         mmDAGB0_CNTL_MISC2,
 543                                         i * MMHUB_INSTANCE_REGISTER_OFFSET +
 544                                         j * dist);
 545                         if (enable &&
 546                             (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) {
 547                                 data1 &=
 548                                     ~(DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 549                                     DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 550                                     DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 551                                     DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 552                                     DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 553                                     DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 554                         } else {
 555                                 data1 |=
 556                                     (DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 557                                     DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 558                                     DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 559                                     DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 560                                     DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 561                                     DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 562                         }
 563
 564                         if (def1 != data1)
 565                                 WREG32_SOC15_OFFSET(MMHUB, 0,
 566                                         mmDAGB0_CNTL_MISC2,
 567                                         i * MMHUB_INSTANCE_REGISTER_OFFSET +
 568                                         j * dist, data1);
 569
 570                         if (i == 1 && j == 3)
 571                                 break;
 572                 }
 573         }
 574 }
 575
 576 static void mmhub_v9_4_update_medium_grain_light_sleep(struct amdgpu_device *adev,
 577                                                        bool enable)
 578 {
 579         uint32_t def, data;
 580         int i;
 581
 582         for (i = 0; i < MMHUB_NUM_INSTANCES; i++) {
 583                 def = data = RREG32_SOC15_OFFSET(MMHUB, 0,
 584                                         mmATCL2_0_ATC_L2_MISC_CG,
 585                                         i * MMHUB_INSTANCE_REGISTER_OFFSET);
 586
 587                 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS))
 588                         data |= ATCL2_0_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
 589                 else
 590                         data &= ~ATCL2_0_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
 591
 592                 if (def != data)
 593                         WREG32_SOC15_OFFSET(MMHUB, 0, mmATCL2_0_ATC_L2_MISC_CG,
 594                                 i * MMHUB_INSTANCE_REGISTER_OFFSET, data);
 595         }
 596 }
 597
 598 int mmhub_v9_4_set_clockgating(struct amdgpu_device *adev,
 599                                enum amd_clockgating_state state)
 600 {
 601         if (amdgpu_sriov_vf(adev))
 602                 return 0;
 603
 604         switch (adev->asic_type) {
 605         case CHIP_ARCTURUS:
 606                 mmhub_v9_4_update_medium_grain_clock_gating(adev,
 607                                 state == AMD_CG_STATE_GATE ? true : false);
 608                 mmhub_v9_4_update_medium_grain_light_sleep(adev,
 609                                 state == AMD_CG_STATE_GATE ? true : false);
 610                 break;
 611         default:
 612                 break;
 613         }
 614
 615         return 0;
 616 }
 617
 618 void mmhub_v9_4_get_clockgating(struct amdgpu_device *adev, u32 *flags)
 619 {
 620         int data, data1;
 621
 622         if (amdgpu_sriov_vf(adev))
 623                 *flags = 0;
 624
 625         /* AMD_CG_SUPPORT_MC_MGCG */
 626         data = RREG32_SOC15(MMHUB, 0, mmATCL2_0_ATC_L2_MISC_CG);
 627
 628         data1 = RREG32_SOC15(MMHUB, 0, mmATCL2_0_ATC_L2_MISC_CG);
 629
 630         if ((data & ATCL2_0_ATC_L2_MISC_CG__ENABLE_MASK) &&
 631             !(data1 & (DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 632                        DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 633                        DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 634                        DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 635                        DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 636                        DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK)))
 637                 *flags |= AMD_CG_SUPPORT_MC_MGCG;
 638
 639         /* AMD_CG_SUPPORT_MC_LS */
 640         if (data & ATCL2_0_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK)
 641                 *flags |= AMD_CG_SUPPORT_MC_LS;
 642 }