drivers/gpu/drm/amd/amdgpu/mmhub_v9_4.c

   1 /*
   2  * Copyright 2018 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include "amdgpu.h"
  24 #include "mmhub_v9_4.h"
  25
  26 #include "mmhub/mmhub_9_4_1_offset.h"
  27 #include "mmhub/mmhub_9_4_1_sh_mask.h"
  28 #include "mmhub/mmhub_9_4_1_default.h"
  29 #include "athub/athub_1_0_offset.h"
  30 #include "athub/athub_1_0_sh_mask.h"
  31 #include "vega10_enum.h"
  32
  33 #include "soc15_common.h"
  34
  35 #define MMHUB_NUM_INSTANCES                     2
  36 #define MMHUB_INSTANCE_REGISTER_OFFSET          0x3000
  37
  38 u64 mmhub_v9_4_get_fb_location(struct amdgpu_device *adev)
  39 {
  40         /* The base should be same b/t 2 mmhubs on Acrturus. Read one here. */
  41         u64 base = RREG32_SOC15(MMHUB, 0, mmVMSHAREDVC0_MC_VM_FB_LOCATION_BASE);
  42         u64 top = RREG32_SOC15(MMHUB, 0, mmVMSHAREDVC0_MC_VM_FB_LOCATION_TOP);
  43
  44         base &= VMSHAREDVC0_MC_VM_FB_LOCATION_BASE__FB_BASE_MASK;
  45         base <<= 24;
  46
  47         top &= VMSHAREDVC0_MC_VM_FB_LOCATION_TOP__FB_TOP_MASK;
  48         top <<= 24;
  49
  50         adev->gmc.fb_start = base;
  51         adev->gmc.fb_end = top;
  52
  53         return base;
  54 }
  55
  56 void mmhub_v9_4_setup_vm_pt_regs(struct amdgpu_device *adev, int hubid,
  57                                 uint32_t vmid, uint64_t value)
  58 {
  59         /* two registers distance between mmVML2VC0_VM_CONTEXT0_* to
  60          * mmVML2VC0_VM_CONTEXT1_*
  61          */
  62         int dist = mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32
  63                         - mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
  64
  65         WREG32_SOC15_OFFSET(MMHUB, 0,
  66                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
  67                             dist * vmid + hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  68                             lower_32_bits(value));
  69
  70         WREG32_SOC15_OFFSET(MMHUB, 0,
  71                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
  72                             dist * vmid + hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  73                             upper_32_bits(value));
  74
  75 }
  76
  77 static void mmhub_v9_4_init_gart_aperture_regs(struct amdgpu_device *adev,
  78                                                int hubid)
  79 {
  80         uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
  81
  82         mmhub_v9_4_setup_vm_pt_regs(adev, hubid, 0, pt_base);
  83
  84         WREG32_SOC15_OFFSET(MMHUB, 0,
  85                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
  86                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  87                             (u32)(adev->gmc.gart_start >> 12));
  88         WREG32_SOC15_OFFSET(MMHUB, 0,
  89                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
  90                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  91                             (u32)(adev->gmc.gart_start >> 44));
  92
  93         WREG32_SOC15_OFFSET(MMHUB, 0,
  94                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
  95                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
  96                             (u32)(adev->gmc.gart_end >> 12));
  97         WREG32_SOC15_OFFSET(MMHUB, 0,
  98                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
  99                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 100                             (u32)(adev->gmc.gart_end >> 44));
 101 }
 102
 103 static void mmhub_v9_4_init_system_aperture_regs(struct amdgpu_device *adev,
 104                                                  int hubid)
 105 {
 106         uint64_t value;
 107         uint32_t tmp;
 108
 109         /* Program the AGP BAR */
 110         WREG32_SOC15_OFFSET(MMHUB, 0, mmVMSHAREDVC0_MC_VM_AGP_BASE,
 111                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 112                             0);
 113         WREG32_SOC15_OFFSET(MMHUB, 0, mmVMSHAREDVC0_MC_VM_AGP_TOP,
 114                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 115                             adev->gmc.agp_end >> 24);
 116         WREG32_SOC15_OFFSET(MMHUB, 0, mmVMSHAREDVC0_MC_VM_AGP_BOT,
 117                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 118                             adev->gmc.agp_start >> 24);
 119
 120         /* Program the system aperture low logical page number. */
 121         WREG32_SOC15_OFFSET(MMHUB, 0,
 122                             mmVMSHAREDVC0_MC_VM_SYSTEM_APERTURE_LOW_ADDR,
 123                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 124                             min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18);
 125         WREG32_SOC15_OFFSET(MMHUB, 0,
 126                             mmVMSHAREDVC0_MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
 127                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 128                             max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18);
 129
 130         /* Set default page address. */
 131         value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start +
 132                 adev->vm_manager.vram_base_offset;
 133         WREG32_SOC15_OFFSET(MMHUB, 0,
 134                         mmVMSHAREDPF0_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
 135                         hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 136                         (u32)(value >> 12));
 137         WREG32_SOC15_OFFSET(MMHUB, 0,
 138                         mmVMSHAREDPF0_MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
 139                         hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 140                         (u32)(value >> 44));
 141
 142         /* Program "protection fault". */
 143         WREG32_SOC15_OFFSET(MMHUB, 0,
 144                             mmVML2PF0_VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
 145                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 146                             (u32)(adev->dummy_page_addr >> 12));
 147         WREG32_SOC15_OFFSET(MMHUB, 0,
 148                             mmVML2PF0_VM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
 149                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET,
 150                             (u32)((u64)adev->dummy_page_addr >> 44));
 151
 152         tmp = RREG32_SOC15_OFFSET(MMHUB, 0,
 153                                   mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL2,
 154                                   hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 155         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL2,
 156                             ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
 157         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL2,
 158                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 159 }
 160
 161 static void mmhub_v9_4_init_tlb_regs(struct amdgpu_device *adev, int hubid)
 162 {
 163         uint32_t tmp;
 164
 165         /* Setup TLB control */
 166         tmp = RREG32_SOC15_OFFSET(MMHUB, 0,
 167                            mmVMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 168                            hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 169
 170         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 171                             ENABLE_L1_TLB, 1);
 172         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 173                             SYSTEM_ACCESS_MODE, 3);
 174         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 175                             ENABLE_ADVANCED_DRIVER_MODEL, 1);
 176         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 177                             SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
 178         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 179                             ECO_BITS, 0);
 180         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 181                             MTYPE, MTYPE_UC);/* XXX for emulation. */
 182         tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 183                             ATC_EN, 1);
 184
 185         WREG32_SOC15_OFFSET(MMHUB, 0, mmVMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 186                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 187 }
 188
 189 static void mmhub_v9_4_init_cache_regs(struct amdgpu_device *adev, int hubid)
 190 {
 191         uint32_t tmp;
 192
 193         /* Setup L2 cache */
 194         tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL,
 195                                   hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 196         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 197                             ENABLE_L2_CACHE, 1);
 198         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 199                             ENABLE_L2_FRAGMENT_PROCESSING, 1);
 200         /* XXX for emulation, Refer to closed source code.*/
 201         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 202                             L2_PDE0_CACHE_TAG_GENERATION_MODE, 0);
 203         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 204                             PDE_FAULT_CLASSIFICATION, 0);
 205         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 206                             CONTEXT1_IDENTITY_ACCESS_MODE, 1);
 207         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 208                             IDENTITY_MODE_FRAGMENT_SIZE, 0);
 209         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL,
 210                      hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 211
 212         tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL2,
 213                                   hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 214         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL2,
 215                             INVALIDATE_ALL_L1_TLBS, 1);
 216         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL2,
 217                             INVALIDATE_L2_CACHE, 1);
 218         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL2,
 219                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 220
 221         tmp = mmVML2PF0_VM_L2_CNTL3_DEFAULT;
 222         if (adev->gmc.translate_further) {
 223                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL3, BANK_SELECT, 12);
 224                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL3,
 225                                     L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
 226         } else {
 227                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL3, BANK_SELECT, 9);
 228                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL3,
 229                                     L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
 230         }
 231         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL3,
 232                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 233
 234         tmp = mmVML2PF0_VM_L2_CNTL4_DEFAULT;
 235         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL4,
 236                             VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
 237         tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL4,
 238                             VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
 239         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL4,
 240                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 241 }
 242
 243 static void mmhub_v9_4_enable_system_domain(struct amdgpu_device *adev,
 244                                             int hubid)
 245 {
 246         uint32_t tmp;
 247
 248         tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2VC0_VM_CONTEXT0_CNTL,
 249                                   hubid * MMHUB_INSTANCE_REGISTER_OFFSET);
 250         tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
 251         tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
 252         WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2VC0_VM_CONTEXT0_CNTL,
 253                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 254 }
 255
 256 static void mmhub_v9_4_disable_identity_aperture(struct amdgpu_device *adev,
 257                                                  int hubid)
 258 {
 259         WREG32_SOC15_OFFSET(MMHUB, 0,
 260                     mmVML2PF0_VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
 261                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0XFFFFFFFF);
 262         WREG32_SOC15_OFFSET(MMHUB, 0,
 263                     mmVML2PF0_VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
 264                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0x0000000F);
 265
 266         WREG32_SOC15_OFFSET(MMHUB, 0,
 267                     mmVML2PF0_VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
 268                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 269         WREG32_SOC15_OFFSET(MMHUB, 0,
 270                     mmVML2PF0_VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
 271                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 272
 273         WREG32_SOC15_OFFSET(MMHUB, 0,
 274                     mmVML2PF0_VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32,
 275                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 276         WREG32_SOC15_OFFSET(MMHUB, 0,
 277                     mmVML2PF0_VM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32,
 278                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 279 }
 280
 281 static void mmhub_v9_4_setup_vmid_config(struct amdgpu_device *adev, int hubid)
 282 {
 283         uint32_t tmp;
 284         int i;
 285
 286         for (i = 0; i <= 14; i++) {
 287                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2VC0_VM_CONTEXT1_CNTL,
 288                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i);
 289                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 290                                     ENABLE_CONTEXT, 1);
 291                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 292                                     PAGE_TABLE_DEPTH,
 293                                     adev->vm_manager.num_level);
 294                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 295                                     RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 296                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 297                                     DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
 298                                     1);
 299                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 300                                     PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 301                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 302                                     VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 303                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 304                                     READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 305                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 306                                     WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 307                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 308                                     EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 309                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 310                                     PAGE_TABLE_BLOCK_SIZE,
 311                                     adev->vm_manager.block_size - 9);
 312                 /* Send no-retry XNACK on fault to suppress VM fault storm. */
 313                 tmp = REG_SET_FIELD(tmp, VML2VC0_VM_CONTEXT1_CNTL,
 314                                     RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
 315                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2VC0_VM_CONTEXT1_CNTL,
 316                                     hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i,
 317                                     tmp);
 318                 WREG32_SOC15_OFFSET(MMHUB, 0,
 319                             mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
 320                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i*2, 0);
 321                 WREG32_SOC15_OFFSET(MMHUB, 0,
 322                             mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
 323                             hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i*2, 0);
 324                 WREG32_SOC15_OFFSET(MMHUB, 0,
 325                                 mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
 326                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i*2,
 327                                 lower_32_bits(adev->vm_manager.max_pfn - 1));
 328                 WREG32_SOC15_OFFSET(MMHUB, 0,
 329                                 mmVML2VC0_VM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
 330                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + i*2,
 331                                 upper_32_bits(adev->vm_manager.max_pfn - 1));
 332         }
 333 }
 334
 335 static void mmhub_v9_4_program_invalidation(struct amdgpu_device *adev,
 336                                             int hubid)
 337 {
 338         unsigned i;
 339
 340         for (i = 0; i < 18; ++i) {
 341                 WREG32_SOC15_OFFSET(MMHUB, 0,
 342                                 mmVML2VC0_VM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
 343                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + 2 * i,
 344                                 0xffffffff);
 345                 WREG32_SOC15_OFFSET(MMHUB, 0,
 346                                 mmVML2VC0_VM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
 347                                 hubid * MMHUB_INSTANCE_REGISTER_OFFSET + 2 * i,
 348                                 0x1f);
 349         }
 350 }
 351
 352 int mmhub_v9_4_gart_enable(struct amdgpu_device *adev)
 353 {
 354         int i;
 355
 356         for (i = 0; i < MMHUB_NUM_INSTANCES; i++) {
 357                 if (amdgpu_sriov_vf(adev)) {
 358                         /*
 359                          * MC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase
 360                          * they are VF copy registers so vbios post doesn't
 361                          * program them, for SRIOV driver need to program them
 362                          */
 363                         WREG32_SOC15_OFFSET(MMHUB, 0,
 364                                      mmVMSHAREDVC0_MC_VM_FB_LOCATION_BASE,
 365                                      i * MMHUB_INSTANCE_REGISTER_OFFSET,
 366                                      adev->gmc.vram_start >> 24);
 367                         WREG32_SOC15_OFFSET(MMHUB, 0,
 368                                      mmVMSHAREDVC0_MC_VM_FB_LOCATION_TOP,
 369                                      i * MMHUB_INSTANCE_REGISTER_OFFSET,
 370                                      adev->gmc.vram_end >> 24);
 371                 }
 372
 373                 /* GART Enable. */
 374                 mmhub_v9_4_init_gart_aperture_regs(adev, i);
 375                 mmhub_v9_4_init_system_aperture_regs(adev, i);
 376                 mmhub_v9_4_init_tlb_regs(adev, i);
 377                 mmhub_v9_4_init_cache_regs(adev, i);
 378
 379                 mmhub_v9_4_enable_system_domain(adev, i);
 380                 mmhub_v9_4_disable_identity_aperture(adev, i);
 381                 mmhub_v9_4_setup_vmid_config(adev, i);
 382                 mmhub_v9_4_program_invalidation(adev, i);
 383         }
 384
 385         return 0;
 386 }
 387
 388 void mmhub_v9_4_gart_disable(struct amdgpu_device *adev)
 389 {
 390         u32 tmp;
 391         u32 i, j;
 392
 393         for (j = 0; j < MMHUB_NUM_INSTANCES; j++) {
 394                 /* Disable all tables */
 395                 for (i = 0; i < 16; i++)
 396                         WREG32_SOC15_OFFSET(MMHUB, 0,
 397                                             mmVML2VC0_VM_CONTEXT0_CNTL,
 398                                             j * MMHUB_INSTANCE_REGISTER_OFFSET +
 399                                             i, 0);
 400
 401                 /* Setup TLB control */
 402                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0,
 403                                    mmVMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 404                                    j * MMHUB_INSTANCE_REGISTER_OFFSET);
 405                 tmp = REG_SET_FIELD(tmp, VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 406                                     ENABLE_L1_TLB, 0);
 407                 tmp = REG_SET_FIELD(tmp,
 408                                     VMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 409                                     ENABLE_ADVANCED_DRIVER_MODEL, 0);
 410                 WREG32_SOC15_OFFSET(MMHUB, 0,
 411                                     mmVMSHAREDVC0_MC_VM_MX_L1_TLB_CNTL,
 412                                     j * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 413
 414                 /* Setup L2 cache */
 415                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL,
 416                                           j * MMHUB_INSTANCE_REGISTER_OFFSET);
 417                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_CNTL,
 418                                     ENABLE_L2_CACHE, 0);
 419                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL,
 420                                     j * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 421                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVML2PF0_VM_L2_CNTL3,
 422                                     j * MMHUB_INSTANCE_REGISTER_OFFSET, 0);
 423         }
 424 }
 425
 426 /**
 427  * mmhub_v1_0_set_fault_enable_default - update GART/VM fault handling
 428  *
 429  * @adev: amdgpu_device pointer
 430  * @value: true redirects VM faults to the default page
 431  */
 432 void mmhub_v9_4_set_fault_enable_default(struct amdgpu_device *adev, bool value)
 433 {
 434         u32 tmp;
 435         int i;
 436
 437         for (i = 0; i < MMHUB_NUM_INSTANCES; i++) {
 438                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0,
 439                                           mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 440                                           i * MMHUB_INSTANCE_REGISTER_OFFSET);
 441                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 442                                     RANGE_PROTECTION_FAULT_ENABLE_DEFAULT,
 443                                     value);
 444                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 445                                     PDE0_PROTECTION_FAULT_ENABLE_DEFAULT,
 446                                     value);
 447                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 448                                     PDE1_PROTECTION_FAULT_ENABLE_DEFAULT,
 449                                     value);
 450                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 451                                     PDE2_PROTECTION_FAULT_ENABLE_DEFAULT,
 452                                     value);
 453                 tmp = REG_SET_FIELD(tmp,
 454                             VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 455                             TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
 456                             value);
 457                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 458                                     NACK_PROTECTION_FAULT_ENABLE_DEFAULT,
 459                                     value);
 460                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 461                                     DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
 462                                     value);
 463                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 464                                     VALID_PROTECTION_FAULT_ENABLE_DEFAULT,
 465                                     value);
 466                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 467                                     READ_PROTECTION_FAULT_ENABLE_DEFAULT,
 468                                     value);
 469                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 470                                     WRITE_PROTECTION_FAULT_ENABLE_DEFAULT,
 471                                     value);
 472                 tmp = REG_SET_FIELD(tmp, VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 473                                     EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT,
 474                                     value);
 475                 if (!value) {
 476                         tmp = REG_SET_FIELD(tmp,
 477                                             VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 478                                             CRASH_ON_NO_RETRY_FAULT, 1);
 479                         tmp = REG_SET_FIELD(tmp,
 480                                             VML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 481                                             CRASH_ON_RETRY_FAULT, 1);
 482                 }
 483
 484                 WREG32_SOC15_OFFSET(MMHUB, 0,
 485                                     mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL,
 486                                     i * MMHUB_INSTANCE_REGISTER_OFFSET, tmp);
 487         }
 488 }
 489
 490 void mmhub_v9_4_init(struct amdgpu_device *adev)
 491 {
 492         struct amdgpu_vmhub *hub[MMHUB_NUM_INSTANCES] =
 493                 {&adev->vmhub[AMDGPU_MMHUB_0], &adev->vmhub[AMDGPU_MMHUB_1]};
 494         int i;
 495
 496         for (i = 0; i < MMHUB_NUM_INSTANCES; i++) {
 497                 hub[i]->ctx0_ptb_addr_lo32 =
 498                         SOC15_REG_OFFSET(MMHUB, 0,
 499                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32) +
 500                             i * MMHUB_INSTANCE_REGISTER_OFFSET;
 501                 hub[i]->ctx0_ptb_addr_hi32 =
 502                         SOC15_REG_OFFSET(MMHUB, 0,
 503                             mmVML2VC0_VM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32) +
 504                             i * MMHUB_INSTANCE_REGISTER_OFFSET;
 505                 hub[i]->vm_inv_eng0_req =
 506                         SOC15_REG_OFFSET(MMHUB, 0,
 507                                          mmVML2VC0_VM_INVALIDATE_ENG0_REQ) +
 508                                          i * MMHUB_INSTANCE_REGISTER_OFFSET;
 509                 hub[i]->vm_inv_eng0_ack =
 510                         SOC15_REG_OFFSET(MMHUB, 0,
 511                                          mmVML2VC0_VM_INVALIDATE_ENG0_ACK) +
 512                                          i * MMHUB_INSTANCE_REGISTER_OFFSET;
 513                 hub[i]->vm_context0_cntl =
 514                         SOC15_REG_OFFSET(MMHUB, 0,
 515                                          mmVML2VC0_VM_CONTEXT0_CNTL) +
 516                                          i * MMHUB_INSTANCE_REGISTER_OFFSET;
 517                 hub[i]->vm_l2_pro_fault_status =
 518                         SOC15_REG_OFFSET(MMHUB, 0,
 519                                     mmVML2PF0_VM_L2_PROTECTION_FAULT_STATUS) +
 520                                     i * MMHUB_INSTANCE_REGISTER_OFFSET;
 521                 hub[i]->vm_l2_pro_fault_cntl =
 522                         SOC15_REG_OFFSET(MMHUB, 0,
 523                                     mmVML2PF0_VM_L2_PROTECTION_FAULT_CNTL) +
 524                                     i * MMHUB_INSTANCE_REGISTER_OFFSET;
 525         }
 526 }
 527
 528 static void mmhub_v9_4_update_medium_grain_clock_gating(struct amdgpu_device *adev,
 529                                                         bool enable)
 530 {
 531         uint32_t def, data, def1, data1;
 532         int i, j;
 533         int dist = mmDAGB1_CNTL_MISC2 - mmDAGB0_CNTL_MISC2;
 534
 535         for (i = 0; i < MMHUB_NUM_INSTANCES; i++) {
 536                 def = data = RREG32_SOC15_OFFSET(MMHUB, 0,
 537                                         mmATCL2_0_ATC_L2_MISC_CG,
 538                                         i * MMHUB_INSTANCE_REGISTER_OFFSET);
 539
 540                 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG))
 541                         data |= ATCL2_0_ATC_L2_MISC_CG__ENABLE_MASK;
 542                 else
 543                         data &= ~ATCL2_0_ATC_L2_MISC_CG__ENABLE_MASK;
 544
 545                 if (def != data)
 546                         WREG32_SOC15_OFFSET(MMHUB, 0, mmATCL2_0_ATC_L2_MISC_CG,
 547                                 i * MMHUB_INSTANCE_REGISTER_OFFSET, data);
 548
 549                 for (j = 0; j < 5; j++) {
 550                         def1 = data1 = RREG32_SOC15_OFFSET(MMHUB, 0,
 551                                         mmDAGB0_CNTL_MISC2,
 552                                         i * MMHUB_INSTANCE_REGISTER_OFFSET +
 553                                         j * dist);
 554                         if (enable &&
 555                             (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) {
 556                                 data1 &=
 557                                     ~(DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 558                                     DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 559                                     DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 560                                     DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 561                                     DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 562                                     DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 563                         } else {
 564                                 data1 |=
 565                                     (DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 566                                     DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 567                                     DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 568                                     DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 569                                     DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 570                                     DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 571                         }
 572
 573                         if (def1 != data1)
 574                                 WREG32_SOC15_OFFSET(MMHUB, 0,
 575                                         mmDAGB0_CNTL_MISC2,
 576                                         i * MMHUB_INSTANCE_REGISTER_OFFSET +
 577                                         j * dist, data1);
 578
 579                         if (i == 1 && j == 3)
 580                                 break;
 581                 }
 582         }
 583 }
 584
 585 static void mmhub_v9_4_update_medium_grain_light_sleep(struct amdgpu_device *adev,
 586                                                        bool enable)
 587 {
 588         uint32_t def, data;
 589         int i;
 590
 591         for (i = 0; i < MMHUB_NUM_INSTANCES; i++) {
 592                 def = data = RREG32_SOC15_OFFSET(MMHUB, 0,
 593                                         mmATCL2_0_ATC_L2_MISC_CG,
 594                                         i * MMHUB_INSTANCE_REGISTER_OFFSET);
 595
 596                 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS))
 597                         data |= ATCL2_0_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
 598                 else
 599                         data &= ~ATCL2_0_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
 600
 601                 if (def != data)
 602                         WREG32_SOC15_OFFSET(MMHUB, 0, mmATCL2_0_ATC_L2_MISC_CG,
 603                                 i * MMHUB_INSTANCE_REGISTER_OFFSET, data);
 604         }
 605 }
 606
 607 int mmhub_v9_4_set_clockgating(struct amdgpu_device *adev,
 608                                enum amd_clockgating_state state)
 609 {
 610         if (amdgpu_sriov_vf(adev))
 611                 return 0;
 612
 613         switch (adev->asic_type) {
 614         case CHIP_ARCTURUS:
 615                 mmhub_v9_4_update_medium_grain_clock_gating(adev,
 616                                 state == AMD_CG_STATE_GATE ? true : false);
 617                 mmhub_v9_4_update_medium_grain_light_sleep(adev,
 618                                 state == AMD_CG_STATE_GATE ? true : false);
 619                 break;
 620         default:
 621                 break;
 622         }
 623
 624         return 0;
 625 }
 626
 627 void mmhub_v9_4_get_clockgating(struct amdgpu_device *adev, u32 *flags)
 628 {
 629         int data, data1;
 630
 631         if (amdgpu_sriov_vf(adev))
 632                 *flags = 0;
 633
 634         /* AMD_CG_SUPPORT_MC_MGCG */
 635         data = RREG32_SOC15(MMHUB, 0, mmATCL2_0_ATC_L2_MISC_CG);
 636
 637         data1 = RREG32_SOC15(MMHUB, 0, mmATCL2_0_ATC_L2_MISC_CG);
 638
 639         if ((data & ATCL2_0_ATC_L2_MISC_CG__ENABLE_MASK) &&
 640             !(data1 & (DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 641                        DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 642                        DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 643                        DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 644                        DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 645                        DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK)))
 646                 *flags |= AMD_CG_SUPPORT_MC_MGCG;
 647
 648         /* AMD_CG_SUPPORT_MC_LS */
 649         if (data & ATCL2_0_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK)
 650                 *flags |= AMD_CG_SUPPORT_MC_LS;
 651 }