projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'armsoc-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm/arm-soc
[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / gmc_v9_0.c
CommitLineData
e60f8db5
AX		 1/*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23#include <linux/firmware.h>
fd5fd480 24#include <drm/drm_cache.h>
e60f8db5
AX		 25#include "amdgpu.h"
26#include "gmc_v9_0.h"
8d6a5230 27#include "amdgpu_atomfirmware.h"
e60f8db5 28
75199b8c
FX		 29#include "hdp/hdp_4_0_offset.h"
30#include "hdp/hdp_4_0_sh_mask.h"
cde5c34f 31#include "gc/gc_9_0_sh_mask.h"
135d4b10
FX		 32#include "dce/dce_12_0_offset.h"
33#include "dce/dce_12_0_sh_mask.h"
fb960bd2 34#include "vega10_enum.h"
65417d9f 35#include "mmhub/mmhub_1_0_offset.h"
6ce68225 36#include "athub/athub_1_0_offset.h"
250b4228 37#include "oss/osssys_4_0_offset.h"
e60f8db5 38
946a4d5b 39#include "soc15.h"
e60f8db5 40#include "soc15_common.h"
90c7a935 41#include "umc/umc_6_0_sh_mask.h"
e60f8db5 42
e60f8db5
AX		 43#include "gfxhub_v1_0.h"
44#include "mmhub_v1_0.h"
45
44a99b65
AG		 46#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
47
ebdef28e
AD		 48/* add these here since we already include dce12 headers and these are for DCN */
49#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d
50#define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2
51#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0
52#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10
53#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL
54#define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L
55
e60f8db5
AX		 56/* XXX Move this macro to VEGA10 header file, which is like vid.h for VI.*/
57#define AMDGPU_NUM_OF_VMIDS 8
58
59static const u32 golden_settings_vega10_hdp[] =
60{
61 0xf64, 0x0fffffff, 0x00000000,
62 0xf65, 0x0fffffff, 0x00000000,
63 0xf66, 0x0fffffff, 0x00000000,
64 0xf67, 0x0fffffff, 0x00000000,
65 0xf68, 0x0fffffff, 0x00000000,
66 0xf6a, 0x0fffffff, 0x00000000,
67 0xf6b, 0x0fffffff, 0x00000000,
68 0xf6c, 0x0fffffff, 0x00000000,
69 0xf6d, 0x0fffffff, 0x00000000,
70 0xf6e, 0x0fffffff, 0x00000000,
71};
72
946a4d5b 73static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] =
5c583018 74{
946a4d5b
SL		 75 SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
76 SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
5c583018
EQ		 77};
78
946a4d5b 79static const struct soc15_reg_golden golden_settings_athub_1_0_0[] =
5c583018 80{
946a4d5b
SL		 81 SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
82 SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
5c583018
EQ		 83};
84
02bab923
DP		 85/* Ecc related register addresses, (BASE + reg offset) */
86/* Universal Memory Controller caps (may be fused). */
87/* UMCCH:UmcLocalCap */
88#define UMCLOCALCAPS_ADDR0 (0x00014306 + 0x00000000)
89#define UMCLOCALCAPS_ADDR1 (0x00014306 + 0x00000800)
90#define UMCLOCALCAPS_ADDR2 (0x00014306 + 0x00001000)
91#define UMCLOCALCAPS_ADDR3 (0x00014306 + 0x00001800)
92#define UMCLOCALCAPS_ADDR4 (0x00054306 + 0x00000000)
93#define UMCLOCALCAPS_ADDR5 (0x00054306 + 0x00000800)
94#define UMCLOCALCAPS_ADDR6 (0x00054306 + 0x00001000)
95#define UMCLOCALCAPS_ADDR7 (0x00054306 + 0x00001800)
96#define UMCLOCALCAPS_ADDR8 (0x00094306 + 0x00000000)
97#define UMCLOCALCAPS_ADDR9 (0x00094306 + 0x00000800)
98#define UMCLOCALCAPS_ADDR10 (0x00094306 + 0x00001000)
99#define UMCLOCALCAPS_ADDR11 (0x00094306 + 0x00001800)
100#define UMCLOCALCAPS_ADDR12 (0x000d4306 + 0x00000000)
101#define UMCLOCALCAPS_ADDR13 (0x000d4306 + 0x00000800)
102#define UMCLOCALCAPS_ADDR14 (0x000d4306 + 0x00001000)
103#define UMCLOCALCAPS_ADDR15 (0x000d4306 + 0x00001800)
104
105/* Universal Memory Controller Channel config. */
106/* UMCCH:UMC_CONFIG */
107#define UMCCH_UMC_CONFIG_ADDR0 (0x00014040 + 0x00000000)
108#define UMCCH_UMC_CONFIG_ADDR1 (0x00014040 + 0x00000800)
109#define UMCCH_UMC_CONFIG_ADDR2 (0x00014040 + 0x00001000)
110#define UMCCH_UMC_CONFIG_ADDR3 (0x00014040 + 0x00001800)
111#define UMCCH_UMC_CONFIG_ADDR4 (0x00054040 + 0x00000000)
112#define UMCCH_UMC_CONFIG_ADDR5 (0x00054040 + 0x00000800)
113#define UMCCH_UMC_CONFIG_ADDR6 (0x00054040 + 0x00001000)
114#define UMCCH_UMC_CONFIG_ADDR7 (0x00054040 + 0x00001800)
115#define UMCCH_UMC_CONFIG_ADDR8 (0x00094040 + 0x00000000)
116#define UMCCH_UMC_CONFIG_ADDR9 (0x00094040 + 0x00000800)
117#define UMCCH_UMC_CONFIG_ADDR10 (0x00094040 + 0x00001000)
118#define UMCCH_UMC_CONFIG_ADDR11 (0x00094040 + 0x00001800)
119#define UMCCH_UMC_CONFIG_ADDR12 (0x000d4040 + 0x00000000)
120#define UMCCH_UMC_CONFIG_ADDR13 (0x000d4040 + 0x00000800)
121#define UMCCH_UMC_CONFIG_ADDR14 (0x000d4040 + 0x00001000)
122#define UMCCH_UMC_CONFIG_ADDR15 (0x000d4040 + 0x00001800)
123
124/* Universal Memory Controller Channel Ecc config. */
125/* UMCCH:EccCtrl */
126#define UMCCH_ECCCTRL_ADDR0 (0x00014053 + 0x00000000)
127#define UMCCH_ECCCTRL_ADDR1 (0x00014053 + 0x00000800)
128#define UMCCH_ECCCTRL_ADDR2 (0x00014053 + 0x00001000)
129#define UMCCH_ECCCTRL_ADDR3 (0x00014053 + 0x00001800)
130#define UMCCH_ECCCTRL_ADDR4 (0x00054053 + 0x00000000)
131#define UMCCH_ECCCTRL_ADDR5 (0x00054053 + 0x00000800)
132#define UMCCH_ECCCTRL_ADDR6 (0x00054053 + 0x00001000)
133#define UMCCH_ECCCTRL_ADDR7 (0x00054053 + 0x00001800)
134#define UMCCH_ECCCTRL_ADDR8 (0x00094053 + 0x00000000)
135#define UMCCH_ECCCTRL_ADDR9 (0x00094053 + 0x00000800)
136#define UMCCH_ECCCTRL_ADDR10 (0x00094053 + 0x00001000)
137#define UMCCH_ECCCTRL_ADDR11 (0x00094053 + 0x00001800)
138#define UMCCH_ECCCTRL_ADDR12 (0x000d4053 + 0x00000000)
139#define UMCCH_ECCCTRL_ADDR13 (0x000d4053 + 0x00000800)
140#define UMCCH_ECCCTRL_ADDR14 (0x000d4053 + 0x00001000)
141#define UMCCH_ECCCTRL_ADDR15 (0x000d4053 + 0x00001800)
142
143static const uint32_t ecc_umclocalcap_addrs[] = {
144 UMCLOCALCAPS_ADDR0,
145 UMCLOCALCAPS_ADDR1,
146 UMCLOCALCAPS_ADDR2,
147 UMCLOCALCAPS_ADDR3,
148 UMCLOCALCAPS_ADDR4,
149 UMCLOCALCAPS_ADDR5,
150 UMCLOCALCAPS_ADDR6,
151 UMCLOCALCAPS_ADDR7,
152 UMCLOCALCAPS_ADDR8,
153 UMCLOCALCAPS_ADDR9,
154 UMCLOCALCAPS_ADDR10,
155 UMCLOCALCAPS_ADDR11,
156 UMCLOCALCAPS_ADDR12,
157 UMCLOCALCAPS_ADDR13,
158 UMCLOCALCAPS_ADDR14,
159 UMCLOCALCAPS_ADDR15,
160};
161
162static const uint32_t ecc_umcch_umc_config_addrs[] = {
163 UMCCH_UMC_CONFIG_ADDR0,
164 UMCCH_UMC_CONFIG_ADDR1,
165 UMCCH_UMC_CONFIG_ADDR2,
166 UMCCH_UMC_CONFIG_ADDR3,
167 UMCCH_UMC_CONFIG_ADDR4,
168 UMCCH_UMC_CONFIG_ADDR5,
169 UMCCH_UMC_CONFIG_ADDR6,
170 UMCCH_UMC_CONFIG_ADDR7,
171 UMCCH_UMC_CONFIG_ADDR8,
172 UMCCH_UMC_CONFIG_ADDR9,
173 UMCCH_UMC_CONFIG_ADDR10,
174 UMCCH_UMC_CONFIG_ADDR11,
175 UMCCH_UMC_CONFIG_ADDR12,
176 UMCCH_UMC_CONFIG_ADDR13,
177 UMCCH_UMC_CONFIG_ADDR14,
178 UMCCH_UMC_CONFIG_ADDR15,
179};
180
181static const uint32_t ecc_umcch_eccctrl_addrs[] = {
182 UMCCH_ECCCTRL_ADDR0,
183 UMCCH_ECCCTRL_ADDR1,
184 UMCCH_ECCCTRL_ADDR2,
185 UMCCH_ECCCTRL_ADDR3,
186 UMCCH_ECCCTRL_ADDR4,
187 UMCCH_ECCCTRL_ADDR5,
188 UMCCH_ECCCTRL_ADDR6,
189 UMCCH_ECCCTRL_ADDR7,
190 UMCCH_ECCCTRL_ADDR8,
191 UMCCH_ECCCTRL_ADDR9,
192 UMCCH_ECCCTRL_ADDR10,
193 UMCCH_ECCCTRL_ADDR11,
194 UMCCH_ECCCTRL_ADDR12,
195 UMCCH_ECCCTRL_ADDR13,
196 UMCCH_ECCCTRL_ADDR14,
197 UMCCH_ECCCTRL_ADDR15,
198};
199
e60f8db5
AX		 200static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
201 struct amdgpu_irq_src *src,
202 unsigned type,
203 enum amdgpu_interrupt_state state)
204{
205 struct amdgpu_vmhub *hub;
ae6d1416 206 u32 tmp, reg, bits, i, j;
e60f8db5 207
11250164
CK		 208 bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
209 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
210 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
211 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
212 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
213 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
214 VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
215
e60f8db5
AX		 216 switch (state) {
217 case AMDGPU_IRQ_STATE_DISABLE:
ae6d1416
TSD		 218 for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) {
219 hub = &adev->vmhub[j];
220 for (i = 0; i < 16; i++) {
221 reg = hub->vm_context0_cntl + i;
222 tmp = RREG32(reg);
223 tmp &= ~bits;
224 WREG32(reg, tmp);
225 }
e60f8db5
AX		 226 }
227 break;
228 case AMDGPU_IRQ_STATE_ENABLE:
ae6d1416
TSD		 229 for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) {
230 hub = &adev->vmhub[j];
231 for (i = 0; i < 16; i++) {
232 reg = hub->vm_context0_cntl + i;
233 tmp = RREG32(reg);
234 tmp |= bits;
235 WREG32(reg, tmp);
236 }
e60f8db5 237 }
e60f8db5
AX		 238 default:
239 break;
240 }
241
242 return 0;
243}
244
245static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
246 struct amdgpu_irq_src *source,
247 struct amdgpu_iv_entry *entry)
248{
c4f46f22 249 struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src];
4d6cbde3 250 uint32_t status = 0;
e60f8db5
AX		 251 u64 addr;
252
253 addr = (u64)entry->src_data[0] << 12;
254 addr |= ((u64)entry->src_data[1] & 0xf) << 44;
255
79a0c465 256 if (!amdgpu_sriov_vf(adev)) {
5a9b8e8a
CK		 257 status = RREG32(hub->vm_l2_pro_fault_status);
258 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
4d6cbde3 259 }
e60f8db5 260
4d6cbde3 261 if (printk_ratelimit()) {
efaa9646
AG		 262 struct amdgpu_task_info task_info = { 0 };
263
264 amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
265
4d6cbde3 266 dev_err(adev->dev,
efaa9646 267 "[%s] VMC page fault (src_id:%u ring:%u vmid:%u pasid:%u, for process %s pid %d thread %s pid %d\n)\n",
c4f46f22
CK		 268 entry->vmid_src ? "mmhub" : "gfxhub",
269 entry->src_id, entry->ring_id, entry->vmid,
efaa9646
AG		 270 entry->pasid, task_info.process_name, task_info.tgid,
271 task_info.task_name, task_info.pid);
6cdf4e87 272 dev_err(adev->dev, " at address 0x%016llx from %d\n",
4d6cbde3
FK		 273 addr, entry->client_id);
274 if (!amdgpu_sriov_vf(adev))
275 dev_err(adev->dev,
276 "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
277 status);
79a0c465 278 }
e60f8db5
AX		 279
280 return 0;
281}
282
283static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
284 .set = gmc_v9_0_vm_fault_interrupt_state,
285 .process = gmc_v9_0_process_interrupt,
286};
287
288static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
289{
770d13b1
CK		 290 adev->gmc.vm_fault.num_types = 1;
291 adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
e60f8db5
AX		 292}
293
c4f46f22 294static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid)
03f89feb
CK		 295{
296 u32 req = 0;
297
c4f46f22 298 /* invalidate using legacy mode on vmid*/
03f89feb 299 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
c4f46f22 300 PER_VMID_INVALIDATE_REQ, 1 << vmid);
03f89feb
CK		 301 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, 0);
302 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
303 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
304 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
305 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
306 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
307 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
308 CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
309
310 return req;
311}
312
e60f8db5
AX		 313/*
314 * GART
315 * VMID 0 is the physical GPU addresses as used by the kernel.
316 * VMIDs 1-15 are used for userspace clients and are handled
317 * by the amdgpu vm/hsa code.
318 */
319
320/**
132f34e4 321 * gmc_v9_0_flush_gpu_tlb - gart tlb flush callback
e60f8db5
AX		 322 *
323 * @adev: amdgpu_device pointer
324 * @vmid: vm instance to flush
325 *
326 * Flush the TLB for the requested page table.
327 */
132f34e4 328static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
e60f8db5
AX		 329 uint32_t vmid)
330{
331 /* Use register 17 for GART */
332 const unsigned eng = 17;
333 unsigned i, j;
334
770d13b1 335 spin_lock(&adev->gmc.invalidate_lock);
e60f8db5
AX		 336
337 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
338 struct amdgpu_vmhub *hub = &adev->vmhub[i];
03f89feb 339 u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
e60f8db5 340
c7a7266b 341 WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
e60f8db5
AX		 342
343 /* Busy wait for ACK.*/
344 for (j = 0; j < 100; j++) {
c7a7266b 345 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng);
e60f8db5
AX		 346 tmp &= 1 << vmid;
347 if (tmp)
348 break;
349 cpu_relax();
350 }
351 if (j < 100)
352 continue;
353
354 /* Wait for ACK with a delay.*/
355 for (j = 0; j < adev->usec_timeout; j++) {
c7a7266b 356 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng);
e60f8db5
AX		 357 tmp &= 1 << vmid;
358 if (tmp)
359 break;
360 udelay(1);
361 }
362 if (j < adev->usec_timeout)
363 continue;
364
365 DRM_ERROR("Timeout waiting for VM flush ACK!\n");
366 }
367
770d13b1 368 spin_unlock(&adev->gmc.invalidate_lock);
e60f8db5
AX		 369}
370
9096d6e5 371static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
c633c00b 372 unsigned vmid, uint64_t pd_addr)
9096d6e5 373{
250b4228
CK		 374 struct amdgpu_device *adev = ring->adev;
375 struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub];
9096d6e5
CK		 376 uint32_t req = gmc_v9_0_get_invalidate_req(vmid);
377 uint64_t flags = AMDGPU_PTE_VALID;
378 unsigned eng = ring->vm_inv_eng;
379
c633c00b 380 amdgpu_gmc_get_vm_pde(adev, -1, &pd_addr, &flags);
9096d6e5
CK		 381 pd_addr |= flags;
382
383 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (2 * vmid),
384 lower_32_bits(pd_addr));
385
386 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid),
387 upper_32_bits(pd_addr));
388
f8bc9037
AD		 389 amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng,
390 hub->vm_inv_eng0_ack + eng,
391 req, 1 << vmid);
f732b6b3 392
9096d6e5
CK		 393 return pd_addr;
394}
395
c633c00b
CK		 396static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
397 unsigned pasid)
398{
399 struct amdgpu_device *adev = ring->adev;
400 uint32_t reg;
401
402 if (ring->funcs->vmhub == AMDGPU_GFXHUB)
403 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
404 else
405 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
406
407 amdgpu_ring_emit_wreg(ring, reg, pasid);
408}
409
e60f8db5 410/**
132f34e4 411 * gmc_v9_0_set_pte_pde - update the page tables using MMIO
e60f8db5
AX		 412 *
413 * @adev: amdgpu_device pointer
414 * @cpu_pt_addr: cpu address of the page table
415 * @gpu_page_idx: entry in the page table to update
416 * @addr: dst addr to write into pte/pde
417 * @flags: access flags
418 *
419 * Update the page tables using the CPU.
420 */
132f34e4
CK		 421static int gmc_v9_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
422 uint32_t gpu_page_idx, uint64_t addr,
423 uint64_t flags)
e60f8db5
AX		 424{
425 void __iomem *ptr = (void *)cpu_pt_addr;
426 uint64_t value;
427
428 /*
429 * PTE format on VEGA 10:
430 * 63:59 reserved
431 * 58:57 mtype
432 * 56 F
433 * 55 L
434 * 54 P
435 * 53 SW
436 * 52 T
437 * 50:48 reserved
438 * 47:12 4k physical page base address
439 * 11:7 fragment
440 * 6 write
441 * 5 read
442 * 4 exe
443 * 3 Z
444 * 2 snooped
445 * 1 system
446 * 0 valid
447 *
448 * PDE format on VEGA 10:
449 * 63:59 block fragment size
450 * 58:55 reserved
451 * 54 P
452 * 53:48 reserved
453 * 47:6 physical base address of PD or PTE
454 * 5:3 reserved
455 * 2 C
456 * 1 system
457 * 0 valid
458 */
459
460 /*
461 * The following is for PTE only. GART does not have PDEs.
462 */
463 value = addr & 0x0000FFFFFFFFF000ULL;
464 value |= flags;
465 writeq(value, ptr + (gpu_page_idx * 8));
466 return 0;
467}
468
469static uint64_t gmc_v9_0_get_vm_pte_flags(struct amdgpu_device *adev,
470 uint32_t flags)
471
472{
473 uint64_t pte_flag = 0;
474
475 if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
476 pte_flag |= AMDGPU_PTE_EXECUTABLE;
477 if (flags & AMDGPU_VM_PAGE_READABLE)
478 pte_flag |= AMDGPU_PTE_READABLE;
479 if (flags & AMDGPU_VM_PAGE_WRITEABLE)
480 pte_flag |= AMDGPU_PTE_WRITEABLE;
481
482 switch (flags & AMDGPU_VM_MTYPE_MASK) {
483 case AMDGPU_VM_MTYPE_DEFAULT:
484 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC);
485 break;
486 case AMDGPU_VM_MTYPE_NC:
487 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC);
488 break;
489 case AMDGPU_VM_MTYPE_WC:
490 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_WC);
491 break;
492 case AMDGPU_VM_MTYPE_CC:
493 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_CC);
494 break;
495 case AMDGPU_VM_MTYPE_UC:
496 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_UC);
497 break;
498 default:
499 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC);
500 break;
501 }
502
503 if (flags & AMDGPU_VM_PAGE_PRT)
504 pte_flag |= AMDGPU_PTE_PRT;
505
506 return pte_flag;
507}
508
3de676d8
CK		 509static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
510 uint64_t *addr, uint64_t *flags)
e60f8db5 511{
3de676d8
CK		 512 if (!(*flags & AMDGPU_PDE_PTE))
513 *addr = adev->vm_manager.vram_base_offset + *addr -
770d13b1 514 adev->gmc.vram_start;
3de676d8 515 BUG_ON(*addr & 0xFFFF00000000003FULL);
6a42fd6f 516
770d13b1 517 if (!adev->gmc.translate_further)
6a42fd6f
CK		 518 return;
519
520 if (level == AMDGPU_VM_PDB1) {
521 /* Set the block fragment size */
522 if (!(*flags & AMDGPU_PDE_PTE))
523 *flags |= AMDGPU_PDE_BFS(0x9);
524
525 } else if (level == AMDGPU_VM_PDB0) {
526 if (*flags & AMDGPU_PDE_PTE)
527 *flags &= ~AMDGPU_PDE_PTE;
528 else
529 *flags |= AMDGPU_PTE_TF;
530 }
e60f8db5
AX		 531}
532
132f34e4
CK		 533static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
534 .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
9096d6e5 535 .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
c633c00b 536 .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
132f34e4 537 .set_pte_pde = gmc_v9_0_set_pte_pde,
b1166325
CK		 538 .get_vm_pte_flags = gmc_v9_0_get_vm_pte_flags,
539 .get_vm_pde = gmc_v9_0_get_vm_pde
e60f8db5
AX		 540};
541
132f34e4 542static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
e60f8db5 543{
132f34e4
CK		 544 if (adev->gmc.gmc_funcs == NULL)
545 adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
e60f8db5
AX		 546}
547
548static int gmc_v9_0_early_init(void *handle)
549{
550 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
551
132f34e4 552 gmc_v9_0_set_gmc_funcs(adev);
e60f8db5
AX		 553 gmc_v9_0_set_irq_funcs(adev);
554
770d13b1
CK		 555 adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
556 adev->gmc.shared_aperture_end =
557 adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
bfa8eea2 558 adev->gmc.private_aperture_start = 0x1000000000000000ULL;
770d13b1
CK		 559 adev->gmc.private_aperture_end =
560 adev->gmc.private_aperture_start + (4ULL << 30) - 1;
a7ea6548 561
e60f8db5
AX		 562 return 0;
563}
564
02bab923
DP		 565static int gmc_v9_0_ecc_available(struct amdgpu_device *adev)
566{
567 uint32_t reg_val;
568 uint32_t reg_addr;
569 uint32_t field_val;
570 size_t i;
571 uint32_t fv2;
572 size_t lost_sheep;
573
574 DRM_DEBUG("ecc: gmc_v9_0_ecc_available()\n");
575
576 lost_sheep = 0;
577 for (i = 0; i < ARRAY_SIZE(ecc_umclocalcap_addrs); ++i) {
578 reg_addr = ecc_umclocalcap_addrs[i];
579 DRM_DEBUG("ecc: "
580 "UMCCH_UmcLocalCap[%zu]: reg_addr: 0x%08x\n",
581 i, reg_addr);
582 reg_val = RREG32(reg_addr);
583 field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UmcLocalCap,
584 EccDis);
585 DRM_DEBUG("ecc: "
586 "reg_val: 0x%08x, "
587 "EccDis: 0x%08x, ",
588 reg_val, field_val);
589 if (field_val) {
590 DRM_ERROR("ecc: UmcLocalCap:EccDis is set.\n");
591 ++lost_sheep;
592 }
593 }
594
595 for (i = 0; i < ARRAY_SIZE(ecc_umcch_umc_config_addrs); ++i) {
596 reg_addr = ecc_umcch_umc_config_addrs[i];
597 DRM_DEBUG("ecc: "
598 "UMCCH0_0_UMC_CONFIG[%zu]: reg_addr: 0x%08x",
599 i, reg_addr);
600 reg_val = RREG32(reg_addr);
601 field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UMC_CONFIG,
602 DramReady);
603 DRM_DEBUG("ecc: "
604 "reg_val: 0x%08x, "
605 "DramReady: 0x%08x\n",
606 reg_val, field_val);
607
608 if (!field_val) {
609 DRM_ERROR("ecc: UMC_CONFIG:DramReady is not set.\n");
610 ++lost_sheep;
611 }
612 }
613
614 for (i = 0; i < ARRAY_SIZE(ecc_umcch_eccctrl_addrs); ++i) {
615 reg_addr = ecc_umcch_eccctrl_addrs[i];
616 DRM_DEBUG("ecc: "
617 "UMCCH_EccCtrl[%zu]: reg_addr: 0x%08x, ",
618 i, reg_addr);
619 reg_val = RREG32(reg_addr);
620 field_val = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl,
621 WrEccEn);
622 fv2 = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl,
623 RdEccEn);
624 DRM_DEBUG("ecc: "
625 "reg_val: 0x%08x, "
626 "WrEccEn: 0x%08x, "
627 "RdEccEn: 0x%08x\n",
628 reg_val, field_val, fv2);
629
630 if (!field_val) {
5a16008f 631 DRM_DEBUG("ecc: WrEccEn is not set\n");
02bab923
DP		 632 ++lost_sheep;
633 }
634 if (!fv2) {
5a16008f 635 DRM_DEBUG("ecc: RdEccEn is not set\n");
02bab923
DP		 636 ++lost_sheep;
637 }
638 }
639
640 DRM_DEBUG("ecc: lost_sheep: %zu\n", lost_sheep);
641 return lost_sheep == 0;
642}
643
e60f8db5
AX		 644static int gmc_v9_0_late_init(void *handle)
645{
646 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
c5066129 647 /*
648 * The latest engine allocation on gfx9 is:
649 * Engine 0, 1: idle
650 * Engine 2, 3: firmware
651 * Engine 4~13: amdgpu ring, subject to change when ring number changes
652 * Engine 14~15: idle
653 * Engine 16: kfd tlb invalidation
654 * Engine 17: Gart flushes
655 */
656 unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 4, 4 };
4789c463 657 unsigned i;
02bab923 658 int r;
4789c463 659
6f752ec2
AG		 660 /*
661 * TODO - Uncomment once GART corruption issue is fixed.
662 */
663 /* amdgpu_bo_late_init(adev); */
664
4789c463
CK		 665 for(i = 0; i < adev->num_rings; ++i) {
666 struct amdgpu_ring *ring = adev->rings[i];
667 unsigned vmhub = ring->funcs->vmhub;
668
669 ring->vm_inv_eng = vm_inv_eng[vmhub]++;
775f55f1
TSD		 670 dev_info(adev->dev, "ring %u(%s) uses VM inv eng %u on hub %u\n",
671 ring->idx, ring->name, ring->vm_inv_eng,
672 ring->funcs->vmhub);
4789c463
CK		 673 }
674
c5066129 675 /* Engine 16 is used for KFD and 17 for GART flushes */
4789c463 676 for(i = 0; i < AMDGPU_MAX_VMHUBS; ++i)
c5066129 677 BUG_ON(vm_inv_eng[i] > 16);
4789c463 678
7b6cbae2 679 if (adev->asic_type == CHIP_VEGA10 && !amdgpu_sriov_vf(adev)) {
5ba4fa35
AD		 680 r = gmc_v9_0_ecc_available(adev);
681 if (r == 1) {
682 DRM_INFO("ECC is active.\n");
683 } else if (r == 0) {
684 DRM_INFO("ECC is not present.\n");
e1d1a772 685 adev->df_funcs->enable_ecc_force_par_wr_rmw(adev, false);
5ba4fa35
AD		 686 } else {
687 DRM_ERROR("gmc_v9_0_ecc_available() failed. r: %d\n", r);
688 return r;
689 }
02bab923
DP		 690 }
691
770d13b1 692 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
e60f8db5
AX		 693}
694
695static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
770d13b1 696 struct amdgpu_gmc *mc)
e60f8db5 697{
eeb2487d
ML		 698 u64 base = 0;
699 if (!amdgpu_sriov_vf(adev))
700 base = mmhub_v1_0_get_fb_location(adev);
770d13b1 701 amdgpu_device_vram_location(adev, &adev->gmc, base);
2543e28a 702 amdgpu_device_gart_location(adev, mc);
bc099ee9 703 /* base offset of vram pages */
b6110c00 704 adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);
e60f8db5
AX		 705}
706
707/**
708 * gmc_v9_0_mc_init - initialize the memory controller driver params
709 *
710 * @adev: amdgpu_device pointer
711 *
712 * Look up the amount of vram, vram width, and decide how to place
713 * vram and gart within the GPU's physical address space.
714 * Returns 0 for success.
715 */
716static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
717{
e60f8db5 718 int chansize, numchan;
d6895ad3 719 int r;
e60f8db5 720
3d918c0e
SL		 721 if (amdgpu_emu_mode != 1)
722 adev->gmc.vram_width = amdgpu_atomfirmware_get_vram_width(adev);
770d13b1 723 if (!adev->gmc.vram_width) {
8d6a5230 724 /* hbm memory channel size */
585b7f16
TSD		 725 if (adev->flags & AMD_IS_APU)
726 chansize = 64;
727 else
728 chansize = 128;
8d6a5230 729
070706c0 730 numchan = adev->df_funcs->get_hbm_channel_number(adev);
770d13b1 731 adev->gmc.vram_width = numchan * chansize;
e60f8db5 732 }
e60f8db5 733
e60f8db5 734 /* size in MB on si */
770d13b1 735 adev->gmc.mc_vram_size =
bf383fb6 736 adev->nbio_funcs->get_memsize(adev) * 1024ULL * 1024ULL;
770d13b1 737 adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
d6895ad3
CK		 738
739 if (!(adev->flags & AMD_IS_APU)) {
740 r = amdgpu_device_resize_fb_bar(adev);
741 if (r)
742 return r;
743 }
770d13b1
CK		 744 adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
745 adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
e60f8db5 746
156a81be
CZ		 747#ifdef CONFIG_X86_64
748 if (adev->flags & AMD_IS_APU) {
749 adev->gmc.aper_base = gfxhub_v1_0_get_mc_fb_offset(adev);
750 adev->gmc.aper_size = adev->gmc.real_vram_size;
751 }
752#endif
e60f8db5 753 /* In case the PCI BAR is larger than the actual amount of vram */
770d13b1
CK		 754 adev->gmc.visible_vram_size = adev->gmc.aper_size;
755 if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
756 adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
e60f8db5 757
c3db7b5a
AD		 758 /* set the gart size */
759 if (amdgpu_gart_size == -1) {
760 switch (adev->asic_type) {
761 case CHIP_VEGA10: /* all engines support GPUVM */
273a14cd 762 case CHIP_VEGA12: /* all engines support GPUVM */
d96b428c 763 case CHIP_VEGA20:
c3db7b5a 764 default:
fe19b862 765 adev->gmc.gart_size = 512ULL << 20;
c3db7b5a
AD		 766 break;
767 case CHIP_RAVEN: /* DCE SG support */
770d13b1 768 adev->gmc.gart_size = 1024ULL << 20;
c3db7b5a
AD		 769 break;
770 }
771 } else {
770d13b1 772 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
c3db7b5a
AD		 773 }
774
770d13b1 775 gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
e60f8db5
AX		 776
777 return 0;
778}
779
780static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
781{
782 int r;
783
784 if (adev->gart.robj) {
785 WARN(1, "VEGA10 PCIE GART already initialized\n");
786 return 0;
787 }
788 /* Initialize common gart structure */
789 r = amdgpu_gart_init(adev);
790 if (r)
791 return r;
792 adev->gart.table_size = adev->gart.num_gpu_pages * 8;
793 adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE(MTYPE_UC) |
794 AMDGPU_PTE_EXECUTABLE;
795 return amdgpu_gart_table_vram_alloc(adev);
796}
797
ebdef28e
AD		 798static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
799{
800#if 0
801 u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
802#endif
803 unsigned size;
804
6f752ec2
AG		 805 /*
806 * TODO Remove once GART corruption is resolved
807 * Check related code in gmc_v9_0_sw_fini
808 * */
809 size = 9 * 1024 * 1024;
810
811#if 0
ebdef28e
AD		 812 if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
813 size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */
814 } else {
815 u32 viewport;
816
817 switch (adev->asic_type) {
818 case CHIP_RAVEN:
819 viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
820 size = (REG_GET_FIELD(viewport,
821 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
822 REG_GET_FIELD(viewport,
823 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
824 4);
825 break;
826 case CHIP_VEGA10:
827 case CHIP_VEGA12:
828 default:
829 viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
830 size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
831 REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
832 4);
833 break;
834 }
835 }
836 /* return 0 if the pre-OS buffer uses up most of vram */
837 if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))
838 return 0;
6f752ec2
AG		 839
840#endif
ebdef28e
AD		 841 return size;
842}
843
e60f8db5
AX		 844static int gmc_v9_0_sw_init(void *handle)
845{
846 int r;
847 int dma_bits;
848 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
849
0c8c0847 850 gfxhub_v1_0_init(adev);
77f6c763 851 mmhub_v1_0_init(adev);
0c8c0847 852
770d13b1 853 spin_lock_init(&adev->gmc.invalidate_lock);
e60f8db5 854
1e09b053 855 adev->gmc.vram_type = amdgpu_atomfirmware_get_vram_type(adev);
fd66560b
HZ		 856 switch (adev->asic_type) {
857 case CHIP_RAVEN:
6a42fd6f 858 if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
f3368128 859 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
6a42fd6f
CK		 860 } else {
861 /* vm_size is 128TB + 512GB for legacy 3-level page support */
862 amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
770d13b1 863 adev->gmc.translate_further =
6a42fd6f
CK		 864 adev->vm_manager.num_level > 1;
865 }
fd66560b
HZ		 866 break;
867 case CHIP_VEGA10:
273a14cd 868 case CHIP_VEGA12:
d96b428c 869 case CHIP_VEGA20:
36b32a68
ZJ		 870 /*
871 * To fulfill 4-level page support,
872 * vm size is 256TB (48bit), maximum size of Vega10,
873 * block size 512 (9bit)
874 */
f3368128 875 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
fd66560b
HZ		 876 break;
877 default:
878 break;
e60f8db5
AX		 879 }
880
881 /* This interrupt is VMC page fault.*/
44a99b65 882 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
770d13b1 883 &adev->gmc.vm_fault);
44a99b65 884 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
770d13b1 885 &adev->gmc.vm_fault);
e60f8db5
AX		 886
887 if (r)
888 return r;
889
e60f8db5
AX		 890 /* Set the internal MC address mask
891 * This is the max address of the GPU's
892 * internal address space.
893 */
770d13b1 894 adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
e60f8db5
AX		 895
896 /* set DMA mask + need_dma32 flags.
897 * PCIE - can handle 44-bits.
898 * IGP - can handle 44-bits
899 * PCI - dma32 for legacy pci gart, 44 bits on vega10
900 */
901 adev->need_dma32 = false;
902 dma_bits = adev->need_dma32 ? 32 : 44;
903 r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
904 if (r) {
905 adev->need_dma32 = true;
906 dma_bits = 32;
907 printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
908 }
909 r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
910 if (r) {
911 pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
912 printk(KERN_WARNING "amdgpu: No coherent DMA available.\n");
913 }
fd5fd480 914 adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits);
e60f8db5
AX		 915
916 r = gmc_v9_0_mc_init(adev);
917 if (r)
918 return r;
919
ebdef28e
AD		 920 adev->gmc.stolen_size = gmc_v9_0_get_vbios_fb_size(adev);
921
e60f8db5
AX		 922 /* Memory manager */
923 r = amdgpu_bo_init(adev);
924 if (r)
925 return r;
926
927 r = gmc_v9_0_gart_init(adev);
928 if (r)
929 return r;
930
05ec3eda
CK		 931 /*
932 * number of VMs
933 * VMID 0 is reserved for System
934 * amdgpu graphics/compute will use VMIDs 1-7
935 * amdkfd will use VMIDs 8-15
936 */
937 adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids = AMDGPU_NUM_OF_VMIDS;
938 adev->vm_manager.id_mgr[AMDGPU_MMHUB].num_ids = AMDGPU_NUM_OF_VMIDS;
939
05ec3eda
CK		 940 amdgpu_vm_manager_init(adev);
941
942 return 0;
e60f8db5
AX		 943}
944
e60f8db5
AX		 945static int gmc_v9_0_sw_fini(void *handle)
946{
947 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
948
f59548c8 949 amdgpu_gem_force_release(adev);
05ec3eda 950 amdgpu_vm_manager_fini(adev);
6f752ec2
AG		 951
952 /*
953 * TODO:
954 * Currently there is a bug where some memory client outside
955 * of the driver writes to first 8M of VRAM on S3 resume,
956 * this overrides GART which by default gets placed in first 8M and
957 * causes VM_FAULTS once GTT is accessed.
958 * Keep the stolen memory reservation until the while this is not solved.
959 * Also check code in gmc_v9_0_get_vbios_fb_size and gmc_v9_0_late_init
960 */
961 amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL);
962
eb7e5cfc 963 amdgpu_gart_table_vram_free(adev);
e60f8db5 964 amdgpu_bo_fini(adev);
eb7e5cfc 965 amdgpu_gart_fini(adev);
e60f8db5
AX		 966
967 return 0;
968}
969
970static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
971{
946a4d5b 972
e60f8db5
AX		 973 switch (adev->asic_type) {
974 case CHIP_VEGA10:
d96b428c 975 case CHIP_VEGA20:
946a4d5b 976 soc15_program_register_sequence(adev,
5c583018 977 golden_settings_mmhub_1_0_0,
c47b41a7 978 ARRAY_SIZE(golden_settings_mmhub_1_0_0));
946a4d5b 979 soc15_program_register_sequence(adev,
5c583018 980 golden_settings_athub_1_0_0,
c47b41a7 981 ARRAY_SIZE(golden_settings_athub_1_0_0));
e60f8db5 982 break;
273a14cd
AD		 983 case CHIP_VEGA12:
984 break;
e4f3abaa 985 case CHIP_RAVEN:
946a4d5b 986 soc15_program_register_sequence(adev,
5c583018 987 golden_settings_athub_1_0_0,
c47b41a7 988 ARRAY_SIZE(golden_settings_athub_1_0_0));
e4f3abaa 989 break;
e60f8db5
AX		 990 default:
991 break;
992 }
993}
994
995/**
996 * gmc_v9_0_gart_enable - gart enable
997 *
998 * @adev: amdgpu_device pointer
999 */
1000static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
1001{
1002 int r;
1003 bool value;
1004 u32 tmp;
1005
9c3f2b54
AD		 1006 amdgpu_device_program_register_sequence(adev,
1007 golden_settings_vega10_hdp,
1008 ARRAY_SIZE(golden_settings_vega10_hdp));
e60f8db5
AX		 1009
1010 if (adev->gart.robj == NULL) {
1011 dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
1012 return -EINVAL;
1013 }
ce1b1b66
ML		 1014 r = amdgpu_gart_table_vram_pin(adev);
1015 if (r)
1016 return r;
e60f8db5 1017
2fcd43ce
HZ		 1018 switch (adev->asic_type) {
1019 case CHIP_RAVEN:
1020 mmhub_v1_0_initialize_power_gating(adev);
f8386b35 1021 mmhub_v1_0_update_power_gating(adev, true);
2fcd43ce
HZ		 1022 break;
1023 default:
1024 break;
1025 }
1026
e60f8db5
AX		 1027 r = gfxhub_v1_0_gart_enable(adev);
1028 if (r)
1029 return r;
1030
1031 r = mmhub_v1_0_gart_enable(adev);
1032 if (r)
1033 return r;
1034
846347c9 1035 WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1);
e60f8db5 1036
b9509c80
HR		 1037 tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL);
1038 WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp);
e60f8db5 1039
1d4e0a8c 1040 /* After HDP is initialized, flush HDP.*/
69882565 1041 adev->nbio_funcs->hdp_flush(adev, NULL);
1d4e0a8c 1042
e60f8db5
AX		 1043 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
1044 value = false;
1045 else
1046 value = true;
1047
1048 gfxhub_v1_0_set_fault_enable_default(adev, value);
1049 mmhub_v1_0_set_fault_enable_default(adev, value);
132f34e4 1050 gmc_v9_0_flush_gpu_tlb(adev, 0);
e60f8db5
AX		 1051
1052 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
770d13b1 1053 (unsigned)(adev->gmc.gart_size >> 20),
e60f8db5
AX		 1054 (unsigned long long)adev->gart.table_addr);
1055 adev->gart.ready = true;
1056 return 0;
1057}
1058
1059static int gmc_v9_0_hw_init(void *handle)
1060{
1061 int r;
1062 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1063
1064 /* The sequence of these two function calls matters.*/
1065 gmc_v9_0_init_golden_registers(adev);
1066
edca2d05 1067 if (adev->mode_info.num_crtc) {
edca2d05 1068 /* Lockout access through VGA aperture*/
4d9c333a 1069 WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
edca2d05
AD		 1070
1071 /* disable VGA render */
4d9c333a 1072 WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
edca2d05
AD		 1073 }
1074
e60f8db5
AX		 1075 r = gmc_v9_0_gart_enable(adev);
1076
1077 return r;
1078}
1079
1080/**
1081 * gmc_v9_0_gart_disable - gart disable
1082 *
1083 * @adev: amdgpu_device pointer
1084 *
1085 * This disables all VM page table.
1086 */
1087static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
1088{
1089 gfxhub_v1_0_gart_disable(adev);
1090 mmhub_v1_0_gart_disable(adev);
ce1b1b66 1091 amdgpu_gart_table_vram_unpin(adev);
e60f8db5
AX		 1092}
1093
1094static int gmc_v9_0_hw_fini(void *handle)
1095{
1096 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1097
5dd696ae
TH		 1098 if (amdgpu_sriov_vf(adev)) {
1099 /* full access mode, so don't touch any GMC register */
1100 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
1101 return 0;
1102 }
1103
770d13b1 1104 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
e60f8db5
AX		 1105 gmc_v9_0_gart_disable(adev);
1106
1107 return 0;
1108}
1109
1110static int gmc_v9_0_suspend(void *handle)
1111{
1112 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1113
f053cd47 1114 return gmc_v9_0_hw_fini(adev);
e60f8db5
AX		 1115}
1116
1117static int gmc_v9_0_resume(void *handle)
1118{
1119 int r;
1120 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1121
1122 r = gmc_v9_0_hw_init(adev);
1123 if (r)
1124 return r;
1125
620f774f 1126 amdgpu_vmid_reset_all(adev);
e60f8db5 1127
32601d48 1128 return 0;
e60f8db5
AX		 1129}
1130
1131static bool gmc_v9_0_is_idle(void *handle)
1132{
1133 /* MC is always ready in GMC v9.*/
1134 return true;
1135}
1136
1137static int gmc_v9_0_wait_for_idle(void *handle)
1138{
1139 /* There is no need to wait for MC idle in GMC v9.*/
1140 return 0;
1141}
1142
1143static int gmc_v9_0_soft_reset(void *handle)
1144{
1145 /* XXX for emulation.*/
1146 return 0;
1147}
1148
1149static int gmc_v9_0_set_clockgating_state(void *handle,
1150 enum amd_clockgating_state state)
1151{
d5583d4f
HR		 1152 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1153
1154 return mmhub_v1_0_set_clockgating(adev, state);
e60f8db5
AX		 1155}
1156
13052be5
HR		 1157static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags)
1158{
1159 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1160
1161 mmhub_v1_0_get_clockgating(adev, flags);
1162}
1163
e60f8db5
AX		 1164static int gmc_v9_0_set_powergating_state(void *handle,
1165 enum amd_powergating_state state)
1166{
1167 return 0;
1168}
1169
1170const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
1171 .name = "gmc_v9_0",
1172 .early_init = gmc_v9_0_early_init,
1173 .late_init = gmc_v9_0_late_init,
1174 .sw_init = gmc_v9_0_sw_init,
1175 .sw_fini = gmc_v9_0_sw_fini,
1176 .hw_init = gmc_v9_0_hw_init,
1177 .hw_fini = gmc_v9_0_hw_fini,
1178 .suspend = gmc_v9_0_suspend,
1179 .resume = gmc_v9_0_resume,
1180 .is_idle = gmc_v9_0_is_idle,
1181 .wait_for_idle = gmc_v9_0_wait_for_idle,
1182 .soft_reset = gmc_v9_0_soft_reset,
1183 .set_clockgating_state = gmc_v9_0_set_clockgating_state,
1184 .set_powergating_state = gmc_v9_0_set_powergating_state,
13052be5 1185 .get_clockgating_state = gmc_v9_0_get_clockgating_state,
e60f8db5
AX		 1186};
1187
1188const struct amdgpu_ip_block_version gmc_v9_0_ip_block =
1189{
1190 .type = AMD_IP_BLOCK_TYPE_GMC,
1191 .major = 9,
1192 .minor = 0,
1193 .rev = 0,
1194 .funcs = &gmc_v9_0_ip_funcs,
1195};
Linux 6.x block layer and io_uring tree(s)