projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/ttm: make TTM_MAX_BO_PRIORITY unsigned
[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / gfx_v8_0.c
CommitLineData
aaa36a97
AD		 1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23#include <linux/firmware.h>
24#include "drmP.h"
25#include "amdgpu.h"
26#include "amdgpu_gfx.h"
27#include "vi.h"
aeab2032 28#include "vi_structs.h"
aaa36a97
AD		 29#include "vid.h"
30#include "amdgpu_ucode.h"
68182d90 31#include "amdgpu_atombios.h"
eeade25a 32#include "atombios_i2c.h"
aaa36a97
AD		 33#include "clearstate_vi.h"
34
35#include "gmc/gmc_8_2_d.h"
36#include "gmc/gmc_8_2_sh_mask.h"
37
38#include "oss/oss_3_0_d.h"
39#include "oss/oss_3_0_sh_mask.h"
40
41#include "bif/bif_5_0_d.h"
42#include "bif/bif_5_0_sh_mask.h"
43
44#include "gca/gfx_8_0_d.h"
45#include "gca/gfx_8_0_enum.h"
46#include "gca/gfx_8_0_sh_mask.h"
47#include "gca/gfx_8_0_enum.h"
48
aaa36a97
AD		 49#include "dce/dce_10_0_d.h"
50#include "dce/dce_10_0_sh_mask.h"
51
d9d533c1
KW		 52#include "smu/smu_7_1_3_d.h"
53
aaa36a97
AD		 54#define GFX8_NUM_GFX_RINGS 1
55#define GFX8_NUM_COMPUTE_RINGS 8
56
57#define TOPAZ_GB_ADDR_CONFIG_GOLDEN 0x22010001
58#define CARRIZO_GB_ADDR_CONFIG_GOLDEN 0x22010001
2cc0c0b5 59#define POLARIS11_GB_ADDR_CONFIG_GOLDEN 0x22011002
aaa36a97
AD		 60#define TONGA_GB_ADDR_CONFIG_GOLDEN 0x22011003
61
62#define ARRAY_MODE(x) ((x) << GB_TILE_MODE0__ARRAY_MODE__SHIFT)
63#define PIPE_CONFIG(x) ((x) << GB_TILE_MODE0__PIPE_CONFIG__SHIFT)
64#define TILE_SPLIT(x) ((x) << GB_TILE_MODE0__TILE_SPLIT__SHIFT)
65#define MICRO_TILE_MODE_NEW(x) ((x) << GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT)
66#define SAMPLE_SPLIT(x) ((x) << GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT)
67#define BANK_WIDTH(x) ((x) << GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT)
68#define BANK_HEIGHT(x) ((x) << GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT)
69#define MACRO_TILE_ASPECT(x) ((x) << GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT)
70#define NUM_BANKS(x) ((x) << GB_MACROTILE_MODE0__NUM_BANKS__SHIFT)
71
6e378858
EH		 72#define RLC_CGTT_MGCG_OVERRIDE__CPF_MASK 0x00000001L
73#define RLC_CGTT_MGCG_OVERRIDE__RLC_MASK 0x00000002L
74#define RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK 0x00000004L
75#define RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK 0x00000008L
76#define RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK 0x00000010L
77#define RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK 0x00000020L
78
79/* BPM SERDES CMD */
80#define SET_BPM_SERDES_CMD 1
81#define CLE_BPM_SERDES_CMD 0
82
83/* BPM Register Address*/
84enum {
85 BPM_REG_CGLS_EN = 0, /* Enable/Disable CGLS */
86 BPM_REG_CGLS_ON, /* ON/OFF CGLS: shall be controlled by RLC FW */
87 BPM_REG_CGCG_OVERRIDE, /* Set/Clear CGCG Override */
88 BPM_REG_MGCG_OVERRIDE, /* Set/Clear MGCG Override */
89 BPM_REG_FGCG_OVERRIDE, /* Set/Clear FGCG Override */
90 BPM_REG_FGCG_MAX
91};
92
2b6cd977
EH		 93#define RLC_FormatDirectRegListLength 14
94
c65444fe
JZ		 95MODULE_FIRMWARE("amdgpu/carrizo_ce.bin");
96MODULE_FIRMWARE("amdgpu/carrizo_pfp.bin");
97MODULE_FIRMWARE("amdgpu/carrizo_me.bin");
98MODULE_FIRMWARE("amdgpu/carrizo_mec.bin");
99MODULE_FIRMWARE("amdgpu/carrizo_mec2.bin");
100MODULE_FIRMWARE("amdgpu/carrizo_rlc.bin");
101
e3c7656c
SL		 102MODULE_FIRMWARE("amdgpu/stoney_ce.bin");
103MODULE_FIRMWARE("amdgpu/stoney_pfp.bin");
104MODULE_FIRMWARE("amdgpu/stoney_me.bin");
105MODULE_FIRMWARE("amdgpu/stoney_mec.bin");
106MODULE_FIRMWARE("amdgpu/stoney_rlc.bin");
107
c65444fe
JZ		 108MODULE_FIRMWARE("amdgpu/tonga_ce.bin");
109MODULE_FIRMWARE("amdgpu/tonga_pfp.bin");
110MODULE_FIRMWARE("amdgpu/tonga_me.bin");
111MODULE_FIRMWARE("amdgpu/tonga_mec.bin");
112MODULE_FIRMWARE("amdgpu/tonga_mec2.bin");
113MODULE_FIRMWARE("amdgpu/tonga_rlc.bin");
114
115MODULE_FIRMWARE("amdgpu/topaz_ce.bin");
116MODULE_FIRMWARE("amdgpu/topaz_pfp.bin");
117MODULE_FIRMWARE("amdgpu/topaz_me.bin");
118MODULE_FIRMWARE("amdgpu/topaz_mec.bin");
c65444fe 119MODULE_FIRMWARE("amdgpu/topaz_rlc.bin");
aaa36a97 120
af15a2d5
DZ		 121MODULE_FIRMWARE("amdgpu/fiji_ce.bin");
122MODULE_FIRMWARE("amdgpu/fiji_pfp.bin");
123MODULE_FIRMWARE("amdgpu/fiji_me.bin");
124MODULE_FIRMWARE("amdgpu/fiji_mec.bin");
125MODULE_FIRMWARE("amdgpu/fiji_mec2.bin");
126MODULE_FIRMWARE("amdgpu/fiji_rlc.bin");
127
2cc0c0b5
FC		 128MODULE_FIRMWARE("amdgpu/polaris11_ce.bin");
129MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin");
130MODULE_FIRMWARE("amdgpu/polaris11_me.bin");
131MODULE_FIRMWARE("amdgpu/polaris11_mec.bin");
132MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin");
133MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin");
68182d90 134
2cc0c0b5
FC		 135MODULE_FIRMWARE("amdgpu/polaris10_ce.bin");
136MODULE_FIRMWARE("amdgpu/polaris10_pfp.bin");
137MODULE_FIRMWARE("amdgpu/polaris10_me.bin");
138MODULE_FIRMWARE("amdgpu/polaris10_mec.bin");
139MODULE_FIRMWARE("amdgpu/polaris10_mec2.bin");
140MODULE_FIRMWARE("amdgpu/polaris10_rlc.bin");
68182d90 141
c4642a47
JZ		 142MODULE_FIRMWARE("amdgpu/polaris12_ce.bin");
143MODULE_FIRMWARE("amdgpu/polaris12_pfp.bin");
144MODULE_FIRMWARE("amdgpu/polaris12_me.bin");
145MODULE_FIRMWARE("amdgpu/polaris12_mec.bin");
146MODULE_FIRMWARE("amdgpu/polaris12_mec2.bin");
147MODULE_FIRMWARE("amdgpu/polaris12_rlc.bin");
148
aaa36a97
AD		 149static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
150{
151 {mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0},
152 {mmGDS_VMID1_BASE, mmGDS_VMID1_SIZE, mmGDS_GWS_VMID1, mmGDS_OA_VMID1},
153 {mmGDS_VMID2_BASE, mmGDS_VMID2_SIZE, mmGDS_GWS_VMID2, mmGDS_OA_VMID2},
154 {mmGDS_VMID3_BASE, mmGDS_VMID3_SIZE, mmGDS_GWS_VMID3, mmGDS_OA_VMID3},
155 {mmGDS_VMID4_BASE, mmGDS_VMID4_SIZE, mmGDS_GWS_VMID4, mmGDS_OA_VMID4},
156 {mmGDS_VMID5_BASE, mmGDS_VMID5_SIZE, mmGDS_GWS_VMID5, mmGDS_OA_VMID5},
157 {mmGDS_VMID6_BASE, mmGDS_VMID6_SIZE, mmGDS_GWS_VMID6, mmGDS_OA_VMID6},
158 {mmGDS_VMID7_BASE, mmGDS_VMID7_SIZE, mmGDS_GWS_VMID7, mmGDS_OA_VMID7},
159 {mmGDS_VMID8_BASE, mmGDS_VMID8_SIZE, mmGDS_GWS_VMID8, mmGDS_OA_VMID8},
160 {mmGDS_VMID9_BASE, mmGDS_VMID9_SIZE, mmGDS_GWS_VMID9, mmGDS_OA_VMID9},
161 {mmGDS_VMID10_BASE, mmGDS_VMID10_SIZE, mmGDS_GWS_VMID10, mmGDS_OA_VMID10},
162 {mmGDS_VMID11_BASE, mmGDS_VMID11_SIZE, mmGDS_GWS_VMID11, mmGDS_OA_VMID11},
163 {mmGDS_VMID12_BASE, mmGDS_VMID12_SIZE, mmGDS_GWS_VMID12, mmGDS_OA_VMID12},
164 {mmGDS_VMID13_BASE, mmGDS_VMID13_SIZE, mmGDS_GWS_VMID13, mmGDS_OA_VMID13},
165 {mmGDS_VMID14_BASE, mmGDS_VMID14_SIZE, mmGDS_GWS_VMID14, mmGDS_OA_VMID14},
166 {mmGDS_VMID15_BASE, mmGDS_VMID15_SIZE, mmGDS_GWS_VMID15, mmGDS_OA_VMID15}
167};
168
169static const u32 golden_settings_tonga_a11[] =
170{
171 mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
172 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
173 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
174 mmGB_GPU_ID, 0x0000000f, 0x00000000,
175 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
176 mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
177 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
ff9d6460 178 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
6a00a09e 179 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
aaa36a97
AD		 180 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
181 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
6a00a09e 182 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
aaa36a97
AD		 183 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
184 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
185 mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
6a00a09e 186 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
aaa36a97
AD		 187};
188
189static const u32 tonga_golden_common_all[] =
190{
191 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
192 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012,
193 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A,
194 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
195 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
196 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
197 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
198 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF
199};
200
201static const u32 tonga_mgcg_cgcg_init[] =
202{
203 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
204 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
205 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
206 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
207 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
208 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
209 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
210 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
211 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
212 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
213 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
214 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
215 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
216 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
217 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
218 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
219 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
220 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
221 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
222 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
223 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
224 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
225 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
226 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
227 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
228 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
229 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
230 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
231 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
232 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
233 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
234 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
235 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
236 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
237 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
238 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
239 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
240 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
241 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
242 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
243 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
244 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
245 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
246 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
247 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
248 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
249 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
250 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
251 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
252 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
253 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
254 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
255 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
256 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
257 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
258 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
259 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
260 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
261 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
262 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
263 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
264 mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000,
265 mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
266 mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007,
267 mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005,
268 mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
269 mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000,
270 mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
271 mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007,
272 mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005,
273 mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
274 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
275 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
276 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
277 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
278};
279
2cc0c0b5 280static const u32 golden_settings_polaris11_a11[] =
68182d90 281{
9761bc53
HR		 282 mmCB_HW_CONTROL, 0x0000f3cf, 0x00007208,
283 mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,
68182d90
FC		 284 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
285 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
286 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
287 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
b9934878
FC		 288 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012,
289 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,
68182d90
FC		 290 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
291 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
9761bc53 292 mmSQ_CONFIG, 0x07f80000, 0x01180000,
68182d90
FC		 293 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
294 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
295 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
296 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
297 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
795c2109 298 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
68182d90
FC		 299};
300
2cc0c0b5 301static const u32 polaris11_golden_common_all[] =
68182d90
FC		 302{
303 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
68182d90
FC		 304 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011002,
305 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
306 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
307 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
308 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
309};
310
2cc0c0b5 311static const u32 golden_settings_polaris10_a11[] =
68182d90
FC		 312{
313 mmATC_MISC_CG, 0x000c0fc0, 0x000c0200,
a5a5e308
HR		 314 mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,
315 mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,
68182d90
FC		 316 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
317 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
318 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
319 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
320 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012,
321 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002a,
322 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
323 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
324 mmSQ_CONFIG, 0x07f80000, 0x07180000,
325 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
326 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
327 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
328 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
795c2109 329 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
68182d90
FC		 330};
331
2cc0c0b5 332static const u32 polaris10_golden_common_all[] =
68182d90
FC		 333{
334 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
335 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012,
336 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A,
337 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
338 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
339 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
340 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
341 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
342};
343
af15a2d5
DZ		 344static const u32 fiji_golden_common_all[] =
345{
346 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
347 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
348 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
a7ca8ef9 349 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
af15a2d5
DZ		 350 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
351 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
352 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
a7ca8ef9
FC		 353 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
354 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
355 mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009,
af15a2d5
DZ		 356};
357
358static const u32 golden_settings_fiji_a10[] =
359{
360 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
361 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
362 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
af15a2d5 363 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
a7ca8ef9
FC		 364 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
365 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
af15a2d5 366 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
a7ca8ef9
FC		 367 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
368 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
af15a2d5 369 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
a7ca8ef9 370 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
af15a2d5
DZ		 371};
372
373static const u32 fiji_mgcg_cgcg_init[] =
374{
a7ca8ef9 375 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
af15a2d5
DZ		 376 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
377 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
378 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
379 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
380 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
381 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
382 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
383 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
384 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
385 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
386 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
387 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
388 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
389 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
390 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
391 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
392 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
393 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
394 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
395 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
396 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
397 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
398 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
399 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
400 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
401 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
402 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
403 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
404 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
405 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
406 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
407 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
408 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
409 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
410};
411
aaa36a97
AD		 412static const u32 golden_settings_iceland_a11[] =
413{
414 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
415 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
416 mmDB_DEBUG3, 0xc0000000, 0xc0000000,
417 mmGB_GPU_ID, 0x0000000f, 0x00000000,
418 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
419 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
420 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x00000002,
421 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,
fe85f07f 422 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
6a00a09e 423 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
aaa36a97
AD		 424 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
425 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
6a00a09e 426 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
aaa36a97
AD		 427 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
428 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
429 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010,
430};
431
432static const u32 iceland_golden_common_all[] =
433{
434 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
435 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002,
436 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
437 mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,
438 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
439 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
440 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
441 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF
442};
443
444static const u32 iceland_mgcg_cgcg_init[] =
445{
446 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
447 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
448 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
449 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
450 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0xc0000100,
451 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0xc0000100,
452 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0xc0000100,
453 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
454 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
455 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
456 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
457 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
458 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
459 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
460 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
461 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
462 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
463 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
464 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
465 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
466 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
467 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
468 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0xff000100,
469 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
470 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
471 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
472 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
473 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
474 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
475 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
476 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
477 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
478 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
479 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87,
480 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
481 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
482 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
483 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
484 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
485 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
486 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
487 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
488 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
489 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
490 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
491 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
492 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
493 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
494 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
495 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
496 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
497 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
498 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
499 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87,
500 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
501 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
502 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
503 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
504 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
505 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
506 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
507 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
508 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
509 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
510};
511
512static const u32 cz_golden_settings_a11[] =
513{
514 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
515 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
516 mmGB_GPU_ID, 0x0000000f, 0x00000000,
517 mmPA_SC_ENHANCE, 0xffffffff, 0x00000001,
518 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
3a494b58 519 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
6a00a09e 520 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
aaa36a97 521 mmTA_CNTL_AUX, 0x000f000f, 0x00010000,
3a494b58 522 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
6a00a09e 523 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
aaa36a97
AD		 524 mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f3,
525 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001302
526};
527
528static const u32 cz_golden_common_all[] =
529{
530 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
531 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002,
532 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
533 mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,
534 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
535 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
536 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
537 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF
538};
539
540static const u32 cz_mgcg_cgcg_init[] =
541{
542 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
543 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
544 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
545 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
546 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
547 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
548 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x00000100,
549 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
550 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
551 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
552 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
553 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
554 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
555 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
556 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
557 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
558 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
559 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
560 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
561 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
562 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
563 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
564 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
565 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
566 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
567 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
568 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
569 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
570 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
571 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
572 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
573 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
574 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
575 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
576 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
577 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
578 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
579 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
580 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
581 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
582 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
583 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
584 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
585 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
586 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
587 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
588 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
589 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
590 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
591 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
592 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
593 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
594 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
595 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
596 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
597 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
598 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
599 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
600 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
601 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
602 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
603 mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000,
604 mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
605 mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007,
606 mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005,
607 mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
608 mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000,
609 mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
610 mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007,
611 mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005,
612 mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
613 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
614 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
615 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
616 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
617};
618
e3c7656c
SL		 619static const u32 stoney_golden_settings_a11[] =
620{
621 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
622 mmGB_GPU_ID, 0x0000000f, 0x00000000,
623 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
624 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
625 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
626 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
edf600da 627 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
e3c7656c
SL		 628 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
629 mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f1,
630 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x10101010,
631};
632
633static const u32 stoney_golden_common_all[] =
634{
635 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
636 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000000,
637 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
638 mmGB_ADDR_CONFIG, 0xffffffff, 0x12010001,
639 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
640 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
641 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00007FBF,
642 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00007FAF,
643};
644
645static const u32 stoney_mgcg_cgcg_init[] =
646{
647 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
648 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
649 mmCP_MEM_SLP_CNTL, 0xffffffff, 0x00020201,
650 mmRLC_MEM_SLP_CNTL, 0xffffffff, 0x00020201,
651 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
e3c7656c
SL		 652};
653
aaa36a97
AD		 654static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev);
655static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev);
656static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev);
dbff57bc 657static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev);
2b6cd977 658static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev);
7dae69a2 659static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev);
acad2b2a
ML		 660static void gfx_v8_0_ring_emit_ce_meta_init(struct amdgpu_ring *ring, uint64_t addr);
661static void gfx_v8_0_ring_emit_de_meta_init(struct amdgpu_ring *ring, uint64_t addr);
aaa36a97
AD		 662
663static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
664{
665 switch (adev->asic_type) {
666 case CHIP_TOPAZ:
667 amdgpu_program_register_sequence(adev,
668 iceland_mgcg_cgcg_init,
669 (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init));
670 amdgpu_program_register_sequence(adev,
671 golden_settings_iceland_a11,
672 (const u32)ARRAY_SIZE(golden_settings_iceland_a11));
673 amdgpu_program_register_sequence(adev,
674 iceland_golden_common_all,
675 (const u32)ARRAY_SIZE(iceland_golden_common_all));
676 break;
af15a2d5
DZ		 677 case CHIP_FIJI:
678 amdgpu_program_register_sequence(adev,
679 fiji_mgcg_cgcg_init,
680 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
681 amdgpu_program_register_sequence(adev,
682 golden_settings_fiji_a10,
683 (const u32)ARRAY_SIZE(golden_settings_fiji_a10));
684 amdgpu_program_register_sequence(adev,
685 fiji_golden_common_all,
686 (const u32)ARRAY_SIZE(fiji_golden_common_all));
687 break;
688
aaa36a97
AD		 689 case CHIP_TONGA:
690 amdgpu_program_register_sequence(adev,
691 tonga_mgcg_cgcg_init,
692 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init));
693 amdgpu_program_register_sequence(adev,
694 golden_settings_tonga_a11,
695 (const u32)ARRAY_SIZE(golden_settings_tonga_a11));
696 amdgpu_program_register_sequence(adev,
697 tonga_golden_common_all,
698 (const u32)ARRAY_SIZE(tonga_golden_common_all));
699 break;
2cc0c0b5 700 case CHIP_POLARIS11:
c4642a47 701 case CHIP_POLARIS12:
68182d90 702 amdgpu_program_register_sequence(adev,
2cc0c0b5
FC		 703 golden_settings_polaris11_a11,
704 (const u32)ARRAY_SIZE(golden_settings_polaris11_a11));
68182d90 705 amdgpu_program_register_sequence(adev,
2cc0c0b5
FC		 706 polaris11_golden_common_all,
707 (const u32)ARRAY_SIZE(polaris11_golden_common_all));
68182d90 708 break;
2cc0c0b5 709 case CHIP_POLARIS10:
68182d90 710 amdgpu_program_register_sequence(adev,
2cc0c0b5
FC		 711 golden_settings_polaris10_a11,
712 (const u32)ARRAY_SIZE(golden_settings_polaris10_a11));
68182d90 713 amdgpu_program_register_sequence(adev,
2cc0c0b5
FC		 714 polaris10_golden_common_all,
715 (const u32)ARRAY_SIZE(polaris10_golden_common_all));
d9d533c1 716 WREG32_SMC(ixCG_ACLK_CNTL, 0x0000001C);
5765a36d
RZ		 717 if (adev->pdev->revision == 0xc7 &&
718 ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) ||
719 (adev->pdev->subsystem_device == 0x4a8 && adev->pdev->subsystem_vendor == 0x1043) ||
720 (adev->pdev->subsystem_device == 0x9480 && adev->pdev->subsystem_vendor == 0x1682))) {
eeade25a
KW		 721 amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1E, 0xDD);
722 amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1F, 0xD0);
723 }
68182d90 724 break;
aaa36a97
AD		 725 case CHIP_CARRIZO:
726 amdgpu_program_register_sequence(adev,
727 cz_mgcg_cgcg_init,
728 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init));
729 amdgpu_program_register_sequence(adev,
730 cz_golden_settings_a11,
731 (const u32)ARRAY_SIZE(cz_golden_settings_a11));
732 amdgpu_program_register_sequence(adev,
733 cz_golden_common_all,
734 (const u32)ARRAY_SIZE(cz_golden_common_all));
735 break;
e3c7656c
SL		 736 case CHIP_STONEY:
737 amdgpu_program_register_sequence(adev,
738 stoney_mgcg_cgcg_init,
739 (const u32)ARRAY_SIZE(stoney_mgcg_cgcg_init));
740 amdgpu_program_register_sequence(adev,
741 stoney_golden_settings_a11,
742 (const u32)ARRAY_SIZE(stoney_golden_settings_a11));
743 amdgpu_program_register_sequence(adev,
744 stoney_golden_common_all,
745 (const u32)ARRAY_SIZE(stoney_golden_common_all));
746 break;
aaa36a97
AD		 747 default:
748 break;
749 }
750}
751
752static void gfx_v8_0_scratch_init(struct amdgpu_device *adev)
753{
aaa36a97
AD		 754 adev->gfx.scratch.num_reg = 7;
755 adev->gfx.scratch.reg_base = mmSCRATCH_REG0;
50261151 756 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
aaa36a97
AD		 757}
758
759static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring)
760{
761 struct amdgpu_device *adev = ring->adev;
762 uint32_t scratch;
763 uint32_t tmp = 0;
764 unsigned i;
765 int r;
766
767 r = amdgpu_gfx_scratch_get(adev, &scratch);
768 if (r) {
769 DRM_ERROR("amdgpu: cp failed to get scratch reg (%d).\n", r);
770 return r;
771 }
772 WREG32(scratch, 0xCAFEDEAD);
a27de35c 773 r = amdgpu_ring_alloc(ring, 3);
aaa36a97
AD		 774 if (r) {
775 DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
776 ring->idx, r);
777 amdgpu_gfx_scratch_free(adev, scratch);
778 return r;
779 }
780 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
781 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
782 amdgpu_ring_write(ring, 0xDEADBEEF);
a27de35c 783 amdgpu_ring_commit(ring);
aaa36a97
AD		 784
785 for (i = 0; i < adev->usec_timeout; i++) {
786 tmp = RREG32(scratch);
787 if (tmp == 0xDEADBEEF)
788 break;
789 DRM_UDELAY(1);
790 }
791 if (i < adev->usec_timeout) {
792 DRM_INFO("ring test on %d succeeded in %d usecs\n",
793 ring->idx, i);
794 } else {
795 DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
796 ring->idx, scratch, tmp);
797 r = -EINVAL;
798 }
799 amdgpu_gfx_scratch_free(adev, scratch);
800 return r;
801}
802
bbec97aa 803static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
aaa36a97
AD		 804{
805 struct amdgpu_device *adev = ring->adev;
806 struct amdgpu_ib ib;
f54d1867 807 struct dma_fence *f = NULL;
aaa36a97
AD		 808 uint32_t scratch;
809 uint32_t tmp = 0;
bbec97aa 810 long r;
aaa36a97
AD		 811
812 r = amdgpu_gfx_scratch_get(adev, &scratch);
813 if (r) {
bbec97aa 814 DRM_ERROR("amdgpu: failed to get scratch reg (%ld).\n", r);
aaa36a97
AD		 815 return r;
816 }
817 WREG32(scratch, 0xCAFEDEAD);
b203dd95 818 memset(&ib, 0, sizeof(ib));
b07c60c0 819 r = amdgpu_ib_get(adev, NULL, 256, &ib);
aaa36a97 820 if (r) {
bbec97aa 821 DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
42d13693 822 goto err1;
aaa36a97
AD		 823 }
824 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
825 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START));
826 ib.ptr[2] = 0xDEADBEEF;
827 ib.length_dw = 3;
42d13693 828
50ddc75e 829 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
42d13693
CZ		 830 if (r)
831 goto err2;
832
f54d1867 833 r = dma_fence_wait_timeout(f, false, timeout);
bbec97aa
CK		 834 if (r == 0) {
835 DRM_ERROR("amdgpu: IB test timed out.\n");
836 r = -ETIMEDOUT;
837 goto err2;
838 } else if (r < 0) {
839 DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
42d13693 840 goto err2;
aaa36a97 841 }
6d44565d
CK		 842 tmp = RREG32(scratch);
843 if (tmp == 0xDEADBEEF) {
844 DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
bbec97aa 845 r = 0;
aaa36a97
AD		 846 } else {
847 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
848 scratch, tmp);
849 r = -EINVAL;
850 }
42d13693 851err2:
cc55c45d 852 amdgpu_ib_free(adev, &ib, NULL);
f54d1867 853 dma_fence_put(f);
42d13693
CZ		 854err1:
855 amdgpu_gfx_scratch_free(adev, scratch);
aaa36a97
AD		 856 return r;
857}
858
13331ac3
ML		 859
860static void gfx_v8_0_free_microcode(struct amdgpu_device *adev) {
861 release_firmware(adev->gfx.pfp_fw);
862 adev->gfx.pfp_fw = NULL;
863 release_firmware(adev->gfx.me_fw);
864 adev->gfx.me_fw = NULL;
865 release_firmware(adev->gfx.ce_fw);
866 adev->gfx.ce_fw = NULL;
867 release_firmware(adev->gfx.rlc_fw);
868 adev->gfx.rlc_fw = NULL;
869 release_firmware(adev->gfx.mec_fw);
870 adev->gfx.mec_fw = NULL;
871 if ((adev->asic_type != CHIP_STONEY) &&
872 (adev->asic_type != CHIP_TOPAZ))
873 release_firmware(adev->gfx.mec2_fw);
874 adev->gfx.mec2_fw = NULL;
875
876 kfree(adev->gfx.rlc.register_list_format);
877}
878
aaa36a97
AD		 879static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
880{
881 const char *chip_name;
882 char fw_name[30];
883 int err;
884 struct amdgpu_firmware_info *info = NULL;
885 const struct common_firmware_header *header = NULL;
595fd013 886 const struct gfx_firmware_header_v1_0 *cp_hdr;
2b6cd977
EH		 887 const struct rlc_firmware_header_v2_0 *rlc_hdr;
888 unsigned int *tmp = NULL, i;
aaa36a97
AD		 889
890 DRM_DEBUG("\n");
891
892 switch (adev->asic_type) {
893 case CHIP_TOPAZ:
894 chip_name = "topaz";
895 break;
896 case CHIP_TONGA:
897 chip_name = "tonga";
898 break;
899 case CHIP_CARRIZO:
900 chip_name = "carrizo";
901 break;
af15a2d5
DZ		 902 case CHIP_FIJI:
903 chip_name = "fiji";
904 break;
2cc0c0b5
FC		 905 case CHIP_POLARIS11:
906 chip_name = "polaris11";
68182d90 907 break;
2cc0c0b5
FC		 908 case CHIP_POLARIS10:
909 chip_name = "polaris10";
68182d90 910 break;
c4642a47
JZ		 911 case CHIP_POLARIS12:
912 chip_name = "polaris12";
913 break;
e3c7656c
SL		 914 case CHIP_STONEY:
915 chip_name = "stoney";
916 break;
aaa36a97
AD		 917 default:
918 BUG();
919 }
920
c65444fe 921 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
aaa36a97
AD		 922 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
923 if (err)
924 goto out;
925 err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
926 if (err)
927 goto out;
595fd013
JZ		 928 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
929 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
930 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
aaa36a97 931
c65444fe 932 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
aaa36a97
AD		 933 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
934 if (err)
935 goto out;
936 err = amdgpu_ucode_validate(adev->gfx.me_fw);
937 if (err)
938 goto out;
595fd013
JZ		 939 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
940 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
ae65a26d
ML		 941
942 /* chain ib ucode isn't formal released, just disable it by far
943 * TODO: when ucod ready we should use ucode version to judge if
944 * chain-ib support or not.
945 */
946 adev->virt.chained_ib_support = false;
947
595fd013 948 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
aaa36a97 949
c65444fe 950 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
aaa36a97
AD		 951 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
952 if (err)
953 goto out;
954 err = amdgpu_ucode_validate(adev->gfx.ce_fw);
955 if (err)
956 goto out;
595fd013
JZ		 957 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
958 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
959 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
aaa36a97 960
c65444fe 961 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
aaa36a97
AD		 962 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
963 if (err)
964 goto out;
965 err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
2b6cd977
EH		 966 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
967 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
968 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
969
970 adev->gfx.rlc.save_and_restore_offset =
971 le32_to_cpu(rlc_hdr->save_and_restore_offset);
972 adev->gfx.rlc.clear_state_descriptor_offset =
973 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
974 adev->gfx.rlc.avail_scratch_ram_locations =
975 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
976 adev->gfx.rlc.reg_restore_list_size =
977 le32_to_cpu(rlc_hdr->reg_restore_list_size);
978 adev->gfx.rlc.reg_list_format_start =
979 le32_to_cpu(rlc_hdr->reg_list_format_start);
980 adev->gfx.rlc.reg_list_format_separate_start =
981 le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
982 adev->gfx.rlc.starting_offsets_start =
983 le32_to_cpu(rlc_hdr->starting_offsets_start);
984 adev->gfx.rlc.reg_list_format_size_bytes =
985 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
986 adev->gfx.rlc.reg_list_size_bytes =
987 le32_to_cpu(rlc_hdr->reg_list_size_bytes);
988
989 adev->gfx.rlc.register_list_format =
990 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
991 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
992
993 if (!adev->gfx.rlc.register_list_format) {
994 err = -ENOMEM;
995 goto out;
996 }
997
ae17c999 998 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
2b6cd977
EH		 999 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1000 for (i = 0 ; i < (rlc_hdr->reg_list_format_size_bytes >> 2); i++)
1001 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
1002
1003 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1004
ae17c999 1005 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
2b6cd977
EH		 1006 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1007 for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++)
1008 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
aaa36a97 1009
c65444fe 1010 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
aaa36a97
AD		 1011 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1012 if (err)
1013 goto out;
1014 err = amdgpu_ucode_validate(adev->gfx.mec_fw);
1015 if (err)
1016 goto out;
595fd013
JZ		 1017 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1018 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1019 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
aaa36a97 1020
97dde76a
AD		 1021 if ((adev->asic_type != CHIP_STONEY) &&
1022 (adev->asic_type != CHIP_TOPAZ)) {
e3c7656c
SL		 1023 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1024 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1025 if (!err) {
1026 err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
1027 if (err)
1028 goto out;
1029 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1030 adev->gfx.mec2_fw->data;
1031 adev->gfx.mec2_fw_version =
1032 le32_to_cpu(cp_hdr->header.ucode_version);
1033 adev->gfx.mec2_feature_version =
1034 le32_to_cpu(cp_hdr->ucode_feature_version);
1035 } else {
1036 err = 0;
1037 adev->gfx.mec2_fw = NULL;
1038 }
aaa36a97
AD		 1039 }
1040
1041 if (adev->firmware.smu_load) {
1042 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1043 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1044 info->fw = adev->gfx.pfp_fw;
1045 header = (const struct common_firmware_header *)info->fw->data;
1046 adev->firmware.fw_size +=
1047 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1048
1049 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1050 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1051 info->fw = adev->gfx.me_fw;
1052 header = (const struct common_firmware_header *)info->fw->data;
1053 adev->firmware.fw_size +=
1054 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1055
1056 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1057 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1058 info->fw = adev->gfx.ce_fw;
1059 header = (const struct common_firmware_header *)info->fw->data;
1060 adev->firmware.fw_size +=
1061 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1062
1063 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1064 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1065 info->fw = adev->gfx.rlc_fw;
1066 header = (const struct common_firmware_header *)info->fw->data;
1067 adev->firmware.fw_size +=
1068 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1069
1070 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1071 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1072 info->fw = adev->gfx.mec_fw;
1073 header = (const struct common_firmware_header *)info->fw->data;
1074 adev->firmware.fw_size +=
1075 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1076
4c2b2453
ML		 1077 /* we need account JT in */
1078 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1079 adev->firmware.fw_size +=
1080 ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE);
1081
bed5712e
ML		 1082 if (amdgpu_sriov_vf(adev)) {
1083 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_STORAGE];
1084 info->ucode_id = AMDGPU_UCODE_ID_STORAGE;
1085 info->fw = adev->gfx.mec_fw;
1086 adev->firmware.fw_size +=
1087 ALIGN(le32_to_cpu(64 * PAGE_SIZE), PAGE_SIZE);
1088 }
1089
aaa36a97
AD		 1090 if (adev->gfx.mec2_fw) {
1091 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1092 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1093 info->fw = adev->gfx.mec2_fw;
1094 header = (const struct common_firmware_header *)info->fw->data;
1095 adev->firmware.fw_size +=
1096 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1097 }
1098
1099 }
1100
1101out:
1102 if (err) {
1103 dev_err(adev->dev,
1104 "gfx8: Failed to load firmware \"%s\"\n",
1105 fw_name);
1106 release_firmware(adev->gfx.pfp_fw);
1107 adev->gfx.pfp_fw = NULL;
1108 release_firmware(adev->gfx.me_fw);
1109 adev->gfx.me_fw = NULL;
1110 release_firmware(adev->gfx.ce_fw);
1111 adev->gfx.ce_fw = NULL;
1112 release_firmware(adev->gfx.rlc_fw);
1113 adev->gfx.rlc_fw = NULL;
1114 release_firmware(adev->gfx.mec_fw);
1115 adev->gfx.mec_fw = NULL;
1116 release_firmware(adev->gfx.mec2_fw);
1117 adev->gfx.mec2_fw = NULL;
1118 }
1119 return err;
1120}
1121
2b6cd977
EH		 1122static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev,
1123 volatile u32 *buffer)
1124{
1125 u32 count = 0, i;
1126 const struct cs_section_def *sect = NULL;
1127 const struct cs_extent_def *ext = NULL;
1128
1129 if (adev->gfx.rlc.cs_data == NULL)
1130 return;
1131 if (buffer == NULL)
1132 return;
1133
1134 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1135 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1136
1137 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1138 buffer[count++] = cpu_to_le32(0x80000000);
1139 buffer[count++] = cpu_to_le32(0x80000000);
1140
1141 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1142 for (ext = sect->section; ext->extent != NULL; ++ext) {
1143 if (sect->id == SECT_CONTEXT) {
1144 buffer[count++] =
1145 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1146 buffer[count++] = cpu_to_le32(ext->reg_index -
1147 PACKET3_SET_CONTEXT_REG_START);
1148 for (i = 0; i < ext->reg_count; i++)
1149 buffer[count++] = cpu_to_le32(ext->extent[i]);
1150 } else {
1151 return;
1152 }
1153 }
1154 }
1155
1156 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
1157 buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG -
1158 PACKET3_SET_CONTEXT_REG_START);
34817db6
AD		 1159 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config);
1160 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config_1);
2b6cd977
EH		 1161
1162 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1163 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1164
1165 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1166 buffer[count++] = cpu_to_le32(0);
1167}
1168
fb16007b
AD		 1169static void cz_init_cp_jump_table(struct amdgpu_device *adev)
1170{
1171 const __le32 *fw_data;
1172 volatile u32 *dst_ptr;
1173 int me, i, max_me = 4;
1174 u32 bo_offset = 0;
1175 u32 table_offset, table_size;
1176
1177 if (adev->asic_type == CHIP_CARRIZO)
1178 max_me = 5;
1179
1180 /* write the cp table buffer */
1181 dst_ptr = adev->gfx.rlc.cp_table_ptr;
1182 for (me = 0; me < max_me; me++) {
1183 if (me == 0) {
1184 const struct gfx_firmware_header_v1_0 *hdr =
1185 (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1186 fw_data = (const __le32 *)
1187 (adev->gfx.ce_fw->data +
1188 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1189 table_offset = le32_to_cpu(hdr->jt_offset);
1190 table_size = le32_to_cpu(hdr->jt_size);
1191 } else if (me == 1) {
1192 const struct gfx_firmware_header_v1_0 *hdr =
1193 (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1194 fw_data = (const __le32 *)
1195 (adev->gfx.pfp_fw->data +
1196 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1197 table_offset = le32_to_cpu(hdr->jt_offset);
1198 table_size = le32_to_cpu(hdr->jt_size);
1199 } else if (me == 2) {
1200 const struct gfx_firmware_header_v1_0 *hdr =
1201 (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1202 fw_data = (const __le32 *)
1203 (adev->gfx.me_fw->data +
1204 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1205 table_offset = le32_to_cpu(hdr->jt_offset);
1206 table_size = le32_to_cpu(hdr->jt_size);
1207 } else if (me == 3) {
1208 const struct gfx_firmware_header_v1_0 *hdr =
1209 (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1210 fw_data = (const __le32 *)
1211 (adev->gfx.mec_fw->data +
1212 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1213 table_offset = le32_to_cpu(hdr->jt_offset);
1214 table_size = le32_to_cpu(hdr->jt_size);
1215 } else if (me == 4) {
1216 const struct gfx_firmware_header_v1_0 *hdr =
1217 (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec2_fw->data;
1218 fw_data = (const __le32 *)
1219 (adev->gfx.mec2_fw->data +
1220 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1221 table_offset = le32_to_cpu(hdr->jt_offset);
1222 table_size = le32_to_cpu(hdr->jt_size);
1223 }
1224
1225 for (i = 0; i < table_size; i ++) {
1226 dst_ptr[bo_offset + i] =
1227 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
1228 }
1229
1230 bo_offset += table_size;
1231 }
1232}
1233
2b6cd977
EH		 1234static void gfx_v8_0_rlc_fini(struct amdgpu_device *adev)
1235{
1236 int r;
1237
1238 /* clear state block */
1239 if (adev->gfx.rlc.clear_state_obj) {
1240 r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
1241 if (unlikely(r != 0))
62d2ce4b 1242 dev_warn(adev->dev, "(%d) reserve RLC cbs bo failed\n", r);
2b6cd977
EH		 1243 amdgpu_bo_unpin(adev->gfx.rlc.clear_state_obj);
1244 amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
2b6cd977
EH		 1245 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
1246 adev->gfx.rlc.clear_state_obj = NULL;
1247 }
fb16007b
AD		 1248
1249 /* jump table block */
1250 if (adev->gfx.rlc.cp_table_obj) {
1251 r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, false);
1252 if (unlikely(r != 0))
1253 dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r);
1254 amdgpu_bo_unpin(adev->gfx.rlc.cp_table_obj);
1255 amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
fb16007b
AD		 1256 amdgpu_bo_unref(&adev->gfx.rlc.cp_table_obj);
1257 adev->gfx.rlc.cp_table_obj = NULL;
1258 }
2b6cd977
EH		 1259}
1260
1261static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
1262{
1263 volatile u32 *dst_ptr;
1264 u32 dws;
1265 const struct cs_section_def *cs_data;
1266 int r;
1267
1268 adev->gfx.rlc.cs_data = vi_cs_data;
1269
1270 cs_data = adev->gfx.rlc.cs_data;
1271
1272 if (cs_data) {
1273 /* clear state block */
1274 adev->gfx.rlc.clear_state_size = dws = gfx_v8_0_get_csb_size(adev);
1275
1276 if (adev->gfx.rlc.clear_state_obj == NULL) {
1277 r = amdgpu_bo_create(adev, dws * 4, PAGE_SIZE, true,
1278 AMDGPU_GEM_DOMAIN_VRAM,
03f48dd5
CK		 1279 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
1280 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
2b6cd977
EH		 1281 NULL, NULL,
1282 &adev->gfx.rlc.clear_state_obj);
1283 if (r) {
1284 dev_warn(adev->dev, "(%d) create RLC c bo failed\n", r);
1285 gfx_v8_0_rlc_fini(adev);
1286 return r;
1287 }
1288 }
1289 r = amdgpu_bo_reserve(adev->gfx.rlc.clear_state_obj, false);
1290 if (unlikely(r != 0)) {
1291 gfx_v8_0_rlc_fini(adev);
1292 return r;
1293 }
1294 r = amdgpu_bo_pin(adev->gfx.rlc.clear_state_obj, AMDGPU_GEM_DOMAIN_VRAM,
1295 &adev->gfx.rlc.clear_state_gpu_addr);
1296 if (r) {
1297 amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
62d2ce4b 1298 dev_warn(adev->dev, "(%d) pin RLC cbs bo failed\n", r);
2b6cd977
EH		 1299 gfx_v8_0_rlc_fini(adev);
1300 return r;
1301 }
1302
1303 r = amdgpu_bo_kmap(adev->gfx.rlc.clear_state_obj, (void **)&adev->gfx.rlc.cs_ptr);
1304 if (r) {
62d2ce4b 1305 dev_warn(adev->dev, "(%d) map RLC cbs bo failed\n", r);
2b6cd977
EH		 1306 gfx_v8_0_rlc_fini(adev);
1307 return r;
1308 }
1309 /* set up the cs buffer */
1310 dst_ptr = adev->gfx.rlc.cs_ptr;
1311 gfx_v8_0_get_csb_buffer(adev, dst_ptr);
1312 amdgpu_bo_kunmap(adev->gfx.rlc.clear_state_obj);
1313 amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
1314 }
1315
fb16007b
AD		 1316 if ((adev->asic_type == CHIP_CARRIZO) ||
1317 (adev->asic_type == CHIP_STONEY)) {
07cf1a0b 1318 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
fb16007b
AD		 1319 if (adev->gfx.rlc.cp_table_obj == NULL) {
1320 r = amdgpu_bo_create(adev, adev->gfx.rlc.cp_table_size, PAGE_SIZE, true,
1321 AMDGPU_GEM_DOMAIN_VRAM,
03f48dd5
CK		 1322 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
1323 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS,
fb16007b
AD		 1324 NULL, NULL,
1325 &adev->gfx.rlc.cp_table_obj);
1326 if (r) {
1327 dev_warn(adev->dev, "(%d) create RLC cp table bo failed\n", r);
1328 return r;
1329 }
1330 }
1331
1332 r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, false);
1333 if (unlikely(r != 0)) {
1334 dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r);
1335 return r;
1336 }
1337 r = amdgpu_bo_pin(adev->gfx.rlc.cp_table_obj, AMDGPU_GEM_DOMAIN_VRAM,
1338 &adev->gfx.rlc.cp_table_gpu_addr);
1339 if (r) {
1340 amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
62d2ce4b 1341 dev_warn(adev->dev, "(%d) pin RLC cp table bo failed\n", r);
fb16007b
AD		 1342 return r;
1343 }
1344 r = amdgpu_bo_kmap(adev->gfx.rlc.cp_table_obj, (void **)&adev->gfx.rlc.cp_table_ptr);
1345 if (r) {
1346 dev_warn(adev->dev, "(%d) map RLC cp table bo failed\n", r);
1347 return r;
1348 }
1349
1350 cz_init_cp_jump_table(adev);
1351
1352 amdgpu_bo_kunmap(adev->gfx.rlc.cp_table_obj);
1353 amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
fb16007b
AD		 1354 }
1355
2b6cd977
EH		 1356 return 0;
1357}
1358
aaa36a97
AD		 1359static void gfx_v8_0_mec_fini(struct amdgpu_device *adev)
1360{
1361 int r;
1362
1363 if (adev->gfx.mec.hpd_eop_obj) {
1364 r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, false);
1365 if (unlikely(r != 0))
1366 dev_warn(adev->dev, "(%d) reserve HPD EOP bo failed\n", r);
1367 amdgpu_bo_unpin(adev->gfx.mec.hpd_eop_obj);
1368 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
aaa36a97
AD		 1369 amdgpu_bo_unref(&adev->gfx.mec.hpd_eop_obj);
1370 adev->gfx.mec.hpd_eop_obj = NULL;
1371 }
1372}
1373
4e638ae9
XY		 1374static int gfx_v8_0_kiq_init_ring(struct amdgpu_device *adev,
1375 struct amdgpu_ring *ring,
1376 struct amdgpu_irq_src *irq)
1377{
1378 int r = 0;
1379
880e87e3
XY		 1380 if (amdgpu_sriov_vf(adev)) {
1381 r = amdgpu_wb_get(adev, &adev->virt.reg_val_offs);
1382 if (r)
1383 return r;
1384 }
1385
4e638ae9
XY		 1386 ring->adev = NULL;
1387 ring->ring_obj = NULL;
1388 ring->use_doorbell = true;
1389 ring->doorbell_index = AMDGPU_DOORBELL_KIQ;
1390 if (adev->gfx.mec2_fw) {
1391 ring->me = 2;
1392 ring->pipe = 0;
1393 } else {
1394 ring->me = 1;
1395 ring->pipe = 1;
1396 }
1397
1398 irq->data = ring;
1399 ring->queue = 0;
1400 sprintf(ring->name, "kiq %d.%d.%d", ring->me, ring->pipe, ring->queue);
1401 r = amdgpu_ring_init(adev, ring, 1024,
1402 irq, AMDGPU_CP_KIQ_IRQ_DRIVER0);
1403 if (r)
1404 dev_warn(adev->dev, "(%d) failed to init kiq ring\n", r);
1405
1406 return r;
1407}
1408
1409static void gfx_v8_0_kiq_free_ring(struct amdgpu_ring *ring,
1410 struct amdgpu_irq_src *irq)
1411{
880e87e3
XY		 1412 if (amdgpu_sriov_vf(ring->adev))
1413 amdgpu_wb_free(ring->adev, ring->adev->virt.reg_val_offs);
1414
4e638ae9
XY		 1415 amdgpu_ring_fini(ring);
1416 irq->data = NULL;
1417}
1418
aaa36a97
AD		 1419#define MEC_HPD_SIZE 2048
1420
1421static int gfx_v8_0_mec_init(struct amdgpu_device *adev)
1422{
1423 int r;
1424 u32 *hpd;
1425
1426 /*
1427 * we assign only 1 pipe because all other pipes will
1428 * be handled by KFD
1429 */
1430 adev->gfx.mec.num_mec = 1;
1431 adev->gfx.mec.num_pipe = 1;
1432 adev->gfx.mec.num_queue = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe * 8;
1433
1434 if (adev->gfx.mec.hpd_eop_obj == NULL) {
1435 r = amdgpu_bo_create(adev,
ad3b9614 1436 adev->gfx.mec.num_queue * MEC_HPD_SIZE,
aaa36a97 1437 PAGE_SIZE, true,
72d7668b 1438 AMDGPU_GEM_DOMAIN_GTT, 0, NULL, NULL,
aaa36a97
AD		 1439 &adev->gfx.mec.hpd_eop_obj);
1440 if (r) {
1441 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1442 return r;
1443 }
1444 }
1445
1446 r = amdgpu_bo_reserve(adev->gfx.mec.hpd_eop_obj, false);
1447 if (unlikely(r != 0)) {
1448 gfx_v8_0_mec_fini(adev);
1449 return r;
1450 }
1451 r = amdgpu_bo_pin(adev->gfx.mec.hpd_eop_obj, AMDGPU_GEM_DOMAIN_GTT,
1452 &adev->gfx.mec.hpd_eop_gpu_addr);
1453 if (r) {
1454 dev_warn(adev->dev, "(%d) pin HDP EOP bo failed\n", r);
1455 gfx_v8_0_mec_fini(adev);
1456 return r;
1457 }
1458 r = amdgpu_bo_kmap(adev->gfx.mec.hpd_eop_obj, (void **)&hpd);
1459 if (r) {
1460 dev_warn(adev->dev, "(%d) map HDP EOP bo failed\n", r);
1461 gfx_v8_0_mec_fini(adev);
1462 return r;
1463 }
1464
ad3b9614 1465 memset(hpd, 0, adev->gfx.mec.num_queue * MEC_HPD_SIZE);
aaa36a97
AD		 1466
1467 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1468 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1469
1470 return 0;
1471}
1472
4e638ae9
XY		 1473static void gfx_v8_0_kiq_fini(struct amdgpu_device *adev)
1474{
1475 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
1476
1477 amdgpu_bo_free_kernel(&kiq->eop_obj, &kiq->eop_gpu_addr, NULL);
1478 kiq->eop_obj = NULL;
1479}
1480
1481static int gfx_v8_0_kiq_init(struct amdgpu_device *adev)
1482{
1483 int r;
1484 u32 *hpd;
1485 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
1486
1487 r = amdgpu_bo_create_kernel(adev, MEC_HPD_SIZE, PAGE_SIZE,
1488 AMDGPU_GEM_DOMAIN_GTT, &kiq->eop_obj,
1489 &kiq->eop_gpu_addr, (void **)&hpd);
1490 if (r) {
1491 dev_warn(adev->dev, "failed to create KIQ bo (%d).\n", r);
1492 return r;
1493 }
1494
1495 memset(hpd, 0, MEC_HPD_SIZE);
1496
1497 amdgpu_bo_kunmap(kiq->eop_obj);
1498
1499 return 0;
1500}
1501
ccba7691
AD		 1502static const u32 vgpr_init_compute_shader[] =
1503{
1504 0x7e000209, 0x7e020208,
1505 0x7e040207, 0x7e060206,
1506 0x7e080205, 0x7e0a0204,
1507 0x7e0c0203, 0x7e0e0202,
1508 0x7e100201, 0x7e120200,
1509 0x7e140209, 0x7e160208,
1510 0x7e180207, 0x7e1a0206,
1511 0x7e1c0205, 0x7e1e0204,
1512 0x7e200203, 0x7e220202,
1513 0x7e240201, 0x7e260200,
1514 0x7e280209, 0x7e2a0208,
1515 0x7e2c0207, 0x7e2e0206,
1516 0x7e300205, 0x7e320204,
1517 0x7e340203, 0x7e360202,
1518 0x7e380201, 0x7e3a0200,
1519 0x7e3c0209, 0x7e3e0208,
1520 0x7e400207, 0x7e420206,
1521 0x7e440205, 0x7e460204,
1522 0x7e480203, 0x7e4a0202,
1523 0x7e4c0201, 0x7e4e0200,
1524 0x7e500209, 0x7e520208,
1525 0x7e540207, 0x7e560206,
1526 0x7e580205, 0x7e5a0204,
1527 0x7e5c0203, 0x7e5e0202,
1528 0x7e600201, 0x7e620200,
1529 0x7e640209, 0x7e660208,
1530 0x7e680207, 0x7e6a0206,
1531 0x7e6c0205, 0x7e6e0204,
1532 0x7e700203, 0x7e720202,
1533 0x7e740201, 0x7e760200,
1534 0x7e780209, 0x7e7a0208,
1535 0x7e7c0207, 0x7e7e0206,
1536 0xbf8a0000, 0xbf810000,
1537};
1538
1539static const u32 sgpr_init_compute_shader[] =
1540{
1541 0xbe8a0100, 0xbe8c0102,
1542 0xbe8e0104, 0xbe900106,
1543 0xbe920108, 0xbe940100,
1544 0xbe960102, 0xbe980104,
1545 0xbe9a0106, 0xbe9c0108,
1546 0xbe9e0100, 0xbea00102,
1547 0xbea20104, 0xbea40106,
1548 0xbea60108, 0xbea80100,
1549 0xbeaa0102, 0xbeac0104,
1550 0xbeae0106, 0xbeb00108,
1551 0xbeb20100, 0xbeb40102,
1552 0xbeb60104, 0xbeb80106,
1553 0xbeba0108, 0xbebc0100,
1554 0xbebe0102, 0xbec00104,
1555 0xbec20106, 0xbec40108,
1556 0xbec60100, 0xbec80102,
1557 0xbee60004, 0xbee70005,
1558 0xbeea0006, 0xbeeb0007,
1559 0xbee80008, 0xbee90009,
1560 0xbefc0000, 0xbf8a0000,
1561 0xbf810000, 0x00000000,
1562};
1563
1564static const u32 vgpr_init_regs[] =
1565{
1566 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xffffffff,
1567 mmCOMPUTE_RESOURCE_LIMITS, 0,
1568 mmCOMPUTE_NUM_THREAD_X, 256*4,
1569 mmCOMPUTE_NUM_THREAD_Y, 1,
1570 mmCOMPUTE_NUM_THREAD_Z, 1,
1571 mmCOMPUTE_PGM_RSRC2, 20,
1572 mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1573 mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1574 mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1575 mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1576 mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1577 mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1578 mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1579 mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1580 mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1581 mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1582};
1583
1584static const u32 sgpr1_init_regs[] =
1585{
1586 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0x0f,
1587 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000,
1588 mmCOMPUTE_NUM_THREAD_X, 256*5,
1589 mmCOMPUTE_NUM_THREAD_Y, 1,
1590 mmCOMPUTE_NUM_THREAD_Z, 1,
1591 mmCOMPUTE_PGM_RSRC2, 20,
1592 mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1593 mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1594 mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1595 mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1596 mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1597 mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1598 mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1599 mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1600 mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1601 mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1602};
1603
1604static const u32 sgpr2_init_regs[] =
1605{
1606 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xf0,
1607 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000,
1608 mmCOMPUTE_NUM_THREAD_X, 256*5,
1609 mmCOMPUTE_NUM_THREAD_Y, 1,
1610 mmCOMPUTE_NUM_THREAD_Z, 1,
1611 mmCOMPUTE_PGM_RSRC2, 20,
1612 mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1613 mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1614 mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1615 mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1616 mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1617 mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1618 mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1619 mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1620 mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1621 mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1622};
1623
1624static const u32 sec_ded_counter_registers[] =
1625{
1626 mmCPC_EDC_ATC_CNT,
1627 mmCPC_EDC_SCRATCH_CNT,
1628 mmCPC_EDC_UCODE_CNT,
1629 mmCPF_EDC_ATC_CNT,
1630 mmCPF_EDC_ROQ_CNT,
1631 mmCPF_EDC_TAG_CNT,
1632 mmCPG_EDC_ATC_CNT,
1633 mmCPG_EDC_DMA_CNT,
1634 mmCPG_EDC_TAG_CNT,
1635 mmDC_EDC_CSINVOC_CNT,
1636 mmDC_EDC_RESTORE_CNT,
1637 mmDC_EDC_STATE_CNT,
1638 mmGDS_EDC_CNT,
1639 mmGDS_EDC_GRBM_CNT,
1640 mmGDS_EDC_OA_DED,
1641 mmSPI_EDC_CNT,
1642 mmSQC_ATC_EDC_GATCL1_CNT,
1643 mmSQC_EDC_CNT,
1644 mmSQ_EDC_DED_CNT,
1645 mmSQ_EDC_INFO,
1646 mmSQ_EDC_SEC_CNT,
1647 mmTCC_EDC_CNT,
1648 mmTCP_ATC_EDC_GATCL1_CNT,
1649 mmTCP_EDC_CNT,
1650 mmTD_EDC_CNT
1651};
1652
1653static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
1654{
1655 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
1656 struct amdgpu_ib ib;
f54d1867 1657 struct dma_fence *f = NULL;
ccba7691
AD		 1658 int r, i;
1659 u32 tmp;
1660 unsigned total_size, vgpr_offset, sgpr_offset;
1661 u64 gpu_addr;
1662
1663 /* only supported on CZ */
1664 if (adev->asic_type != CHIP_CARRIZO)
1665 return 0;
1666
1667 /* bail if the compute ring is not ready */
1668 if (!ring->ready)
1669 return 0;
1670
1671 tmp = RREG32(mmGB_EDC_MODE);
1672 WREG32(mmGB_EDC_MODE, 0);
1673
1674 total_size =
1675 (((ARRAY_SIZE(vgpr_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1676 total_size +=
1677 (((ARRAY_SIZE(sgpr1_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1678 total_size +=
1679 (((ARRAY_SIZE(sgpr2_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1680 total_size = ALIGN(total_size, 256);
1681 vgpr_offset = total_size;
1682 total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256);
1683 sgpr_offset = total_size;
1684 total_size += sizeof(sgpr_init_compute_shader);
1685
1686 /* allocate an indirect buffer to put the commands in */
1687 memset(&ib, 0, sizeof(ib));
b07c60c0 1688 r = amdgpu_ib_get(adev, NULL, total_size, &ib);
ccba7691
AD		 1689 if (r) {
1690 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
1691 return r;
1692 }
1693
1694 /* load the compute shaders */
1695 for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++)
1696 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i];
1697
1698 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
1699 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
1700
1701 /* init the ib length to 0 */
1702 ib.length_dw = 0;
1703
1704 /* VGPR */
1705 /* write the register state for the compute dispatch */
1706 for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i += 2) {
1707 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1708 ib.ptr[ib.length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START;
1709 ib.ptr[ib.length_dw++] = vgpr_init_regs[i + 1];
1710 }
1711 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1712 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
1713 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1714 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1715 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1716 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1717
1718 /* write dispatch packet */
1719 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1720 ib.ptr[ib.length_dw++] = 8; /* x */
1721 ib.ptr[ib.length_dw++] = 1; /* y */
1722 ib.ptr[ib.length_dw++] = 1; /* z */
1723 ib.ptr[ib.length_dw++] =
1724 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1725
1726 /* write CS partial flush packet */
1727 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1728 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1729
1730 /* SGPR1 */
1731 /* write the register state for the compute dispatch */
1732 for (i = 0; i < ARRAY_SIZE(sgpr1_init_regs); i += 2) {
1733 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1734 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START;
1735 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i + 1];
1736 }
1737 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1738 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
1739 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1740 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1741 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1742 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1743
1744 /* write dispatch packet */
1745 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1746 ib.ptr[ib.length_dw++] = 8; /* x */
1747 ib.ptr[ib.length_dw++] = 1; /* y */
1748 ib.ptr[ib.length_dw++] = 1; /* z */
1749 ib.ptr[ib.length_dw++] =
1750 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1751
1752 /* write CS partial flush packet */
1753 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1754 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1755
1756 /* SGPR2 */
1757 /* write the register state for the compute dispatch */
1758 for (i = 0; i < ARRAY_SIZE(sgpr2_init_regs); i += 2) {
1759 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1760 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START;
1761 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i + 1];
1762 }
1763 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1764 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
1765 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1766 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1767 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1768 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1769
1770 /* write dispatch packet */
1771 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1772 ib.ptr[ib.length_dw++] = 8; /* x */
1773 ib.ptr[ib.length_dw++] = 1; /* y */
1774 ib.ptr[ib.length_dw++] = 1; /* z */
1775 ib.ptr[ib.length_dw++] =
1776 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1777
1778 /* write CS partial flush packet */
1779 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1780 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1781
1782 /* shedule the ib on the ring */
50ddc75e 1783 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
ccba7691
AD		 1784 if (r) {
1785 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
1786 goto fail;
1787 }
1788
1789 /* wait for the GPU to finish processing the IB */
f54d1867 1790 r = dma_fence_wait(f, false);
ccba7691
AD		 1791 if (r) {
1792 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
1793 goto fail;
1794 }
1795
1796 tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, DED_MODE, 2);
1797 tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, PROP_FED, 1);
1798 WREG32(mmGB_EDC_MODE, tmp);
1799
1800 tmp = RREG32(mmCC_GC_EDC_CONFIG);
1801 tmp = REG_SET_FIELD(tmp, CC_GC_EDC_CONFIG, DIS_EDC, 0) | 1;
1802 WREG32(mmCC_GC_EDC_CONFIG, tmp);
1803
1804
1805 /* read back registers to clear the counters */
1806 for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++)
1807 RREG32(sec_ded_counter_registers[i]);
1808
1809fail:
cc55c45d 1810 amdgpu_ib_free(adev, &ib, NULL);
f54d1867 1811 dma_fence_put(f);
ccba7691
AD		 1812
1813 return r;
1814}
1815
68182d90 1816static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev)
0bde3a95
AD		 1817{
1818 u32 gb_addr_config;
1819 u32 mc_shared_chmap, mc_arb_ramcfg;
1820 u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map;
1821 u32 tmp;
68182d90 1822 int ret;
0bde3a95
AD		 1823
1824 switch (adev->asic_type) {
1825 case CHIP_TOPAZ:
1826 adev->gfx.config.max_shader_engines = 1;
1827 adev->gfx.config.max_tile_pipes = 2;
1828 adev->gfx.config.max_cu_per_sh = 6;
1829 adev->gfx.config.max_sh_per_se = 1;
1830 adev->gfx.config.max_backends_per_se = 2;
1831 adev->gfx.config.max_texture_channel_caches = 2;
1832 adev->gfx.config.max_gprs = 256;
1833 adev->gfx.config.max_gs_threads = 32;
1834 adev->gfx.config.max_hw_contexts = 8;
1835
1836 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1837 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1838 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1839 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1840 gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN;
1841 break;
1842 case CHIP_FIJI:
1843 adev->gfx.config.max_shader_engines = 4;
1844 adev->gfx.config.max_tile_pipes = 16;
1845 adev->gfx.config.max_cu_per_sh = 16;
1846 adev->gfx.config.max_sh_per_se = 1;
1847 adev->gfx.config.max_backends_per_se = 4;
5f2e816b 1848 adev->gfx.config.max_texture_channel_caches = 16;
0bde3a95
AD		 1849 adev->gfx.config.max_gprs = 256;
1850 adev->gfx.config.max_gs_threads = 32;
1851 adev->gfx.config.max_hw_contexts = 8;
1852
68182d90
FC		 1853 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1854 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1855 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1856 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1857 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1858 break;
2cc0c0b5 1859 case CHIP_POLARIS11:
c4642a47 1860 case CHIP_POLARIS12:
68182d90
FC		 1861 ret = amdgpu_atombios_get_gfx_info(adev);
1862 if (ret)
1863 return ret;
1864 adev->gfx.config.max_gprs = 256;
1865 adev->gfx.config.max_gs_threads = 32;
1866 adev->gfx.config.max_hw_contexts = 8;
1867
1868 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1869 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1870 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1871 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
2cc0c0b5 1872 gb_addr_config = POLARIS11_GB_ADDR_CONFIG_GOLDEN;
68182d90 1873 break;
2cc0c0b5 1874 case CHIP_POLARIS10:
68182d90
FC		 1875 ret = amdgpu_atombios_get_gfx_info(adev);
1876 if (ret)
1877 return ret;
1878 adev->gfx.config.max_gprs = 256;
1879 adev->gfx.config.max_gs_threads = 32;
1880 adev->gfx.config.max_hw_contexts = 8;
1881
0bde3a95
AD		 1882 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1883 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1884 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1885 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1886 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1887 break;
1888 case CHIP_TONGA:
1889 adev->gfx.config.max_shader_engines = 4;
1890 adev->gfx.config.max_tile_pipes = 8;
1891 adev->gfx.config.max_cu_per_sh = 8;
1892 adev->gfx.config.max_sh_per_se = 1;
1893 adev->gfx.config.max_backends_per_se = 2;
1894 adev->gfx.config.max_texture_channel_caches = 8;
1895 adev->gfx.config.max_gprs = 256;
1896 adev->gfx.config.max_gs_threads = 32;
1897 adev->gfx.config.max_hw_contexts = 8;
1898
1899 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1900 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1901 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1902 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1903 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1904 break;
1905 case CHIP_CARRIZO:
1906 adev->gfx.config.max_shader_engines = 1;
1907 adev->gfx.config.max_tile_pipes = 2;
1908 adev->gfx.config.max_sh_per_se = 1;
1909 adev->gfx.config.max_backends_per_se = 2;
1910
1911 switch (adev->pdev->revision) {
1912 case 0xc4:
1913 case 0x84:
1914 case 0xc8:
1915 case 0xcc:
b8b339ea
AD		 1916 case 0xe1:
1917 case 0xe3:
0bde3a95
AD		 1918 /* B10 */
1919 adev->gfx.config.max_cu_per_sh = 8;
1920 break;
1921 case 0xc5:
1922 case 0x81:
1923 case 0x85:
1924 case 0xc9:
1925 case 0xcd:
b8b339ea
AD		 1926 case 0xe2:
1927 case 0xe4:
0bde3a95
AD		 1928 /* B8 */
1929 adev->gfx.config.max_cu_per_sh = 6;
1930 break;
1931 case 0xc6:
1932 case 0xca:
1933 case 0xce:
b8b339ea 1934 case 0x88:
0bde3a95
AD		 1935 /* B6 */
1936 adev->gfx.config.max_cu_per_sh = 6;
1937 break;
1938 case 0xc7:
1939 case 0x87:
1940 case 0xcb:
b8b339ea
AD		 1941 case 0xe5:
1942 case 0x89:
0bde3a95
AD		 1943 default:
1944 /* B4 */
1945 adev->gfx.config.max_cu_per_sh = 4;
1946 break;
1947 }
1948
1949 adev->gfx.config.max_texture_channel_caches = 2;
1950 adev->gfx.config.max_gprs = 256;
1951 adev->gfx.config.max_gs_threads = 32;
1952 adev->gfx.config.max_hw_contexts = 8;
1953
e3c7656c
SL		 1954 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1955 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1956 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1957 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1958 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
1959 break;
1960 case CHIP_STONEY:
1961 adev->gfx.config.max_shader_engines = 1;
1962 adev->gfx.config.max_tile_pipes = 2;
1963 adev->gfx.config.max_sh_per_se = 1;
1964 adev->gfx.config.max_backends_per_se = 1;
1965
1966 switch (adev->pdev->revision) {
1967 case 0xc0:
1968 case 0xc1:
1969 case 0xc2:
1970 case 0xc4:
1971 case 0xc8:
1972 case 0xc9:
1973 adev->gfx.config.max_cu_per_sh = 3;
1974 break;
1975 case 0xd0:
1976 case 0xd1:
1977 case 0xd2:
1978 default:
1979 adev->gfx.config.max_cu_per_sh = 2;
1980 break;
1981 }
1982
1983 adev->gfx.config.max_texture_channel_caches = 2;
1984 adev->gfx.config.max_gprs = 256;
1985 adev->gfx.config.max_gs_threads = 16;
1986 adev->gfx.config.max_hw_contexts = 8;
1987
0bde3a95
AD		 1988 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1989 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1990 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1991 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1992 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
1993 break;
1994 default:
1995 adev->gfx.config.max_shader_engines = 2;
1996 adev->gfx.config.max_tile_pipes = 4;
1997 adev->gfx.config.max_cu_per_sh = 2;
1998 adev->gfx.config.max_sh_per_se = 1;
1999 adev->gfx.config.max_backends_per_se = 2;
2000 adev->gfx.config.max_texture_channel_caches = 4;
2001 adev->gfx.config.max_gprs = 256;
2002 adev->gfx.config.max_gs_threads = 32;
2003 adev->gfx.config.max_hw_contexts = 8;
2004
2005 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2006 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2007 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2008 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
2009 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
2010 break;
2011 }
2012
2013 mc_shared_chmap = RREG32(mmMC_SHARED_CHMAP);
2014 adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG);
2015 mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg;
2016
2017 adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
2018 adev->gfx.config.mem_max_burst_length_bytes = 256;
2019 if (adev->flags & AMD_IS_APU) {
2020 /* Get memory bank mapping mode. */
2021 tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING);
2022 dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
2023 dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
2024
2025 tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING);
2026 dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
2027 dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
2028
2029 /* Validate settings in case only one DIMM installed. */
2030 if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12))
2031 dimm00_addr_map = 0;
2032 if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12))
2033 dimm01_addr_map = 0;
2034 if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12))
2035 dimm10_addr_map = 0;
2036 if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12))
2037 dimm11_addr_map = 0;
2038
2039 /* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */
2040 /* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */
2041 if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11))
2042 adev->gfx.config.mem_row_size_in_kb = 2;
2043 else
2044 adev->gfx.config.mem_row_size_in_kb = 1;
2045 } else {
2046 tmp = REG_GET_FIELD(mc_arb_ramcfg, MC_ARB_RAMCFG, NOOFCOLS);
2047 adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2048 if (adev->gfx.config.mem_row_size_in_kb > 4)
2049 adev->gfx.config.mem_row_size_in_kb = 4;
2050 }
2051
2052 adev->gfx.config.shader_engine_tile_size = 32;
2053 adev->gfx.config.num_gpus = 1;
2054 adev->gfx.config.multi_gpu_tile_size = 64;
2055
2056 /* fix up row size */
2057 switch (adev->gfx.config.mem_row_size_in_kb) {
2058 case 1:
2059 default:
2060 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 0);
2061 break;
2062 case 2:
2063 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 1);
2064 break;
2065 case 4:
2066 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 2);
2067 break;
2068 }
2069 adev->gfx.config.gb_addr_config = gb_addr_config;
68182d90
FC		 2070
2071 return 0;
0bde3a95
AD		 2072}
2073
5fc3aeeb 2074static int gfx_v8_0_sw_init(void *handle)
aaa36a97
AD		 2075{
2076 int i, r;
2077 struct amdgpu_ring *ring;
4e638ae9 2078 struct amdgpu_kiq *kiq;
5fc3aeeb 2079 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
aaa36a97 2080
4e638ae9
XY		 2081 /* KIQ event */
2082 r = amdgpu_irq_add_id(adev, 178, &adev->gfx.kiq.irq);
2083 if (r)
2084 return r;
2085
aaa36a97
AD		 2086 /* EOP Event */
2087 r = amdgpu_irq_add_id(adev, 181, &adev->gfx.eop_irq);
2088 if (r)
2089 return r;
2090
2091 /* Privileged reg */
2092 r = amdgpu_irq_add_id(adev, 184, &adev->gfx.priv_reg_irq);
2093 if (r)
2094 return r;
2095
2096 /* Privileged inst */
2097 r = amdgpu_irq_add_id(adev, 185, &adev->gfx.priv_inst_irq);
2098 if (r)
2099 return r;
2100
2101 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2102
2103 gfx_v8_0_scratch_init(adev);
2104
2105 r = gfx_v8_0_init_microcode(adev);
2106 if (r) {
2107 DRM_ERROR("Failed to load gfx firmware!\n");
2108 return r;
2109 }
2110
2b6cd977
EH		 2111 r = gfx_v8_0_rlc_init(adev);
2112 if (r) {
2113 DRM_ERROR("Failed to init rlc BOs!\n");
2114 return r;
2115 }
2116
aaa36a97
AD		 2117 r = gfx_v8_0_mec_init(adev);
2118 if (r) {
2119 DRM_ERROR("Failed to init MEC BOs!\n");
2120 return r;
2121 }
2122
4e638ae9
XY		 2123 r = gfx_v8_0_kiq_init(adev);
2124 if (r) {
2125 DRM_ERROR("Failed to init KIQ BOs!\n");
2126 return r;
2127 }
2128
2129 kiq = &adev->gfx.kiq;
2130 r = gfx_v8_0_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2131 if (r)
2132 return r;
2133
aaa36a97
AD		 2134 /* set up the gfx ring */
2135 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2136 ring = &adev->gfx.gfx_ring[i];
2137 ring->ring_obj = NULL;
2138 sprintf(ring->name, "gfx");
2139 /* no gfx doorbells on iceland */
2140 if (adev->asic_type != CHIP_TOPAZ) {
2141 ring->use_doorbell = true;
2142 ring->doorbell_index = AMDGPU_DOORBELL_GFX_RING0;
2143 }
2144
79887142
CK		 2145 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2146 AMDGPU_CP_IRQ_GFX_EOP);
aaa36a97
AD		 2147 if (r)
2148 return r;
2149 }
2150
2151 /* set up the compute queues */
2152 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
2153 unsigned irq_type;
2154
2155 /* max 32 queues per MEC */
2156 if ((i >= 32) || (i >= AMDGPU_MAX_COMPUTE_RINGS)) {
2157 DRM_ERROR("Too many (%d) compute rings!\n", i);
2158 break;
2159 }
2160 ring = &adev->gfx.compute_ring[i];
2161 ring->ring_obj = NULL;
2162 ring->use_doorbell = true;
2163 ring->doorbell_index = AMDGPU_DOORBELL_MEC_RING0 + i;
2164 ring->me = 1; /* first MEC */
2165 ring->pipe = i / 8;
2166 ring->queue = i % 8;
771c8ec1 2167 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
aaa36a97
AD		 2168 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP + ring->pipe;
2169 /* type-2 packets are deprecated on MEC, use type-3 instead */
79887142
CK		 2170 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2171 irq_type);
aaa36a97
AD		 2172 if (r)
2173 return r;
2174 }
2175
2176 /* reserve GDS, GWS and OA resource for gfx */
78bbbd9c
CK		 2177 r = amdgpu_bo_create_kernel(adev, adev->gds.mem.gfx_partition_size,
2178 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GDS,
2179 &adev->gds.gds_gfx_bo, NULL, NULL);
aaa36a97
AD		 2180 if (r)
2181 return r;
2182
78bbbd9c
CK		 2183 r = amdgpu_bo_create_kernel(adev, adev->gds.gws.gfx_partition_size,
2184 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GWS,
2185 &adev->gds.gws_gfx_bo, NULL, NULL);
aaa36a97
AD		 2186 if (r)
2187 return r;
2188
78bbbd9c
CK		 2189 r = amdgpu_bo_create_kernel(adev, adev->gds.oa.gfx_partition_size,
2190 PAGE_SIZE, AMDGPU_GEM_DOMAIN_OA,
2191 &adev->gds.oa_gfx_bo, NULL, NULL);
aaa36a97
AD		 2192 if (r)
2193 return r;
2194
a101a899
KW		 2195 adev->gfx.ce_ram_size = 0x8000;
2196
68182d90
FC		 2197 r = gfx_v8_0_gpu_early_init(adev);
2198 if (r)
2199 return r;
0bde3a95 2200
aaa36a97
AD		 2201 return 0;
2202}
2203
5fc3aeeb 2204static int gfx_v8_0_sw_fini(void *handle)
aaa36a97
AD		 2205{
2206 int i;
5fc3aeeb 2207 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
aaa36a97 2208
8640faed
JZ		 2209 amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
2210 amdgpu_bo_free_kernel(&adev->gds.gws_gfx_bo, NULL, NULL);
2211 amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
aaa36a97
AD		 2212
2213 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2214 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2215 for (i = 0; i < adev->gfx.num_compute_rings; i++)
2216 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
4e638ae9 2217 gfx_v8_0_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
aaa36a97 2218
4e638ae9 2219 gfx_v8_0_kiq_fini(adev);
aaa36a97 2220 gfx_v8_0_mec_fini(adev);
2b6cd977 2221 gfx_v8_0_rlc_fini(adev);
13331ac3 2222 gfx_v8_0_free_microcode(adev);
2b6cd977 2223
aaa36a97
AD		 2224 return 0;
2225}
2226
2227static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev)
2228{
90bea0ab 2229 uint32_t *modearray, *mod2array;
eb64526f
TSD		 2230 const u32 num_tile_mode_states = ARRAY_SIZE(adev->gfx.config.tile_mode_array);
2231 const u32 num_secondary_tile_mode_states = ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
90bea0ab 2232 u32 reg_offset;
aaa36a97 2233
90bea0ab
TSD		 2234 modearray = adev->gfx.config.tile_mode_array;
2235 mod2array = adev->gfx.config.macrotile_mode_array;
2236
2237 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2238 modearray[reg_offset] = 0;
2239
2240 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2241 mod2array[reg_offset] = 0;
aaa36a97
AD		 2242
2243 switch (adev->asic_type) {
2244 case CHIP_TOPAZ:
90bea0ab
TSD		 2245 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2246 PIPE_CONFIG(ADDR_SURF_P2) |
2247 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2248 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2249 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2250 PIPE_CONFIG(ADDR_SURF_P2) |
2251 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2252 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2253 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2254 PIPE_CONFIG(ADDR_SURF_P2) |
2255 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2256 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2257 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2258 PIPE_CONFIG(ADDR_SURF_P2) |
2259 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2260 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2261 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2262 PIPE_CONFIG(ADDR_SURF_P2) |
2263 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2264 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2265 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2266 PIPE_CONFIG(ADDR_SURF_P2) |
2267 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2268 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2269 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2270 PIPE_CONFIG(ADDR_SURF_P2) |
2271 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2272 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2273 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2274 PIPE_CONFIG(ADDR_SURF_P2));
2275 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2276 PIPE_CONFIG(ADDR_SURF_P2) |
2277 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2278 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2279 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2280 PIPE_CONFIG(ADDR_SURF_P2) |
2281 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2282 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2283 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2284 PIPE_CONFIG(ADDR_SURF_P2) |
2285 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2286 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2287 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2288 PIPE_CONFIG(ADDR_SURF_P2) |
2289 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2290 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2291 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2292 PIPE_CONFIG(ADDR_SURF_P2) |
2293 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2294 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2295 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2296 PIPE_CONFIG(ADDR_SURF_P2) |
2297 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2298 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2299 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2300 PIPE_CONFIG(ADDR_SURF_P2) |
2301 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2302 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2303 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2304 PIPE_CONFIG(ADDR_SURF_P2) |
2305 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2306 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2307 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2308 PIPE_CONFIG(ADDR_SURF_P2) |
2309 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2310 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2311 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2312 PIPE_CONFIG(ADDR_SURF_P2) |
2313 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2314 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2315 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2316 PIPE_CONFIG(ADDR_SURF_P2) |
2317 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2318 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2319 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2320 PIPE_CONFIG(ADDR_SURF_P2) |
2321 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2322 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2323 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2324 PIPE_CONFIG(ADDR_SURF_P2) |
2325 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2326 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2327 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2328 PIPE_CONFIG(ADDR_SURF_P2) |
2329 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2330 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2331 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2332 PIPE_CONFIG(ADDR_SURF_P2) |
2333 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2334 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2335 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2336 PIPE_CONFIG(ADDR_SURF_P2) |
2337 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2338 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2339 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2340 PIPE_CONFIG(ADDR_SURF_P2) |
2341 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2342 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2343 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2344 PIPE_CONFIG(ADDR_SURF_P2) |
2345 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2346 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2347
2348 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2349 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2350 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2351 NUM_BANKS(ADDR_SURF_8_BANK));
2352 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2353 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2354 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2355 NUM_BANKS(ADDR_SURF_8_BANK));
2356 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2357 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2358 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2359 NUM_BANKS(ADDR_SURF_8_BANK));
2360 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2361 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2362 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2363 NUM_BANKS(ADDR_SURF_8_BANK));
2364 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2365 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2366 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2367 NUM_BANKS(ADDR_SURF_8_BANK));
2368 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2369 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2370 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2371 NUM_BANKS(ADDR_SURF_8_BANK));
2372 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2373 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2374 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2375 NUM_BANKS(ADDR_SURF_8_BANK));
2376 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2377 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2378 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2379 NUM_BANKS(ADDR_SURF_16_BANK));
2380 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2381 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2382 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2383 NUM_BANKS(ADDR_SURF_16_BANK));
2384 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2385 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2386 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2387 NUM_BANKS(ADDR_SURF_16_BANK));
2388 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2389 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2390 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2391 NUM_BANKS(ADDR_SURF_16_BANK));
2392 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2393 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2394 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2395 NUM_BANKS(ADDR_SURF_16_BANK));
2396 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2397 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2398 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2399 NUM_BANKS(ADDR_SURF_16_BANK));
2400 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2401 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2402 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2403 NUM_BANKS(ADDR_SURF_8_BANK));
2404
2405 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2406 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
2407 reg_offset != 23)
2408 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2409
2410 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2411 if (reg_offset != 7)
2412 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2413
8cdacf44 2414 break;
af15a2d5 2415 case CHIP_FIJI:
90bea0ab
TSD		 2416 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2417 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2418 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2419 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2420 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2421 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2422 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2423 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2424 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2425 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2426 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2427 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2428 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2429 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2430 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2431 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2432 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2433 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2434 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2435 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2436 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2437 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2438 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2439 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2440 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2441 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2442 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2443 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2444 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2445 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2446 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2447 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2448 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2449 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2450 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2451 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2452 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2453 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2454 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2455 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2456 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2457 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2458 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2459 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2460 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2461 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2462 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2463 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2464 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2465 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2466 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2467 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2468 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2469 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2470 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2471 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2472 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2473 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2474 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2475 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2476 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2477 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2478 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2479 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2480 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2481 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2482 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2483 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2484 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2485 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2486 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2487 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2488 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2489 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2490 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2491 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2492 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2493 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2494 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2495 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2496 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2497 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2498 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2499 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2500 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2501 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2502 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2503 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2504 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2505 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2506 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2507 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2508 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2509 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2510 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2511 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2512 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2513 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2514 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2515 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2516 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2517 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2518 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2519 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2520 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2521 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2522 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2523 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2524 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2525 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2526 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2527 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2528 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2529 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2530 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2531 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2532 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2533 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2534 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2535 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2536 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2537 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2538
2539 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2540 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2541 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2542 NUM_BANKS(ADDR_SURF_8_BANK));
2543 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2544 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2545 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2546 NUM_BANKS(ADDR_SURF_8_BANK));
2547 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2548 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2549 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2550 NUM_BANKS(ADDR_SURF_8_BANK));
2551 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2552 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2553 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2554 NUM_BANKS(ADDR_SURF_8_BANK));
2555 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2556 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2557 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2558 NUM_BANKS(ADDR_SURF_8_BANK));
2559 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2560 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2561 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2562 NUM_BANKS(ADDR_SURF_8_BANK));
2563 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2564 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2565 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2566 NUM_BANKS(ADDR_SURF_8_BANK));
2567 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2568 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2569 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2570 NUM_BANKS(ADDR_SURF_8_BANK));
2571 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2572 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2573 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2574 NUM_BANKS(ADDR_SURF_8_BANK));
2575 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2576 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2577 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2578 NUM_BANKS(ADDR_SURF_8_BANK));
2579 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2580 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2581 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2582 NUM_BANKS(ADDR_SURF_8_BANK));
2583 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2584 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2585 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2586 NUM_BANKS(ADDR_SURF_8_BANK));
2587 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2588 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2589 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2590 NUM_BANKS(ADDR_SURF_8_BANK));
2591 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2593 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2594 NUM_BANKS(ADDR_SURF_4_BANK));
2595
2596 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2597 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2598
2599 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2600 if (reg_offset != 7)
2601 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2602
5f2e816b 2603 break;
aaa36a97 2604 case CHIP_TONGA:
90bea0ab
TSD		 2605 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2606 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2607 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2608 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2609 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2610 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2611 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2612 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2613 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2614 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2615 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2616 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2617 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2618 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2619 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2620 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2621 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2622 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2623 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2624 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2625 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2626 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2627 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2628 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2629 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2630 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2631 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2632 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2633 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2634 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2635 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2636 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2637 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2638 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2639 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2640 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2641 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2642 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2643 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2644 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2645 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2646 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2647 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2648 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2649 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2650 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2651 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2652 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2653 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2654 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2655 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2656 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2657 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2658 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2659 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2660 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2661 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2662 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2663 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2664 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2665 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2666 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2667 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2668 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2669 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2670 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2671 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2672 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2673 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2674 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2675 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2676 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2677 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2678 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2679 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2680 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2681 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2682 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2683 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2684 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2685 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2686 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2687 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2688 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2689 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2690 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2691 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2692 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2693 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2694 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2695 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2696 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2697 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2698 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2699 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2700 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2701 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2702 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2703 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2704 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2705 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2706 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2707 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2708 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2709 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2710 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2711 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2712 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2713 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2714 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2715 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2716 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2717 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2718 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2719 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2720 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2721 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2722 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2723 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2724 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2725 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2726 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2727
2728 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2729 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2730 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2731 NUM_BANKS(ADDR_SURF_16_BANK));
2732 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2735 NUM_BANKS(ADDR_SURF_16_BANK));
2736 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2737 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2738 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2739 NUM_BANKS(ADDR_SURF_16_BANK));
2740 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2741 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2742 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2743 NUM_BANKS(ADDR_SURF_16_BANK));
2744 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2745 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2746 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2747 NUM_BANKS(ADDR_SURF_16_BANK));
2748 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2749 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2750 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2751 NUM_BANKS(ADDR_SURF_16_BANK));
2752 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2753 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2754 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2755 NUM_BANKS(ADDR_SURF_16_BANK));
2756 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2757 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2758 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2759 NUM_BANKS(ADDR_SURF_16_BANK));
2760 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2761 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2762 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2763 NUM_BANKS(ADDR_SURF_16_BANK));
2764 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2765 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2766 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2767 NUM_BANKS(ADDR_SURF_16_BANK));
2768 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2769 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2770 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2771 NUM_BANKS(ADDR_SURF_16_BANK));
2772 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2773 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2774 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2775 NUM_BANKS(ADDR_SURF_8_BANK));
2776 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2777 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2778 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2779 NUM_BANKS(ADDR_SURF_4_BANK));
2780 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2781 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2782 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2783 NUM_BANKS(ADDR_SURF_4_BANK));
2784
2785 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2786 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2787
2788 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2789 if (reg_offset != 7)
2790 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2791
68182d90 2792 break;
2cc0c0b5 2793 case CHIP_POLARIS11:
c4642a47 2794 case CHIP_POLARIS12:
68182d90
FC		 2795 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2796 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2797 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2798 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2799 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2800 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2801 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2802 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2803 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2804 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2805 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2806 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2807 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2808 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2809 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2810 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2811 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2812 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2813 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2814 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2815 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2816 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2817 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2818 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2819 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2820 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2821 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2822 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2823 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2824 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2825 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2826 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2827 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2828 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2829 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2830 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2831 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2832 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2833 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2834 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2835 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2836 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2837 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2838 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2839 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2840 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2841 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2842 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2843 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2844 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2845 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2846 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2847 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2848 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2849 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2850 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2851 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2852 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2853 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2854 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2855 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2856 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2857 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2858 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2859 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2861 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2862 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2863 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2864 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2865 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2866 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2867 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2868 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2869 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2870 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2871 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2872 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2873 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2874 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2875 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2876 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2877 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2878 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2879 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2880 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2881 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2882 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2883 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2884 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2885 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2886 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2887 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2888 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2889 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2890 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2891 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2892 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2893 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2894 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2895 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2896 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2897 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2898 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2899 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2900 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2901 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2902 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2903 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2904 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2905 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2906 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2907 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2908 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2909 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2910 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2911 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2912 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2913 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2914 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2915 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2916 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2917
2918 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2919 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2920 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2921 NUM_BANKS(ADDR_SURF_16_BANK));
2922
2923 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2924 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2925 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2926 NUM_BANKS(ADDR_SURF_16_BANK));
2927
2928 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2929 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2930 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2931 NUM_BANKS(ADDR_SURF_16_BANK));
2932
2933 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2934 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2935 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2936 NUM_BANKS(ADDR_SURF_16_BANK));
2937
2938 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2939 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2940 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2941 NUM_BANKS(ADDR_SURF_16_BANK));
2942
2943 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2944 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2945 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2946 NUM_BANKS(ADDR_SURF_16_BANK));
2947
2948 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2949 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2950 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2951 NUM_BANKS(ADDR_SURF_16_BANK));
2952
2953 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2954 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2955 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2956 NUM_BANKS(ADDR_SURF_16_BANK));
2957
2958 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2959 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2960 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2961 NUM_BANKS(ADDR_SURF_16_BANK));
2962
2963 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2964 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2965 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2966 NUM_BANKS(ADDR_SURF_16_BANK));
2967
2968 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2969 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2970 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2971 NUM_BANKS(ADDR_SURF_16_BANK));
2972
2973 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2974 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2975 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2976 NUM_BANKS(ADDR_SURF_16_BANK));
2977
2978 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2979 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2980 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2981 NUM_BANKS(ADDR_SURF_8_BANK));
2982
2983 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2984 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2985 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2986 NUM_BANKS(ADDR_SURF_4_BANK));
2987
2988 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2989 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2990
2991 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2992 if (reg_offset != 7)
2993 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2994
2995 break;
2cc0c0b5 2996 case CHIP_POLARIS10:
68182d90
FC		 2997 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2998 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2999 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
3000 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3001 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3002 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3003 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
3004 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3005 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3006 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3007 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
3008 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3009 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3010 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3011 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
3012 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3013 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3014 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3015 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3016 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3017 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3018 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3019 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3020 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3021 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3022 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3023 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3024 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3025 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3026 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
3027 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3028 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3029 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3030 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
3031 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3032 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3033 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3034 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3035 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3036 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3037 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3038 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3039 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3040 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3041 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3042 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3043 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3044 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
3045 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3046 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3047 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3048 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3049 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3050 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3051 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3052 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3053 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3054 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3055 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
3056 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3057 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3058 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3059 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3060 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3061 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3062 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3063 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3064 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
3065 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3066 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3067 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3068 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3069 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3070 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3071 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3072 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3073 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3074 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3075 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3076 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3077 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3078 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3079 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
3080 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3081 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3082 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3083 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3084 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3085 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3086 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3087 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3088 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
3089 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3090 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3091 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3092 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3093 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3094 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3095 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
3096 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3097 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3098 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3099 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
3100 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3101 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3102 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3103 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3104 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3105 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3106 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3107 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3108 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3109 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3110 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3111 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3112 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3113 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3114 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3115 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3116 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
3117 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3118 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3119
3120 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3121 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3122 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3123 NUM_BANKS(ADDR_SURF_16_BANK));
3124
3125 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3126 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3127 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3128 NUM_BANKS(ADDR_SURF_16_BANK));
3129
3130 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3131 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3132 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3133 NUM_BANKS(ADDR_SURF_16_BANK));
3134
3135 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3136 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3137 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3138 NUM_BANKS(ADDR_SURF_16_BANK));
3139
3140 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3141 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3142 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3143 NUM_BANKS(ADDR_SURF_16_BANK));
3144
3145 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3146 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3147 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3148 NUM_BANKS(ADDR_SURF_16_BANK));
3149
3150 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3151 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3152 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3153 NUM_BANKS(ADDR_SURF_16_BANK));
3154
3155 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3156 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3157 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3158 NUM_BANKS(ADDR_SURF_16_BANK));
3159
3160 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3161 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3162 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3163 NUM_BANKS(ADDR_SURF_16_BANK));
3164
3165 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3166 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3167 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3168 NUM_BANKS(ADDR_SURF_16_BANK));
3169
3170 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3171 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3172 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3173 NUM_BANKS(ADDR_SURF_16_BANK));
3174
3175 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3176 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3177 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3178 NUM_BANKS(ADDR_SURF_8_BANK));
3179
3180 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3181 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3182 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3183 NUM_BANKS(ADDR_SURF_4_BANK));
3184
3185 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3186 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3187 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3188 NUM_BANKS(ADDR_SURF_4_BANK));
3189
3190 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3191 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3192
3193 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3194 if (reg_offset != 7)
3195 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3196
aaa36a97 3197 break;
e3c7656c 3198 case CHIP_STONEY:
90bea0ab
TSD		 3199 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3200 PIPE_CONFIG(ADDR_SURF_P2) |
3201 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
3202 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3203 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3204 PIPE_CONFIG(ADDR_SURF_P2) |
3205 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
3206 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3207 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3208 PIPE_CONFIG(ADDR_SURF_P2) |
3209 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
3210 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3211 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3212 PIPE_CONFIG(ADDR_SURF_P2) |
3213 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
3214 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3215 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3216 PIPE_CONFIG(ADDR_SURF_P2) |
3217 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3218 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3219 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3220 PIPE_CONFIG(ADDR_SURF_P2) |
3221 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3222 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3223 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3224 PIPE_CONFIG(ADDR_SURF_P2) |
3225 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3226 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3227 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3228 PIPE_CONFIG(ADDR_SURF_P2));
3229 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3230 PIPE_CONFIG(ADDR_SURF_P2) |
3231 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3232 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3233 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3234 PIPE_CONFIG(ADDR_SURF_P2) |
3235 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3236 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3237 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3238 PIPE_CONFIG(ADDR_SURF_P2) |
3239 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3240 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3241 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3242 PIPE_CONFIG(ADDR_SURF_P2) |
3243 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3244 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3245 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3246 PIPE_CONFIG(ADDR_SURF_P2) |
3247 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3248 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3249 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
3250 PIPE_CONFIG(ADDR_SURF_P2) |
3251 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3252 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3253 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3254 PIPE_CONFIG(ADDR_SURF_P2) |
3255 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3256 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3257 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3258 PIPE_CONFIG(ADDR_SURF_P2) |
3259 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3260 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3261 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3262 PIPE_CONFIG(ADDR_SURF_P2) |
3263 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3264 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3265 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3266 PIPE_CONFIG(ADDR_SURF_P2) |
3267 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3268 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3269 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
3270 PIPE_CONFIG(ADDR_SURF_P2) |
3271 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3272 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3273 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3274 PIPE_CONFIG(ADDR_SURF_P2) |
3275 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3276 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3277 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3278 PIPE_CONFIG(ADDR_SURF_P2) |
3279 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3280 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3281 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
3282 PIPE_CONFIG(ADDR_SURF_P2) |
3283 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3284 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3285 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
3286 PIPE_CONFIG(ADDR_SURF_P2) |
3287 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3288 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3289 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3290 PIPE_CONFIG(ADDR_SURF_P2) |
3291 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3292 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3293 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3294 PIPE_CONFIG(ADDR_SURF_P2) |
3295 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3296 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3297 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3298 PIPE_CONFIG(ADDR_SURF_P2) |
3299 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3300 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3301
3302 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3303 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3304 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3305 NUM_BANKS(ADDR_SURF_8_BANK));
3306 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3309 NUM_BANKS(ADDR_SURF_8_BANK));
3310 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3311 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3312 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3313 NUM_BANKS(ADDR_SURF_8_BANK));
3314 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3315 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3316 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3317 NUM_BANKS(ADDR_SURF_8_BANK));
3318 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3319 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3320 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3321 NUM_BANKS(ADDR_SURF_8_BANK));
3322 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3323 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3324 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3325 NUM_BANKS(ADDR_SURF_8_BANK));
3326 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3327 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3328 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3329 NUM_BANKS(ADDR_SURF_8_BANK));
3330 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3331 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3332 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3333 NUM_BANKS(ADDR_SURF_16_BANK));
3334 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3335 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3336 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3337 NUM_BANKS(ADDR_SURF_16_BANK));
3338 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3339 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3340 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3341 NUM_BANKS(ADDR_SURF_16_BANK));
3342 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3343 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3344 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3345 NUM_BANKS(ADDR_SURF_16_BANK));
3346 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3347 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3348 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3349 NUM_BANKS(ADDR_SURF_16_BANK));
3350 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3351 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3352 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3353 NUM_BANKS(ADDR_SURF_16_BANK));
3354 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3355 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3356 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3357 NUM_BANKS(ADDR_SURF_8_BANK));
3358
3359 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3360 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
3361 reg_offset != 23)
3362 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3363
3364 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3365 if (reg_offset != 7)
3366 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3367
e3c7656c 3368 break;
aaa36a97 3369 default:
90bea0ab
TSD		 3370 dev_warn(adev->dev,
3371 "Unknown chip type (%d) in function gfx_v8_0_tiling_mode_table_init() falling through to CHIP_CARRIZO\n",
3372 adev->asic_type);
3373
3374 case CHIP_CARRIZO:
3375 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3376 PIPE_CONFIG(ADDR_SURF_P2) |
3377 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
3378 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3379 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3380 PIPE_CONFIG(ADDR_SURF_P2) |
3381 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
3382 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3383 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3384 PIPE_CONFIG(ADDR_SURF_P2) |
3385 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
3386 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3387 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3388 PIPE_CONFIG(ADDR_SURF_P2) |
3389 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
3390 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3391 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3392 PIPE_CONFIG(ADDR_SURF_P2) |
3393 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3394 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3395 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3396 PIPE_CONFIG(ADDR_SURF_P2) |
3397 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3398 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3399 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3400 PIPE_CONFIG(ADDR_SURF_P2) |
3401 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3402 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3403 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3404 PIPE_CONFIG(ADDR_SURF_P2));
3405 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3406 PIPE_CONFIG(ADDR_SURF_P2) |
3407 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3408 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3409 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3410 PIPE_CONFIG(ADDR_SURF_P2) |
3411 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3412 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3413 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3414 PIPE_CONFIG(ADDR_SURF_P2) |
3415 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3416 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3417 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3418 PIPE_CONFIG(ADDR_SURF_P2) |
3419 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3420 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3421 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3422 PIPE_CONFIG(ADDR_SURF_P2) |
3423 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3424 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3425 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
3426 PIPE_CONFIG(ADDR_SURF_P2) |
3427 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3428 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3429 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3430 PIPE_CONFIG(ADDR_SURF_P2) |
3431 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3432 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3433 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3434 PIPE_CONFIG(ADDR_SURF_P2) |
3435 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3436 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3437 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3438 PIPE_CONFIG(ADDR_SURF_P2) |
3439 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3440 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3441 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3442 PIPE_CONFIG(ADDR_SURF_P2) |
3443 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3444 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3445 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
3446 PIPE_CONFIG(ADDR_SURF_P2) |
3447 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3448 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3449 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3450 PIPE_CONFIG(ADDR_SURF_P2) |
3451 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3452 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3453 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3454 PIPE_CONFIG(ADDR_SURF_P2) |
3455 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3456 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3457 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
3458 PIPE_CONFIG(ADDR_SURF_P2) |
3459 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3460 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3461 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
3462 PIPE_CONFIG(ADDR_SURF_P2) |
3463 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3464 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3465 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3466 PIPE_CONFIG(ADDR_SURF_P2) |
3467 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3468 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3469 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3470 PIPE_CONFIG(ADDR_SURF_P2) |
3471 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3472 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3473 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3474 PIPE_CONFIG(ADDR_SURF_P2) |
3475 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3476 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3477
3478 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3479 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3480 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3481 NUM_BANKS(ADDR_SURF_8_BANK));
3482 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3483 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3484 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3485 NUM_BANKS(ADDR_SURF_8_BANK));
3486 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3487 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3488 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3489 NUM_BANKS(ADDR_SURF_8_BANK));
3490 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3491 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3492 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3493 NUM_BANKS(ADDR_SURF_8_BANK));
3494 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3495 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3496 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3497 NUM_BANKS(ADDR_SURF_8_BANK));
3498 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3499 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3500 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3501 NUM_BANKS(ADDR_SURF_8_BANK));
3502 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3503 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3504 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3505 NUM_BANKS(ADDR_SURF_8_BANK));
3506 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3507 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3508 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3509 NUM_BANKS(ADDR_SURF_16_BANK));
3510 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3511 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3512 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3513 NUM_BANKS(ADDR_SURF_16_BANK));
3514 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3515 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3516 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3517 NUM_BANKS(ADDR_SURF_16_BANK));
3518 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3519 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3520 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3521 NUM_BANKS(ADDR_SURF_16_BANK));
3522 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3523 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3524 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3525 NUM_BANKS(ADDR_SURF_16_BANK));
3526 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3527 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3528 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3529 NUM_BANKS(ADDR_SURF_16_BANK));
3530 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3531 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3532 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3533 NUM_BANKS(ADDR_SURF_8_BANK));
3534
3535 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3536 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
3537 reg_offset != 23)
3538 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3539
3540 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3541 if (reg_offset != 7)
3542 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3543
3544 break;
aaa36a97
AD		 3545 }
3546}
3547
05fb7291 3548static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev,
9559ef5b 3549 u32 se_num, u32 sh_num, u32 instance)
aaa36a97 3550{
9559ef5b
TSD		 3551 u32 data;
3552
3553 if (instance == 0xffffffff)
3554 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
3555 else
3556 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
aaa36a97 3557
5003f278 3558 if (se_num == 0xffffffff)
aaa36a97 3559 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
5003f278 3560 else
aaa36a97 3561 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
5003f278
TSD		 3562
3563 if (sh_num == 0xffffffff)
3564 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
3565 else
aaa36a97 3566 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
5003f278 3567
aaa36a97
AD		 3568 WREG32(mmGRBM_GFX_INDEX, data);
3569}
3570
8f8e00c1
AD		 3571static u32 gfx_v8_0_create_bitmask(u32 bit_width)
3572{
3573 return (u32)((1ULL << bit_width) - 1);
3574}
3575
3576static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev)
aaa36a97
AD		 3577{
3578 u32 data, mask;
3579
5003f278
TSD		 3580 data = RREG32(mmCC_RB_BACKEND_DISABLE) |
3581 RREG32(mmGC_USER_RB_BACKEND_DISABLE);
aaa36a97 3582
5003f278 3583 data = REG_GET_FIELD(data, GC_USER_RB_BACKEND_DISABLE, BACKEND_DISABLE);
aaa36a97 3584
8f8e00c1
AD		 3585 mask = gfx_v8_0_create_bitmask(adev->gfx.config.max_backends_per_se /
3586 adev->gfx.config.max_sh_per_se);
aaa36a97 3587
8f8e00c1 3588 return (~data) & mask;
aaa36a97
AD		 3589}
3590
167ac573
HR		 3591static void
3592gfx_v8_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1)
3593{
3594 switch (adev->asic_type) {
3595 case CHIP_FIJI:
3596 *rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) |
3597 RB_XSEL2(1) | PKR_MAP(2) |
3598 PKR_XSEL(1) | PKR_YSEL(1) |
3599 SE_MAP(2) | SE_XSEL(2) | SE_YSEL(3);
3600 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) |
3601 SE_PAIR_YSEL(2);
3602 break;
3603 case CHIP_TONGA:
3604 case CHIP_POLARIS10:
3605 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
3606 SE_XSEL(1) | SE_YSEL(1);
3607 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(2) |
3608 SE_PAIR_YSEL(2);
3609 break;
3610 case CHIP_TOPAZ:
3611 case CHIP_CARRIZO:
3612 *rconf |= RB_MAP_PKR0(2);
3613 *rconf1 |= 0x0;
3614 break;
3615 case CHIP_POLARIS11:
c4642a47 3616 case CHIP_POLARIS12:
167ac573
HR		 3617 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
3618 SE_XSEL(1) | SE_YSEL(1);
3619 *rconf1 |= 0x0;
3620 break;
3621 case CHIP_STONEY:
3622 *rconf |= 0x0;
3623 *rconf1 |= 0x0;
3624 break;
3625 default:
3626 DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
3627 break;
3628 }
3629}
3630
3631static void
3632gfx_v8_0_write_harvested_raster_configs(struct amdgpu_device *adev,
3633 u32 raster_config, u32 raster_config_1,
3634 unsigned rb_mask, unsigned num_rb)
3635{
3636 unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1);
3637 unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1);
3638 unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2);
3639 unsigned rb_per_se = num_rb / num_se;
3640 unsigned se_mask[4];
3641 unsigned se;
3642
3643 se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask;
3644 se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask;
3645 se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask;
3646 se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask;
3647
3648 WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4));
3649 WARN_ON(!(sh_per_se == 1 || sh_per_se == 2));
3650 WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2));
3651
3652 if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) ||
3653 (!se_mask[2] && !se_mask[3]))) {
3654 raster_config_1 &= ~SE_PAIR_MAP_MASK;
3655
3656 if (!se_mask[0] && !se_mask[1]) {
3657 raster_config_1 |=
3658 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3);
3659 } else {
3660 raster_config_1 |=
3661 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0);
3662 }
3663 }
3664
3665 for (se = 0; se < num_se; se++) {
3666 unsigned raster_config_se = raster_config;
3667 unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se);
3668 unsigned pkr1_mask = pkr0_mask << rb_per_pkr;
3669 int idx = (se / 2) * 2;
3670
3671 if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {
3672 raster_config_se &= ~SE_MAP_MASK;
3673
3674 if (!se_mask[idx]) {
3675 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3);
3676 } else {
3677 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0);
3678 }
3679 }
3680
3681 pkr0_mask &= rb_mask;
3682 pkr1_mask &= rb_mask;
3683 if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {
3684 raster_config_se &= ~PKR_MAP_MASK;
3685
3686 if (!pkr0_mask) {
3687 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3);
3688 } else {
3689 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0);
3690 }
3691 }
3692
3693 if (rb_per_se >= 2) {
3694 unsigned rb0_mask = 1 << (se * rb_per_se);
3695 unsigned rb1_mask = rb0_mask << 1;
3696
3697 rb0_mask &= rb_mask;
3698 rb1_mask &= rb_mask;
3699 if (!rb0_mask || !rb1_mask) {
3700 raster_config_se &= ~RB_MAP_PKR0_MASK;
3701
3702 if (!rb0_mask) {
3703 raster_config_se |=
3704 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3);
3705 } else {
3706 raster_config_se |=
3707 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0);
3708 }
3709 }
3710
3711 if (rb_per_se > 2) {
3712 rb0_mask = 1 << (se * rb_per_se + rb_per_pkr);
3713 rb1_mask = rb0_mask << 1;
3714 rb0_mask &= rb_mask;
3715 rb1_mask &= rb_mask;
3716 if (!rb0_mask || !rb1_mask) {
3717 raster_config_se &= ~RB_MAP_PKR1_MASK;
3718
3719 if (!rb0_mask) {
3720 raster_config_se |=
3721 RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3);
3722 } else {
3723 raster_config_se |=
3724 RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0);
3725 }
3726 }
3727 }
3728 }
3729
3730 /* GRBM_GFX_INDEX has a different offset on VI */
3731 gfx_v8_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff);
3732 WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se);
3733 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
3734 }
3735
3736 /* GRBM_GFX_INDEX has a different offset on VI */
3737 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3738}
3739
8f8e00c1 3740static void gfx_v8_0_setup_rb(struct amdgpu_device *adev)
aaa36a97
AD		 3741{
3742 int i, j;
aac1e3ca 3743 u32 data;
167ac573 3744 u32 raster_config = 0, raster_config_1 = 0;
8f8e00c1 3745 u32 active_rbs = 0;
6157bd7a
FC		 3746 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
3747 adev->gfx.config.max_sh_per_se;
167ac573 3748 unsigned num_rb_pipes;
aaa36a97
AD		 3749
3750 mutex_lock(&adev->grbm_idx_mutex);
8f8e00c1
AD		 3751 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3752 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
9559ef5b 3753 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
8f8e00c1
AD		 3754 data = gfx_v8_0_get_rb_active_bitmap(adev);
3755 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
6157bd7a 3756 rb_bitmap_width_per_sh);
aaa36a97
AD		 3757 }
3758 }
9559ef5b 3759 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
aaa36a97 3760
8f8e00c1 3761 adev->gfx.config.backend_enable_mask = active_rbs;
aac1e3ca 3762 adev->gfx.config.num_rbs = hweight32(active_rbs);
167ac573
HR		 3763
3764 num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se *
3765 adev->gfx.config.max_shader_engines, 16);
3766
3767 gfx_v8_0_raster_config(adev, &raster_config, &raster_config_1);
3768
3769 if (!adev->gfx.config.backend_enable_mask ||
3770 adev->gfx.config.num_rbs >= num_rb_pipes) {
3771 WREG32(mmPA_SC_RASTER_CONFIG, raster_config);
3772 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
3773 } else {
3774 gfx_v8_0_write_harvested_raster_configs(adev, raster_config, raster_config_1,
3775 adev->gfx.config.backend_enable_mask,
3776 num_rb_pipes);
3777 }
3778
392f0c77
AD		 3779 /* cache the values for userspace */
3780 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3781 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3782 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
3783 adev->gfx.config.rb_config[i][j].rb_backend_disable =
3784 RREG32(mmCC_RB_BACKEND_DISABLE);
3785 adev->gfx.config.rb_config[i][j].user_rb_backend_disable =
3786 RREG32(mmGC_USER_RB_BACKEND_DISABLE);
3787 adev->gfx.config.rb_config[i][j].raster_config =
3788 RREG32(mmPA_SC_RASTER_CONFIG);
3789 adev->gfx.config.rb_config[i][j].raster_config_1 =
3790 RREG32(mmPA_SC_RASTER_CONFIG_1);
3791 }
3792 }
3793 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
167ac573 3794 mutex_unlock(&adev->grbm_idx_mutex);
aaa36a97
AD		 3795}
3796
cd06bf68 3797/**
35c7a952 3798 * gfx_v8_0_init_compute_vmid - gart enable
cd06bf68
BG		 3799 *
3800 * @rdev: amdgpu_device pointer
3801 *
3802 * Initialize compute vmid sh_mem registers
3803 *
3804 */
3805#define DEFAULT_SH_MEM_BASES (0x6000)
3806#define FIRST_COMPUTE_VMID (8)
3807#define LAST_COMPUTE_VMID (16)
35c7a952 3808static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)
cd06bf68
BG		 3809{
3810 int i;
3811 uint32_t sh_mem_config;
3812 uint32_t sh_mem_bases;
3813
3814 /*
3815 * Configure apertures:
3816 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
3817 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
3818 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
3819 */
3820 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
3821
3822 sh_mem_config = SH_MEM_ADDRESS_MODE_HSA64 <<
3823 SH_MEM_CONFIG__ADDRESS_MODE__SHIFT |
3824 SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
3825 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT |
3826 MTYPE_CC << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT |
3827 SH_MEM_CONFIG__PRIVATE_ATC_MASK;
3828
3829 mutex_lock(&adev->srbm_mutex);
3830 for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
3831 vi_srbm_select(adev, 0, 0, 0, i);
3832 /* CP and shaders */
3833 WREG32(mmSH_MEM_CONFIG, sh_mem_config);
3834 WREG32(mmSH_MEM_APE1_BASE, 1);
3835 WREG32(mmSH_MEM_APE1_LIMIT, 0);
3836 WREG32(mmSH_MEM_BASES, sh_mem_bases);
3837 }
3838 vi_srbm_select(adev, 0, 0, 0, 0);
3839 mutex_unlock(&adev->srbm_mutex);
3840}
3841
aaa36a97
AD		 3842static void gfx_v8_0_gpu_init(struct amdgpu_device *adev)
3843{
aaa36a97
AD		 3844 u32 tmp;
3845 int i;
3846
61cb8cef 3847 WREG32_FIELD(GRBM_CNTL, READ_TIMEOUT, 0xFF);
0bde3a95
AD		 3848 WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
3849 WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
3850 WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config);
aaa36a97
AD		 3851
3852 gfx_v8_0_tiling_mode_table_init(adev);
8f8e00c1 3853 gfx_v8_0_setup_rb(adev);
7dae69a2 3854 gfx_v8_0_get_cu_info(adev);
aaa36a97
AD		 3855
3856 /* XXX SH_MEM regs */
3857 /* where to put LDS, scratch, GPUVM in FSA64 space */
3858 mutex_lock(&adev->srbm_mutex);
3859 for (i = 0; i < 16; i++) {
3860 vi_srbm_select(adev, 0, 0, 0, i);
3861 /* CP and shaders */
3862 if (i == 0) {
3863 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_UC);
3864 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC);
0bde3a95 3865 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
74a5d165 3866 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
aaa36a97
AD		 3867 WREG32(mmSH_MEM_CONFIG, tmp);
3868 } else {
3869 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_NC);
3870 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_NC);
0bde3a95 3871 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
74a5d165 3872 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
aaa36a97
AD		 3873 WREG32(mmSH_MEM_CONFIG, tmp);
3874 }
3875
3876 WREG32(mmSH_MEM_APE1_BASE, 1);
3877 WREG32(mmSH_MEM_APE1_LIMIT, 0);
3878 WREG32(mmSH_MEM_BASES, 0);
3879 }
3880 vi_srbm_select(adev, 0, 0, 0, 0);
3881 mutex_unlock(&adev->srbm_mutex);
3882
35c7a952 3883 gfx_v8_0_init_compute_vmid(adev);
cd06bf68 3884
aaa36a97
AD		 3885 mutex_lock(&adev->grbm_idx_mutex);
3886 /*
3887 * making sure that the following register writes will be broadcasted
3888 * to all the shaders
3889 */
9559ef5b 3890 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
aaa36a97
AD		 3891
3892 WREG32(mmPA_SC_FIFO_SIZE,
3893 (adev->gfx.config.sc_prim_fifo_size_frontend <<
3894 PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |
3895 (adev->gfx.config.sc_prim_fifo_size_backend <<
3896 PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) |
3897 (adev->gfx.config.sc_hiz_tile_fifo_size <<
3898 PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) |
3899 (adev->gfx.config.sc_earlyz_tile_fifo_size <<
3900 PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT));
d2383267 3901
3902 tmp = RREG32(mmSPI_ARB_PRIORITY);
3903 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS0, 2);
3904 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS1, 2);
3905 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS2, 2);
3906 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS3, 2);
3907 WREG32(mmSPI_ARB_PRIORITY, tmp);
3908
aaa36a97
AD		 3909 mutex_unlock(&adev->grbm_idx_mutex);
3910
3911}
3912
3913static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
3914{
3915 u32 i, j, k;
3916 u32 mask;
3917
3918 mutex_lock(&adev->grbm_idx_mutex);
3919 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3920 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
9559ef5b 3921 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
aaa36a97
AD		 3922 for (k = 0; k < adev->usec_timeout; k++) {
3923 if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0)
3924 break;
3925 udelay(1);
3926 }
3927 }
3928 }
9559ef5b 3929 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
aaa36a97
AD		 3930 mutex_unlock(&adev->grbm_idx_mutex);
3931
3932 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
3933 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
3934 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
3935 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
3936 for (k = 0; k < adev->usec_timeout; k++) {
3937 if ((RREG32(mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
3938 break;
3939 udelay(1);
3940 }
3941}
3942
3943static void gfx_v8_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
3944 bool enable)
3945{
3946 u32 tmp = RREG32(mmCP_INT_CNTL_RING0);
3947
0d07db7e
TSD		 3948 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
3949 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
3950 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
3951 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
3952
aaa36a97
AD		 3953 WREG32(mmCP_INT_CNTL_RING0, tmp);
3954}
3955
2b6cd977
EH		 3956static void gfx_v8_0_init_csb(struct amdgpu_device *adev)
3957{
3958 /* csib */
3959 WREG32(mmRLC_CSIB_ADDR_HI,
3960 adev->gfx.rlc.clear_state_gpu_addr >> 32);
3961 WREG32(mmRLC_CSIB_ADDR_LO,
3962 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
3963 WREG32(mmRLC_CSIB_LENGTH,
3964 adev->gfx.rlc.clear_state_size);
3965}
3966
3967static void gfx_v8_0_parse_ind_reg_list(int *register_list_format,
3968 int ind_offset,
3969 int list_size,
3970 int *unique_indices,
3971 int *indices_count,
3972 int max_indices,
3973 int *ind_start_offsets,
3974 int *offset_count,
3975 int max_offset)
3976{
3977 int indices;
3978 bool new_entry = true;
3979
3980 for (; ind_offset < list_size; ind_offset++) {
3981
3982 if (new_entry) {
3983 new_entry = false;
3984 ind_start_offsets[*offset_count] = ind_offset;
3985 *offset_count = *offset_count + 1;
3986 BUG_ON(*offset_count >= max_offset);
3987 }
3988
3989 if (register_list_format[ind_offset] == 0xFFFFFFFF) {
3990 new_entry = true;
3991 continue;
3992 }
3993
3994 ind_offset += 2;
3995
3996 /* look for the matching indice */
3997 for (indices = 0;
3998 indices < *indices_count;
3999 indices++) {
4000 if (unique_indices[indices] ==
4001 register_list_format[ind_offset])
4002 break;
4003 }
4004
4005 if (indices >= *indices_count) {
4006 unique_indices[*indices_count] =
4007 register_list_format[ind_offset];
4008 indices = *indices_count;
4009 *indices_count = *indices_count + 1;
4010 BUG_ON(*indices_count >= max_indices);
4011 }
4012
4013 register_list_format[ind_offset] = indices;
4014 }
4015}
4016
4017static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
4018{
4019 int i, temp, data;
4020 int unique_indices[] = {0, 0, 0, 0, 0, 0, 0, 0};
4021 int indices_count = 0;
4022 int indirect_start_offsets[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
4023 int offset_count = 0;
4024
4025 int list_size;
4026 unsigned int *register_list_format =
4027 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
3f12325a 4028 if (!register_list_format)
2b6cd977
EH		 4029 return -ENOMEM;
4030 memcpy(register_list_format, adev->gfx.rlc.register_list_format,
4031 adev->gfx.rlc.reg_list_format_size_bytes);
4032
4033 gfx_v8_0_parse_ind_reg_list(register_list_format,
4034 RLC_FormatDirectRegListLength,
4035 adev->gfx.rlc.reg_list_format_size_bytes >> 2,
4036 unique_indices,
4037 &indices_count,
4038 sizeof(unique_indices) / sizeof(int),
4039 indirect_start_offsets,
4040 &offset_count,
4041 sizeof(indirect_start_offsets)/sizeof(int));
4042
4043 /* save and restore list */
61cb8cef 4044 WREG32_FIELD(RLC_SRM_CNTL, AUTO_INCR_ADDR, 1);
2b6cd977
EH		 4045
4046 WREG32(mmRLC_SRM_ARAM_ADDR, 0);
4047 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
4048 WREG32(mmRLC_SRM_ARAM_DATA, adev->gfx.rlc.register_restore[i]);
4049
4050 /* indirect list */
4051 WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_list_format_start);
4052 for (i = 0; i < adev->gfx.rlc.reg_list_format_size_bytes >> 2; i++)
4053 WREG32(mmRLC_GPM_SCRATCH_DATA, register_list_format[i]);
4054
4055 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
4056 list_size = list_size >> 1;
4057 WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_restore_list_size);
4058 WREG32(mmRLC_GPM_SCRATCH_DATA, list_size);
4059
4060 /* starting offsets starts */
4061 WREG32(mmRLC_GPM_SCRATCH_ADDR,
4062 adev->gfx.rlc.starting_offsets_start);
4063 for (i = 0; i < sizeof(indirect_start_offsets)/sizeof(int); i++)
4064 WREG32(mmRLC_GPM_SCRATCH_DATA,
4065 indirect_start_offsets[i]);
4066
4067 /* unique indices */
4068 temp = mmRLC_SRM_INDEX_CNTL_ADDR_0;
4069 data = mmRLC_SRM_INDEX_CNTL_DATA_0;
4070 for (i = 0; i < sizeof(unique_indices) / sizeof(int); i++) {
202e0b22
RZ		 4071 if (unique_indices[i] != 0) {
4072 amdgpu_mm_wreg(adev, temp + i,
4073 unique_indices[i] & 0x3FFFF, false);
4074 amdgpu_mm_wreg(adev, data + i,
4075 unique_indices[i] >> 20, false);
4076 }
2b6cd977
EH		 4077 }
4078 kfree(register_list_format);
4079
4080 return 0;
4081}
4082
4083static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev)
4084{
61cb8cef 4085 WREG32_FIELD(RLC_SRM_CNTL, SRM_ENABLE, 1);
2b6cd977
EH		 4086}
4087
fb16007b 4088static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev)
f4bfffdd
EH		 4089{
4090 uint32_t data;
4091
c4d17b81
RZ		 4092 WREG32_FIELD(CP_RB_WPTR_POLL_CNTL, IDLE_POLL_COUNT, 0x60);
4093
4094 data = REG_SET_FIELD(0, RLC_PG_DELAY, POWER_UP_DELAY, 0x10);
4095 data = REG_SET_FIELD(data, RLC_PG_DELAY, POWER_DOWN_DELAY, 0x10);
4096 data = REG_SET_FIELD(data, RLC_PG_DELAY, CMD_PROPAGATE_DELAY, 0x10);
4097 data = REG_SET_FIELD(data, RLC_PG_DELAY, MEM_SLEEP_DELAY, 0x10);
4098 WREG32(mmRLC_PG_DELAY, data);
4099
4100 WREG32_FIELD(RLC_PG_DELAY_2, SERDES_CMD_DELAY, 0x3);
4101 WREG32_FIELD(RLC_AUTO_PG_CTRL, GRBM_REG_SAVE_GFX_IDLE_THRESHOLD, 0x55f0);
4102
f4bfffdd
EH		 4103}
4104
2c547165
AD		 4105static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
4106 bool enable)
4107{
61cb8cef 4108 WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PU_ENABLE, enable ? 1 : 0);
2c547165
AD		 4109}
4110
4111static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
4112 bool enable)
4113{
61cb8cef 4114 WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PD_ENABLE, enable ? 1 : 0);
2c547165
AD		 4115}
4116
4117static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable)
4118{
eb584241 4119 WREG32_FIELD(RLC_PG_CNTL, CP_PG_DISABLE, enable ? 0 : 1);
2c547165
AD		 4120}
4121
2b6cd977
EH		 4122static void gfx_v8_0_init_pg(struct amdgpu_device *adev)
4123{
c4d17b81
RZ		 4124 if ((adev->asic_type == CHIP_CARRIZO) ||
4125 (adev->asic_type == CHIP_STONEY)) {
2b6cd977
EH		 4126 gfx_v8_0_init_csb(adev);
4127 gfx_v8_0_init_save_restore_list(adev);
4128 gfx_v8_0_enable_save_restore_machine(adev);
c4d17b81
RZ		 4129 WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8);
4130 gfx_v8_0_init_power_gating(adev);
4131 WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask);
c4642a47
JZ		 4132 } else if ((adev->asic_type == CHIP_POLARIS11) ||
4133 (adev->asic_type == CHIP_POLARIS12)) {
c4d17b81
RZ		 4134 gfx_v8_0_init_csb(adev);
4135 gfx_v8_0_init_save_restore_list(adev);
4136 gfx_v8_0_enable_save_restore_machine(adev);
4137 gfx_v8_0_init_power_gating(adev);
2b6cd977 4138 }
c4d17b81 4139
2b6cd977
EH		 4140}
4141
761c2e82 4142static void gfx_v8_0_rlc_stop(struct amdgpu_device *adev)
aaa36a97 4143{
61cb8cef 4144 WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 0);
aaa36a97
AD		 4145
4146 gfx_v8_0_enable_gui_idle_interrupt(adev, false);
aaa36a97
AD		 4147 gfx_v8_0_wait_for_rlc_serdes(adev);
4148}
4149
4150static void gfx_v8_0_rlc_reset(struct amdgpu_device *adev)
4151{
61cb8cef 4152 WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
aaa36a97 4153 udelay(50);
61cb8cef
TSD		 4154
4155 WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
aaa36a97
AD		 4156 udelay(50);
4157}
4158
4159static void gfx_v8_0_rlc_start(struct amdgpu_device *adev)
4160{
61cb8cef 4161 WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 1);
aaa36a97
AD		 4162
4163 /* carrizo do enable cp interrupt after cp inited */
e3c7656c 4164 if (!(adev->flags & AMD_IS_APU))
aaa36a97
AD		 4165 gfx_v8_0_enable_gui_idle_interrupt(adev, true);
4166
4167 udelay(50);
4168}
4169
4170static int gfx_v8_0_rlc_load_microcode(struct amdgpu_device *adev)
4171{
4172 const struct rlc_firmware_header_v2_0 *hdr;
4173 const __le32 *fw_data;
4174 unsigned i, fw_size;
4175
4176 if (!adev->gfx.rlc_fw)
4177 return -EINVAL;
4178
4179 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
4180 amdgpu_ucode_print_rlc_hdr(&hdr->header);
aaa36a97
AD		 4181
4182 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
4183 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
4184 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
4185
4186 WREG32(mmRLC_GPM_UCODE_ADDR, 0);
4187 for (i = 0; i < fw_size; i++)
4188 WREG32(mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
4189 WREG32(mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
4190
4191 return 0;
4192}
4193
4194static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)
4195{
4196 int r;
6ae81452 4197 u32 tmp;
aaa36a97
AD		 4198
4199 gfx_v8_0_rlc_stop(adev);
4200
4201 /* disable CG */
6ae81452
AD		 4202 tmp = RREG32(mmRLC_CGCG_CGLS_CTRL);
4203 tmp &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK |
4204 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
4205 WREG32(mmRLC_CGCG_CGLS_CTRL, tmp);
2cc0c0b5 4206 if (adev->asic_type == CHIP_POLARIS11 ||
c4642a47
JZ		 4207 adev->asic_type == CHIP_POLARIS10 ||
4208 adev->asic_type == CHIP_POLARIS12) {
6ae81452
AD		 4209 tmp = RREG32(mmRLC_CGCG_CGLS_CTRL_3D);
4210 tmp &= ~0x3;
4211 WREG32(mmRLC_CGCG_CGLS_CTRL_3D, tmp);
4212 }
aaa36a97
AD		 4213
4214 /* disable PG */
4215 WREG32(mmRLC_PG_CNTL, 0);
4216
4217 gfx_v8_0_rlc_reset(adev);
2b6cd977
EH		 4218 gfx_v8_0_init_pg(adev);
4219
e61710c5 4220 if (!adev->pp_enabled) {
ba5c2a87
RZ		 4221 if (!adev->firmware.smu_load) {
4222 /* legacy rlc firmware loading */
4223 r = gfx_v8_0_rlc_load_microcode(adev);
4224 if (r)
4225 return r;
4226 } else {
4227 r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
4228 AMDGPU_UCODE_ID_RLC_G);
4229 if (r)
4230 return -EINVAL;
4231 }
aaa36a97
AD		 4232 }
4233
4234 gfx_v8_0_rlc_start(adev);
4235
4236 return 0;
4237}
4238
4239static void gfx_v8_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
4240{
4241 int i;
4242 u32 tmp = RREG32(mmCP_ME_CNTL);
4243
4244 if (enable) {
4245 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 0);
4246 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 0);
4247 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 0);
4248 } else {
4249 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 1);
4250 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 1);
4251 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 1);
4252 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
4253 adev->gfx.gfx_ring[i].ready = false;
4254 }
4255 WREG32(mmCP_ME_CNTL, tmp);
4256 udelay(50);
4257}
4258
4259static int gfx_v8_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
4260{
4261 const struct gfx_firmware_header_v1_0 *pfp_hdr;
4262 const struct gfx_firmware_header_v1_0 *ce_hdr;
4263 const struct gfx_firmware_header_v1_0 *me_hdr;
4264 const __le32 *fw_data;
4265 unsigned i, fw_size;
4266
4267 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
4268 return -EINVAL;
4269
4270 pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
4271 adev->gfx.pfp_fw->data;
4272 ce_hdr = (const struct gfx_firmware_header_v1_0 *)
4273 adev->gfx.ce_fw->data;
4274 me_hdr = (const struct gfx_firmware_header_v1_0 *)
4275 adev->gfx.me_fw->data;
4276
4277 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
4278 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
4279 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
aaa36a97
AD		 4280
4281 gfx_v8_0_cp_gfx_enable(adev, false);
4282
4283 /* PFP */
4284 fw_data = (const __le32 *)
4285 (adev->gfx.pfp_fw->data +
4286 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4287 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4288 WREG32(mmCP_PFP_UCODE_ADDR, 0);
4289 for (i = 0; i < fw_size; i++)
4290 WREG32(mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
4291 WREG32(mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
4292
4293 /* CE */
4294 fw_data = (const __le32 *)
4295 (adev->gfx.ce_fw->data +
4296 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4297 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4298 WREG32(mmCP_CE_UCODE_ADDR, 0);
4299 for (i = 0; i < fw_size; i++)
4300 WREG32(mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
4301 WREG32(mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
4302
4303 /* ME */
4304 fw_data = (const __le32 *)
4305 (adev->gfx.me_fw->data +
4306 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4307 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4308 WREG32(mmCP_ME_RAM_WADDR, 0);
4309 for (i = 0; i < fw_size; i++)
4310 WREG32(mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4311 WREG32(mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
4312
4313 return 0;
4314}
4315
4316static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev)
4317{
4318 u32 count = 0;
4319 const struct cs_section_def *sect = NULL;
4320 const struct cs_extent_def *ext = NULL;
4321
4322 /* begin clear state */
4323 count += 2;
4324 /* context control state */
4325 count += 3;
4326
4327 for (sect = vi_cs_data; sect->section != NULL; ++sect) {
4328 for (ext = sect->section; ext->extent != NULL; ++ext) {
4329 if (sect->id == SECT_CONTEXT)
4330 count += 2 + ext->reg_count;
4331 else
4332 return 0;
4333 }
4334 }
4335 /* pa_sc_raster_config/pa_sc_raster_config1 */
4336 count += 4;
4337 /* end clear state */
4338 count += 2;
4339 /* clear state */
4340 count += 2;
4341
4342 return count;
4343}
4344
4345static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev)
4346{
4347 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
4348 const struct cs_section_def *sect = NULL;
4349 const struct cs_extent_def *ext = NULL;
4350 int r, i;
4351
4352 /* init the CP */
4353 WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
4354 WREG32(mmCP_ENDIAN_SWAP, 0);
4355 WREG32(mmCP_DEVICE_ID, 1);
4356
4357 gfx_v8_0_cp_gfx_enable(adev, true);
4358
a27de35c 4359 r = amdgpu_ring_alloc(ring, gfx_v8_0_get_csb_size(adev) + 4);
aaa36a97
AD		 4360 if (r) {
4361 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
4362 return r;
4363 }
4364
4365 /* clear state buffer */
4366 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4367 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4368
4369 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4370 amdgpu_ring_write(ring, 0x80000000);
4371 amdgpu_ring_write(ring, 0x80000000);
4372
4373 for (sect = vi_cs_data; sect->section != NULL; ++sect) {
4374 for (ext = sect->section; ext->extent != NULL; ++ext) {
4375 if (sect->id == SECT_CONTEXT) {
4376 amdgpu_ring_write(ring,
4377 PACKET3(PACKET3_SET_CONTEXT_REG,
4378 ext->reg_count));
4379 amdgpu_ring_write(ring,
4380 ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
4381 for (i = 0; i < ext->reg_count; i++)
4382 amdgpu_ring_write(ring, ext->extent[i]);
4383 }
4384 }
4385 }
4386
4387 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4388 amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
4389 switch (adev->asic_type) {
4390 case CHIP_TONGA:
2cc0c0b5 4391 case CHIP_POLARIS10:
aaa36a97
AD		 4392 amdgpu_ring_write(ring, 0x16000012);
4393 amdgpu_ring_write(ring, 0x0000002A);
4394 break;
2cc0c0b5 4395 case CHIP_POLARIS11:
c4642a47 4396 case CHIP_POLARIS12:
68182d90
FC		 4397 amdgpu_ring_write(ring, 0x16000012);
4398 amdgpu_ring_write(ring, 0x00000000);
4399 break;
fa676048
FC		 4400 case CHIP_FIJI:
4401 amdgpu_ring_write(ring, 0x3a00161a);
4402 amdgpu_ring_write(ring, 0x0000002e);
4403 break;
aaa36a97
AD		 4404 case CHIP_CARRIZO:
4405 amdgpu_ring_write(ring, 0x00000002);
4406 amdgpu_ring_write(ring, 0x00000000);
4407 break;
d1a7f7aa
KW		 4408 case CHIP_TOPAZ:
4409 amdgpu_ring_write(ring, adev->gfx.config.num_rbs == 1 ?
4410 0x00000000 : 0x00000002);
4411 amdgpu_ring_write(ring, 0x00000000);
4412 break;
e3c7656c
SL		 4413 case CHIP_STONEY:
4414 amdgpu_ring_write(ring, 0x00000000);
4415 amdgpu_ring_write(ring, 0x00000000);
4416 break;
aaa36a97
AD		 4417 default:
4418 BUG();
4419 }
4420
4421 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4422 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4423
4424 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4425 amdgpu_ring_write(ring, 0);
4426
4427 /* init the CE partitions */
4428 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4429 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4430 amdgpu_ring_write(ring, 0x8000);
4431 amdgpu_ring_write(ring, 0x8000);
4432
a27de35c 4433 amdgpu_ring_commit(ring);
aaa36a97
AD		 4434
4435 return 0;
4436}
4437
4438static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev)
4439{
4440 struct amdgpu_ring *ring;
4441 u32 tmp;
4442 u32 rb_bufsz;
42e8cb50 4443 u64 rb_addr, rptr_addr, wptr_gpu_addr;
aaa36a97
AD		 4444 int r;
4445
4446 /* Set the write pointer delay */
4447 WREG32(mmCP_RB_WPTR_DELAY, 0);
4448
4449 /* set the RB to use vmid 0 */
4450 WREG32(mmCP_RB_VMID, 0);
4451
4452 /* Set ring buffer size */
4453 ring = &adev->gfx.gfx_ring[0];
4454 rb_bufsz = order_base_2(ring->ring_size / 8);
4455 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
4456 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
4457 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MTYPE, 3);
4458 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MIN_IB_AVAILSZ, 1);
4459#ifdef __BIG_ENDIAN
4460 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
4461#endif
4462 WREG32(mmCP_RB0_CNTL, tmp);
4463
4464 /* Initialize the ring buffer's read and write pointers */
4465 WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK);
4466 ring->wptr = 0;
4467 WREG32(mmCP_RB0_WPTR, ring->wptr);
4468
4469 /* set the wb address wether it's enabled or not */
4470 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
4471 WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
4472 WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
4473
42e8cb50
FM		 4474 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
4475 WREG32(mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
4476 WREG32(mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
aaa36a97
AD		 4477 mdelay(1);
4478 WREG32(mmCP_RB0_CNTL, tmp);
4479
4480 rb_addr = ring->gpu_addr >> 8;
4481 WREG32(mmCP_RB0_BASE, rb_addr);
4482 WREG32(mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
4483
4484 /* no gfx doorbells on iceland */
4485 if (adev->asic_type != CHIP_TOPAZ) {
4486 tmp = RREG32(mmCP_RB_DOORBELL_CONTROL);
4487 if (ring->use_doorbell) {
4488 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4489 DOORBELL_OFFSET, ring->doorbell_index);
68182d90
FC		 4490 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4491 DOORBELL_HIT, 0);
aaa36a97
AD		 4492 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4493 DOORBELL_EN, 1);
4494 } else {
4495 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4496 DOORBELL_EN, 0);
4497 }
4498 WREG32(mmCP_RB_DOORBELL_CONTROL, tmp);
4499
4500 if (adev->asic_type == CHIP_TONGA) {
4501 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
4502 DOORBELL_RANGE_LOWER,
4503 AMDGPU_DOORBELL_GFX_RING0);
4504 WREG32(mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
4505
4506 WREG32(mmCP_RB_DOORBELL_RANGE_UPPER,
4507 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
4508 }
4509
4510 }
4511
4512 /* start the ring */
4513 gfx_v8_0_cp_gfx_start(adev);
4514 ring->ready = true;
4515 r = amdgpu_ring_test_ring(ring);
5003f278 4516 if (r)
aaa36a97 4517 ring->ready = false;
aaa36a97 4518
5003f278 4519 return r;
aaa36a97
AD		 4520}
4521
4522static void gfx_v8_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
4523{
4524 int i;
4525
4526 if (enable) {
4527 WREG32(mmCP_MEC_CNTL, 0);
4528 } else {
4529 WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
4530 for (i = 0; i < adev->gfx.num_compute_rings; i++)
4531 adev->gfx.compute_ring[i].ready = false;
4532 }
4533 udelay(50);
4534}
4535
aaa36a97
AD		 4536static int gfx_v8_0_cp_compute_load_microcode(struct amdgpu_device *adev)
4537{
4538 const struct gfx_firmware_header_v1_0 *mec_hdr;
4539 const __le32 *fw_data;
4540 unsigned i, fw_size;
4541
4542 if (!adev->gfx.mec_fw)
4543 return -EINVAL;
4544
4545 gfx_v8_0_cp_compute_enable(adev, false);
4546
4547 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
4548 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
aaa36a97
AD		 4549
4550 fw_data = (const __le32 *)
4551 (adev->gfx.mec_fw->data +
4552 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4553 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4554
4555 /* MEC1 */
4556 WREG32(mmCP_MEC_ME1_UCODE_ADDR, 0);
4557 for (i = 0; i < fw_size; i++)
4558 WREG32(mmCP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data+i));
4559 WREG32(mmCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version);
4560
4561 /* Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
4562 if (adev->gfx.mec2_fw) {
4563 const struct gfx_firmware_header_v1_0 *mec2_hdr;
4564
4565 mec2_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec2_fw->data;
4566 amdgpu_ucode_print_gfx_hdr(&mec2_hdr->header);
aaa36a97
AD		 4567
4568 fw_data = (const __le32 *)
4569 (adev->gfx.mec2_fw->data +
4570 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4571 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4572
4573 WREG32(mmCP_MEC_ME2_UCODE_ADDR, 0);
4574 for (i = 0; i < fw_size; i++)
4575 WREG32(mmCP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data+i));
4576 WREG32(mmCP_MEC_ME2_UCODE_ADDR, adev->gfx.mec2_fw_version);
4577 }
4578
4579 return 0;
4580}
4581
aaa36a97
AD		 4582static void gfx_v8_0_cp_compute_fini(struct amdgpu_device *adev)
4583{
4584 int i, r;
4585
4586 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4587 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
4588
4589 if (ring->mqd_obj) {
4590 r = amdgpu_bo_reserve(ring->mqd_obj, false);
4591 if (unlikely(r != 0))
4592 dev_warn(adev->dev, "(%d) reserve MQD bo failed\n", r);
4593
4594 amdgpu_bo_unpin(ring->mqd_obj);
4595 amdgpu_bo_unreserve(ring->mqd_obj);
4596
4597 amdgpu_bo_unref(&ring->mqd_obj);
4598 ring->mqd_obj = NULL;
4599 }
4600 }
4601}
4602
4e638ae9
XY		 4603/* KIQ functions */
4604static void gfx_v8_0_kiq_setting(struct amdgpu_ring *ring)
4605{
4606 uint32_t tmp;
4607 struct amdgpu_device *adev = ring->adev;
4608
4609 /* tell RLC which is KIQ queue */
4610 tmp = RREG32(mmRLC_CP_SCHEDULERS);
4611 tmp &= 0xffffff00;
4612 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
4613 WREG32(mmRLC_CP_SCHEDULERS, tmp);
4614 tmp |= 0x80;
4615 WREG32(mmRLC_CP_SCHEDULERS, tmp);
4616}
4617
4618static void gfx_v8_0_kiq_enable(struct amdgpu_ring *ring)
4619{
4620 amdgpu_ring_alloc(ring, 8);
4621 /* set resources */
4622 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_RESOURCES, 6));
4623 amdgpu_ring_write(ring, 0); /* vmid_mask:0 queue_type:0 (KIQ) */
4624 amdgpu_ring_write(ring, 0x000000FF); /* queue mask lo */
4625 amdgpu_ring_write(ring, 0); /* queue mask hi */
4626 amdgpu_ring_write(ring, 0); /* gws mask lo */
4627 amdgpu_ring_write(ring, 0); /* gws mask hi */
4628 amdgpu_ring_write(ring, 0); /* oac mask */
4629 amdgpu_ring_write(ring, 0); /* gds heap base:0, gds heap size:0 */
4630 amdgpu_ring_commit(ring);
4631 udelay(50);
4632}
4633
4634static void gfx_v8_0_map_queue_enable(struct amdgpu_ring *kiq_ring,
4635 struct amdgpu_ring *ring)
4636{
4637 struct amdgpu_device *adev = kiq_ring->adev;
4638 uint64_t mqd_addr, wptr_addr;
4639
4640 mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
4641 wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
4642 amdgpu_ring_alloc(kiq_ring, 8);
4643
4644 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
4645 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
4646 amdgpu_ring_write(kiq_ring, 0x21010000);
4647 amdgpu_ring_write(kiq_ring, (ring->doorbell_index << 2) |
4648 (ring->queue << 26) |
4649 (ring->pipe << 29) |
4650 ((ring->me == 1 ? 0 : 1) << 31)); /* doorbell */
4651 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
4652 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
4653 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
4654 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
4655 amdgpu_ring_commit(kiq_ring);
4656 udelay(50);
4657}
4658
4659static int gfx_v8_0_mqd_init(struct amdgpu_device *adev,
4660 struct vi_mqd *mqd,
4661 uint64_t mqd_gpu_addr,
4662 uint64_t eop_gpu_addr,
4663 struct amdgpu_ring *ring)
4664{
4665 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
4666 uint32_t tmp;
4667
4668 mqd->header = 0xC0310800;
4669 mqd->compute_pipelinestat_enable = 0x00000001;
4670 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
4671 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
4672 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
4673 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
4674 mqd->compute_misc_reserved = 0x00000003;
4675
4676 eop_base_addr = eop_gpu_addr >> 8;
4677 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
4678 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
4679
4680 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4681 tmp = RREG32(mmCP_HQD_EOP_CONTROL);
4682 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
4683 (order_base_2(MEC_HPD_SIZE / 4) - 1));
4684
4685 mqd->cp_hqd_eop_control = tmp;
4686
4687 /* enable doorbell? */
4688 tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
4689
4690 if (ring->use_doorbell)
4691 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4692 DOORBELL_EN, 1);
4693 else
4694 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4695 DOORBELL_EN, 0);
4696
4697 mqd->cp_hqd_pq_doorbell_control = tmp;
4698
4699 /* disable the queue if it's active */
4700 mqd->cp_hqd_dequeue_request = 0;
4701 mqd->cp_hqd_pq_rptr = 0;
4702 mqd->cp_hqd_pq_wptr = 0;
4703
4704 /* set the pointer to the MQD */
4705 mqd->cp_mqd_base_addr_lo = mqd_gpu_addr & 0xfffffffc;
4706 mqd->cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4707
4708 /* set MQD vmid to 0 */
4709 tmp = RREG32(mmCP_MQD_CONTROL);
4710 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
4711 mqd->cp_mqd_control = tmp;
4712
4713 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4714 hqd_gpu_addr = ring->gpu_addr >> 8;
4715 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
4716 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4717
4718 /* set up the HQD, this is similar to CP_RB0_CNTL */
4719 tmp = RREG32(mmCP_HQD_PQ_CONTROL);
4720 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
4721 (order_base_2(ring->ring_size / 4) - 1));
4722 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
4723 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
4724#ifdef __BIG_ENDIAN
4725 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
4726#endif
4727 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
4728 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
4729 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
4730 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
4731 mqd->cp_hqd_pq_control = tmp;
4732
4733 /* set the wb address whether it's enabled or not */
4734 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
4735 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
4736 mqd->cp_hqd_pq_rptr_report_addr_hi =
4737 upper_32_bits(wb_gpu_addr) & 0xffff;
4738
4739 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4740 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
4741 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4742 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4743
4744 tmp = 0;
4745 /* enable the doorbell if requested */
4746 if (ring->use_doorbell) {
4747 tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
4748 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4749 DOORBELL_OFFSET, ring->doorbell_index);
4750
4751 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4752 DOORBELL_EN, 1);
4753 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4754 DOORBELL_SOURCE, 0);
4755 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4756 DOORBELL_HIT, 0);
4757 }
4758
4759 mqd->cp_hqd_pq_doorbell_control = tmp;
4760
4761 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4762 ring->wptr = 0;
4763 mqd->cp_hqd_pq_wptr = ring->wptr;
4764 mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
4765
4766 /* set the vmid for the queue */
4767 mqd->cp_hqd_vmid = 0;
4768
4769 tmp = RREG32(mmCP_HQD_PERSISTENT_STATE);
4770 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
4771 mqd->cp_hqd_persistent_state = tmp;
4772
4773 /* activate the queue */
4774 mqd->cp_hqd_active = 1;
4775
4776 return 0;
4777}
4778
4779static int gfx_v8_0_kiq_init_register(struct amdgpu_device *adev,
4780 struct vi_mqd *mqd,
4781 struct amdgpu_ring *ring)
4782{
4783 uint32_t tmp;
4784 int j;
4785
4786 /* disable wptr polling */
4787 tmp = RREG32(mmCP_PQ_WPTR_POLL_CNTL);
4788 tmp = REG_SET_FIELD(tmp, CP_PQ_WPTR_POLL_CNTL, EN, 0);
4789 WREG32(mmCP_PQ_WPTR_POLL_CNTL, tmp);
4790
4791 WREG32(mmCP_HQD_EOP_BASE_ADDR, mqd->cp_hqd_eop_base_addr_lo);
4792 WREG32(mmCP_HQD_EOP_BASE_ADDR_HI, mqd->cp_hqd_eop_base_addr_hi);
4793
4794 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4795 WREG32(mmCP_HQD_EOP_CONTROL, mqd->cp_hqd_eop_control);
4796
4797 /* enable doorbell? */
4798 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, mqd->cp_hqd_pq_doorbell_control);
4799
4800 /* disable the queue if it's active */
4801 if (RREG32(mmCP_HQD_ACTIVE) & 1) {
4802 WREG32(mmCP_HQD_DEQUEUE_REQUEST, 1);
4803 for (j = 0; j < adev->usec_timeout; j++) {
4804 if (!(RREG32(mmCP_HQD_ACTIVE) & 1))
4805 break;
4806 udelay(1);
4807 }
4808 WREG32(mmCP_HQD_DEQUEUE_REQUEST, mqd->cp_hqd_dequeue_request);
4809 WREG32(mmCP_HQD_PQ_RPTR, mqd->cp_hqd_pq_rptr);
4810 WREG32(mmCP_HQD_PQ_WPTR, mqd->cp_hqd_pq_wptr);
4811 }
4812
4813 /* set the pointer to the MQD */
4814 WREG32(mmCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);
4815 WREG32(mmCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
4816
4817 /* set MQD vmid to 0 */
4818 WREG32(mmCP_MQD_CONTROL, mqd->cp_mqd_control);
4819
4820 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4821 WREG32(mmCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);
4822 WREG32(mmCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);
4823
4824 /* set up the HQD, this is similar to CP_RB0_CNTL */
4825 WREG32(mmCP_HQD_PQ_CONTROL, mqd->cp_hqd_pq_control);
4826
4827 /* set the wb address whether it's enabled or not */
4828 WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR,
4829 mqd->cp_hqd_pq_rptr_report_addr_lo);
4830 WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4831 mqd->cp_hqd_pq_rptr_report_addr_hi);
4832
4833 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4834 WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR, mqd->cp_hqd_pq_wptr_poll_addr_lo);
4835 WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, mqd->cp_hqd_pq_wptr_poll_addr_hi);
4836
4837 /* enable the doorbell if requested */
4838 if (ring->use_doorbell) {
4839 if ((adev->asic_type == CHIP_CARRIZO) ||
4840 (adev->asic_type == CHIP_FIJI) ||
4841 (adev->asic_type == CHIP_STONEY)) {
4842 WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER,
4843 AMDGPU_DOORBELL_KIQ << 2);
4844 WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER,
4845 AMDGPU_DOORBELL_MEC_RING7 << 2);
4846 }
4847 }
4848 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, mqd->cp_hqd_pq_doorbell_control);
4849
4850 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4851 WREG32(mmCP_HQD_PQ_WPTR, mqd->cp_hqd_pq_wptr);
4852
4853 /* set the vmid for the queue */
4854 WREG32(mmCP_HQD_VMID, mqd->cp_hqd_vmid);
4855
4856 WREG32(mmCP_HQD_PERSISTENT_STATE, mqd->cp_hqd_persistent_state);
4857
4858 /* activate the queue */
4859 WREG32(mmCP_HQD_ACTIVE, mqd->cp_hqd_active);
4860
4861 if (ring->use_doorbell) {
4862 tmp = RREG32(mmCP_PQ_STATUS);
4863 tmp = REG_SET_FIELD(tmp, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4864 WREG32(mmCP_PQ_STATUS, tmp);
4865 }
4866
4867 return 0;
4868}
4869
4870static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring,
4871 struct vi_mqd *mqd,
4872 u64 mqd_gpu_addr)
4873{
4874 struct amdgpu_device *adev = ring->adev;
4875 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
4876 uint64_t eop_gpu_addr;
4877 bool is_kiq = false;
4878
4879 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
4880 is_kiq = true;
4881
4882 if (is_kiq) {
4883 eop_gpu_addr = kiq->eop_gpu_addr;
4884 gfx_v8_0_kiq_setting(&kiq->ring);
4885 } else
4886 eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr +
4887 ring->queue * MEC_HPD_SIZE;
4888
4889 mutex_lock(&adev->srbm_mutex);
4890 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4891
4892 gfx_v8_0_mqd_init(adev, mqd, mqd_gpu_addr, eop_gpu_addr, ring);
4893
4894 if (is_kiq)
4895 gfx_v8_0_kiq_init_register(adev, mqd, ring);
4896
4897 vi_srbm_select(adev, 0, 0, 0, 0);
4898 mutex_unlock(&adev->srbm_mutex);
4899
4900 if (is_kiq)
4901 gfx_v8_0_kiq_enable(ring);
4902 else
4903 gfx_v8_0_map_queue_enable(&kiq->ring, ring);
4904
4905 return 0;
4906}
4907
4908static void gfx_v8_0_kiq_free_queue(struct amdgpu_device *adev)
4909{
4910 struct amdgpu_ring *ring = NULL;
4911 int i;
4912
4913 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4914 ring = &adev->gfx.compute_ring[i];
4915 amdgpu_bo_free_kernel(&ring->mqd_obj, NULL, NULL);
4916 ring->mqd_obj = NULL;
4917 }
4918
4919 ring = &adev->gfx.kiq.ring;
4920 amdgpu_bo_free_kernel(&ring->mqd_obj, NULL, NULL);
4921 ring->mqd_obj = NULL;
4922}
4923
4924static int gfx_v8_0_kiq_setup_queue(struct amdgpu_device *adev,
4925 struct amdgpu_ring *ring)
4926{
4927 struct vi_mqd *mqd;
4928 u64 mqd_gpu_addr;
4929 u32 *buf;
4930 int r = 0;
4931
4932 r = amdgpu_bo_create_kernel(adev, sizeof(struct vi_mqd), PAGE_SIZE,
4933 AMDGPU_GEM_DOMAIN_GTT, &ring->mqd_obj,
4934 &mqd_gpu_addr, (void **)&buf);
4935 if (r) {
4936 dev_warn(adev->dev, "failed to create ring mqd ob (%d)", r);
4937 return r;
4938 }
4939
4940 /* init the mqd struct */
4941 memset(buf, 0, sizeof(struct vi_mqd));
4942 mqd = (struct vi_mqd *)buf;
4943
4944 r = gfx_v8_0_kiq_init_queue(ring, mqd, mqd_gpu_addr);
4945 if (r)
4946 return r;
4947
4948 amdgpu_bo_kunmap(ring->mqd_obj);
4949
4950 return 0;
4951}
4952
4953static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev)
4954{
4955 struct amdgpu_ring *ring = NULL;
4956 int r, i;
4957
4958 ring = &adev->gfx.kiq.ring;
4959 r = gfx_v8_0_kiq_setup_queue(adev, ring);
4960 if (r)
4961 return r;
4962
4963 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4964 ring = &adev->gfx.compute_ring[i];
4965 r = gfx_v8_0_kiq_setup_queue(adev, ring);
4966 if (r)
4967 return r;
4968 }
4969
4970 gfx_v8_0_cp_compute_enable(adev, true);
4971
4972 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4973 ring = &adev->gfx.compute_ring[i];
4974
4975 ring->ready = true;
4976 r = amdgpu_ring_test_ring(ring);
4977 if (r)
4978 ring->ready = false;
4979 }
4980
4981 ring = &adev->gfx.kiq.ring;
4982 ring->ready = true;
4983 r = amdgpu_ring_test_ring(ring);
4984 if (r)
4985 ring->ready = false;
4986
4987 return 0;
4988}
4989
aaa36a97
AD		 4990static int gfx_v8_0_cp_compute_resume(struct amdgpu_device *adev)
4991{
4992 int r, i, j;
4993 u32 tmp;
4994 bool use_doorbell = true;
4995 u64 hqd_gpu_addr;
4996 u64 mqd_gpu_addr;
4997 u64 eop_gpu_addr;
4998 u64 wb_gpu_addr;
4999 u32 *buf;
5000 struct vi_mqd *mqd;
5001
ad3b9614 5002 /* init the queues. */
aaa36a97
AD		 5003 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5004 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
5005
5006 if (ring->mqd_obj == NULL) {
5007 r = amdgpu_bo_create(adev,
5008 sizeof(struct vi_mqd),
5009 PAGE_SIZE, true,
5010 AMDGPU_GEM_DOMAIN_GTT, 0, NULL,
72d7668b 5011 NULL, &ring->mqd_obj);
aaa36a97
AD		 5012 if (r) {
5013 dev_warn(adev->dev, "(%d) create MQD bo failed\n", r);
5014 return r;
5015 }
5016 }
5017
5018 r = amdgpu_bo_reserve(ring->mqd_obj, false);
5019 if (unlikely(r != 0)) {
5020 gfx_v8_0_cp_compute_fini(adev);
5021 return r;
5022 }
5023 r = amdgpu_bo_pin(ring->mqd_obj, AMDGPU_GEM_DOMAIN_GTT,
5024 &mqd_gpu_addr);
5025 if (r) {
5026 dev_warn(adev->dev, "(%d) pin MQD bo failed\n", r);
5027 gfx_v8_0_cp_compute_fini(adev);
5028 return r;
5029 }
5030 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&buf);
5031 if (r) {
5032 dev_warn(adev->dev, "(%d) map MQD bo failed\n", r);
5033 gfx_v8_0_cp_compute_fini(adev);
5034 return r;
5035 }
5036
5037 /* init the mqd struct */
5038 memset(buf, 0, sizeof(struct vi_mqd));
5039
5040 mqd = (struct vi_mqd *)buf;
5041 mqd->header = 0xC0310800;
5042 mqd->compute_pipelinestat_enable = 0x00000001;
5043 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
5044 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
5045 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
5046 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
5047 mqd->compute_misc_reserved = 0x00000003;
5048
5049 mutex_lock(&adev->srbm_mutex);
5050 vi_srbm_select(adev, ring->me,
5051 ring->pipe,
5052 ring->queue, 0);
5053
ad3b9614
AD		 5054 eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE);
5055 eop_gpu_addr >>= 8;
5056
5057 /* write the EOP addr */
5058 WREG32(mmCP_HQD_EOP_BASE_ADDR, eop_gpu_addr);
5059 WREG32(mmCP_HQD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr));
5060
5061 /* set the VMID assigned */
5062 WREG32(mmCP_HQD_VMID, 0);
5063
5064 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
5065 tmp = RREG32(mmCP_HQD_EOP_CONTROL);
5066 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
5067 (order_base_2(MEC_HPD_SIZE / 4) - 1));
5068 WREG32(mmCP_HQD_EOP_CONTROL, tmp);
5069
aaa36a97
AD		 5070 /* disable wptr polling */
5071 tmp = RREG32(mmCP_PQ_WPTR_POLL_CNTL);
5072 tmp = REG_SET_FIELD(tmp, CP_PQ_WPTR_POLL_CNTL, EN, 0);
5073 WREG32(mmCP_PQ_WPTR_POLL_CNTL, tmp);
5074
5075 mqd->cp_hqd_eop_base_addr_lo =
5076 RREG32(mmCP_HQD_EOP_BASE_ADDR);
5077 mqd->cp_hqd_eop_base_addr_hi =
5078 RREG32(mmCP_HQD_EOP_BASE_ADDR_HI);
5079
5080 /* enable doorbell? */
5081 tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
5082 if (use_doorbell) {
5083 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
5084 } else {
5085 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 0);
5086 }
5087 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, tmp);
5088 mqd->cp_hqd_pq_doorbell_control = tmp;
5089
5090 /* disable the queue if it's active */
5091 mqd->cp_hqd_dequeue_request = 0;
5092 mqd->cp_hqd_pq_rptr = 0;
5093 mqd->cp_hqd_pq_wptr= 0;
5094 if (RREG32(mmCP_HQD_ACTIVE) & 1) {
5095 WREG32(mmCP_HQD_DEQUEUE_REQUEST, 1);
5096 for (j = 0; j < adev->usec_timeout; j++) {
5097 if (!(RREG32(mmCP_HQD_ACTIVE) & 1))
5098 break;
5099 udelay(1);
5100 }
5101 WREG32(mmCP_HQD_DEQUEUE_REQUEST, mqd->cp_hqd_dequeue_request);
5102 WREG32(mmCP_HQD_PQ_RPTR, mqd->cp_hqd_pq_rptr);
5103 WREG32(mmCP_HQD_PQ_WPTR, mqd->cp_hqd_pq_wptr);
5104 }
5105
5106 /* set the pointer to the MQD */
5107 mqd->cp_mqd_base_addr_lo = mqd_gpu_addr & 0xfffffffc;
5108 mqd->cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
5109 WREG32(mmCP_MQD_BASE_ADDR, mqd->cp_mqd_base_addr_lo);
5110 WREG32(mmCP_MQD_BASE_ADDR_HI, mqd->cp_mqd_base_addr_hi);
5111
5112 /* set MQD vmid to 0 */
5113 tmp = RREG32(mmCP_MQD_CONTROL);
5114 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
5115 WREG32(mmCP_MQD_CONTROL, tmp);
5116 mqd->cp_mqd_control = tmp;
5117
5118 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
5119 hqd_gpu_addr = ring->gpu_addr >> 8;
5120 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
5121 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
5122 WREG32(mmCP_HQD_PQ_BASE, mqd->cp_hqd_pq_base_lo);
5123 WREG32(mmCP_HQD_PQ_BASE_HI, mqd->cp_hqd_pq_base_hi);
5124
5125 /* set up the HQD, this is similar to CP_RB0_CNTL */
5126 tmp = RREG32(mmCP_HQD_PQ_CONTROL);
5127 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
5128 (order_base_2(ring->ring_size / 4) - 1));
5129 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
5130 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
5131#ifdef __BIG_ENDIAN
5132 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
5133#endif
5134 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
5135 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
5136 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
5137 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
5138 WREG32(mmCP_HQD_PQ_CONTROL, tmp);
5139 mqd->cp_hqd_pq_control = tmp;
5140
5141 /* set the wb address wether it's enabled or not */
5142 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
5143 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
5144 mqd->cp_hqd_pq_rptr_report_addr_hi =
5145 upper_32_bits(wb_gpu_addr) & 0xffff;
5146 WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR,
5147 mqd->cp_hqd_pq_rptr_report_addr_lo);
5148 WREG32(mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
5149 mqd->cp_hqd_pq_rptr_report_addr_hi);
5150
5151 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
5152 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
aeab2032 5153 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
aaa36a97 5154 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
aeab2032 5155 WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR, mqd->cp_hqd_pq_wptr_poll_addr_lo);
aaa36a97
AD		 5156 WREG32(mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
5157 mqd->cp_hqd_pq_wptr_poll_addr_hi);
5158
5159 /* enable the doorbell if requested */
5160 if (use_doorbell) {
bddf8026 5161 if ((adev->asic_type == CHIP_CARRIZO) ||
e3c7656c 5162 (adev->asic_type == CHIP_FIJI) ||
68182d90 5163 (adev->asic_type == CHIP_STONEY) ||
2cc0c0b5 5164 (adev->asic_type == CHIP_POLARIS11) ||
c4642a47
JZ		 5165 (adev->asic_type == CHIP_POLARIS10) ||
5166 (adev->asic_type == CHIP_POLARIS12)) {
aaa36a97
AD		 5167 WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER,
5168 AMDGPU_DOORBELL_KIQ << 2);
5169 WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER,
b8826b0c 5170 AMDGPU_DOORBELL_MEC_RING7 << 2);
aaa36a97
AD		 5171 }
5172 tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
5173 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
5174 DOORBELL_OFFSET, ring->doorbell_index);
5175 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_EN, 1);
5176 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_SOURCE, 0);
5177 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, DOORBELL_HIT, 0);
5178 mqd->cp_hqd_pq_doorbell_control = tmp;
5179
5180 } else {
5181 mqd->cp_hqd_pq_doorbell_control = 0;
5182 }
5183 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL,
5184 mqd->cp_hqd_pq_doorbell_control);
5185
845253e7
SJ		 5186 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5187 ring->wptr = 0;
5188 mqd->cp_hqd_pq_wptr = ring->wptr;
5189 WREG32(mmCP_HQD_PQ_WPTR, mqd->cp_hqd_pq_wptr);
5190 mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
5191
aaa36a97
AD		 5192 /* set the vmid for the queue */
5193 mqd->cp_hqd_vmid = 0;
5194 WREG32(mmCP_HQD_VMID, mqd->cp_hqd_vmid);
5195
5196 tmp = RREG32(mmCP_HQD_PERSISTENT_STATE);
5197 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
5198 WREG32(mmCP_HQD_PERSISTENT_STATE, tmp);
5199 mqd->cp_hqd_persistent_state = tmp;
68182d90 5200 if (adev->asic_type == CHIP_STONEY ||
2cc0c0b5 5201 adev->asic_type == CHIP_POLARIS11 ||
c4642a47
JZ		 5202 adev->asic_type == CHIP_POLARIS10 ||
5203 adev->asic_type == CHIP_POLARIS12) {
3b55ddad
FC		 5204 tmp = RREG32(mmCP_ME1_PIPE3_INT_CNTL);
5205 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE3_INT_CNTL, GENERIC2_INT_ENABLE, 1);
5206 WREG32(mmCP_ME1_PIPE3_INT_CNTL, tmp);
5207 }
aaa36a97
AD		 5208
5209 /* activate the queue */
5210 mqd->cp_hqd_active = 1;
5211 WREG32(mmCP_HQD_ACTIVE, mqd->cp_hqd_active);
5212
5213 vi_srbm_select(adev, 0, 0, 0, 0);
5214 mutex_unlock(&adev->srbm_mutex);
5215
5216 amdgpu_bo_kunmap(ring->mqd_obj);
5217 amdgpu_bo_unreserve(ring->mqd_obj);
5218 }
5219
5220 if (use_doorbell) {
5221 tmp = RREG32(mmCP_PQ_STATUS);
5222 tmp = REG_SET_FIELD(tmp, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
5223 WREG32(mmCP_PQ_STATUS, tmp);
5224 }
5225
6e9821b2 5226 gfx_v8_0_cp_compute_enable(adev, true);
aaa36a97
AD		 5227
5228 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5229 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
5230
5231 ring->ready = true;
5232 r = amdgpu_ring_test_ring(ring);
5233 if (r)
5234 ring->ready = false;
5235 }
5236
5237 return 0;
5238}
5239
5240static int gfx_v8_0_cp_resume(struct amdgpu_device *adev)
5241{
5242 int r;
5243
e3c7656c 5244 if (!(adev->flags & AMD_IS_APU))
aaa36a97
AD		 5245 gfx_v8_0_enable_gui_idle_interrupt(adev, false);
5246
e61710c5 5247 if (!adev->pp_enabled) {
ba5c2a87
RZ		 5248 if (!adev->firmware.smu_load) {
5249 /* legacy firmware loading */
5250 r = gfx_v8_0_cp_gfx_load_microcode(adev);
5251 if (r)
5252 return r;
aaa36a97 5253
ba5c2a87
RZ		 5254 r = gfx_v8_0_cp_compute_load_microcode(adev);
5255 if (r)
5256 return r;
5257 } else {
5258 r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
5259 AMDGPU_UCODE_ID_CP_CE);
5260 if (r)
5261 return -EINVAL;
5262
5263 r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
5264 AMDGPU_UCODE_ID_CP_PFP);
5265 if (r)
5266 return -EINVAL;
5267
5268 r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
5269 AMDGPU_UCODE_ID_CP_ME);
5270 if (r)
5271 return -EINVAL;
5272
951e0962
AD		 5273 if (adev->asic_type == CHIP_TOPAZ) {
5274 r = gfx_v8_0_cp_compute_load_microcode(adev);
5275 if (r)
5276 return r;
5277 } else {
5278 r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
5279 AMDGPU_UCODE_ID_CP_MEC1);
5280 if (r)
5281 return -EINVAL;
5282 }
ba5c2a87 5283 }
aaa36a97
AD		 5284 }
5285
5286 r = gfx_v8_0_cp_gfx_resume(adev);
5287 if (r)
5288 return r;
5289
4e638ae9
XY		 5290 if (amdgpu_sriov_vf(adev))
5291 r = gfx_v8_0_kiq_resume(adev);
5292 else
5293 r = gfx_v8_0_cp_compute_resume(adev);
aaa36a97
AD		 5294 if (r)
5295 return r;
5296
5297 gfx_v8_0_enable_gui_idle_interrupt(adev, true);
5298
5299 return 0;
5300}
5301
5302static void gfx_v8_0_cp_enable(struct amdgpu_device *adev, bool enable)
5303{
5304 gfx_v8_0_cp_gfx_enable(adev, enable);
5305 gfx_v8_0_cp_compute_enable(adev, enable);
5306}
5307
5fc3aeeb 5308static int gfx_v8_0_hw_init(void *handle)
aaa36a97
AD		 5309{
5310 int r;
5fc3aeeb 5311 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
aaa36a97
AD		 5312
5313 gfx_v8_0_init_golden_registers(adev);
aaa36a97
AD		 5314 gfx_v8_0_gpu_init(adev);
5315
5316 r = gfx_v8_0_rlc_resume(adev);
5317 if (r)
5318 return r;
5319
5320 r = gfx_v8_0_cp_resume(adev);
aaa36a97
AD		 5321
5322 return r;
5323}
5324
5fc3aeeb 5325static int gfx_v8_0_hw_fini(void *handle)
aaa36a97 5326{
5fc3aeeb 5327 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5328
1d22a454
AD		 5329 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
5330 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
84f3f05b 5331 if (amdgpu_sriov_vf(adev)) {
4e638ae9 5332 gfx_v8_0_kiq_free_queue(adev);
84f3f05b
XY		 5333 pr_debug("For SRIOV client, shouldn't do anything.\n");
5334 return 0;
5335 }
aaa36a97
AD		 5336 gfx_v8_0_cp_enable(adev, false);
5337 gfx_v8_0_rlc_stop(adev);
5338 gfx_v8_0_cp_compute_fini(adev);
5339
62a86fc2
EH		 5340 amdgpu_set_powergating_state(adev,
5341 AMD_IP_BLOCK_TYPE_GFX, AMD_PG_STATE_UNGATE);
5342
aaa36a97
AD		 5343 return 0;
5344}
5345
5fc3aeeb 5346static int gfx_v8_0_suspend(void *handle)
aaa36a97 5347{
5fc3aeeb 5348 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5349
aaa36a97
AD		 5350 return gfx_v8_0_hw_fini(adev);
5351}
5352
5fc3aeeb 5353static int gfx_v8_0_resume(void *handle)
aaa36a97 5354{
5fc3aeeb 5355 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5356
aaa36a97
AD		 5357 return gfx_v8_0_hw_init(adev);
5358}
5359
5fc3aeeb 5360static bool gfx_v8_0_is_idle(void *handle)
aaa36a97 5361{
5fc3aeeb 5362 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5363
aaa36a97
AD		 5364 if (REG_GET_FIELD(RREG32(mmGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE))
5365 return false;
5366 else
5367 return true;
5368}
5369
5fc3aeeb 5370static int gfx_v8_0_wait_for_idle(void *handle)
aaa36a97
AD		 5371{
5372 unsigned i;
5fc3aeeb 5373 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
aaa36a97
AD		 5374
5375 for (i = 0; i < adev->usec_timeout; i++) {
5003f278 5376 if (gfx_v8_0_is_idle(handle))
aaa36a97 5377 return 0;
5003f278 5378
aaa36a97
AD		 5379 udelay(1);
5380 }
5381 return -ETIMEDOUT;
5382}
5383
da146d3b 5384static bool gfx_v8_0_check_soft_reset(void *handle)
aaa36a97 5385{
3d7c6384 5386 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
aaa36a97
AD		 5387 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5388 u32 tmp;
5389
5390 /* GRBM_STATUS */
5391 tmp = RREG32(mmGRBM_STATUS);
5392 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
5393 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
5394 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
5395 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
5396 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
3d7c6384
CZ		 5397 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK |
5398 GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
aaa36a97
AD		 5399 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
5400 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
5401 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
5402 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
aaa36a97
AD		 5403 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
5404 SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
5405 }
5406
5407 /* GRBM_STATUS2 */
5408 tmp = RREG32(mmGRBM_STATUS2);
5409 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
5410 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
5411 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
5412
3d7c6384
CZ		 5413 if (REG_GET_FIELD(tmp, GRBM_STATUS2, CPF_BUSY) ||
5414 REG_GET_FIELD(tmp, GRBM_STATUS2, CPC_BUSY) ||
5415 REG_GET_FIELD(tmp, GRBM_STATUS2, CPG_BUSY)) {
5416 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5417 SOFT_RESET_CPF, 1);
5418 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5419 SOFT_RESET_CPC, 1);
5420 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5421 SOFT_RESET_CPG, 1);
5422 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,
5423 SOFT_RESET_GRBM, 1);
5424 }
5425
aaa36a97
AD		 5426 /* SRBM_STATUS */
5427 tmp = RREG32(mmSRBM_STATUS);
5428 if (REG_GET_FIELD(tmp, SRBM_STATUS, GRBM_RQ_PENDING))
5429 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
5430 SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
3d7c6384
CZ		 5431 if (REG_GET_FIELD(tmp, SRBM_STATUS, SEM_BUSY))
5432 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
5433 SRBM_SOFT_RESET, SOFT_RESET_SEM, 1);
aaa36a97
AD		 5434
5435 if (grbm_soft_reset || srbm_soft_reset) {
3d7c6384
CZ		 5436 adev->gfx.grbm_soft_reset = grbm_soft_reset;
5437 adev->gfx.srbm_soft_reset = srbm_soft_reset;
da146d3b 5438 return true;
3d7c6384 5439 } else {
3d7c6384
CZ		 5440 adev->gfx.grbm_soft_reset = 0;
5441 adev->gfx.srbm_soft_reset = 0;
da146d3b 5442 return false;
3d7c6384 5443 }
3d7c6384 5444}
aaa36a97 5445
1057f20c
CZ		 5446static void gfx_v8_0_inactive_hqd(struct amdgpu_device *adev,
5447 struct amdgpu_ring *ring)
5448{
5449 int i;
5450
5451 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
5452 if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) {
5453 u32 tmp;
5454 tmp = RREG32(mmCP_HQD_DEQUEUE_REQUEST);
5455 tmp = REG_SET_FIELD(tmp, CP_HQD_DEQUEUE_REQUEST,
5456 DEQUEUE_REQ, 2);
5457 WREG32(mmCP_HQD_DEQUEUE_REQUEST, tmp);
5458 for (i = 0; i < adev->usec_timeout; i++) {
5459 if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK))
5460 break;
5461 udelay(1);
5462 }
5463 }
5464}
5465
5466static int gfx_v8_0_pre_soft_reset(void *handle)
5467{
5468 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5469 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5470
da146d3b
AD		 5471 if ((!adev->gfx.grbm_soft_reset) &&
5472 (!adev->gfx.srbm_soft_reset))
1057f20c
CZ		 5473 return 0;
5474
5475 grbm_soft_reset = adev->gfx.grbm_soft_reset;
5476 srbm_soft_reset = adev->gfx.srbm_soft_reset;
5477
5478 /* stop the rlc */
5479 gfx_v8_0_rlc_stop(adev);
5480
5481 if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5482 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
aaa36a97
AD		 5483 /* Disable GFX parsing/prefetching */
5484 gfx_v8_0_cp_gfx_enable(adev, false);
5485
1057f20c
CZ		 5486 if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5487 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
5488 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
5489 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
5490 int i;
5491
5492 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5493 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
5494
5495 gfx_v8_0_inactive_hqd(adev, ring);
5496 }
aaa36a97 5497 /* Disable MEC parsing/prefetching */
7776a693 5498 gfx_v8_0_cp_compute_enable(adev, false);
1057f20c 5499 }
7776a693 5500
1057f20c
CZ		 5501 return 0;
5502}
7776a693 5503
3d7c6384
CZ		 5504static int gfx_v8_0_soft_reset(void *handle)
5505{
5506 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5507 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5508 u32 tmp;
aaa36a97 5509
da146d3b
AD		 5510 if ((!adev->gfx.grbm_soft_reset) &&
5511 (!adev->gfx.srbm_soft_reset))
3d7c6384 5512 return 0;
aaa36a97 5513
3d7c6384
CZ		 5514 grbm_soft_reset = adev->gfx.grbm_soft_reset;
5515 srbm_soft_reset = adev->gfx.srbm_soft_reset;
aaa36a97 5516
3d7c6384
CZ		 5517 if (grbm_soft_reset || srbm_soft_reset) {
5518 tmp = RREG32(mmGMCON_DEBUG);
5519 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 1);
5520 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 1);
5521 WREG32(mmGMCON_DEBUG, tmp);
5522 udelay(50);
5523 }
aaa36a97 5524
3d7c6384
CZ		 5525 if (grbm_soft_reset) {
5526 tmp = RREG32(mmGRBM_SOFT_RESET);
5527 tmp |= grbm_soft_reset;
5528 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5529 WREG32(mmGRBM_SOFT_RESET, tmp);
5530 tmp = RREG32(mmGRBM_SOFT_RESET);
aaa36a97 5531
3d7c6384 5532 udelay(50);
aaa36a97 5533
3d7c6384
CZ		 5534 tmp &= ~grbm_soft_reset;
5535 WREG32(mmGRBM_SOFT_RESET, tmp);
5536 tmp = RREG32(mmGRBM_SOFT_RESET);
5537 }
7776a693 5538
3d7c6384
CZ		 5539 if (srbm_soft_reset) {
5540 tmp = RREG32(mmSRBM_SOFT_RESET);
5541 tmp |= srbm_soft_reset;
5542 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5543 WREG32(mmSRBM_SOFT_RESET, tmp);
5544 tmp = RREG32(mmSRBM_SOFT_RESET);
7776a693 5545
aaa36a97 5546 udelay(50);
7776a693 5547
3d7c6384
CZ		 5548 tmp &= ~srbm_soft_reset;
5549 WREG32(mmSRBM_SOFT_RESET, tmp);
5550 tmp = RREG32(mmSRBM_SOFT_RESET);
aaa36a97 5551 }
7776a693 5552
3d7c6384
CZ		 5553 if (grbm_soft_reset || srbm_soft_reset) {
5554 tmp = RREG32(mmGMCON_DEBUG);
5555 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 0);
5556 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 0);
5557 WREG32(mmGMCON_DEBUG, tmp);
aaa36a97 5558 }
3d7c6384
CZ		 5559
5560 /* Wait a little for things to settle down */
5561 udelay(50);
5562
aaa36a97
AD		 5563 return 0;
5564}
5565
e4ae0fc3
CZ		 5566static void gfx_v8_0_init_hqd(struct amdgpu_device *adev,
5567 struct amdgpu_ring *ring)
5568{
5569 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
5570 WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0);
5571 WREG32(mmCP_HQD_PQ_RPTR, 0);
5572 WREG32(mmCP_HQD_PQ_WPTR, 0);
5573 vi_srbm_select(adev, 0, 0, 0, 0);
5574}
5575
5576static int gfx_v8_0_post_soft_reset(void *handle)
5577{
5578 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5579 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5580
da146d3b
AD		 5581 if ((!adev->gfx.grbm_soft_reset) &&
5582 (!adev->gfx.srbm_soft_reset))
e4ae0fc3
CZ		 5583 return 0;
5584
5585 grbm_soft_reset = adev->gfx.grbm_soft_reset;
5586 srbm_soft_reset = adev->gfx.srbm_soft_reset;
5587
5588 if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5589 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
5590 gfx_v8_0_cp_gfx_resume(adev);
5591
5592 if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5593 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
5594 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
5595 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
5596 int i;
5597
5598 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5599 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
5600
5601 gfx_v8_0_init_hqd(adev, ring);
5602 }
5603 gfx_v8_0_cp_compute_resume(adev);
5604 }
5605 gfx_v8_0_rlc_start(adev);
5606
aaa36a97
AD		 5607 return 0;
5608}
5609
5610/**
5611 * gfx_v8_0_get_gpu_clock_counter - return GPU clock counter snapshot
5612 *
5613 * @adev: amdgpu_device pointer
5614 *
5615 * Fetches a GPU clock counter snapshot.
5616 * Returns the 64 bit clock counter snapshot.
5617 */
b95e31fd 5618static uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev)
aaa36a97
AD		 5619{
5620 uint64_t clock;
5621
5622 mutex_lock(&adev->gfx.gpu_clock_mutex);
5623 WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
5624 clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) |
5625 ((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
5626 mutex_unlock(&adev->gfx.gpu_clock_mutex);
5627 return clock;
5628}
5629
5630static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
5631 uint32_t vmid,
5632 uint32_t gds_base, uint32_t gds_size,
5633 uint32_t gws_base, uint32_t gws_size,
5634 uint32_t oa_base, uint32_t oa_size)
5635{
5636 gds_base = gds_base >> AMDGPU_GDS_SHIFT;
5637 gds_size = gds_size >> AMDGPU_GDS_SHIFT;
5638
5639 gws_base = gws_base >> AMDGPU_GWS_SHIFT;
5640 gws_size = gws_size >> AMDGPU_GWS_SHIFT;
5641
5642 oa_base = oa_base >> AMDGPU_OA_SHIFT;
5643 oa_size = oa_size >> AMDGPU_OA_SHIFT;
5644
5645 /* GDS Base */
5646 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5647 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5648 WRITE_DATA_DST_SEL(0)));
5649 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_base);
5650 amdgpu_ring_write(ring, 0);
5651 amdgpu_ring_write(ring, gds_base);
5652
5653 /* GDS Size */
5654 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5655 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5656 WRITE_DATA_DST_SEL(0)));
5657 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_size);
5658 amdgpu_ring_write(ring, 0);
5659 amdgpu_ring_write(ring, gds_size);
5660
5661 /* GWS */
5662 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5663 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5664 WRITE_DATA_DST_SEL(0)));
5665 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].gws);
5666 amdgpu_ring_write(ring, 0);
5667 amdgpu_ring_write(ring, gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
5668
5669 /* OA */
5670 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5671 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5672 WRITE_DATA_DST_SEL(0)));
5673 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].oa);
5674 amdgpu_ring_write(ring, 0);
5675 amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base));
5676}
5677
472259f0
TSD		 5678static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
5679{
bc24fbe9
TSD		 5680 WREG32(mmSQ_IND_INDEX,
5681 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
5682 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
5683 (address << SQ_IND_INDEX__INDEX__SHIFT) |
5684 (SQ_IND_INDEX__FORCE_READ_MASK));
472259f0
TSD		 5685 return RREG32(mmSQ_IND_DATA);
5686}
5687
c5a60ce8
TSD		 5688static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
5689 uint32_t wave, uint32_t thread,
5690 uint32_t regno, uint32_t num, uint32_t *out)
5691{
5692 WREG32(mmSQ_IND_INDEX,
5693 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
5694 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
5695 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
5696 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
5697 (SQ_IND_INDEX__FORCE_READ_MASK) |
5698 (SQ_IND_INDEX__AUTO_INCR_MASK));
5699 while (num--)
5700 *(out++) = RREG32(mmSQ_IND_DATA);
5701}
5702
472259f0
TSD		 5703static void gfx_v8_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
5704{
5705 /* type 0 wave data */
5706 dst[(*no_fields)++] = 0;
5707 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
5708 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
5709 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
5710 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
5711 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
5712 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
5713 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
5714 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
5715 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
5716 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
5717 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
5718 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
74f3ce31
TSD		 5719 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO);
5720 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI);
5721 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO);
5722 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI);
5723 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
5724 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
472259f0
TSD		 5725}
5726
c5a60ce8
TSD		 5727static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
5728 uint32_t wave, uint32_t start,
5729 uint32_t size, uint32_t *dst)
5730{
5731 wave_read_regs(
5732 adev, simd, wave, 0,
5733 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
5734}
5735
472259f0 5736
b95e31fd
AD		 5737static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = {
5738 .get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter,
05fb7291 5739 .select_se_sh = &gfx_v8_0_select_se_sh,
472259f0 5740 .read_wave_data = &gfx_v8_0_read_wave_data,
c5a60ce8 5741 .read_wave_sgprs = &gfx_v8_0_read_wave_sgprs,
b95e31fd
AD		 5742};
5743
5fc3aeeb 5744static int gfx_v8_0_early_init(void *handle)
aaa36a97 5745{
5fc3aeeb 5746 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
aaa36a97
AD		 5747
5748 adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS;
5749 adev->gfx.num_compute_rings = GFX8_NUM_COMPUTE_RINGS;
b95e31fd 5750 adev->gfx.funcs = &gfx_v8_0_gfx_funcs;
aaa36a97
AD		 5751 gfx_v8_0_set_ring_funcs(adev);
5752 gfx_v8_0_set_irq_funcs(adev);
5753 gfx_v8_0_set_gds_init(adev);
dbff57bc 5754 gfx_v8_0_set_rlc_funcs(adev);
aaa36a97
AD		 5755
5756 return 0;
5757}
5758
ccba7691
AD		 5759static int gfx_v8_0_late_init(void *handle)
5760{
5761 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5762 int r;
5763
1d22a454
AD		 5764 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
5765 if (r)
5766 return r;
5767
5768 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
5769 if (r)
5770 return r;
5771
ccba7691
AD		 5772 /* requires IBs so do in late init after IB pool is initialized */
5773 r = gfx_v8_0_do_edc_gpr_workarounds(adev);
5774 if (r)
5775 return r;
5776
62a86fc2
EH		 5777 amdgpu_set_powergating_state(adev,
5778 AMD_IP_BLOCK_TYPE_GFX, AMD_PG_STATE_GATE);
5779
ccba7691
AD		 5780 return 0;
5781}
5782
c2546f55
AD		 5783static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
5784 bool enable)
62a86fc2 5785{
c4642a47
JZ		 5786 if ((adev->asic_type == CHIP_POLARIS11) ||
5787 (adev->asic_type == CHIP_POLARIS12))
c2546f55
AD		 5788 /* Send msg to SMU via Powerplay */
5789 amdgpu_set_powergating_state(adev,
5790 AMD_IP_BLOCK_TYPE_SMC,
5791 enable ?
5792 AMD_PG_STATE_GATE : AMD_PG_STATE_UNGATE);
62a86fc2 5793
61cb8cef 5794 WREG32_FIELD(RLC_PG_CNTL, STATIC_PER_CU_PG_ENABLE, enable ? 1 : 0);
62a86fc2
EH		 5795}
5796
c2546f55
AD		 5797static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
5798 bool enable)
62a86fc2 5799{
61cb8cef 5800 WREG32_FIELD(RLC_PG_CNTL, DYN_PER_CU_PG_ENABLE, enable ? 1 : 0);
62a86fc2
EH		 5801}
5802
2cc0c0b5 5803static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev,
62a86fc2
EH		 5804 bool enable)
5805{
61cb8cef 5806 WREG32_FIELD(RLC_PG_CNTL, QUICK_PG_ENABLE, enable ? 1 : 0);
62a86fc2
EH		 5807}
5808
2c547165
AD		 5809static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
5810 bool enable)
5811{
61cb8cef 5812 WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_ENABLE, enable ? 1 : 0);
2c547165
AD		 5813}
5814
5815static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev,
5816 bool enable)
5817{
61cb8cef 5818 WREG32_FIELD(RLC_PG_CNTL, GFX_PIPELINE_PG_ENABLE, enable ? 1 : 0);
2c547165
AD		 5819
5820 /* Read any GFX register to wake up GFX. */
5821 if (!enable)
61cb8cef 5822 RREG32(mmDB_RENDER_CONTROL);
2c547165
AD		 5823}
5824
5825static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev,
5826 bool enable)
5827{
5828 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
5829 cz_enable_gfx_cg_power_gating(adev, true);
5830 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
5831 cz_enable_gfx_pipeline_power_gating(adev, true);
5832 } else {
5833 cz_enable_gfx_cg_power_gating(adev, false);
5834 cz_enable_gfx_pipeline_power_gating(adev, false);
5835 }
5836}
5837
5fc3aeeb 5838static int gfx_v8_0_set_powergating_state(void *handle,
5839 enum amd_powergating_state state)
aaa36a97 5840{
62a86fc2 5841 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2c547165 5842 bool enable = (state == AMD_PG_STATE_GATE) ? true : false;
62a86fc2 5843
62a86fc2 5844 switch (adev->asic_type) {
2c547165
AD		 5845 case CHIP_CARRIZO:
5846 case CHIP_STONEY:
ad1830d5 5847
5c964221
RZ		 5848 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5849 cz_enable_sck_slow_down_on_power_up(adev, true);
5850 cz_enable_sck_slow_down_on_power_down(adev, true);
5851 } else {
5852 cz_enable_sck_slow_down_on_power_up(adev, false);
5853 cz_enable_sck_slow_down_on_power_down(adev, false);
5854 }
5855 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5856 cz_enable_cp_power_gating(adev, true);
5857 else
5858 cz_enable_cp_power_gating(adev, false);
5859
ad1830d5 5860 cz_update_gfx_cg_power_gating(adev, enable);
2c547165
AD		 5861
5862 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
5863 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
5864 else
5865 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);
5866
5867 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
5868 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);
5869 else
5870 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);
5871 break;
2cc0c0b5 5872 case CHIP_POLARIS11:
c4642a47 5873 case CHIP_POLARIS12:
7ba0eb6d
AD		 5874 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
5875 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
5876 else
5877 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);
5878
5879 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
5880 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);
5881 else
5882 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);
5883
5884 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_QUICK_MG) && enable)
5885 polaris11_enable_gfx_quick_mg_power_gating(adev, true);
62a86fc2 5886 else
7ba0eb6d 5887 polaris11_enable_gfx_quick_mg_power_gating(adev, false);
62a86fc2
EH		 5888 break;
5889 default:
5890 break;
5891 }
5892
aaa36a97
AD		 5893 return 0;
5894}
5895
ebd843d6
HR		 5896static void gfx_v8_0_get_clockgating_state(void *handle, u32 *flags)
5897{
5898 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5899 int data;
5900
5901 /* AMD_CG_SUPPORT_GFX_MGCG */
5902 data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5903 if (!(data & RLC_CGTT_MGCG_OVERRIDE__CPF_MASK))
5904 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5905
5906 /* AMD_CG_SUPPORT_GFX_CGLG */
5907 data = RREG32(mmRLC_CGCG_CGLS_CTRL);
5908 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5909 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5910
5911 /* AMD_CG_SUPPORT_GFX_CGLS */
5912 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5913 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5914
5915 /* AMD_CG_SUPPORT_GFX_CGTS */
5916 data = RREG32(mmCGTS_SM_CTRL_REG);
5917 if (!(data & CGTS_SM_CTRL_REG__OVERRIDE_MASK))
5918 *flags |= AMD_CG_SUPPORT_GFX_CGTS;
5919
5920 /* AMD_CG_SUPPORT_GFX_CGTS_LS */
5921 if (!(data & CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK))
5922 *flags |= AMD_CG_SUPPORT_GFX_CGTS_LS;
5923
5924 /* AMD_CG_SUPPORT_GFX_RLC_LS */
5925 data = RREG32(mmRLC_MEM_SLP_CNTL);
5926 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5927 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5928
5929 /* AMD_CG_SUPPORT_GFX_CP_LS */
5930 data = RREG32(mmCP_MEM_SLP_CNTL);
5931 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5932 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5933}
5934
79deaaf4 5935static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev,
14698b6c 5936 uint32_t reg_addr, uint32_t cmd)
6e378858
EH		 5937{
5938 uint32_t data;
5939
9559ef5b 5940 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6e378858
EH		 5941
5942 WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5943 WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5944
5945 data = RREG32(mmRLC_SERDES_WR_CTRL);
146f256f 5946 if (adev->asic_type == CHIP_STONEY)
62d2ce4b
TSD		 5947 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
5948 RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
5949 RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
5950 RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
5951 RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
5952 RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
5953 RLC_SERDES_WR_CTRL__POWER_UP_MASK |
5954 RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
5955 RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
146f256f
AD		 5956 else
5957 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
5958 RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
5959 RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
5960 RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
5961 RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
5962 RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
5963 RLC_SERDES_WR_CTRL__POWER_UP_MASK |
5964 RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
5965 RLC_SERDES_WR_CTRL__BPM_DATA_MASK |
5966 RLC_SERDES_WR_CTRL__REG_ADDR_MASK |
5967 RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
6e378858 5968 data |= (RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK |
146f256f
AD		 5969 (cmd << RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT) |
5970 (reg_addr << RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT) |
5971 (0xff << RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT));
6e378858
EH		 5972
5973 WREG32(mmRLC_SERDES_WR_CTRL, data);
5974}
5975
dbff57bc
AD		 5976#define MSG_ENTER_RLC_SAFE_MODE 1
5977#define MSG_EXIT_RLC_SAFE_MODE 0
61cb8cef
TSD		 5978#define RLC_GPR_REG2__REQ_MASK 0x00000001
5979#define RLC_GPR_REG2__REQ__SHIFT 0
5980#define RLC_GPR_REG2__MESSAGE__SHIFT 0x00000001
5981#define RLC_GPR_REG2__MESSAGE_MASK 0x0000001e
dbff57bc 5982
dbff57bc
AD		 5983static void iceland_enter_rlc_safe_mode(struct amdgpu_device *adev)
5984{
5985 u32 data;
5986 unsigned i;
5987
5988 data = RREG32(mmRLC_CNTL);
5989 if (!(data & RLC_CNTL__RLC_ENABLE_F32_MASK))
5990 return;
5991
5992 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_MGCG)) {
5993 data |= RLC_SAFE_MODE__CMD_MASK;
5994 data &= ~RLC_SAFE_MODE__MESSAGE_MASK;
5995 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
5996 WREG32(mmRLC_SAFE_MODE, data);
5997
5998 for (i = 0; i < adev->usec_timeout; i++) {
5999 if ((RREG32(mmRLC_GPM_STAT) &
6000 (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |
6001 RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) ==
6002 (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |
6003 RLC_GPM_STAT__GFX_POWER_STATUS_MASK))
6004 break;
6005 udelay(1);
6006 }
6007
6008 for (i = 0; i < adev->usec_timeout; i++) {
61cb8cef 6009 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
dbff57bc
AD		 6010 break;
6011 udelay(1);
6012 }
6013 adev->gfx.rlc.in_safe_mode = true;
6014 }
6015}
6016
6017static void iceland_exit_rlc_safe_mode(struct amdgpu_device *adev)
6018{
6019 u32 data = 0;
6020 unsigned i;
6021
6022 data = RREG32(mmRLC_CNTL);
6023 if (!(data & RLC_CNTL__RLC_ENABLE_F32_MASK))
6024 return;
6025
6026 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_MGCG)) {
6027 if (adev->gfx.rlc.in_safe_mode) {
6028 data |= RLC_SAFE_MODE__CMD_MASK;
6029 data &= ~RLC_SAFE_MODE__MESSAGE_MASK;
6030 WREG32(mmRLC_SAFE_MODE, data);
6031 adev->gfx.rlc.in_safe_mode = false;
6032 }
6033 }
6034
6035 for (i = 0; i < adev->usec_timeout; i++) {
61cb8cef 6036 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
dbff57bc
AD		 6037 break;
6038 udelay(1);
6039 }
6040}
6041
dbff57bc
AD		 6042static const struct amdgpu_rlc_funcs iceland_rlc_funcs = {
6043 .enter_safe_mode = iceland_enter_rlc_safe_mode,
6044 .exit_safe_mode = iceland_exit_rlc_safe_mode
6045};
6046
dbff57bc
AD		 6047static void gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
6048 bool enable)
6e378858
EH		 6049{
6050 uint32_t temp, data;
6051
dbff57bc
AD		 6052 adev->gfx.rlc.funcs->enter_safe_mode(adev);
6053
6e378858 6054 /* It is disabled by HW by default */
14698b6c
AD		 6055 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
6056 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
61cb8cef 6057 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS)
14698b6c 6058 /* 1 - RLC memory Light sleep */
61cb8cef 6059 WREG32_FIELD(RLC_MEM_SLP_CNTL, RLC_MEM_LS_EN, 1);
6e378858 6060
61cb8cef
TSD		 6061 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS)
6062 WREG32_FIELD(CP_MEM_SLP_CNTL, CP_MEM_LS_EN, 1);
14698b6c 6063 }
6e378858
EH		 6064
6065 /* 3 - RLC_CGTT_MGCG_OVERRIDE */
6066 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
dbff57bc
AD		 6067 if (adev->flags & AMD_IS_APU)
6068 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
6069 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
6070 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK);
6071 else
6072 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
6073 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
6074 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |
6075 RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);
6e378858
EH		 6076
6077 if (temp != data)
6078 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
6079
6080 /* 4 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
6081 gfx_v8_0_wait_for_rlc_serdes(adev);
6082
6083 /* 5 - clear mgcg override */
79deaaf4 6084 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
6e378858 6085
14698b6c
AD		 6086 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) {
6087 /* 6 - Enable CGTS(Tree Shade) MGCG /MGLS */
6088 temp = data = RREG32(mmCGTS_SM_CTRL_REG);
6089 data &= ~(CGTS_SM_CTRL_REG__SM_MODE_MASK);
6090 data |= (0x2 << CGTS_SM_CTRL_REG__SM_MODE__SHIFT);
6091 data |= CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK;
6092 data &= ~CGTS_SM_CTRL_REG__OVERRIDE_MASK;
6093 if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) &&
6094 (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS))
6095 data &= ~CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK;
6096 data |= CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK;
6097 data |= (0x96 << CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT);
6098 if (temp != data)
6099 WREG32(mmCGTS_SM_CTRL_REG, data);
6100 }
6e378858
EH		 6101 udelay(50);
6102
6103 /* 7 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
6104 gfx_v8_0_wait_for_rlc_serdes(adev);
6105 } else {
6106 /* 1 - MGCG_OVERRIDE[0] for CP and MGCG_OVERRIDE[1] for RLC */
6107 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
6108 data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
6109 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
6110 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |
6111 RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);
6112 if (temp != data)
6113 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
6114
6115 /* 2 - disable MGLS in RLC */
6116 data = RREG32(mmRLC_MEM_SLP_CNTL);
6117 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
6118 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
6119 WREG32(mmRLC_MEM_SLP_CNTL, data);
6120 }
6121
6122 /* 3 - disable MGLS in CP */
6123 data = RREG32(mmCP_MEM_SLP_CNTL);
6124 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
6125 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
6126 WREG32(mmCP_MEM_SLP_CNTL, data);
6127 }
6128
6129 /* 4 - Disable CGTS(Tree Shade) MGCG and MGLS */
6130 temp = data = RREG32(mmCGTS_SM_CTRL_REG);
6131 data |= (CGTS_SM_CTRL_REG__OVERRIDE_MASK |
6132 CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK);
6133 if (temp != data)
6134 WREG32(mmCGTS_SM_CTRL_REG, data);
6135
6136 /* 5 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
6137 gfx_v8_0_wait_for_rlc_serdes(adev);
6138
6139 /* 6 - set mgcg override */
79deaaf4 6140 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, SET_BPM_SERDES_CMD);
6e378858
EH		 6141
6142 udelay(50);
6143
6144 /* 7- wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
6145 gfx_v8_0_wait_for_rlc_serdes(adev);
6146 }
dbff57bc
AD		 6147
6148 adev->gfx.rlc.funcs->exit_safe_mode(adev);
6e378858
EH		 6149}
6150
dbff57bc
AD		 6151static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
6152 bool enable)
6e378858
EH		 6153{
6154 uint32_t temp, temp1, data, data1;
6155
6156 temp = data = RREG32(mmRLC_CGCG_CGLS_CTRL);
6157
dbff57bc
AD		 6158 adev->gfx.rlc.funcs->enter_safe_mode(adev);
6159
14698b6c 6160 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
6e378858
EH		 6161 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
6162 data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK;
6163 if (temp1 != data1)
6164 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
6165
dd31ae9a 6166 /* : wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
6e378858
EH		 6167 gfx_v8_0_wait_for_rlc_serdes(adev);
6168
dd31ae9a 6169 /* 2 - clear cgcg override */
79deaaf4 6170 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
6e378858
EH		 6171
6172 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
6173 gfx_v8_0_wait_for_rlc_serdes(adev);
6174
dd31ae9a 6175 /* 3 - write cmd to set CGLS */
79deaaf4 6176 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, SET_BPM_SERDES_CMD);
6e378858 6177
dd31ae9a 6178 /* 4 - enable cgcg */
6e378858
EH		 6179 data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
6180
14698b6c
AD		 6181 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
6182 /* enable cgls*/
6183 data |= RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
6e378858 6184
14698b6c
AD		 6185 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
6186 data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK;
6e378858 6187
14698b6c
AD		 6188 if (temp1 != data1)
6189 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
6190 } else {
6191 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
6192 }
6e378858
EH		 6193
6194 if (temp != data)
6195 WREG32(mmRLC_CGCG_CGLS_CTRL, data);
dd31ae9a
AN		 6196
6197 /* 5 enable cntx_empty_int_enable/cntx_busy_int_enable/
6198 * Cmp_busy/GFX_Idle interrupts
6199 */
6200 gfx_v8_0_enable_gui_idle_interrupt(adev, true);
6e378858
EH		 6201 } else {
6202 /* disable cntx_empty_int_enable & GFX Idle interrupt */
6203 gfx_v8_0_enable_gui_idle_interrupt(adev, false);
6204
6205 /* TEST CGCG */
6206 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
6207 data1 |= (RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK |
6208 RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK);
6209 if (temp1 != data1)
6210 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
6211
6212 /* read gfx register to wake up cgcg */
6213 RREG32(mmCB_CGTT_SCLK_CTRL);
6214 RREG32(mmCB_CGTT_SCLK_CTRL);
6215 RREG32(mmCB_CGTT_SCLK_CTRL);
6216 RREG32(mmCB_CGTT_SCLK_CTRL);
6217
6218 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
6219 gfx_v8_0_wait_for_rlc_serdes(adev);
6220
6221 /* write cmd to Set CGCG Overrride */
79deaaf4 6222 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, SET_BPM_SERDES_CMD);
6e378858
EH		 6223
6224 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
6225 gfx_v8_0_wait_for_rlc_serdes(adev);
6226
6227 /* write cmd to Clear CGLS */
79deaaf4 6228 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, CLE_BPM_SERDES_CMD);
6e378858
EH		 6229
6230 /* disable cgcg, cgls should be disabled too. */
6231 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK |
14698b6c 6232 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
6e378858
EH		 6233 if (temp != data)
6234 WREG32(mmRLC_CGCG_CGLS_CTRL, data);
6235 }
dbff57bc 6236
7894745a
TSD		 6237 gfx_v8_0_wait_for_rlc_serdes(adev);
6238
dbff57bc 6239 adev->gfx.rlc.funcs->exit_safe_mode(adev);
6e378858 6240}
dbff57bc
AD		 6241static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev,
6242 bool enable)
6e378858
EH		 6243{
6244 if (enable) {
6245 /* CGCG/CGLS should be enabled after MGCG/MGLS/TS(CG/LS)
6246 * === MGCG + MGLS + TS(CG/LS) ===
6247 */
dbff57bc
AD		 6248 gfx_v8_0_update_medium_grain_clock_gating(adev, enable);
6249 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable);
6e378858
EH		 6250 } else {
6251 /* CGCG/CGLS should be disabled before MGCG/MGLS/TS(CG/LS)
6252 * === CGCG + CGLS ===
6253 */
dbff57bc
AD		 6254 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable);
6255 gfx_v8_0_update_medium_grain_clock_gating(adev, enable);
6e378858
EH		 6256 }
6257 return 0;
6258}
6259
a8ca3413
RZ		 6260static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
6261 enum amd_clockgating_state state)
6262{
8a19e7fa
RZ		 6263 uint32_t msg_id, pp_state = 0;
6264 uint32_t pp_support_state = 0;
a8ca3413
RZ		 6265 void *pp_handle = adev->powerplay.pp_handle;
6266
8a19e7fa
RZ		 6267 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
6268 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
6269 pp_support_state = PP_STATE_SUPPORT_LS;
6270 pp_state = PP_STATE_LS;
6271 }
6272 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
6273 pp_support_state |= PP_STATE_SUPPORT_CG;
6274 pp_state |= PP_STATE_CG;
6275 }
6276 if (state == AMD_CG_STATE_UNGATE)
6277 pp_state = 0;
6278
6279 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
6280 PP_BLOCK_GFX_CG,
6281 pp_support_state,
6282 pp_state);
6283 amd_set_clockgating_by_smu(pp_handle, msg_id);
6284 }
a8ca3413 6285
8a19e7fa
RZ		 6286 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
6287 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
6288 pp_support_state = PP_STATE_SUPPORT_LS;
6289 pp_state = PP_STATE_LS;
6290 }
a8ca3413 6291
8a19e7fa
RZ		 6292 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
6293 pp_support_state |= PP_STATE_SUPPORT_CG;
6294 pp_state |= PP_STATE_CG;
6295 }
6296
6297 if (state == AMD_CG_STATE_UNGATE)
6298 pp_state = 0;
6299
6300 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
6301 PP_BLOCK_GFX_MG,
6302 pp_support_state,
6303 pp_state);
6304 amd_set_clockgating_by_smu(pp_handle, msg_id);
6305 }
a8ca3413
RZ		 6306
6307 return 0;
6308}
6309
6310static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
6311 enum amd_clockgating_state state)
6312{
8a19e7fa
RZ		 6313
6314 uint32_t msg_id, pp_state = 0;
6315 uint32_t pp_support_state = 0;
a8ca3413
RZ		 6316 void *pp_handle = adev->powerplay.pp_handle;
6317
8a19e7fa
RZ		 6318 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
6319 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
6320 pp_support_state = PP_STATE_SUPPORT_LS;
6321 pp_state = PP_STATE_LS;
6322 }
6323 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
6324 pp_support_state |= PP_STATE_SUPPORT_CG;
6325 pp_state |= PP_STATE_CG;
6326 }
6327 if (state == AMD_CG_STATE_UNGATE)
6328 pp_state = 0;
6329
6330 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
6331 PP_BLOCK_GFX_CG,
6332 pp_support_state,
6333 pp_state);
6334 amd_set_clockgating_by_smu(pp_handle, msg_id);
6335 }
a8ca3413 6336
8a19e7fa
RZ		 6337 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) {
6338 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
6339 pp_support_state = PP_STATE_SUPPORT_LS;
6340 pp_state = PP_STATE_LS;
6341 }
6342 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
6343 pp_support_state |= PP_STATE_SUPPORT_CG;
6344 pp_state |= PP_STATE_CG;
6345 }
6346 if (state == AMD_CG_STATE_UNGATE)
6347 pp_state = 0;
6348
6349 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
6350 PP_BLOCK_GFX_3D,
6351 pp_support_state,
6352 pp_state);
6353 amd_set_clockgating_by_smu(pp_handle, msg_id);
6354 }
a8ca3413 6355
8a19e7fa
RZ		 6356 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
6357 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
6358 pp_support_state = PP_STATE_SUPPORT_LS;
6359 pp_state = PP_STATE_LS;
6360 }
a8ca3413 6361
8a19e7fa
RZ		 6362 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
6363 pp_support_state |= PP_STATE_SUPPORT_CG;
6364 pp_state |= PP_STATE_CG;
6365 }
a8ca3413 6366
8a19e7fa
RZ		 6367 if (state == AMD_CG_STATE_UNGATE)
6368 pp_state = 0;
a8ca3413 6369
8a19e7fa
RZ		 6370 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
6371 PP_BLOCK_GFX_MG,
6372 pp_support_state,
6373 pp_state);
6374 amd_set_clockgating_by_smu(pp_handle, msg_id);
6375 }
6376
6377 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
6378 pp_support_state = PP_STATE_SUPPORT_LS;
6379
6380 if (state == AMD_CG_STATE_UNGATE)
6381 pp_state = 0;
6382 else
6383 pp_state = PP_STATE_LS;
6384
6385 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
6386 PP_BLOCK_GFX_RLC,
6387 pp_support_state,
6388 pp_state);
6389 amd_set_clockgating_by_smu(pp_handle, msg_id);
6390 }
6391
6392 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
6393 pp_support_state = PP_STATE_SUPPORT_LS;
6394
6395 if (state == AMD_CG_STATE_UNGATE)
6396 pp_state = 0;
6397 else
6398 pp_state = PP_STATE_LS;
6399 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
a8ca3413 6400 PP_BLOCK_GFX_CP,
8a19e7fa 6401 pp_support_state,
a8ca3413 6402 pp_state);
8a19e7fa
RZ		 6403 amd_set_clockgating_by_smu(pp_handle, msg_id);
6404 }
a8ca3413
RZ		 6405
6406 return 0;
6407}
6408
5fc3aeeb 6409static int gfx_v8_0_set_clockgating_state(void *handle,
6410 enum amd_clockgating_state state)
aaa36a97 6411{
6e378858
EH		 6412 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6413
6414 switch (adev->asic_type) {
6415 case CHIP_FIJI:
dbff57bc
AD		 6416 case CHIP_CARRIZO:
6417 case CHIP_STONEY:
6418 gfx_v8_0_update_gfx_clock_gating(adev,
6419 state == AMD_CG_STATE_GATE ? true : false);
6e378858 6420 break;
a8ca3413
RZ		 6421 case CHIP_TONGA:
6422 gfx_v8_0_tonga_update_gfx_clock_gating(adev, state);
6423 break;
6424 case CHIP_POLARIS10:
6425 case CHIP_POLARIS11:
6426 gfx_v8_0_polaris_update_gfx_clock_gating(adev, state);
6427 break;
6e378858
EH		 6428 default:
6429 break;
6430 }
aaa36a97
AD		 6431 return 0;
6432}
6433
e7706b42 6434static u32 gfx_v8_0_ring_get_rptr(struct amdgpu_ring *ring)
aaa36a97 6435{
5003f278 6436 return ring->adev->wb.wb[ring->rptr_offs];
aaa36a97
AD		 6437}
6438
6439static u32 gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
6440{
6441 struct amdgpu_device *adev = ring->adev;
aaa36a97
AD		 6442
6443 if (ring->use_doorbell)
6444 /* XXX check if swapping is necessary on BE */
5003f278 6445 return ring->adev->wb.wb[ring->wptr_offs];
aaa36a97 6446 else
5003f278 6447 return RREG32(mmCP_RB0_WPTR);
aaa36a97
AD		 6448}
6449
6450static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
6451{
6452 struct amdgpu_device *adev = ring->adev;
6453
6454 if (ring->use_doorbell) {
6455 /* XXX check if swapping is necessary on BE */
6456 adev->wb.wb[ring->wptr_offs] = ring->wptr;
6457 WDOORBELL32(ring->doorbell_index, ring->wptr);
6458 } else {
6459 WREG32(mmCP_RB0_WPTR, ring->wptr);
6460 (void)RREG32(mmCP_RB0_WPTR);
6461 }
6462}
6463
d2edb07b 6464static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
aaa36a97
AD		 6465{
6466 u32 ref_and_mask, reg_mem_engine;
6467
4e638ae9
XY		 6468 if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) ||
6469 (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) {
aaa36a97
AD		 6470 switch (ring->me) {
6471 case 1:
6472 ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
6473 break;
6474 case 2:
6475 ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe;
6476 break;
6477 default:
6478 return;
6479 }
6480 reg_mem_engine = 0;
6481 } else {
6482 ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK;
6483 reg_mem_engine = WAIT_REG_MEM_ENGINE(1); /* pfp */
6484 }
6485
6486 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6487 amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
6488 WAIT_REG_MEM_FUNCTION(3) | /* == */
6489 reg_mem_engine));
6490 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ);
6491 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE);
6492 amdgpu_ring_write(ring, ref_and_mask);
6493 amdgpu_ring_write(ring, ref_and_mask);
6494 amdgpu_ring_write(ring, 0x20); /* poll interval */
6495}
6496
45682886
ML		 6497static void gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)
6498{
6499 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
6500 amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) |
6501 EVENT_INDEX(4));
6502
6503 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
6504 amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) |
6505 EVENT_INDEX(0));
6506}
6507
6508
d35db561
CZ		 6509static void gfx_v8_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
6510{
6511 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6512 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6513 WRITE_DATA_DST_SEL(0) |
6514 WR_CONFIRM));
6515 amdgpu_ring_write(ring, mmHDP_DEBUG0);
6516 amdgpu_ring_write(ring, 0);
6517 amdgpu_ring_write(ring, 1);
6518
6519}
6520
93323131 6521static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
d88bf583
CK		 6522 struct amdgpu_ib *ib,
6523 unsigned vm_id, bool ctx_switch)
aaa36a97
AD		 6524{
6525 u32 header, control = 0;
aaa36a97 6526
de807f81 6527 if (ib->flags & AMDGPU_IB_FLAG_CE)
aaa36a97
AD		 6528 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
6529 else
6530 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
6531
d88bf583 6532 control |= ib->length_dw | (vm_id << 24);
aaa36a97
AD		 6533
6534 amdgpu_ring_write(ring, header);
6535 amdgpu_ring_write(ring,
6536#ifdef __BIG_ENDIAN
6537 (2 << 0) |
6538#endif
6539 (ib->gpu_addr & 0xFFFFFFFC));
6540 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
6541 amdgpu_ring_write(ring, control);
6542}
6543
93323131 6544static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
d88bf583
CK		 6545 struct amdgpu_ib *ib,
6546 unsigned vm_id, bool ctx_switch)
93323131 6547{
33b7ed01 6548 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vm_id << 24);
93323131 6549
33b7ed01 6550 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
93323131 6551 amdgpu_ring_write(ring,
6552#ifdef __BIG_ENDIAN
62d2ce4b 6553 (2 << 0) |
93323131 6554#endif
62d2ce4b 6555 (ib->gpu_addr & 0xFFFFFFFC));
93323131 6556 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
6557 amdgpu_ring_write(ring, control);
6558}
6559
aaa36a97 6560static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr,
890ee23f 6561 u64 seq, unsigned flags)
aaa36a97 6562{
890ee23f
CZ		 6563 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
6564 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
6565
aaa36a97
AD		 6566 /* EVENT_WRITE_EOP - flush caches, send int */
6567 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
6568 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6569 EOP_TC_ACTION_EN |
f84e63f2 6570 EOP_TC_WB_ACTION_EN |
aaa36a97
AD		 6571 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6572 EVENT_INDEX(5)));
6573 amdgpu_ring_write(ring, addr & 0xfffffffc);
90bea0ab 6574 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
890ee23f 6575 DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
aaa36a97
AD		 6576 amdgpu_ring_write(ring, lower_32_bits(seq));
6577 amdgpu_ring_write(ring, upper_32_bits(seq));
22c01cc4 6578
aaa36a97
AD		 6579}
6580
b8c7b39e 6581static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
aaa36a97 6582{
21cd942e 6583 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5907a0d8 6584 uint32_t seq = ring->fence_drv.sync_seq;
22c01cc4
AA		 6585 uint64_t addr = ring->fence_drv.gpu_addr;
6586
6587 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6588 amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */
9cac5373
CZ		 6589 WAIT_REG_MEM_FUNCTION(3) | /* equal */
6590 WAIT_REG_MEM_ENGINE(usepfp))); /* pfp or me */
22c01cc4
AA		 6591 amdgpu_ring_write(ring, addr & 0xfffffffc);
6592 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
6593 amdgpu_ring_write(ring, seq);
6594 amdgpu_ring_write(ring, 0xffffffff);
6595 amdgpu_ring_write(ring, 4); /* poll interval */
b8c7b39e
CK		 6596}
6597
6598static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
6599 unsigned vm_id, uint64_t pd_addr)
6600{
21cd942e 6601 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5c3422b0 6602
aaa36a97
AD		 6603 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6604 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
20a85ff8
CK		 6605 WRITE_DATA_DST_SEL(0)) |
6606 WR_CONFIRM);
aaa36a97
AD		 6607 if (vm_id < 8) {
6608 amdgpu_ring_write(ring,
6609 (mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id));
6610 } else {
6611 amdgpu_ring_write(ring,
6612 (mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vm_id - 8));
6613 }
6614 amdgpu_ring_write(ring, 0);
6615 amdgpu_ring_write(ring, pd_addr >> 12);
6616
aaa36a97
AD		 6617 /* bits 0-15 are the VM contexts0-15 */
6618 /* invalidate the cache */
6619 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6620 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6621 WRITE_DATA_DST_SEL(0)));
6622 amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
6623 amdgpu_ring_write(ring, 0);
6624 amdgpu_ring_write(ring, 1 << vm_id);
6625
6626 /* wait for the invalidate to complete */
6627 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6628 amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
6629 WAIT_REG_MEM_FUNCTION(0) | /* always */
6630 WAIT_REG_MEM_ENGINE(0))); /* me */
6631 amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
6632 amdgpu_ring_write(ring, 0);
6633 amdgpu_ring_write(ring, 0); /* ref */
6634 amdgpu_ring_write(ring, 0); /* mask */
6635 amdgpu_ring_write(ring, 0x20); /* poll interval */
6636
6637 /* compute doesn't have PFP */
6638 if (usepfp) {
6639 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6640 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6641 amdgpu_ring_write(ring, 0x0);
d4946ccf
ML		 6642 /* GFX8 emits 128 dw nop to prevent CE access VM before vm_flush finish */
6643 amdgpu_ring_insert_nop(ring, 128);
aaa36a97
AD		 6644 }
6645}
6646
aaa36a97
AD		 6647static u32 gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
6648{
6649 return ring->adev->wb.wb[ring->wptr_offs];
6650}
6651
6652static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
6653{
6654 struct amdgpu_device *adev = ring->adev;
6655
6656 /* XXX check if swapping is necessary on BE */
6657 adev->wb.wb[ring->wptr_offs] = ring->wptr;
6658 WDOORBELL32(ring->doorbell_index, ring->wptr);
6659}
6660
6661static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
6662 u64 addr, u64 seq,
890ee23f 6663 unsigned flags)
aaa36a97 6664{
890ee23f
CZ		 6665 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
6666 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
6667
aaa36a97
AD		 6668 /* RELEASE_MEM - flush caches, send int */
6669 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
6670 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6671 EOP_TC_ACTION_EN |
a3d5aaa8 6672 EOP_TC_WB_ACTION_EN |
aaa36a97
AD		 6673 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6674 EVENT_INDEX(5)));
890ee23f 6675 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
aaa36a97
AD		 6676 amdgpu_ring_write(ring, addr & 0xfffffffc);
6677 amdgpu_ring_write(ring, upper_32_bits(addr));
6678 amdgpu_ring_write(ring, lower_32_bits(seq));
6679 amdgpu_ring_write(ring, upper_32_bits(seq));
6680}
6681
4e638ae9
XY		 6682static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
6683 u64 seq, unsigned int flags)
6684{
6685 /* we only allocate 32bit for each seq wb address */
f10b478d 6686 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
4e638ae9
XY		 6687
6688 /* write fence seq to the "addr" */
6689 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6690 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6691 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
6692 amdgpu_ring_write(ring, lower_32_bits(addr));
6693 amdgpu_ring_write(ring, upper_32_bits(addr));
6694 amdgpu_ring_write(ring, lower_32_bits(seq));
6695
6696 if (flags & AMDGPU_FENCE_FLAG_INT) {
6697 /* set register to trigger INT */
6698 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6699 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6700 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
6701 amdgpu_ring_write(ring, mmCPC_INT_STATUS);
6702 amdgpu_ring_write(ring, 0);
6703 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
6704 }
6705}
6706
c2167a65
ML		 6707static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring)
6708{
6709 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
6710 amdgpu_ring_write(ring, 0);
6711}
6712
753ad49c
ML		 6713static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
6714{
6715 uint32_t dw2 = 0;
6716
c2ce92fc
ML		 6717 if (amdgpu_sriov_vf(ring->adev))
6718 gfx_v8_0_ring_emit_ce_meta_init(ring,
6719 (flags & AMDGPU_VM_DOMAIN) ? AMDGPU_CSA_VADDR : ring->adev->virt.csa_vmid0_addr);
6720
753ad49c
ML		 6721 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
6722 if (flags & AMDGPU_HAVE_CTX_SWITCH) {
45682886 6723 gfx_v8_0_ring_emit_vgt_flush(ring);
753ad49c
ML		 6724 /* set load_global_config & load_global_uconfig */
6725 dw2 |= 0x8001;
6726 /* set load_cs_sh_regs */
6727 dw2 |= 0x01000000;
6728 /* set load_per_context_state & load_gfx_sh_regs for GFX */
6729 dw2 |= 0x10002;
6730
6731 /* set load_ce_ram if preamble presented */
6732 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
6733 dw2 |= 0x10000000;
6734 } else {
6735 /* still load_ce_ram if this is the first time preamble presented
6736 * although there is no context switch happens.
6737 */
6738 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
6739 dw2 |= 0x10000000;
6740 }
6741
6742 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6743 amdgpu_ring_write(ring, dw2);
6744 amdgpu_ring_write(ring, 0);
c2ce92fc
ML		 6745
6746 if (amdgpu_sriov_vf(ring->adev))
6747 gfx_v8_0_ring_emit_de_meta_init(ring,
6748 (flags & AMDGPU_VM_DOMAIN) ? AMDGPU_CSA_VADDR : ring->adev->virt.csa_vmid0_addr);
753ad49c
ML		 6749}
6750
880e87e3
XY		 6751static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
6752{
6753 struct amdgpu_device *adev = ring->adev;
6754
6755 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
6756 amdgpu_ring_write(ring, 0 | /* src: register*/
6757 (5 << 8) | /* dst: memory */
6758 (1 << 20)); /* write confirm */
6759 amdgpu_ring_write(ring, reg);
6760 amdgpu_ring_write(ring, 0);
6761 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
6762 adev->virt.reg_val_offs * 4));
6763 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
6764 adev->virt.reg_val_offs * 4));
6765}
6766
6767static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
6768 uint32_t val)
6769{
6770 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6771 amdgpu_ring_write(ring, (1 << 16)); /* no inc addr */
6772 amdgpu_ring_write(ring, reg);
6773 amdgpu_ring_write(ring, 0);
6774 amdgpu_ring_write(ring, val);
6775}
6776
aaa36a97
AD		 6777static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
6778 enum amdgpu_interrupt_state state)
6779{
61cb8cef
TSD		 6780 WREG32_FIELD(CP_INT_CNTL_RING0, TIME_STAMP_INT_ENABLE,
6781 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
aaa36a97
AD		 6782}
6783
6784static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
6785 int me, int pipe,
6786 enum amdgpu_interrupt_state state)
6787{
aaa36a97
AD		 6788 /*
6789 * amdgpu controls only pipe 0 of MEC1. That's why this function only
6790 * handles the setting of interrupts for this specific pipe. All other
6791 * pipes' interrupts are set by amdkfd.
6792 */
6793
6794 if (me == 1) {
6795 switch (pipe) {
6796 case 0:
aaa36a97
AD		 6797 break;
6798 default:
6799 DRM_DEBUG("invalid pipe %d\n", pipe);
6800 return;
6801 }
6802 } else {
6803 DRM_DEBUG("invalid me %d\n", me);
6804 return;
6805 }
6806
61cb8cef
TSD		 6807 WREG32_FIELD(CP_ME1_PIPE0_INT_CNTL, TIME_STAMP_INT_ENABLE,
6808 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
aaa36a97
AD		 6809}
6810
6811static int gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6812 struct amdgpu_irq_src *source,
6813 unsigned type,
6814 enum amdgpu_interrupt_state state)
6815{
61cb8cef
TSD		 6816 WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_REG_INT_ENABLE,
6817 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
aaa36a97
AD		 6818
6819 return 0;
6820}
6821
6822static int gfx_v8_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6823 struct amdgpu_irq_src *source,
6824 unsigned type,
6825 enum amdgpu_interrupt_state state)
6826{
61cb8cef
TSD		 6827 WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_INSTR_INT_ENABLE,
6828 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
aaa36a97
AD		 6829
6830 return 0;
6831}
6832
6833static int gfx_v8_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6834 struct amdgpu_irq_src *src,
6835 unsigned type,
6836 enum amdgpu_interrupt_state state)
6837{
6838 switch (type) {
6839 case AMDGPU_CP_IRQ_GFX_EOP:
6840 gfx_v8_0_set_gfx_eop_interrupt_state(adev, state);
6841 break;
6842 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6843 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6844 break;
6845 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6846 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6847 break;
6848 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6849 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6850 break;
6851 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6852 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6853 break;
6854 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
6855 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
6856 break;
6857 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
6858 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
6859 break;
6860 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
6861 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
6862 break;
6863 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
6864 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
6865 break;
6866 default:
6867 break;
6868 }
6869 return 0;
6870}
6871
6872static int gfx_v8_0_eop_irq(struct amdgpu_device *adev,
6873 struct amdgpu_irq_src *source,
6874 struct amdgpu_iv_entry *entry)
6875{
6876 int i;
6877 u8 me_id, pipe_id, queue_id;
6878 struct amdgpu_ring *ring;
6879
6880 DRM_DEBUG("IH: CP EOP\n");
6881 me_id = (entry->ring_id & 0x0c) >> 2;
6882 pipe_id = (entry->ring_id & 0x03) >> 0;
6883 queue_id = (entry->ring_id & 0x70) >> 4;
6884
6885 switch (me_id) {
6886 case 0:
6887 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6888 break;
6889 case 1:
6890 case 2:
6891 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6892 ring = &adev->gfx.compute_ring[i];
6893 /* Per-queue interrupt is supported for MEC starting from VI.
6894 * The interrupt can only be enabled/disabled per pipe instead of per queue.
6895 */
6896 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
6897 amdgpu_fence_process(ring);
6898 }
6899 break;
6900 }
6901 return 0;
6902}
6903
6904static int gfx_v8_0_priv_reg_irq(struct amdgpu_device *adev,
6905 struct amdgpu_irq_src *source,
6906 struct amdgpu_iv_entry *entry)
6907{
6908 DRM_ERROR("Illegal register access in command stream\n");
6909 schedule_work(&adev->reset_work);
6910 return 0;
6911}
6912
6913static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev,
6914 struct amdgpu_irq_src *source,
6915 struct amdgpu_iv_entry *entry)
6916{
6917 DRM_ERROR("Illegal instruction in command stream\n");
6918 schedule_work(&adev->reset_work);
6919 return 0;
6920}
6921
4e638ae9
XY		 6922static int gfx_v8_0_kiq_set_interrupt_state(struct amdgpu_device *adev,
6923 struct amdgpu_irq_src *src,
6924 unsigned int type,
6925 enum amdgpu_interrupt_state state)
6926{
6927 uint32_t tmp, target;
6928 struct amdgpu_ring *ring = (struct amdgpu_ring *)src->data;
6929
6930 BUG_ON(!ring || (ring->funcs->type != AMDGPU_RING_TYPE_KIQ));
6931
6932 if (ring->me == 1)
6933 target = mmCP_ME1_PIPE0_INT_CNTL;
6934 else
6935 target = mmCP_ME2_PIPE0_INT_CNTL;
6936 target += ring->pipe;
6937
6938 switch (type) {
6939 case AMDGPU_CP_KIQ_IRQ_DRIVER0:
6940 if (state == AMDGPU_IRQ_STATE_DISABLE) {
6941 tmp = RREG32(mmCPC_INT_CNTL);
6942 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6943 GENERIC2_INT_ENABLE, 0);
6944 WREG32(mmCPC_INT_CNTL, tmp);
6945
6946 tmp = RREG32(target);
6947 tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
6948 GENERIC2_INT_ENABLE, 0);
6949 WREG32(target, tmp);
6950 } else {
6951 tmp = RREG32(mmCPC_INT_CNTL);
6952 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL,
6953 GENERIC2_INT_ENABLE, 1);
6954 WREG32(mmCPC_INT_CNTL, tmp);
6955
6956 tmp = RREG32(target);
6957 tmp = REG_SET_FIELD(tmp, CP_ME2_PIPE0_INT_CNTL,
6958 GENERIC2_INT_ENABLE, 1);
6959 WREG32(target, tmp);
6960 }
6961 break;
6962 default:
6963 BUG(); /* kiq only support GENERIC2_INT now */
6964 break;
6965 }
6966 return 0;
6967}
6968
6969static int gfx_v8_0_kiq_irq(struct amdgpu_device *adev,
6970 struct amdgpu_irq_src *source,
6971 struct amdgpu_iv_entry *entry)
6972{
6973 u8 me_id, pipe_id, queue_id;
6974 struct amdgpu_ring *ring = (struct amdgpu_ring *)source->data;
6975
6976 BUG_ON(!ring || (ring->funcs->type != AMDGPU_RING_TYPE_KIQ));
6977
6978 me_id = (entry->ring_id & 0x0c) >> 2;
6979 pipe_id = (entry->ring_id & 0x03) >> 0;
6980 queue_id = (entry->ring_id & 0x70) >> 4;
6981 DRM_DEBUG("IH: CPC GENERIC2_INT, me:%d, pipe:%d, queue:%d\n",
6982 me_id, pipe_id, queue_id);
6983
6984 amdgpu_fence_process(ring);
6985 return 0;
6986}
6987
a1255107 6988static const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
88a907d6 6989 .name = "gfx_v8_0",
aaa36a97 6990 .early_init = gfx_v8_0_early_init,
ccba7691 6991 .late_init = gfx_v8_0_late_init,
aaa36a97
AD		 6992 .sw_init = gfx_v8_0_sw_init,
6993 .sw_fini = gfx_v8_0_sw_fini,
6994 .hw_init = gfx_v8_0_hw_init,
6995 .hw_fini = gfx_v8_0_hw_fini,
6996 .suspend = gfx_v8_0_suspend,
6997 .resume = gfx_v8_0_resume,
6998 .is_idle = gfx_v8_0_is_idle,
6999 .wait_for_idle = gfx_v8_0_wait_for_idle,
3d7c6384 7000 .check_soft_reset = gfx_v8_0_check_soft_reset,
1057f20c 7001 .pre_soft_reset = gfx_v8_0_pre_soft_reset,
aaa36a97 7002 .soft_reset = gfx_v8_0_soft_reset,
e4ae0fc3 7003 .post_soft_reset = gfx_v8_0_post_soft_reset,
aaa36a97
AD		 7004 .set_clockgating_state = gfx_v8_0_set_clockgating_state,
7005 .set_powergating_state = gfx_v8_0_set_powergating_state,
ebd843d6 7006 .get_clockgating_state = gfx_v8_0_get_clockgating_state,
aaa36a97
AD		 7007};
7008
7009static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
21cd942e 7010 .type = AMDGPU_RING_TYPE_GFX,
79887142
CK		 7011 .align_mask = 0xff,
7012 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
e7706b42 7013 .get_rptr = gfx_v8_0_ring_get_rptr,
aaa36a97
AD		 7014 .get_wptr = gfx_v8_0_ring_get_wptr_gfx,
7015 .set_wptr = gfx_v8_0_ring_set_wptr_gfx,
e12f3d7a
CK		 7016 .emit_frame_size =
7017 20 + /* gfx_v8_0_ring_emit_gds_switch */
7018 7 + /* gfx_v8_0_ring_emit_hdp_flush */
7019 5 + /* gfx_v8_0_ring_emit_hdp_invalidate */
7020 6 + 6 + 6 +/* gfx_v8_0_ring_emit_fence_gfx x3 for user fence, vm fence */
7021 7 + /* gfx_v8_0_ring_emit_pipeline_sync */
7022 128 + 19 + /* gfx_v8_0_ring_emit_vm_flush */
7023 2 + /* gfx_v8_ring_emit_sb */
c2ce92fc 7024 3 + 4 + 29, /* gfx_v8_ring_emit_cntxcntl including vgt flush/meta-data */
e12f3d7a 7025 .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_gfx */
93323131 7026 .emit_ib = gfx_v8_0_ring_emit_ib_gfx,
aaa36a97 7027 .emit_fence = gfx_v8_0_ring_emit_fence_gfx,
b8c7b39e 7028 .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
aaa36a97
AD		 7029 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,
7030 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,
d2edb07b 7031 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
d35db561 7032 .emit_hdp_invalidate = gfx_v8_0_ring_emit_hdp_invalidate,
aaa36a97
AD		 7033 .test_ring = gfx_v8_0_ring_test_ring,
7034 .test_ib = gfx_v8_0_ring_test_ib,
edff0e28 7035 .insert_nop = amdgpu_ring_insert_nop,
9e5d5309 7036 .pad_ib = amdgpu_ring_generic_pad_ib,
c2167a65 7037 .emit_switch_buffer = gfx_v8_ring_emit_sb,
753ad49c 7038 .emit_cntxcntl = gfx_v8_ring_emit_cntxcntl,
aaa36a97
AD		 7039};
7040
7041static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
21cd942e 7042 .type = AMDGPU_RING_TYPE_COMPUTE,
79887142
CK		 7043 .align_mask = 0xff,
7044 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
e7706b42 7045 .get_rptr = gfx_v8_0_ring_get_rptr,
aaa36a97
AD		 7046 .get_wptr = gfx_v8_0_ring_get_wptr_compute,
7047 .set_wptr = gfx_v8_0_ring_set_wptr_compute,
e12f3d7a
CK		 7048 .emit_frame_size =
7049 20 + /* gfx_v8_0_ring_emit_gds_switch */
7050 7 + /* gfx_v8_0_ring_emit_hdp_flush */
7051 5 + /* gfx_v8_0_ring_emit_hdp_invalidate */
7052 7 + /* gfx_v8_0_ring_emit_pipeline_sync */
7053 17 + /* gfx_v8_0_ring_emit_vm_flush */
7054 7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */
7055 .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_compute */
93323131 7056 .emit_ib = gfx_v8_0_ring_emit_ib_compute,
aaa36a97 7057 .emit_fence = gfx_v8_0_ring_emit_fence_compute,
b8c7b39e 7058 .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
aaa36a97
AD		 7059 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,
7060 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,
35074d2d 7061 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
d35db561 7062 .emit_hdp_invalidate = gfx_v8_0_ring_emit_hdp_invalidate,
aaa36a97
AD		 7063 .test_ring = gfx_v8_0_ring_test_ring,
7064 .test_ib = gfx_v8_0_ring_test_ib,
edff0e28 7065 .insert_nop = amdgpu_ring_insert_nop,
9e5d5309 7066 .pad_ib = amdgpu_ring_generic_pad_ib,
aaa36a97
AD		 7067};
7068
4e638ae9
XY		 7069static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {
7070 .type = AMDGPU_RING_TYPE_KIQ,
7071 .align_mask = 0xff,
7072 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
7073 .get_rptr = gfx_v8_0_ring_get_rptr,
7074 .get_wptr = gfx_v8_0_ring_get_wptr_compute,
7075 .set_wptr = gfx_v8_0_ring_set_wptr_compute,
7076 .emit_frame_size =
7077 20 + /* gfx_v8_0_ring_emit_gds_switch */
7078 7 + /* gfx_v8_0_ring_emit_hdp_flush */
7079 5 + /* gfx_v8_0_ring_emit_hdp_invalidate */
7080 7 + /* gfx_v8_0_ring_emit_pipeline_sync */
7081 17 + /* gfx_v8_0_ring_emit_vm_flush */
7082 7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_kiq x3 for user fence, vm fence */
7083 .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_compute */
7084 .emit_ib = gfx_v8_0_ring_emit_ib_compute,
7085 .emit_fence = gfx_v8_0_ring_emit_fence_kiq,
7086 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
7087 .emit_hdp_invalidate = gfx_v8_0_ring_emit_hdp_invalidate,
7088 .test_ring = gfx_v8_0_ring_test_ring,
7089 .test_ib = gfx_v8_0_ring_test_ib,
7090 .insert_nop = amdgpu_ring_insert_nop,
7091 .pad_ib = amdgpu_ring_generic_pad_ib,
880e87e3
XY		 7092 .emit_rreg = gfx_v8_0_ring_emit_rreg,
7093 .emit_wreg = gfx_v8_0_ring_emit_wreg,
4e638ae9
XY		 7094};
7095
aaa36a97
AD		 7096static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
7097{
7098 int i;
7099
4e638ae9
XY		 7100 adev->gfx.kiq.ring.funcs = &gfx_v8_0_ring_funcs_kiq;
7101
aaa36a97
AD		 7102 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
7103 adev->gfx.gfx_ring[i].funcs = &gfx_v8_0_ring_funcs_gfx;
7104
7105 for (i = 0; i < adev->gfx.num_compute_rings; i++)
7106 adev->gfx.compute_ring[i].funcs = &gfx_v8_0_ring_funcs_compute;
7107}
7108
7109static const struct amdgpu_irq_src_funcs gfx_v8_0_eop_irq_funcs = {
7110 .set = gfx_v8_0_set_eop_interrupt_state,
7111 .process = gfx_v8_0_eop_irq,
7112};
7113
7114static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_reg_irq_funcs = {
7115 .set = gfx_v8_0_set_priv_reg_fault_state,
7116 .process = gfx_v8_0_priv_reg_irq,
7117};
7118
7119static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_inst_irq_funcs = {
7120 .set = gfx_v8_0_set_priv_inst_fault_state,
7121 .process = gfx_v8_0_priv_inst_irq,
7122};
7123
4e638ae9
XY		 7124static const struct amdgpu_irq_src_funcs gfx_v8_0_kiq_irq_funcs = {
7125 .set = gfx_v8_0_kiq_set_interrupt_state,
7126 .process = gfx_v8_0_kiq_irq,
7127};
7128
aaa36a97
AD		 7129static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev)
7130{
7131 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7132 adev->gfx.eop_irq.funcs = &gfx_v8_0_eop_irq_funcs;
7133
7134 adev->gfx.priv_reg_irq.num_types = 1;
7135 adev->gfx.priv_reg_irq.funcs = &gfx_v8_0_priv_reg_irq_funcs;
7136
7137 adev->gfx.priv_inst_irq.num_types = 1;
7138 adev->gfx.priv_inst_irq.funcs = &gfx_v8_0_priv_inst_irq_funcs;
4e638ae9
XY		 7139
7140 adev->gfx.kiq.irq.num_types = AMDGPU_CP_KIQ_IRQ_LAST;
7141 adev->gfx.kiq.irq.funcs = &gfx_v8_0_kiq_irq_funcs;
aaa36a97
AD		 7142}
7143
dbff57bc
AD		 7144static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev)
7145{
ae6a58e4 7146 adev->gfx.rlc.funcs = &iceland_rlc_funcs;
dbff57bc
AD		 7147}
7148
aaa36a97
AD		 7149static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev)
7150{
7151 /* init asci gds info */
7152 adev->gds.mem.total_size = RREG32(mmGDS_VMID0_SIZE);
7153 adev->gds.gws.total_size = 64;
7154 adev->gds.oa.total_size = 16;
7155
7156 if (adev->gds.mem.total_size == 64 * 1024) {
7157 adev->gds.mem.gfx_partition_size = 4096;
7158 adev->gds.mem.cs_partition_size = 4096;
7159
7160 adev->gds.gws.gfx_partition_size = 4;
7161 adev->gds.gws.cs_partition_size = 4;
7162
7163 adev->gds.oa.gfx_partition_size = 4;
7164 adev->gds.oa.cs_partition_size = 1;
7165 } else {
7166 adev->gds.mem.gfx_partition_size = 1024;
7167 adev->gds.mem.cs_partition_size = 1024;
7168
7169 adev->gds.gws.gfx_partition_size = 16;
7170 adev->gds.gws.cs_partition_size = 16;
7171
7172 adev->gds.oa.gfx_partition_size = 4;
7173 adev->gds.oa.cs_partition_size = 4;
7174 }
7175}
7176
9de06de8
NH		 7177static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7178 u32 bitmap)
7179{
7180 u32 data;
7181
7182 if (!bitmap)
7183 return;
7184
7185 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7186 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7187
7188 WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data);
7189}
7190
8f8e00c1 7191static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev)
aaa36a97 7192{
8f8e00c1 7193 u32 data, mask;
aaa36a97 7194
5003f278
TSD		 7195 data = RREG32(mmCC_GC_SHADER_ARRAY_CONFIG) |
7196 RREG32(mmGC_USER_SHADER_ARRAY_CONFIG);
aaa36a97 7197
6157bd7a 7198 mask = gfx_v8_0_create_bitmask(adev->gfx.config.max_cu_per_sh);
aaa36a97 7199
5003f278 7200 return ~REG_GET_FIELD(data, CC_GC_SHADER_ARRAY_CONFIG, INACTIVE_CUS) & mask;
aaa36a97
AD		 7201}
7202
7dae69a2 7203static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)
aaa36a97
AD		 7204{
7205 int i, j, k, counter, active_cu_number = 0;
7206 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7dae69a2 7207 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
9de06de8 7208 unsigned disable_masks[4 * 2];
aaa36a97 7209
6157bd7a
FC		 7210 memset(cu_info, 0, sizeof(*cu_info));
7211
9de06de8
NH		 7212 amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2);
7213
aaa36a97
AD		 7214 mutex_lock(&adev->grbm_idx_mutex);
7215 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7216 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7217 mask = 1;
7218 ao_bitmap = 0;
7219 counter = 0;
9559ef5b 7220 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
9de06de8
NH		 7221 if (i < 4 && j < 2)
7222 gfx_v8_0_set_user_cu_inactive_bitmap(
7223 adev, disable_masks[i * 2 + j]);
8f8e00c1 7224 bitmap = gfx_v8_0_get_cu_active_bitmap(adev);
aaa36a97
AD		 7225 cu_info->bitmap[i][j] = bitmap;
7226
8f8e00c1 7227 for (k = 0; k < 16; k ++) {
aaa36a97
AD		 7228 if (bitmap & mask) {
7229 if (counter < 2)
7230 ao_bitmap |= mask;
7231 counter ++;
7232 }
7233 mask <<= 1;
7234 }
7235 active_cu_number += counter;
7236 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7237 }
7238 }
9559ef5b 7239 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
8f8e00c1 7240 mutex_unlock(&adev->grbm_idx_mutex);
aaa36a97
AD		 7241
7242 cu_info->number = active_cu_number;
7243 cu_info->ao_cu_mask = ao_cu_mask;
aaa36a97 7244}
a1255107
AD		 7245
7246const struct amdgpu_ip_block_version gfx_v8_0_ip_block =
7247{
7248 .type = AMD_IP_BLOCK_TYPE_GFX,
7249 .major = 8,
7250 .minor = 0,
7251 .rev = 0,
7252 .funcs = &gfx_v8_0_ip_funcs,
7253};
7254
7255const struct amdgpu_ip_block_version gfx_v8_1_ip_block =
7256{
7257 .type = AMD_IP_BLOCK_TYPE_GFX,
7258 .major = 8,
7259 .minor = 1,
7260 .rev = 0,
7261 .funcs = &gfx_v8_0_ip_funcs,
7262};
acad2b2a
ML		 7263
7264static void gfx_v8_0_ring_emit_ce_meta_init(struct amdgpu_ring *ring, uint64_t csa_addr)
7265{
7266 uint64_t ce_payload_addr;
7267 int cnt_ce;
7268 static union {
7269 struct amdgpu_ce_ib_state regular;
7270 struct amdgpu_ce_ib_state_chained_ib chained;
7271 } ce_payload = {0};
7272
7273 if (ring->adev->virt.chained_ib_support) {
7274 ce_payload_addr = csa_addr + offsetof(struct amdgpu_gfx_meta_data_chained_ib, ce_payload);
7275 cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2;
7276 } else {
7277 ce_payload_addr = csa_addr + offsetof(struct amdgpu_gfx_meta_data, ce_payload);
7278 cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2;
7279 }
7280
7281 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_ce));
7282 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
7283 WRITE_DATA_DST_SEL(8) |
7284 WR_CONFIRM) |
7285 WRITE_DATA_CACHE_POLICY(0));
7286 amdgpu_ring_write(ring, lower_32_bits(ce_payload_addr));
7287 amdgpu_ring_write(ring, upper_32_bits(ce_payload_addr));
7288 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2);
7289}
7290
7291static void gfx_v8_0_ring_emit_de_meta_init(struct amdgpu_ring *ring, uint64_t csa_addr)
7292{
7293 uint64_t de_payload_addr, gds_addr;
7294 int cnt_de;
7295 static union {
7296 struct amdgpu_de_ib_state regular;
7297 struct amdgpu_de_ib_state_chained_ib chained;
7298 } de_payload = {0};
7299
7300 gds_addr = csa_addr + 4096;
7301 if (ring->adev->virt.chained_ib_support) {
7302 de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr);
7303 de_payload.chained.gds_backup_addrhi = upper_32_bits(gds_addr);
7304 de_payload_addr = csa_addr + offsetof(struct amdgpu_gfx_meta_data_chained_ib, de_payload);
7305 cnt_de = (sizeof(de_payload.chained) >> 2) + 4 - 2;
7306 } else {
7307 de_payload.regular.gds_backup_addrlo = lower_32_bits(gds_addr);
7308 de_payload.regular.gds_backup_addrhi = upper_32_bits(gds_addr);
7309 de_payload_addr = csa_addr + offsetof(struct amdgpu_gfx_meta_data, de_payload);
7310 cnt_de = (sizeof(de_payload.regular) >> 2) + 4 - 2;
7311 }
7312
7313 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_de));
7314 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
7315 WRITE_DATA_DST_SEL(8) |
7316 WR_CONFIRM) |
7317 WRITE_DATA_CACHE_POLICY(0));
7318 amdgpu_ring_write(ring, lower_32_bits(de_payload_addr));
7319 amdgpu_ring_write(ring, upper_32_bits(de_payload_addr));
7320 amdgpu_ring_write_multiple(ring, (void *)&de_payload, cnt_de - 2);
7321}
Linux 6.x block layer and io_uring tree(s)