projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branch 'asoc/topic/pcm5102a' into asoc-next
[linux-2.6-block.git] / drivers / gpu / drm / amd / amdgpu / si.c
CommitLineData
62a37553
KW		 1/*
2 * Copyright 2015 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24#include <linux/firmware.h>
25#include <linux/slab.h>
26#include <linux/module.h>
248a1d6f 27#include <drm/drmP.h>
62a37553
KW		 28#include "amdgpu.h"
29#include "amdgpu_atombios.h"
30#include "amdgpu_ih.h"
31#include "amdgpu_uvd.h"
32#include "amdgpu_vce.h"
33#include "atom.h"
05656e5e 34#include "amd_pcie.h"
62a37553 35#include "amdgpu_powerplay.h"
689957b1 36#include "sid.h"
62a37553
KW		 37#include "si_ih.h"
38#include "gfx_v6_0.h"
39#include "gmc_v6_0.h"
40#include "si_dma.h"
41#include "dce_v6_0.h"
42#include "si.h"
2120df47 43#include "dce_virtual.h"
78bbe771
TSD		 44#include "gca/gfx_6_0_d.h"
45#include "oss/oss_1_0_d.h"
46#include "gmc/gmc_6_0_d.h"
47#include "dce/dce_6_0_d.h"
48#include "uvd/uvd_4_0_d.h"
bbf282d8 49#include "bif/bif_3_0_d.h"
62a37553
KW		 50
51static const u32 tahiti_golden_registers[] =
52{
78bbe771
TSD		 53 mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
54 mmCB_HW_CONTROL, 0x00010000, 0x00018208,
55 mmDB_DEBUG, 0xffffffff, 0x00000000,
56 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
57 mmDB_DEBUG3, 0x0002021c, 0x00020200,
58 mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
7c0a705e
FC		 59 0x340c, 0x000000c0, 0x00800040,
60 0x360c, 0x000000c0, 0x00800040,
78bbe771
TSD		 61 mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
62 mmFBC_MISC, 0x00200000, 0x50100000,
63 mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
64 mmMC_ARB_WTM_CNTL_RD, 0x00000003, 0x000007ff,
65 mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
66 mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
67 mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
68 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
69 mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
70 mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
7c0a705e 71 0x000c, 0xffffffff, 0x0040,
62a37553 72 0x000d, 0x00000040, 0x00004040,
78bbe771
TSD		 73 mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
74 mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
75 mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
76 mmSX_DEBUG_1, 0x0000007f, 0x00000020,
77 mmTA_CNTL_AUX, 0x00010000, 0x00010000,
78 mmTCP_ADDR_CONFIG, 0x00000200, 0x000002fb,
79 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
80 mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
81 mmVGT_FIFO_DEPTHS, 0xffffffff, 0x000fff40,
82 mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
83 mmVM_CONTEXT0_CNTL, 0x20000000, 0x20fffed8,
84 mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
85 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
86 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
87 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
88 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
62a37553
KW		 89};
90
91static const u32 tahiti_golden_registers2[] =
92{
78bbe771 93 mmMCIF_MEM_CONTROL, 0x00000001, 0x00000001,
62a37553
KW		 94};
95
96static const u32 tahiti_golden_rlc_registers[] =
97{
78bbe771
TSD		 98 mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
99 mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
62a37553
KW		 100 0x311f, 0xffffffff, 0x10104040,
101 0x3122, 0xffffffff, 0x0100000a,
78bbe771
TSD		 102 mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
103 mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
104 mmUVD_CGC_GATE, 0x00000008, 0x00000000,
62a37553
KW		 105};
106
107static const u32 pitcairn_golden_registers[] =
108{
78bbe771
TSD		 109 mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
110 mmCB_HW_CONTROL, 0x00010000, 0x00018208,
111 mmDB_DEBUG, 0xffffffff, 0x00000000,
112 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
113 mmDB_DEBUG3, 0x0002021c, 0x00020200,
114 mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
62a37553
KW		 115 0x340c, 0x000300c0, 0x00800040,
116 0x360c, 0x000300c0, 0x00800040,
78bbe771
TSD		 117 mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
118 mmFBC_MISC, 0x00200000, 0x50100000,
119 mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
120 mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
121 mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
122 mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
123 mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
124 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
125 mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
126 mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
1245a694 127 0x000c, 0xffffffff, 0x0040,
62a37553 128 0x000d, 0x00000040, 0x00004040,
78bbe771
TSD		 129 mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
130 mmSX_DEBUG_1, 0x0000007f, 0x00000020,
131 mmTA_CNTL_AUX, 0x00010000, 0x00010000,
132 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
133 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
134 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
135 mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
136 mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
137 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
138 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
139 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
140 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
62a37553
KW		 141};
142
143static const u32 pitcairn_golden_rlc_registers[] =
144{
78bbe771
TSD		 145 mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
146 mmRLC_LB_PARAMS, 0xffffffff, 0x00601004,
62a37553
KW		 147 0x311f, 0xffffffff, 0x10102020,
148 0x3122, 0xffffffff, 0x01000020,
78bbe771
TSD		 149 mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
150 mmRLC_LB_CNTL, 0xffffffff, 0x800000a4,
62a37553
KW		 151};
152
153static const u32 verde_pg_init[] =
154{
78bbe771
TSD		 155 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x40000,
156 mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x200010ff,
157 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
158 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
159 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
160 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
161 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
162 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x7007,
163 mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x300010ff,
164 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
165 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
166 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
167 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
168 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
169 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x400000,
170 mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x100010ff,
171 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
172 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
173 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
174 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
175 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
176 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x120200,
177 mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x500010ff,
178 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
179 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
180 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
181 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
182 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
183 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x1e1e16,
184 mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x600010ff,
185 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
186 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
187 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
188 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
189 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
190 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x171f1e,
191 mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x700010ff,
192 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
193 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
194 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
195 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
196 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
197 mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
198 mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x9ff,
199 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x0,
200 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10000800,
201 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
202 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
203 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4,
204 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1000051e,
205 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
206 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
207 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x8,
208 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x80500,
209 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x12,
210 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x9050c,
211 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1d,
212 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xb052c,
213 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2a,
214 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1053e,
215 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2d,
216 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10546,
217 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x30,
218 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xa054e,
219 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3c,
220 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1055f,
221 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3f,
222 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10567,
223 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x42,
224 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1056f,
225 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x45,
226 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10572,
227 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x48,
228 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20575,
229 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4c,
230 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x190801,
231 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x67,
232 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1082a,
233 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x6a,
234 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1b082d,
235 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x87,
236 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x310851,
237 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xba,
238 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x891,
239 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbc,
240 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x893,
241 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbe,
242 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20895,
243 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc2,
244 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20899,
245 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc6,
246 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2089d,
247 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xca,
248 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a1,
249 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xcc,
250 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a3,
251 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xce,
252 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x308a5,
253 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xd3,
254 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x6d08cd,
255 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x142,
256 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2000095a,
257 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1,
258 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x144,
259 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x301f095b,
260 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x165,
261 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc094d,
262 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x173,
263 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf096d,
264 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x184,
265 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x15097f,
266 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x19b,
267 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc0998,
268 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1a9,
269 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x409a7,
270 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1af,
271 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xcdc,
272 mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1b1,
273 mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x800,
274 mmGMCON_RENG_EXECUTE, 0xffffffff, 0x6c9b2000,
275 mmGMCON_MISC2, 0xfc00, 0x2000,
276 mmGMCON_MISC3, 0xffffffff, 0xfc0,
277 mmMC_PMG_AUTO_CFG, 0x00000100, 0x100,
62a37553
KW		 278};
279
280static const u32 verde_golden_rlc_registers[] =
281{
78bbe771
TSD		 282 mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
283 mmRLC_LB_PARAMS, 0xffffffff, 0x033f1005,
62a37553
KW		 284 0x311f, 0xffffffff, 0x10808020,
285 0x3122, 0xffffffff, 0x00800008,
78bbe771
TSD		 286 mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00001000,
287 mmRLC_LB_CNTL, 0xffffffff, 0x80010014,
62a37553
KW		 288};
289
290static const u32 verde_golden_registers[] =
291{
78bbe771
TSD		 292 mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
293 mmCB_HW_CONTROL, 0x00010000, 0x00018208,
294 mmDB_DEBUG, 0xffffffff, 0x00000000,
295 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
296 mmDB_DEBUG3, 0x0002021c, 0x00020200,
297 mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
62a37553 298 0x340c, 0x000300c0, 0x00800040,
62a37553 299 0x360c, 0x000300c0, 0x00800040,
78bbe771
TSD		 300 mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
301 mmFBC_MISC, 0x00200000, 0x50100000,
302 mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
303 mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
304 mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
305 mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
306 mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
307 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
308 mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
309 mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x0000124a,
dae5c298 310 0x000c, 0xffffffff, 0x0040,
62a37553 311 0x000d, 0x00000040, 0x00004040,
78bbe771
TSD		 312 mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
313 mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
314 mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
315 mmSX_DEBUG_1, 0x0000007f, 0x00000020,
316 mmTA_CNTL_AUX, 0x00010000, 0x00010000,
317 mmTCP_ADDR_CONFIG, 0x000003ff, 0x00000003,
318 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
319 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001032,
320 mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
321 mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
322 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
323 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
324 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
325 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
62a37553
KW		 326};
327
328static const u32 oland_golden_registers[] =
329{
78bbe771
TSD		 330 mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
331 mmCB_HW_CONTROL, 0x00010000, 0x00018208,
332 mmDB_DEBUG, 0xffffffff, 0x00000000,
333 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
334 mmDB_DEBUG3, 0x0002021c, 0x00020200,
335 mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
62a37553
KW		 336 0x340c, 0x000300c0, 0x00800040,
337 0x360c, 0x000300c0, 0x00800040,
78bbe771
TSD		 338 mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
339 mmFBC_MISC, 0x00200000, 0x50100000,
340 mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
341 mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
342 mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
343 mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
344 mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
345 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
346 mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
347 mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000082,
6b7985ef 348 0x000c, 0xffffffff, 0x0040,
62a37553 349 0x000d, 0x00000040, 0x00004040,
78bbe771
TSD		 350 mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
351 mmSX_DEBUG_1, 0x0000007f, 0x00000020,
352 mmTA_CNTL_AUX, 0x00010000, 0x00010000,
353 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
354 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
355 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
356 mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
357 mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
358 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
359 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
360 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
361 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
362
62a37553
KW		 363};
364
365static const u32 oland_golden_rlc_registers[] =
366{
78bbe771
TSD		 367 mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
368 mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
62a37553
KW		 369 0x311f, 0xffffffff, 0x10104040,
370 0x3122, 0xffffffff, 0x0100000a,
78bbe771
TSD		 371 mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
372 mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
62a37553
KW		 373};
374
375static const u32 hainan_golden_registers[] =
376{
bd27b678 377 0x17bc, 0x00000030, 0x00000011,
78bbe771
TSD		 378 mmCB_HW_CONTROL, 0x00010000, 0x00018208,
379 mmDB_DEBUG, 0xffffffff, 0x00000000,
380 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
381 mmDB_DEBUG3, 0x0002021c, 0x00020200,
bd27b678
FC		 382 0x031e, 0x00000080, 0x00000000,
383 0x3430, 0xff000fff, 0x00000100,
62a37553
KW		 384 0x340c, 0x000300c0, 0x00800040,
385 0x3630, 0xff000fff, 0x00000100,
386 0x360c, 0x000300c0, 0x00800040,
bd27b678
FC		 387 0x16ec, 0x000000f0, 0x00000070,
388 0x16f0, 0x00200000, 0x50100000,
389 0x1c0c, 0x31000311, 0x00000011,
78bbe771
TSD		 390 mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
391 mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
392 mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
393 mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
394 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
395 mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
396 mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000000,
bd27b678 397 0x000c, 0xffffffff, 0x0040,
62a37553 398 0x000d, 0x00000040, 0x00004040,
78bbe771
TSD		 399 mmSPI_CONFIG_CNTL, 0x03e00000, 0x03600000,
400 mmSX_DEBUG_1, 0x0000007f, 0x00000020,
401 mmTA_CNTL_AUX, 0x00010000, 0x00010000,
402 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
403 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
404 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
405 mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
406 mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
407 mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
408 mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
409 mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
410 mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
62a37553
KW		 411};
412
413static const u32 hainan_golden_registers2[] =
414{
78bbe771 415 mmGB_ADDR_CONFIG, 0xffffffff, 0x2011003,
62a37553
KW		 416};
417
418static const u32 tahiti_mgcg_cgcg_init[] =
419{
78bbe771
TSD		 420 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
421 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
422 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
423 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
424 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
425 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
426 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
427 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
428 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
429 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
430 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
431 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
432 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
433 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
434 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
435 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
436 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
437 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
438 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
439 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
440 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
441 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
442 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
443 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
444 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
445 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
446 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
62a37553
KW		 447 0x2458, 0xffffffff, 0x00010000,
448 0x2459, 0xffffffff, 0x00030002,
449 0x245a, 0xffffffff, 0x00040007,
450 0x245b, 0xffffffff, 0x00060005,
451 0x245c, 0xffffffff, 0x00090008,
452 0x245d, 0xffffffff, 0x00020001,
453 0x245e, 0xffffffff, 0x00040003,
454 0x245f, 0xffffffff, 0x00000007,
455 0x2460, 0xffffffff, 0x00060005,
456 0x2461, 0xffffffff, 0x00090008,
457 0x2462, 0xffffffff, 0x00030002,
458 0x2463, 0xffffffff, 0x00050004,
459 0x2464, 0xffffffff, 0x00000008,
460 0x2465, 0xffffffff, 0x00070006,
461 0x2466, 0xffffffff, 0x000a0009,
462 0x2467, 0xffffffff, 0x00040003,
463 0x2468, 0xffffffff, 0x00060005,
464 0x2469, 0xffffffff, 0x00000009,
465 0x246a, 0xffffffff, 0x00080007,
466 0x246b, 0xffffffff, 0x000b000a,
467 0x246c, 0xffffffff, 0x00050004,
468 0x246d, 0xffffffff, 0x00070006,
469 0x246e, 0xffffffff, 0x0008000b,
470 0x246f, 0xffffffff, 0x000a0009,
471 0x2470, 0xffffffff, 0x000d000c,
472 0x2471, 0xffffffff, 0x00060005,
473 0x2472, 0xffffffff, 0x00080007,
474 0x2473, 0xffffffff, 0x0000000b,
475 0x2474, 0xffffffff, 0x000a0009,
476 0x2475, 0xffffffff, 0x000d000c,
477 0x2476, 0xffffffff, 0x00070006,
478 0x2477, 0xffffffff, 0x00090008,
479 0x2478, 0xffffffff, 0x0000000c,
480 0x2479, 0xffffffff, 0x000b000a,
481 0x247a, 0xffffffff, 0x000e000d,
482 0x247b, 0xffffffff, 0x00080007,
483 0x247c, 0xffffffff, 0x000a0009,
484 0x247d, 0xffffffff, 0x0000000d,
485 0x247e, 0xffffffff, 0x000c000b,
486 0x247f, 0xffffffff, 0x000f000e,
487 0x2480, 0xffffffff, 0x00090008,
488 0x2481, 0xffffffff, 0x000b000a,
489 0x2482, 0xffffffff, 0x000c000f,
490 0x2483, 0xffffffff, 0x000e000d,
491 0x2484, 0xffffffff, 0x00110010,
492 0x2485, 0xffffffff, 0x000a0009,
493 0x2486, 0xffffffff, 0x000c000b,
494 0x2487, 0xffffffff, 0x0000000f,
495 0x2488, 0xffffffff, 0x000e000d,
496 0x2489, 0xffffffff, 0x00110010,
497 0x248a, 0xffffffff, 0x000b000a,
498 0x248b, 0xffffffff, 0x000d000c,
499 0x248c, 0xffffffff, 0x00000010,
500 0x248d, 0xffffffff, 0x000f000e,
501 0x248e, 0xffffffff, 0x00120011,
502 0x248f, 0xffffffff, 0x000c000b,
503 0x2490, 0xffffffff, 0x000e000d,
504 0x2491, 0xffffffff, 0x00000011,
505 0x2492, 0xffffffff, 0x0010000f,
506 0x2493, 0xffffffff, 0x00130012,
507 0x2494, 0xffffffff, 0x000d000c,
508 0x2495, 0xffffffff, 0x000f000e,
509 0x2496, 0xffffffff, 0x00100013,
510 0x2497, 0xffffffff, 0x00120011,
511 0x2498, 0xffffffff, 0x00150014,
512 0x2499, 0xffffffff, 0x000e000d,
513 0x249a, 0xffffffff, 0x0010000f,
514 0x249b, 0xffffffff, 0x00000013,
515 0x249c, 0xffffffff, 0x00120011,
516 0x249d, 0xffffffff, 0x00150014,
517 0x249e, 0xffffffff, 0x000f000e,
518 0x249f, 0xffffffff, 0x00110010,
519 0x24a0, 0xffffffff, 0x00000014,
520 0x24a1, 0xffffffff, 0x00130012,
521 0x24a2, 0xffffffff, 0x00160015,
522 0x24a3, 0xffffffff, 0x0010000f,
523 0x24a4, 0xffffffff, 0x00120011,
524 0x24a5, 0xffffffff, 0x00000015,
525 0x24a6, 0xffffffff, 0x00140013,
526 0x24a7, 0xffffffff, 0x00170016,
78bbe771
TSD		 527 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
528 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
529 mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
530 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
7c0a705e
FC		 531 0x000c, 0xffffffff, 0x0000001c,
532 0x000d, 0x000f0000, 0x000f0000,
533 0x0583, 0xffffffff, 0x00000100,
78bbe771
TSD		 534 mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
535 mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
536 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
537 mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
538 mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
539 mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
62a37553 540 0x157a, 0x00000001, 0x00000001,
78bbe771
TSD		 541 mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
542 mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
543 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
62a37553 544 0x3430, 0xfffffff0, 0x00000100,
78bbe771 545 0x3630, 0xfffffff0, 0x00000100,
62a37553
KW		 546};
547static const u32 pitcairn_mgcg_cgcg_init[] =
548{
78bbe771
TSD		 549 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
550 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
551 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
552 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
553 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
554 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
555 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
556 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
557 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
558 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
559 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
560 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
561 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
562 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
563 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
564 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
565 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
566 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
567 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
568 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
569 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
570 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
571 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
572 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
573 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
574 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
575 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
62a37553
KW		 576 0x2458, 0xffffffff, 0x00010000,
577 0x2459, 0xffffffff, 0x00030002,
578 0x245a, 0xffffffff, 0x00040007,
579 0x245b, 0xffffffff, 0x00060005,
580 0x245c, 0xffffffff, 0x00090008,
581 0x245d, 0xffffffff, 0x00020001,
582 0x245e, 0xffffffff, 0x00040003,
583 0x245f, 0xffffffff, 0x00000007,
584 0x2460, 0xffffffff, 0x00060005,
585 0x2461, 0xffffffff, 0x00090008,
586 0x2462, 0xffffffff, 0x00030002,
587 0x2463, 0xffffffff, 0x00050004,
588 0x2464, 0xffffffff, 0x00000008,
589 0x2465, 0xffffffff, 0x00070006,
590 0x2466, 0xffffffff, 0x000a0009,
591 0x2467, 0xffffffff, 0x00040003,
592 0x2468, 0xffffffff, 0x00060005,
593 0x2469, 0xffffffff, 0x00000009,
594 0x246a, 0xffffffff, 0x00080007,
595 0x246b, 0xffffffff, 0x000b000a,
596 0x246c, 0xffffffff, 0x00050004,
597 0x246d, 0xffffffff, 0x00070006,
598 0x246e, 0xffffffff, 0x0008000b,
599 0x246f, 0xffffffff, 0x000a0009,
600 0x2470, 0xffffffff, 0x000d000c,
601 0x2480, 0xffffffff, 0x00090008,
602 0x2481, 0xffffffff, 0x000b000a,
603 0x2482, 0xffffffff, 0x000c000f,
604 0x2483, 0xffffffff, 0x000e000d,
605 0x2484, 0xffffffff, 0x00110010,
606 0x2485, 0xffffffff, 0x000a0009,
607 0x2486, 0xffffffff, 0x000c000b,
608 0x2487, 0xffffffff, 0x0000000f,
609 0x2488, 0xffffffff, 0x000e000d,
610 0x2489, 0xffffffff, 0x00110010,
611 0x248a, 0xffffffff, 0x000b000a,
612 0x248b, 0xffffffff, 0x000d000c,
613 0x248c, 0xffffffff, 0x00000010,
614 0x248d, 0xffffffff, 0x000f000e,
615 0x248e, 0xffffffff, 0x00120011,
616 0x248f, 0xffffffff, 0x000c000b,
617 0x2490, 0xffffffff, 0x000e000d,
618 0x2491, 0xffffffff, 0x00000011,
619 0x2492, 0xffffffff, 0x0010000f,
620 0x2493, 0xffffffff, 0x00130012,
621 0x2494, 0xffffffff, 0x000d000c,
622 0x2495, 0xffffffff, 0x000f000e,
623 0x2496, 0xffffffff, 0x00100013,
624 0x2497, 0xffffffff, 0x00120011,
625 0x2498, 0xffffffff, 0x00150014,
78bbe771
TSD		 626 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
627 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
628 mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
629 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
1245a694
FC		 630 0x000c, 0xffffffff, 0x0000001c,
631 0x000d, 0x000f0000, 0x000f0000,
632 0x0583, 0xffffffff, 0x00000100,
78bbe771
TSD		 633 mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
634 mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
635 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
636 mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
62a37553 637 0x157a, 0x00000001, 0x00000001,
78bbe771
TSD		 638 mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
639 mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
640 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
62a37553 641 0x3430, 0xfffffff0, 0x00000100,
78bbe771 642 0x3630, 0xfffffff0, 0x00000100,
62a37553 643};
78bbe771 644
62a37553
KW		 645static const u32 verde_mgcg_cgcg_init[] =
646{
78bbe771
TSD		 647 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
648 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
649 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
650 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
651 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
652 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
653 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
654 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
655 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
656 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
657 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
658 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
659 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
660 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
661 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
662 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
663 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
664 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
665 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
666 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
667 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
668 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
669 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
670 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
671 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
672 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
673 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
62a37553
KW		 674 0x2458, 0xffffffff, 0x00010000,
675 0x2459, 0xffffffff, 0x00030002,
676 0x245a, 0xffffffff, 0x00040007,
677 0x245b, 0xffffffff, 0x00060005,
678 0x245c, 0xffffffff, 0x00090008,
679 0x245d, 0xffffffff, 0x00020001,
680 0x245e, 0xffffffff, 0x00040003,
681 0x245f, 0xffffffff, 0x00000007,
682 0x2460, 0xffffffff, 0x00060005,
683 0x2461, 0xffffffff, 0x00090008,
684 0x2462, 0xffffffff, 0x00030002,
685 0x2463, 0xffffffff, 0x00050004,
686 0x2464, 0xffffffff, 0x00000008,
687 0x2465, 0xffffffff, 0x00070006,
688 0x2466, 0xffffffff, 0x000a0009,
689 0x2467, 0xffffffff, 0x00040003,
690 0x2468, 0xffffffff, 0x00060005,
691 0x2469, 0xffffffff, 0x00000009,
692 0x246a, 0xffffffff, 0x00080007,
693 0x246b, 0xffffffff, 0x000b000a,
694 0x246c, 0xffffffff, 0x00050004,
695 0x246d, 0xffffffff, 0x00070006,
696 0x246e, 0xffffffff, 0x0008000b,
697 0x246f, 0xffffffff, 0x000a0009,
698 0x2470, 0xffffffff, 0x000d000c,
699 0x2480, 0xffffffff, 0x00090008,
700 0x2481, 0xffffffff, 0x000b000a,
701 0x2482, 0xffffffff, 0x000c000f,
702 0x2483, 0xffffffff, 0x000e000d,
703 0x2484, 0xffffffff, 0x00110010,
704 0x2485, 0xffffffff, 0x000a0009,
705 0x2486, 0xffffffff, 0x000c000b,
706 0x2487, 0xffffffff, 0x0000000f,
707 0x2488, 0xffffffff, 0x000e000d,
708 0x2489, 0xffffffff, 0x00110010,
709 0x248a, 0xffffffff, 0x000b000a,
710 0x248b, 0xffffffff, 0x000d000c,
711 0x248c, 0xffffffff, 0x00000010,
712 0x248d, 0xffffffff, 0x000f000e,
713 0x248e, 0xffffffff, 0x00120011,
714 0x248f, 0xffffffff, 0x000c000b,
715 0x2490, 0xffffffff, 0x000e000d,
716 0x2491, 0xffffffff, 0x00000011,
717 0x2492, 0xffffffff, 0x0010000f,
718 0x2493, 0xffffffff, 0x00130012,
719 0x2494, 0xffffffff, 0x000d000c,
720 0x2495, 0xffffffff, 0x000f000e,
721 0x2496, 0xffffffff, 0x00100013,
722 0x2497, 0xffffffff, 0x00120011,
723 0x2498, 0xffffffff, 0x00150014,
78bbe771
TSD		 724 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
725 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
726 mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
727 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
dae5c298
FC		 728 0x000c, 0xffffffff, 0x0000001c,
729 0x000d, 0x000f0000, 0x000f0000,
730 0x0583, 0xffffffff, 0x00000100,
78bbe771
TSD		 731 mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
732 mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
733 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
734 mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
735 mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
736 mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
62a37553 737 0x157a, 0x00000001, 0x00000001,
78bbe771
TSD		 738 mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
739 mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
740 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
62a37553 741 0x3430, 0xfffffff0, 0x00000100,
78bbe771 742 0x3630, 0xfffffff0, 0x00000100,
62a37553 743};
78bbe771 744
62a37553
KW		 745static const u32 oland_mgcg_cgcg_init[] =
746{
78bbe771
TSD		 747 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
748 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
749 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
750 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
751 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
752 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
753 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
754 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
755 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
756 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
757 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
758 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
759 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
760 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
761 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
762 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
763 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
764 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
765 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
766 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
767 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
768 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
769 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
770 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
771 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
772 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
773 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
62a37553
KW		 774 0x2458, 0xffffffff, 0x00010000,
775 0x2459, 0xffffffff, 0x00030002,
776 0x245a, 0xffffffff, 0x00040007,
777 0x245b, 0xffffffff, 0x00060005,
778 0x245c, 0xffffffff, 0x00090008,
779 0x245d, 0xffffffff, 0x00020001,
780 0x245e, 0xffffffff, 0x00040003,
781 0x245f, 0xffffffff, 0x00000007,
782 0x2460, 0xffffffff, 0x00060005,
783 0x2461, 0xffffffff, 0x00090008,
784 0x2462, 0xffffffff, 0x00030002,
785 0x2463, 0xffffffff, 0x00050004,
786 0x2464, 0xffffffff, 0x00000008,
787 0x2465, 0xffffffff, 0x00070006,
788 0x2466, 0xffffffff, 0x000a0009,
789 0x2467, 0xffffffff, 0x00040003,
790 0x2468, 0xffffffff, 0x00060005,
791 0x2469, 0xffffffff, 0x00000009,
792 0x246a, 0xffffffff, 0x00080007,
793 0x246b, 0xffffffff, 0x000b000a,
794 0x246c, 0xffffffff, 0x00050004,
795 0x246d, 0xffffffff, 0x00070006,
796 0x246e, 0xffffffff, 0x0008000b,
797 0x246f, 0xffffffff, 0x000a0009,
798 0x2470, 0xffffffff, 0x000d000c,
799 0x2471, 0xffffffff, 0x00060005,
800 0x2472, 0xffffffff, 0x00080007,
801 0x2473, 0xffffffff, 0x0000000b,
802 0x2474, 0xffffffff, 0x000a0009,
803 0x2475, 0xffffffff, 0x000d000c,
78bbe771
TSD		 804 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
805 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
806 mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
807 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
6b7985ef
FC		 808 0x000c, 0xffffffff, 0x0000001c,
809 0x000d, 0x000f0000, 0x000f0000,
810 0x0583, 0xffffffff, 0x00000100,
78bbe771
TSD		 811 mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
812 mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
813 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
814 mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
815 mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
816 mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
62a37553 817 0x157a, 0x00000001, 0x00000001,
78bbe771
TSD		 818 mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
819 mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
820 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
62a37553 821 0x3430, 0xfffffff0, 0x00000100,
78bbe771 822 0x3630, 0xfffffff0, 0x00000100,
62a37553 823};
78bbe771 824
62a37553
KW		 825static const u32 hainan_mgcg_cgcg_init[] =
826{
78bbe771
TSD		 827 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
828 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
829 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
830 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
831 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
832 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
833 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
834 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
835 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
836 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
837 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
838 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
839 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
840 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
841 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
842 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
843 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
844 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
845 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
846 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
847 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
848 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
849 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
850 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
851 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
852 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
853 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
62a37553
KW		 854 0x2458, 0xffffffff, 0x00010000,
855 0x2459, 0xffffffff, 0x00030002,
856 0x245a, 0xffffffff, 0x00040007,
857 0x245b, 0xffffffff, 0x00060005,
858 0x245c, 0xffffffff, 0x00090008,
859 0x245d, 0xffffffff, 0x00020001,
860 0x245e, 0xffffffff, 0x00040003,
861 0x245f, 0xffffffff, 0x00000007,
862 0x2460, 0xffffffff, 0x00060005,
863 0x2461, 0xffffffff, 0x00090008,
864 0x2462, 0xffffffff, 0x00030002,
865 0x2463, 0xffffffff, 0x00050004,
866 0x2464, 0xffffffff, 0x00000008,
867 0x2465, 0xffffffff, 0x00070006,
868 0x2466, 0xffffffff, 0x000a0009,
869 0x2467, 0xffffffff, 0x00040003,
870 0x2468, 0xffffffff, 0x00060005,
871 0x2469, 0xffffffff, 0x00000009,
872 0x246a, 0xffffffff, 0x00080007,
873 0x246b, 0xffffffff, 0x000b000a,
874 0x246c, 0xffffffff, 0x00050004,
875 0x246d, 0xffffffff, 0x00070006,
876 0x246e, 0xffffffff, 0x0008000b,
877 0x246f, 0xffffffff, 0x000a0009,
878 0x2470, 0xffffffff, 0x000d000c,
879 0x2471, 0xffffffff, 0x00060005,
880 0x2472, 0xffffffff, 0x00080007,
881 0x2473, 0xffffffff, 0x0000000b,
882 0x2474, 0xffffffff, 0x000a0009,
883 0x2475, 0xffffffff, 0x000d000c,
78bbe771
TSD		 884 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
885 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
886 mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
887 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
bd27b678
FC		 888 0x000c, 0xffffffff, 0x0000001c,
889 0x000d, 0x000f0000, 0x000f0000,
890 0x0583, 0xffffffff, 0x00000100,
891 0x0409, 0xffffffff, 0x00000100,
78bbe771
TSD		 892 mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
893 mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
894 mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
895 mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
896 mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
897 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
62a37553 898 0x3430, 0xfffffff0, 0x00000100,
78bbe771 899 0x3630, 0xfffffff0, 0x00000100,
62a37553
KW		 900};
901
902static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
903{
904 unsigned long flags;
905 u32 r;
906
907 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
908 WREG32(AMDGPU_PCIE_INDEX, reg);
909 (void)RREG32(AMDGPU_PCIE_INDEX);
910 r = RREG32(AMDGPU_PCIE_DATA);
911 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
912 return r;
913}
914
915static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
916{
917 unsigned long flags;
918
919 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
920 WREG32(AMDGPU_PCIE_INDEX, reg);
921 (void)RREG32(AMDGPU_PCIE_INDEX);
922 WREG32(AMDGPU_PCIE_DATA, v);
923 (void)RREG32(AMDGPU_PCIE_DATA);
924 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
925}
926
d1936cc2 927static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
36b9a952
HR		 928{
929 unsigned long flags;
930 u32 r;
931
932 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
933 WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
934 (void)RREG32(PCIE_PORT_INDEX);
935 r = RREG32(PCIE_PORT_DATA);
936 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
937 return r;
938}
939
d1936cc2 940static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
36b9a952
HR		 941{
942 unsigned long flags;
943
944 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
945 WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
946 (void)RREG32(PCIE_PORT_INDEX);
947 WREG32(PCIE_PORT_DATA, (v));
948 (void)RREG32(PCIE_PORT_DATA);
949 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
950}
951
62a37553
KW		 952static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
953{
954 unsigned long flags;
955 u32 r;
956
957 spin_lock_irqsave(&adev->smc_idx_lock, flags);
958 WREG32(SMC_IND_INDEX_0, (reg));
959 r = RREG32(SMC_IND_DATA_0);
960 spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
961 return r;
962}
963
964static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
965{
966 unsigned long flags;
967
968 spin_lock_irqsave(&adev->smc_idx_lock, flags);
969 WREG32(SMC_IND_INDEX_0, (reg));
970 WREG32(SMC_IND_DATA_0, (v));
971 spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
972}
973
62a37553 974static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {
97fcc76b
CK		 975 {GRBM_STATUS},
976 {GB_ADDR_CONFIG},
977 {MC_ARB_RAMCFG},
978 {GB_TILE_MODE0},
979 {GB_TILE_MODE1},
980 {GB_TILE_MODE2},
981 {GB_TILE_MODE3},
982 {GB_TILE_MODE4},
983 {GB_TILE_MODE5},
984 {GB_TILE_MODE6},
985 {GB_TILE_MODE7},
986 {GB_TILE_MODE8},
987 {GB_TILE_MODE9},
988 {GB_TILE_MODE10},
989 {GB_TILE_MODE11},
990 {GB_TILE_MODE12},
991 {GB_TILE_MODE13},
992 {GB_TILE_MODE14},
993 {GB_TILE_MODE15},
994 {GB_TILE_MODE16},
995 {GB_TILE_MODE17},
996 {GB_TILE_MODE18},
997 {GB_TILE_MODE19},
998 {GB_TILE_MODE20},
999 {GB_TILE_MODE21},
1000 {GB_TILE_MODE22},
1001 {GB_TILE_MODE23},
1002 {GB_TILE_MODE24},
1003 {GB_TILE_MODE25},
1004 {GB_TILE_MODE26},
1005 {GB_TILE_MODE27},
1006 {GB_TILE_MODE28},
1007 {GB_TILE_MODE29},
1008 {GB_TILE_MODE30},
1009 {GB_TILE_MODE31},
1010 {CC_RB_BACKEND_DISABLE, true},
1011 {GC_USER_RB_BACKEND_DISABLE, true},
1012 {PA_SC_RASTER_CONFIG, true},
62a37553
KW		 1013};
1014
dd5dfa61
FC		 1015static uint32_t si_get_register_value(struct amdgpu_device *adev,
1016 bool indexed, u32 se_num,
1017 u32 sh_num, u32 reg_offset)
62a37553 1018{
dd5dfa61
FC		 1019 if (indexed) {
1020 uint32_t val;
1021 unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1022 unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
62a37553 1023
dd5dfa61
FC		 1024 switch (reg_offset) {
1025 case mmCC_RB_BACKEND_DISABLE:
1026 return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1027 case mmGC_USER_RB_BACKEND_DISABLE:
1028 return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1029 case mmPA_SC_RASTER_CONFIG:
1030 return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1031 }
62a37553 1032
dd5dfa61
FC		 1033 mutex_lock(&adev->grbm_idx_mutex);
1034 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1035 amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
62a37553 1036
dd5dfa61
FC		 1037 val = RREG32(reg_offset);
1038
1039 if (se_num != 0xffffffff || sh_num != 0xffffffff)
1040 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1041 mutex_unlock(&adev->grbm_idx_mutex);
1042 return val;
1043 } else {
1044 unsigned idx;
62a37553 1045
dd5dfa61
FC		 1046 switch (reg_offset) {
1047 case mmGB_ADDR_CONFIG:
1048 return adev->gfx.config.gb_addr_config;
1049 case mmMC_ARB_RAMCFG:
1050 return adev->gfx.config.mc_arb_ramcfg;
1051 case mmGB_TILE_MODE0:
1052 case mmGB_TILE_MODE1:
1053 case mmGB_TILE_MODE2:
1054 case mmGB_TILE_MODE3:
1055 case mmGB_TILE_MODE4:
1056 case mmGB_TILE_MODE5:
1057 case mmGB_TILE_MODE6:
1058 case mmGB_TILE_MODE7:
1059 case mmGB_TILE_MODE8:
1060 case mmGB_TILE_MODE9:
1061 case mmGB_TILE_MODE10:
1062 case mmGB_TILE_MODE11:
1063 case mmGB_TILE_MODE12:
1064 case mmGB_TILE_MODE13:
1065 case mmGB_TILE_MODE14:
1066 case mmGB_TILE_MODE15:
1067 case mmGB_TILE_MODE16:
1068 case mmGB_TILE_MODE17:
1069 case mmGB_TILE_MODE18:
1070 case mmGB_TILE_MODE19:
1071 case mmGB_TILE_MODE20:
1072 case mmGB_TILE_MODE21:
1073 case mmGB_TILE_MODE22:
1074 case mmGB_TILE_MODE23:
1075 case mmGB_TILE_MODE24:
1076 case mmGB_TILE_MODE25:
1077 case mmGB_TILE_MODE26:
1078 case mmGB_TILE_MODE27:
1079 case mmGB_TILE_MODE28:
1080 case mmGB_TILE_MODE29:
1081 case mmGB_TILE_MODE30:
1082 case mmGB_TILE_MODE31:
1083 idx = (reg_offset - mmGB_TILE_MODE0);
1084 return adev->gfx.config.tile_mode_array[idx];
1085 default:
1086 return RREG32(reg_offset);
1087 }
1088 }
1089}
62a37553
KW		 1090static int si_read_register(struct amdgpu_device *adev, u32 se_num,
1091 u32 sh_num, u32 reg_offset, u32 *value)
1092{
1093 uint32_t i;
1094
1095 *value = 0;
1096 for (i = 0; i < ARRAY_SIZE(si_allowed_read_registers); i++) {
97fcc76b
CK		 1097 bool indexed = si_allowed_read_registers[i].grbm_indexed;
1098
62a37553
KW		 1099 if (reg_offset != si_allowed_read_registers[i].reg_offset)
1100 continue;
1101
97fcc76b
CK		 1102 *value = si_get_register_value(adev, indexed, se_num, sh_num,
1103 reg_offset);
62a37553
KW		 1104 return 0;
1105 }
1106 return -EINVAL;
1107}
1108
1109static bool si_read_disabled_bios(struct amdgpu_device *adev)
1110{
1111 u32 bus_cntl;
1112 u32 d1vga_control = 0;
1113 u32 d2vga_control = 0;
1114 u32 vga_render_control = 0;
1115 u32 rom_cntl;
1116 bool r;
1117
1118 bus_cntl = RREG32(R600_BUS_CNTL);
1119 if (adev->mode_info.num_crtc) {
1120 d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
1121 d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
1122 vga_render_control = RREG32(VGA_RENDER_CONTROL);
1123 }
1124 rom_cntl = RREG32(R600_ROM_CNTL);
1125
1126 /* enable the rom */
1127 WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
1128 if (adev->mode_info.num_crtc) {
1129 /* Disable VGA mode */
1130 WREG32(AVIVO_D1VGA_CONTROL,
1131 (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1132 AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1133 WREG32(AVIVO_D2VGA_CONTROL,
1134 (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1135 AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1136 WREG32(VGA_RENDER_CONTROL,
1137 (vga_render_control & C_000300_VGA_VSTATUS_CNTL));
1138 }
1139 WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
1140
1141 r = amdgpu_read_bios(adev);
1142
1143 /* restore regs */
1144 WREG32(R600_BUS_CNTL, bus_cntl);
1145 if (adev->mode_info.num_crtc) {
1146 WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
1147 WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
1148 WREG32(VGA_RENDER_CONTROL, vga_render_control);
1149 }
1150 WREG32(R600_ROM_CNTL, rom_cntl);
1151 return r;
1152}
1153
6d949d24
AD		 1154#define mmROM_INDEX 0x2A
1155#define mmROM_DATA 0x2B
1156
1157static bool si_read_bios_from_rom(struct amdgpu_device *adev,
1158 u8 *bios, u32 length_bytes)
1159{
1160 u32 *dw_ptr;
1161 u32 i, length_dw;
1162
1163 if (bios == NULL)
1164 return false;
1165 if (length_bytes == 0)
1166 return false;
1167 /* APU vbios image is part of sbios image */
1168 if (adev->flags & AMD_IS_APU)
1169 return false;
1170
1171 dw_ptr = (u32 *)bios;
1172 length_dw = ALIGN(length_bytes, 4) / 4;
1173 /* set rom index to 0 */
1174 WREG32(mmROM_INDEX, 0);
1175 for (i = 0; i < length_dw; i++)
1176 dw_ptr[i] = RREG32(mmROM_DATA);
1177
1178 return true;
1179}
1180
62a37553
KW		 1181//xxx: not implemented
1182static int si_asic_reset(struct amdgpu_device *adev)
1183{
1184 return 0;
1185}
1186
bbf282d8
AD		 1187static u32 si_get_config_memsize(struct amdgpu_device *adev)
1188{
1189 return RREG32(mmCONFIG_MEMSIZE);
1190}
1191
62a37553
KW		 1192static void si_vga_set_state(struct amdgpu_device *adev, bool state)
1193{
1194 uint32_t temp;
1195
1196 temp = RREG32(CONFIG_CNTL);
1197 if (state == false) {
1198 temp &= ~(1<<0);
1199 temp |= (1<<1);
1200 } else {
1201 temp &= ~(1<<1);
1202 }
1203 WREG32(CONFIG_CNTL, temp);
1204}
1205
1206static u32 si_get_xclk(struct amdgpu_device *adev)
1207{
1208 u32 reference_clock = adev->clock.spll.reference_freq;
1209 u32 tmp;
1210
1211 tmp = RREG32(CG_CLKPIN_CNTL_2);
1212 if (tmp & MUX_TCLK_TO_XCLK)
1213 return TCLK;
1214
1215 tmp = RREG32(CG_CLKPIN_CNTL);
1216 if (tmp & XTALIN_DIVIDE)
1217 return reference_clock / 4;
1218
1219 return reference_clock;
1220}
1919696e 1221
62a37553
KW		 1222//xxx:not implemented
1223static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1224{
1225 return 0;
1226}
1227
4e99a44e
ML		 1228static void si_detect_hw_virtualization(struct amdgpu_device *adev)
1229{
1230 if (is_virtual_machine()) /* passthrough mode */
5a5099cb 1231 adev->virt.caps |= AMDGPU_PASSTHROUGH_MODE;
4e99a44e
ML		 1232}
1233
62a37553
KW		 1234static const struct amdgpu_asic_funcs si_asic_funcs =
1235{
1236 .read_disabled_bios = &si_read_disabled_bios,
6d949d24 1237 .read_bios_from_rom = &si_read_bios_from_rom,
62a37553
KW		 1238 .read_register = &si_read_register,
1239 .reset = &si_asic_reset,
1240 .set_vga_state = &si_vga_set_state,
1241 .get_xclk = &si_get_xclk,
1242 .set_uvd_clocks = &si_set_uvd_clocks,
1243 .set_vce_clocks = NULL,
bbf282d8 1244 .get_config_memsize = &si_get_config_memsize,
62a37553
KW		 1245};
1246
1247static uint32_t si_get_rev_id(struct amdgpu_device *adev)
1248{
1249 return (RREG32(CC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1250 >> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1251}
1252
1253static int si_common_early_init(void *handle)
1254{
1255 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1256
1257 adev->smc_rreg = &si_smc_rreg;
1258 adev->smc_wreg = &si_smc_wreg;
1259 adev->pcie_rreg = &si_pcie_rreg;
1260 adev->pcie_wreg = &si_pcie_wreg;
36b9a952
HR		 1261 adev->pciep_rreg = &si_pciep_rreg;
1262 adev->pciep_wreg = &si_pciep_wreg;
62a37553
KW		 1263 adev->uvd_ctx_rreg = NULL;
1264 adev->uvd_ctx_wreg = NULL;
1265 adev->didt_rreg = NULL;
1266 adev->didt_wreg = NULL;
1267
1268 adev->asic_funcs = &si_asic_funcs;
1269
1270 adev->rev_id = si_get_rev_id(adev);
1271 adev->external_rev_id = 0xFF;
1272 switch (adev->asic_type) {
1273 case CHIP_TAHITI:
1274 adev->cg_flags =
1275 AMD_CG_SUPPORT_GFX_MGCG |
1276 AMD_CG_SUPPORT_GFX_MGLS |
1277 /*AMD_CG_SUPPORT_GFX_CGCG |*/
1278 AMD_CG_SUPPORT_GFX_CGLS |
1279 AMD_CG_SUPPORT_GFX_CGTS |
1280 AMD_CG_SUPPORT_GFX_CP_LS |
1281 AMD_CG_SUPPORT_MC_MGCG |
1282 AMD_CG_SUPPORT_SDMA_MGCG |
1283 AMD_CG_SUPPORT_BIF_LS |
1284 AMD_CG_SUPPORT_VCE_MGCG |
1285 AMD_CG_SUPPORT_UVD_MGCG |
1286 AMD_CG_SUPPORT_HDP_LS |
1287 AMD_CG_SUPPORT_HDP_MGCG;
1288 adev->pg_flags = 0;
7c0a705e
FC		 1289 adev->external_rev_id = (adev->rev_id == 0) ? 1 :
1290 (adev->rev_id == 1) ? 5 : 6;
62a37553
KW		 1291 break;
1292 case CHIP_PITCAIRN:
1293 adev->cg_flags =
1294 AMD_CG_SUPPORT_GFX_MGCG |
1295 AMD_CG_SUPPORT_GFX_MGLS |
1296 /*AMD_CG_SUPPORT_GFX_CGCG |*/
1297 AMD_CG_SUPPORT_GFX_CGLS |
1298 AMD_CG_SUPPORT_GFX_CGTS |
1299 AMD_CG_SUPPORT_GFX_CP_LS |
1300 AMD_CG_SUPPORT_GFX_RLC_LS |
1301 AMD_CG_SUPPORT_MC_LS |
1302 AMD_CG_SUPPORT_MC_MGCG |
1303 AMD_CG_SUPPORT_SDMA_MGCG |
1304 AMD_CG_SUPPORT_BIF_LS |
1305 AMD_CG_SUPPORT_VCE_MGCG |
1306 AMD_CG_SUPPORT_UVD_MGCG |
1307 AMD_CG_SUPPORT_HDP_LS |
1308 AMD_CG_SUPPORT_HDP_MGCG;
1309 adev->pg_flags = 0;
e285a9a6 1310 adev->external_rev_id = adev->rev_id + 20;
62a37553
KW		 1311 break;
1312
1313 case CHIP_VERDE:
1314 adev->cg_flags =
1315 AMD_CG_SUPPORT_GFX_MGCG |
1316 AMD_CG_SUPPORT_GFX_MGLS |
1317 AMD_CG_SUPPORT_GFX_CGLS |
1318 AMD_CG_SUPPORT_GFX_CGTS |
1319 AMD_CG_SUPPORT_GFX_CGTS_LS |
1320 AMD_CG_SUPPORT_GFX_CP_LS |
1321 AMD_CG_SUPPORT_MC_LS |
1322 AMD_CG_SUPPORT_MC_MGCG |
1323 AMD_CG_SUPPORT_SDMA_MGCG |
1324 AMD_CG_SUPPORT_SDMA_LS |
1325 AMD_CG_SUPPORT_BIF_LS |
1326 AMD_CG_SUPPORT_VCE_MGCG |
1327 AMD_CG_SUPPORT_UVD_MGCG |
1328 AMD_CG_SUPPORT_HDP_LS |
1329 AMD_CG_SUPPORT_HDP_MGCG;
1330 adev->pg_flags = 0;
1331 //???
f815b29c 1332 adev->external_rev_id = adev->rev_id + 40;
62a37553
KW		 1333 break;
1334 case CHIP_OLAND:
1335 adev->cg_flags =
1336 AMD_CG_SUPPORT_GFX_MGCG |
1337 AMD_CG_SUPPORT_GFX_MGLS |
1338 /*AMD_CG_SUPPORT_GFX_CGCG |*/
1339 AMD_CG_SUPPORT_GFX_CGLS |
1340 AMD_CG_SUPPORT_GFX_CGTS |
1341 AMD_CG_SUPPORT_GFX_CP_LS |
1342 AMD_CG_SUPPORT_GFX_RLC_LS |
1343 AMD_CG_SUPPORT_MC_LS |
1344 AMD_CG_SUPPORT_MC_MGCG |
1345 AMD_CG_SUPPORT_SDMA_MGCG |
1346 AMD_CG_SUPPORT_BIF_LS |
1347 AMD_CG_SUPPORT_UVD_MGCG |
1348 AMD_CG_SUPPORT_HDP_LS |
1349 AMD_CG_SUPPORT_HDP_MGCG;
1350 adev->pg_flags = 0;
8fd74cb4 1351 adev->external_rev_id = 60;
62a37553
KW		 1352 break;
1353 case CHIP_HAINAN:
1354 adev->cg_flags =
1355 AMD_CG_SUPPORT_GFX_MGCG |
1356 AMD_CG_SUPPORT_GFX_MGLS |
1357 /*AMD_CG_SUPPORT_GFX_CGCG |*/
1358 AMD_CG_SUPPORT_GFX_CGLS |
1359 AMD_CG_SUPPORT_GFX_CGTS |
1360 AMD_CG_SUPPORT_GFX_CP_LS |
1361 AMD_CG_SUPPORT_GFX_RLC_LS |
1362 AMD_CG_SUPPORT_MC_LS |
1363 AMD_CG_SUPPORT_MC_MGCG |
1364 AMD_CG_SUPPORT_SDMA_MGCG |
1365 AMD_CG_SUPPORT_BIF_LS |
1366 AMD_CG_SUPPORT_HDP_LS |
1367 AMD_CG_SUPPORT_HDP_MGCG;
1368 adev->pg_flags = 0;
05319478 1369 adev->external_rev_id = 70;
62a37553
KW		 1370 break;
1371
1372 default:
1373 return -EINVAL;
1374 }
1375
1376 return 0;
1377}
1378
1379static int si_common_sw_init(void *handle)
1380{
1381 return 0;
1382}
1383
1384static int si_common_sw_fini(void *handle)
1385{
1386 return 0;
1387}
1388
1389
1390static void si_init_golden_registers(struct amdgpu_device *adev)
1391{
1392 switch (adev->asic_type) {
1393 case CHIP_TAHITI:
9c3f2b54
AD		 1394 amdgpu_device_program_register_sequence(adev,
1395 tahiti_golden_registers,
1396 ARRAY_SIZE(tahiti_golden_registers));
1397 amdgpu_device_program_register_sequence(adev,
1398 tahiti_golden_rlc_registers,
1399 ARRAY_SIZE(tahiti_golden_rlc_registers));
1400 amdgpu_device_program_register_sequence(adev,
1401 tahiti_mgcg_cgcg_init,
1402 ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1403 amdgpu_device_program_register_sequence(adev,
1404 tahiti_golden_registers2,
1405 ARRAY_SIZE(tahiti_golden_registers2));
62a37553
KW		 1406 break;
1407 case CHIP_PITCAIRN:
9c3f2b54
AD		 1408 amdgpu_device_program_register_sequence(adev,
1409 pitcairn_golden_registers,
1410 ARRAY_SIZE(pitcairn_golden_registers));
1411 amdgpu_device_program_register_sequence(adev,
1412 pitcairn_golden_rlc_registers,
1413 ARRAY_SIZE(pitcairn_golden_rlc_registers));
1414 amdgpu_device_program_register_sequence(adev,
1415 pitcairn_mgcg_cgcg_init,
1416 ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
5694785c 1417 break;
62a37553 1418 case CHIP_VERDE:
9c3f2b54
AD		 1419 amdgpu_device_program_register_sequence(adev,
1420 verde_golden_registers,
1421 ARRAY_SIZE(verde_golden_registers));
1422 amdgpu_device_program_register_sequence(adev,
1423 verde_golden_rlc_registers,
1424 ARRAY_SIZE(verde_golden_rlc_registers));
1425 amdgpu_device_program_register_sequence(adev,
1426 verde_mgcg_cgcg_init,
1427 ARRAY_SIZE(verde_mgcg_cgcg_init));
1428 amdgpu_device_program_register_sequence(adev,
1429 verde_pg_init,
1430 ARRAY_SIZE(verde_pg_init));
62a37553
KW		 1431 break;
1432 case CHIP_OLAND:
9c3f2b54
AD		 1433 amdgpu_device_program_register_sequence(adev,
1434 oland_golden_registers,
1435 ARRAY_SIZE(oland_golden_registers));
1436 amdgpu_device_program_register_sequence(adev,
1437 oland_golden_rlc_registers,
1438 ARRAY_SIZE(oland_golden_rlc_registers));
1439 amdgpu_device_program_register_sequence(adev,
1440 oland_mgcg_cgcg_init,
1441 ARRAY_SIZE(oland_mgcg_cgcg_init));
5694785c 1442 break;
62a37553 1443 case CHIP_HAINAN:
9c3f2b54
AD		 1444 amdgpu_device_program_register_sequence(adev,
1445 hainan_golden_registers,
1446 ARRAY_SIZE(hainan_golden_registers));
1447 amdgpu_device_program_register_sequence(adev,
1448 hainan_golden_registers2,
1449 ARRAY_SIZE(hainan_golden_registers2));
1450 amdgpu_device_program_register_sequence(adev,
1451 hainan_mgcg_cgcg_init,
1452 ARRAY_SIZE(hainan_mgcg_cgcg_init));
62a37553
KW		 1453 break;
1454
1455
1456 default:
1457 BUG();
1458 }
1459}
1460
62a37553
KW		 1461static void si_pcie_gen3_enable(struct amdgpu_device *adev)
1462{
1463 struct pci_dev *root = adev->pdev->bus->self;
1464 int bridge_pos, gpu_pos;
05656e5e
AD		 1465 u32 speed_cntl, current_data_rate;
1466 int i;
62a37553
KW		 1467 u16 tmp16;
1468
1469 if (pci_is_root_bus(adev->pdev->bus))
1470 return;
1471
1472 if (amdgpu_pcie_gen2 == 0)
1473 return;
1474
1475 if (adev->flags & AMD_IS_APU)
1476 return;
1477
05656e5e
AD		 1478 if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1479 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
62a37553
KW		 1480 return;
1481
36b9a952 1482 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
62a37553
KW		 1483 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
1484 LC_CURRENT_DATA_RATE_SHIFT;
05656e5e 1485 if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
62a37553
KW		 1486 if (current_data_rate == 2) {
1487 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1488 return;
1489 }
1490 DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
05656e5e 1491 } else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
62a37553
KW		 1492 if (current_data_rate == 1) {
1493 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1494 return;
1495 }
1496 DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1497 }
1498
1499 bridge_pos = pci_pcie_cap(root);
1500 if (!bridge_pos)
1501 return;
1502
1503 gpu_pos = pci_pcie_cap(adev->pdev);
1504 if (!gpu_pos)
1505 return;
1506
05656e5e 1507 if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
62a37553
KW		 1508 if (current_data_rate != 2) {
1509 u16 bridge_cfg, gpu_cfg;
1510 u16 bridge_cfg2, gpu_cfg2;
1511 u32 max_lw, current_lw, tmp;
1512
1513 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1514 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1515
1516 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1517 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1518
1519 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1520 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1521
1522 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
1523 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
1524 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
1525
1526 if (current_lw < max_lw) {
36b9a952 1527 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
62a37553
KW		 1528 if (tmp & LC_RENEGOTIATION_SUPPORT) {
1529 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
1530 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
1531 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
36b9a952 1532 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
62a37553
KW		 1533 }
1534 }
1535
1536 for (i = 0; i < 10; i++) {
1537 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1538 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1539 break;
1540
1541 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1542 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1543
1544 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1545 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1546
36b9a952 1547 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
62a37553 1548 tmp |= LC_SET_QUIESCE;
36b9a952 1549 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
62a37553 1550
36b9a952 1551 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
62a37553 1552 tmp |= LC_REDO_EQ;
36b9a952 1553 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
62a37553
KW		 1554
1555 mdelay(100);
1556
1557 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1558 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1559 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1560 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1561
1562 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1563 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1564 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1565 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1566
1567 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1568 tmp16 &= ~((1 << 4) | (7 << 9));
1569 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1570 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1571
1572 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1573 tmp16 &= ~((1 << 4) | (7 << 9));
1574 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1575 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1576
36b9a952 1577 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
62a37553 1578 tmp &= ~LC_SET_QUIESCE;
36b9a952 1579 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
62a37553
KW		 1580 }
1581 }
1582 }
1583
1584 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
1585 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
36b9a952 1586 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
62a37553
KW		 1587
1588 pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1589 tmp16 &= ~0xf;
05656e5e 1590 if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
62a37553 1591 tmp16 |= 3;
05656e5e 1592 else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
62a37553
KW		 1593 tmp16 |= 2;
1594 else
1595 tmp16 |= 1;
1596 pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1597
36b9a952 1598 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
62a37553 1599 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
36b9a952 1600 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
62a37553
KW		 1601
1602 for (i = 0; i < adev->usec_timeout; i++) {
36b9a952 1603 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
62a37553
KW		 1604 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
1605 break;
1606 udelay(1);
1607 }
1608}
1609
1610static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)
1611{
1612 unsigned long flags;
1613 u32 r;
1614
1615 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1616 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
1617 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
1618 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1619 return r;
1620}
1621
1622static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1623{
1624 unsigned long flags;
1625
1626 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1627 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
1628 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
1629 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1630}
1631
1632static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
1633{
1634 unsigned long flags;
1635 u32 r;
1636
1637 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1638 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
1639 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
1640 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1641 return r;
1642}
1643
1644static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1645{
1646 unsigned long flags;
1647
1648 spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1649 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
1650 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
1651 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1652}
1653static void si_program_aspm(struct amdgpu_device *adev)
1654{
1655 u32 data, orig;
1656 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1657 bool disable_clkreq = false;
1658
1659 if (amdgpu_aspm == 0)
1660 return;
1661
1662 if (adev->flags & AMD_IS_APU)
1663 return;
36b9a952 1664 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
62a37553
KW		 1665 data &= ~LC_XMIT_N_FTS_MASK;
1666 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
1667 if (orig != data)
36b9a952 1668 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
62a37553 1669
36b9a952 1670 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
62a37553
KW		 1671 data |= LC_GO_TO_RECOVERY;
1672 if (orig != data)
36b9a952 1673 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
62a37553
KW		 1674
1675 orig = data = RREG32_PCIE(PCIE_P_CNTL);
1676 data |= P_IGNORE_EDB_ERR;
1677 if (orig != data)
1678 WREG32_PCIE(PCIE_P_CNTL, data);
1679
36b9a952 1680 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
62a37553
KW		 1681 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
1682 data |= LC_PMI_TO_L1_DIS;
1683 if (!disable_l0s)
1684 data |= LC_L0S_INACTIVITY(7);
1685
1686 if (!disable_l1) {
1687 data |= LC_L1_INACTIVITY(7);
1688 data &= ~LC_PMI_TO_L1_DIS;
1689 if (orig != data)
36b9a952 1690 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
62a37553
KW		 1691
1692 if (!disable_plloff_in_l1) {
1693 bool clk_req_support;
1694
1695 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
1696 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
1697 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
1698 if (orig != data)
1699 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
1700
1701 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
1702 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
1703 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
1704 if (orig != data)
1705 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
1706
1707 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
1708 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
1709 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
1710 if (orig != data)
1711 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
1712
1713 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
1714 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
1715 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
1716 if (orig != data)
1717 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
1718
1719 if ((adev->family != CHIP_OLAND) && (adev->family != CHIP_HAINAN)) {
1720 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
1721 data &= ~PLL_RAMP_UP_TIME_0_MASK;
1722 if (orig != data)
1723 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
1724
1725 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
1726 data &= ~PLL_RAMP_UP_TIME_1_MASK;
1727 if (orig != data)
1728 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
1729
1730 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_2);
1731 data &= ~PLL_RAMP_UP_TIME_2_MASK;
1732 if (orig != data)
1733 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_2, data);
1734
1735 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_3);
1736 data &= ~PLL_RAMP_UP_TIME_3_MASK;
1737 if (orig != data)
1738 si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_3, data);
1739
1740 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
1741 data &= ~PLL_RAMP_UP_TIME_0_MASK;
1742 if (orig != data)
1743 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
1744
1745 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
1746 data &= ~PLL_RAMP_UP_TIME_1_MASK;
1747 if (orig != data)
1748 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
1749
1750 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_2);
1751 data &= ~PLL_RAMP_UP_TIME_2_MASK;
1752 if (orig != data)
1753 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_2, data);
1754
1755 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_3);
1756 data &= ~PLL_RAMP_UP_TIME_3_MASK;
1757 if (orig != data)
1758 si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_3, data);
1759 }
36b9a952 1760 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
62a37553
KW		 1761 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
1762 data |= LC_DYN_LANES_PWR_STATE(3);
1763 if (orig != data)
36b9a952 1764 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
62a37553
KW		 1765
1766 orig = data = si_pif_phy0_rreg(adev,PB0_PIF_CNTL);
1767 data &= ~LS2_EXIT_TIME_MASK;
1768 if ((adev->family == CHIP_OLAND) || (adev->family == CHIP_HAINAN))
1769 data |= LS2_EXIT_TIME(5);
1770 if (orig != data)
1771 si_pif_phy0_wreg(adev,PB0_PIF_CNTL, data);
1772
1773 orig = data = si_pif_phy1_rreg(adev,PB1_PIF_CNTL);
1774 data &= ~LS2_EXIT_TIME_MASK;
1775 if ((adev->family == CHIP_OLAND) || (adev->family == CHIP_HAINAN))
1776 data |= LS2_EXIT_TIME(5);
1777 if (orig != data)
1778 si_pif_phy1_wreg(adev,PB1_PIF_CNTL, data);
1779
1780 if (!disable_clkreq &&
1781 !pci_is_root_bus(adev->pdev->bus)) {
1782 struct pci_dev *root = adev->pdev->bus->self;
1783 u32 lnkcap;
1784
1785 clk_req_support = false;
1786 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1787 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1788 clk_req_support = true;
1789 } else {
1790 clk_req_support = false;
1791 }
1792
1793 if (clk_req_support) {
36b9a952 1794 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
62a37553
KW		 1795 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
1796 if (orig != data)
36b9a952 1797 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
62a37553
KW		 1798
1799 orig = data = RREG32(THM_CLK_CNTL);
1800 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
1801 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
1802 if (orig != data)
1803 WREG32(THM_CLK_CNTL, data);
1804
1805 orig = data = RREG32(MISC_CLK_CNTL);
1806 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
1807 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
1808 if (orig != data)
1809 WREG32(MISC_CLK_CNTL, data);
1810
1811 orig = data = RREG32(CG_CLKPIN_CNTL);
1812 data &= ~BCLK_AS_XCLK;
1813 if (orig != data)
1814 WREG32(CG_CLKPIN_CNTL, data);
1815
1816 orig = data = RREG32(CG_CLKPIN_CNTL_2);
1817 data &= ~FORCE_BIF_REFCLK_EN;
1818 if (orig != data)
1819 WREG32(CG_CLKPIN_CNTL_2, data);
1820
1821 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
1822 data &= ~MPLL_CLKOUT_SEL_MASK;
1823 data |= MPLL_CLKOUT_SEL(4);
1824 if (orig != data)
1825 WREG32(MPLL_BYPASSCLK_SEL, data);
1826
1827 orig = data = RREG32(SPLL_CNTL_MODE);
1828 data &= ~SPLL_REFCLK_SEL_MASK;
1829 if (orig != data)
1830 WREG32(SPLL_CNTL_MODE, data);
1831 }
1832 }
1833 } else {
1834 if (orig != data)
36b9a952 1835 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
62a37553
KW		 1836 }
1837
1838 orig = data = RREG32_PCIE(PCIE_CNTL2);
1839 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
1840 if (orig != data)
1841 WREG32_PCIE(PCIE_CNTL2, data);
1842
1843 if (!disable_l0s) {
36b9a952 1844 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
62a37553
KW		 1845 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
1846 data = RREG32_PCIE(PCIE_LC_STATUS1);
1847 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
36b9a952 1848 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
62a37553
KW		 1849 data &= ~LC_L0S_INACTIVITY_MASK;
1850 if (orig != data)
36b9a952 1851 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
62a37553
KW		 1852 }
1853 }
1854 }
1855}
1856
1857static void si_fix_pci_max_read_req_size(struct amdgpu_device *adev)
1858{
1859 int readrq;
1860 u16 v;
1861
1862 readrq = pcie_get_readrq(adev->pdev);
1863 v = ffs(readrq) - 8;
1864 if ((v == 0) || (v == 6) || (v == 7))
1865 pcie_set_readrq(adev->pdev, 512);
1866}
1867
1868static int si_common_hw_init(void *handle)
1869{
1870 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1871
1872 si_fix_pci_max_read_req_size(adev);
1873 si_init_golden_registers(adev);
1874 si_pcie_gen3_enable(adev);
1875 si_program_aspm(adev);
1876
1877 return 0;
1878}
1879
1880static int si_common_hw_fini(void *handle)
1881{
1882 return 0;
1883}
1884
1885static int si_common_suspend(void *handle)
1886{
1887 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1888
1889 return si_common_hw_fini(adev);
1890}
1891
1892static int si_common_resume(void *handle)
1893{
1894 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1895
1896 return si_common_hw_init(adev);
1897}
1898
1899static bool si_common_is_idle(void *handle)
1900{
1901 return true;
1902}
1903
1904static int si_common_wait_for_idle(void *handle)
1905{
1906 return 0;
1907}
1908
1909static int si_common_soft_reset(void *handle)
1910{
1911 return 0;
1912}
1913
1914static int si_common_set_clockgating_state(void *handle,
1915 enum amd_clockgating_state state)
1916{
1917 return 0;
1918}
1919
1920static int si_common_set_powergating_state(void *handle,
1921 enum amd_powergating_state state)
1922{
1923 return 0;
1924}
1925
a1255107 1926static const struct amd_ip_funcs si_common_ip_funcs = {
62a37553
KW		 1927 .name = "si_common",
1928 .early_init = si_common_early_init,
1929 .late_init = NULL,
1930 .sw_init = si_common_sw_init,
1931 .sw_fini = si_common_sw_fini,
1932 .hw_init = si_common_hw_init,
1933 .hw_fini = si_common_hw_fini,
1934 .suspend = si_common_suspend,
1935 .resume = si_common_resume,
1936 .is_idle = si_common_is_idle,
1937 .wait_for_idle = si_common_wait_for_idle,
1938 .soft_reset = si_common_soft_reset,
1939 .set_clockgating_state = si_common_set_clockgating_state,
1940 .set_powergating_state = si_common_set_powergating_state,
1941};
1942
a1255107 1943static const struct amdgpu_ip_block_version si_common_ip_block =
62a37553 1944{
a1255107
AD		 1945 .type = AMD_IP_BLOCK_TYPE_COMMON,
1946 .major = 1,
1947 .minor = 0,
1948 .rev = 0,
1949 .funcs = &si_common_ip_funcs,
2120df47
AD		 1950};
1951
62a37553
KW		 1952int si_set_ip_blocks(struct amdgpu_device *adev)
1953{
c8394f38
XY		 1954 si_detect_hw_virtualization(adev);
1955
62a37553
KW		 1956 switch (adev->asic_type) {
1957 case CHIP_VERDE:
1958 case CHIP_TAHITI:
1959 case CHIP_PITCAIRN:
2990a1fc
AD		 1960 amdgpu_device_ip_block_add(adev, &si_common_ip_block);
1961 amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
1962 amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
1963 amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
a1255107 1964 if (adev->enable_virtual_display)
2990a1fc 1965 amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
a1255107 1966 else
2990a1fc
AD		 1967 amdgpu_device_ip_block_add(adev, &dce_v6_0_ip_block);
1968 amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
1969 amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
1970 /* amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block); */
1971 /* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
a1255107 1972 break;
62a37553 1973 case CHIP_OLAND:
2990a1fc
AD		 1974 amdgpu_device_ip_block_add(adev, &si_common_ip_block);
1975 amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
1976 amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
1977 amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
a1255107 1978 if (adev->enable_virtual_display)
2990a1fc 1979 amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
a1255107 1980 else
2990a1fc
AD		 1981 amdgpu_device_ip_block_add(adev, &dce_v6_4_ip_block);
1982 amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
1983 amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
1984 /* amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block); */
1985 /* amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block); */
62a37553
KW		 1986 break;
1987 case CHIP_HAINAN:
2990a1fc
AD		 1988 amdgpu_device_ip_block_add(adev, &si_common_ip_block);
1989 amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
1990 amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
1991 amdgpu_device_ip_block_add(adev, &amdgpu_pp_ip_block);
a1255107 1992 if (adev->enable_virtual_display)
2990a1fc
AD		 1993 amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block);
1994 amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
1995 amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
62a37553
KW		 1996 break;
1997 default:
1998 BUG();
1999 }
2000 return 0;
2001}
2002
Linux 6.x block layer and io_uring tree(s)