projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: add support for ASPM on evergreen asics
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
0f0de06c
AD		 24#include <linux/firmware.h>
25#include <linux/platform_device.h>
26#include <linux/slab.h>
27#include <linux/module.h>
760285e7 28#include <drm/drmP.h>
43b3cd99
AD		 29#include "radeon.h"
30#include "radeon_asic.h"
760285e7 31#include <drm/radeon_drm.h>
43b3cd99
AD		 32#include "sid.h"
33#include "atom.h"
48c0c902 34#include "si_blit_shaders.h"
43b3cd99 35
0f0de06c
AD		 36#define SI_PFP_UCODE_SIZE 2144
37#define SI_PM4_UCODE_SIZE 2144
38#define SI_CE_UCODE_SIZE 2144
39#define SI_RLC_UCODE_SIZE 2048
40#define SI_MC_UCODE_SIZE 7769
bcc7f5d2 41#define OLAND_MC_UCODE_SIZE 7863
0f0de06c
AD		 42
43MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
44MODULE_FIRMWARE("radeon/TAHITI_me.bin");
45MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
46MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
47MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
48MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
49MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
50MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
51MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
52MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
53MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
54MODULE_FIRMWARE("radeon/VERDE_me.bin");
55MODULE_FIRMWARE("radeon/VERDE_ce.bin");
56MODULE_FIRMWARE("radeon/VERDE_mc.bin");
57MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
bcc7f5d2
AD		 58MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
59MODULE_FIRMWARE("radeon/OLAND_me.bin");
60MODULE_FIRMWARE("radeon/OLAND_ce.bin");
61MODULE_FIRMWARE("radeon/OLAND_mc.bin");
62MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
c04c00b4
AD		 63MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
64MODULE_FIRMWARE("radeon/HAINAN_me.bin");
65MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
66MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
67MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
0f0de06c 68
b9d305df 69static void si_pcie_gen3_enable(struct radeon_device *rdev);
25a857fb
AD		 70extern int r600_ih_ring_alloc(struct radeon_device *rdev);
71extern void r600_ih_ring_fini(struct radeon_device *rdev);
0a96d72b 72extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
c476dde2
AD		 73extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
74extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
ca7db22b 75extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
1c534671 76extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
014bb209 77extern bool evergreen_is_display_hung(struct radeon_device *rdev);
0a96d72b 78
205996c0
AD		 79static const u32 tahiti_golden_rlc_registers[] =
80{
81 0xc424, 0xffffffff, 0x00601005,
82 0xc47c, 0xffffffff, 0x10104040,
83 0xc488, 0xffffffff, 0x0100000a,
84 0xc314, 0xffffffff, 0x00000800,
85 0xc30c, 0xffffffff, 0x800000f4,
86 0xf4a8, 0xffffffff, 0x00000000
87};
88
89static const u32 tahiti_golden_registers[] =
90{
91 0x9a10, 0x00010000, 0x00018208,
92 0x9830, 0xffffffff, 0x00000000,
93 0x9834, 0xf00fffff, 0x00000400,
94 0x9838, 0x0002021c, 0x00020200,
95 0xc78, 0x00000080, 0x00000000,
96 0xd030, 0x000300c0, 0x00800040,
97 0xd830, 0x000300c0, 0x00800040,
98 0x5bb0, 0x000000f0, 0x00000070,
99 0x5bc0, 0x00200000, 0x50100000,
100 0x7030, 0x31000311, 0x00000011,
101 0x277c, 0x00000003, 0x000007ff,
102 0x240c, 0x000007ff, 0x00000000,
103 0x8a14, 0xf000001f, 0x00000007,
104 0x8b24, 0xffffffff, 0x00ffffff,
105 0x8b10, 0x0000ff0f, 0x00000000,
106 0x28a4c, 0x07ffffff, 0x4e000000,
107 0x28350, 0x3f3f3fff, 0x2a00126a,
108 0x30, 0x000000ff, 0x0040,
109 0x34, 0x00000040, 0x00004040,
110 0x9100, 0x07ffffff, 0x03000000,
111 0x8e88, 0x01ff1f3f, 0x00000000,
112 0x8e84, 0x01ff1f3f, 0x00000000,
113 0x9060, 0x0000007f, 0x00000020,
114 0x9508, 0x00010000, 0x00010000,
115 0xac14, 0x00000200, 0x000002fb,
116 0xac10, 0xffffffff, 0x0000543b,
117 0xac0c, 0xffffffff, 0xa9210876,
118 0x88d0, 0xffffffff, 0x000fff40,
119 0x88d4, 0x0000001f, 0x00000010,
120 0x1410, 0x20000000, 0x20fffed8,
121 0x15c0, 0x000c0fc0, 0x000c0400
122};
123
124static const u32 tahiti_golden_registers2[] =
125{
126 0xc64, 0x00000001, 0x00000001
127};
128
129static const u32 pitcairn_golden_rlc_registers[] =
130{
131 0xc424, 0xffffffff, 0x00601004,
132 0xc47c, 0xffffffff, 0x10102020,
133 0xc488, 0xffffffff, 0x01000020,
134 0xc314, 0xffffffff, 0x00000800,
135 0xc30c, 0xffffffff, 0x800000a4
136};
137
138static const u32 pitcairn_golden_registers[] =
139{
140 0x9a10, 0x00010000, 0x00018208,
141 0x9830, 0xffffffff, 0x00000000,
142 0x9834, 0xf00fffff, 0x00000400,
143 0x9838, 0x0002021c, 0x00020200,
144 0xc78, 0x00000080, 0x00000000,
145 0xd030, 0x000300c0, 0x00800040,
146 0xd830, 0x000300c0, 0x00800040,
147 0x5bb0, 0x000000f0, 0x00000070,
148 0x5bc0, 0x00200000, 0x50100000,
149 0x7030, 0x31000311, 0x00000011,
150 0x2ae4, 0x00073ffe, 0x000022a2,
151 0x240c, 0x000007ff, 0x00000000,
152 0x8a14, 0xf000001f, 0x00000007,
153 0x8b24, 0xffffffff, 0x00ffffff,
154 0x8b10, 0x0000ff0f, 0x00000000,
155 0x28a4c, 0x07ffffff, 0x4e000000,
156 0x28350, 0x3f3f3fff, 0x2a00126a,
157 0x30, 0x000000ff, 0x0040,
158 0x34, 0x00000040, 0x00004040,
159 0x9100, 0x07ffffff, 0x03000000,
160 0x9060, 0x0000007f, 0x00000020,
161 0x9508, 0x00010000, 0x00010000,
162 0xac14, 0x000003ff, 0x000000f7,
163 0xac10, 0xffffffff, 0x00000000,
164 0xac0c, 0xffffffff, 0x32761054,
165 0x88d4, 0x0000001f, 0x00000010,
166 0x15c0, 0x000c0fc0, 0x000c0400
167};
168
169static const u32 verde_golden_rlc_registers[] =
170{
171 0xc424, 0xffffffff, 0x033f1005,
172 0xc47c, 0xffffffff, 0x10808020,
173 0xc488, 0xffffffff, 0x00800008,
174 0xc314, 0xffffffff, 0x00001000,
175 0xc30c, 0xffffffff, 0x80010014
176};
177
178static const u32 verde_golden_registers[] =
179{
180 0x9a10, 0x00010000, 0x00018208,
181 0x9830, 0xffffffff, 0x00000000,
182 0x9834, 0xf00fffff, 0x00000400,
183 0x9838, 0x0002021c, 0x00020200,
184 0xc78, 0x00000080, 0x00000000,
185 0xd030, 0x000300c0, 0x00800040,
186 0xd030, 0x000300c0, 0x00800040,
187 0xd830, 0x000300c0, 0x00800040,
188 0xd830, 0x000300c0, 0x00800040,
189 0x5bb0, 0x000000f0, 0x00000070,
190 0x5bc0, 0x00200000, 0x50100000,
191 0x7030, 0x31000311, 0x00000011,
192 0x2ae4, 0x00073ffe, 0x000022a2,
193 0x2ae4, 0x00073ffe, 0x000022a2,
194 0x2ae4, 0x00073ffe, 0x000022a2,
195 0x240c, 0x000007ff, 0x00000000,
196 0x240c, 0x000007ff, 0x00000000,
197 0x240c, 0x000007ff, 0x00000000,
198 0x8a14, 0xf000001f, 0x00000007,
199 0x8a14, 0xf000001f, 0x00000007,
200 0x8a14, 0xf000001f, 0x00000007,
201 0x8b24, 0xffffffff, 0x00ffffff,
202 0x8b10, 0x0000ff0f, 0x00000000,
203 0x28a4c, 0x07ffffff, 0x4e000000,
204 0x28350, 0x3f3f3fff, 0x0000124a,
205 0x28350, 0x3f3f3fff, 0x0000124a,
206 0x28350, 0x3f3f3fff, 0x0000124a,
207 0x30, 0x000000ff, 0x0040,
208 0x34, 0x00000040, 0x00004040,
209 0x9100, 0x07ffffff, 0x03000000,
210 0x9100, 0x07ffffff, 0x03000000,
211 0x8e88, 0x01ff1f3f, 0x00000000,
212 0x8e88, 0x01ff1f3f, 0x00000000,
213 0x8e88, 0x01ff1f3f, 0x00000000,
214 0x8e84, 0x01ff1f3f, 0x00000000,
215 0x8e84, 0x01ff1f3f, 0x00000000,
216 0x8e84, 0x01ff1f3f, 0x00000000,
217 0x9060, 0x0000007f, 0x00000020,
218 0x9508, 0x00010000, 0x00010000,
219 0xac14, 0x000003ff, 0x00000003,
220 0xac14, 0x000003ff, 0x00000003,
221 0xac14, 0x000003ff, 0x00000003,
222 0xac10, 0xffffffff, 0x00000000,
223 0xac10, 0xffffffff, 0x00000000,
224 0xac10, 0xffffffff, 0x00000000,
225 0xac0c, 0xffffffff, 0x00001032,
226 0xac0c, 0xffffffff, 0x00001032,
227 0xac0c, 0xffffffff, 0x00001032,
228 0x88d4, 0x0000001f, 0x00000010,
229 0x88d4, 0x0000001f, 0x00000010,
230 0x88d4, 0x0000001f, 0x00000010,
231 0x15c0, 0x000c0fc0, 0x000c0400
232};
233
234static const u32 oland_golden_rlc_registers[] =
235{
236 0xc424, 0xffffffff, 0x00601005,
237 0xc47c, 0xffffffff, 0x10104040,
238 0xc488, 0xffffffff, 0x0100000a,
239 0xc314, 0xffffffff, 0x00000800,
240 0xc30c, 0xffffffff, 0x800000f4
241};
242
243static const u32 oland_golden_registers[] =
244{
245 0x9a10, 0x00010000, 0x00018208,
246 0x9830, 0xffffffff, 0x00000000,
247 0x9834, 0xf00fffff, 0x00000400,
248 0x9838, 0x0002021c, 0x00020200,
249 0xc78, 0x00000080, 0x00000000,
250 0xd030, 0x000300c0, 0x00800040,
251 0xd830, 0x000300c0, 0x00800040,
252 0x5bb0, 0x000000f0, 0x00000070,
253 0x5bc0, 0x00200000, 0x50100000,
254 0x7030, 0x31000311, 0x00000011,
255 0x2ae4, 0x00073ffe, 0x000022a2,
256 0x240c, 0x000007ff, 0x00000000,
257 0x8a14, 0xf000001f, 0x00000007,
258 0x8b24, 0xffffffff, 0x00ffffff,
259 0x8b10, 0x0000ff0f, 0x00000000,
260 0x28a4c, 0x07ffffff, 0x4e000000,
261 0x28350, 0x3f3f3fff, 0x00000082,
262 0x30, 0x000000ff, 0x0040,
263 0x34, 0x00000040, 0x00004040,
264 0x9100, 0x07ffffff, 0x03000000,
265 0x9060, 0x0000007f, 0x00000020,
266 0x9508, 0x00010000, 0x00010000,
267 0xac14, 0x000003ff, 0x000000f3,
268 0xac10, 0xffffffff, 0x00000000,
269 0xac0c, 0xffffffff, 0x00003210,
270 0x88d4, 0x0000001f, 0x00000010,
271 0x15c0, 0x000c0fc0, 0x000c0400
272};
273
fffbdda4
AD		 274static const u32 hainan_golden_registers[] =
275{
276 0x9a10, 0x00010000, 0x00018208,
277 0x9830, 0xffffffff, 0x00000000,
278 0x9834, 0xf00fffff, 0x00000400,
279 0x9838, 0x0002021c, 0x00020200,
280 0xd0c0, 0xff000fff, 0x00000100,
281 0xd030, 0x000300c0, 0x00800040,
282 0xd8c0, 0xff000fff, 0x00000100,
283 0xd830, 0x000300c0, 0x00800040,
284 0x2ae4, 0x00073ffe, 0x000022a2,
285 0x240c, 0x000007ff, 0x00000000,
286 0x8a14, 0xf000001f, 0x00000007,
287 0x8b24, 0xffffffff, 0x00ffffff,
288 0x8b10, 0x0000ff0f, 0x00000000,
289 0x28a4c, 0x07ffffff, 0x4e000000,
290 0x28350, 0x3f3f3fff, 0x00000000,
291 0x30, 0x000000ff, 0x0040,
292 0x34, 0x00000040, 0x00004040,
293 0x9100, 0x03e00000, 0x03600000,
294 0x9060, 0x0000007f, 0x00000020,
295 0x9508, 0x00010000, 0x00010000,
296 0xac14, 0x000003ff, 0x000000f1,
297 0xac10, 0xffffffff, 0x00000000,
298 0xac0c, 0xffffffff, 0x00003210,
299 0x88d4, 0x0000001f, 0x00000010,
300 0x15c0, 0x000c0fc0, 0x000c0400
301};
302
303static const u32 hainan_golden_registers2[] =
304{
305 0x98f8, 0xffffffff, 0x02010001
306};
307
205996c0
AD		 308static const u32 tahiti_mgcg_cgcg_init[] =
309{
310 0xc400, 0xffffffff, 0xfffffffc,
311 0x802c, 0xffffffff, 0xe0000000,
312 0x9a60, 0xffffffff, 0x00000100,
313 0x92a4, 0xffffffff, 0x00000100,
314 0xc164, 0xffffffff, 0x00000100,
315 0x9774, 0xffffffff, 0x00000100,
316 0x8984, 0xffffffff, 0x06000100,
317 0x8a18, 0xffffffff, 0x00000100,
318 0x92a0, 0xffffffff, 0x00000100,
319 0xc380, 0xffffffff, 0x00000100,
320 0x8b28, 0xffffffff, 0x00000100,
321 0x9144, 0xffffffff, 0x00000100,
322 0x8d88, 0xffffffff, 0x00000100,
323 0x8d8c, 0xffffffff, 0x00000100,
324 0x9030, 0xffffffff, 0x00000100,
325 0x9034, 0xffffffff, 0x00000100,
326 0x9038, 0xffffffff, 0x00000100,
327 0x903c, 0xffffffff, 0x00000100,
328 0xad80, 0xffffffff, 0x00000100,
329 0xac54, 0xffffffff, 0x00000100,
330 0x897c, 0xffffffff, 0x06000100,
331 0x9868, 0xffffffff, 0x00000100,
332 0x9510, 0xffffffff, 0x00000100,
333 0xaf04, 0xffffffff, 0x00000100,
334 0xae04, 0xffffffff, 0x00000100,
335 0x949c, 0xffffffff, 0x00000100,
336 0x802c, 0xffffffff, 0xe0000000,
337 0x9160, 0xffffffff, 0x00010000,
338 0x9164, 0xffffffff, 0x00030002,
339 0x9168, 0xffffffff, 0x00040007,
340 0x916c, 0xffffffff, 0x00060005,
341 0x9170, 0xffffffff, 0x00090008,
342 0x9174, 0xffffffff, 0x00020001,
343 0x9178, 0xffffffff, 0x00040003,
344 0x917c, 0xffffffff, 0x00000007,
345 0x9180, 0xffffffff, 0x00060005,
346 0x9184, 0xffffffff, 0x00090008,
347 0x9188, 0xffffffff, 0x00030002,
348 0x918c, 0xffffffff, 0x00050004,
349 0x9190, 0xffffffff, 0x00000008,
350 0x9194, 0xffffffff, 0x00070006,
351 0x9198, 0xffffffff, 0x000a0009,
352 0x919c, 0xffffffff, 0x00040003,
353 0x91a0, 0xffffffff, 0x00060005,
354 0x91a4, 0xffffffff, 0x00000009,
355 0x91a8, 0xffffffff, 0x00080007,
356 0x91ac, 0xffffffff, 0x000b000a,
357 0x91b0, 0xffffffff, 0x00050004,
358 0x91b4, 0xffffffff, 0x00070006,
359 0x91b8, 0xffffffff, 0x0008000b,
360 0x91bc, 0xffffffff, 0x000a0009,
361 0x91c0, 0xffffffff, 0x000d000c,
362 0x91c4, 0xffffffff, 0x00060005,
363 0x91c8, 0xffffffff, 0x00080007,
364 0x91cc, 0xffffffff, 0x0000000b,
365 0x91d0, 0xffffffff, 0x000a0009,
366 0x91d4, 0xffffffff, 0x000d000c,
367 0x91d8, 0xffffffff, 0x00070006,
368 0x91dc, 0xffffffff, 0x00090008,
369 0x91e0, 0xffffffff, 0x0000000c,
370 0x91e4, 0xffffffff, 0x000b000a,
371 0x91e8, 0xffffffff, 0x000e000d,
372 0x91ec, 0xffffffff, 0x00080007,
373 0x91f0, 0xffffffff, 0x000a0009,
374 0x91f4, 0xffffffff, 0x0000000d,
375 0x91f8, 0xffffffff, 0x000c000b,
376 0x91fc, 0xffffffff, 0x000f000e,
377 0x9200, 0xffffffff, 0x00090008,
378 0x9204, 0xffffffff, 0x000b000a,
379 0x9208, 0xffffffff, 0x000c000f,
380 0x920c, 0xffffffff, 0x000e000d,
381 0x9210, 0xffffffff, 0x00110010,
382 0x9214, 0xffffffff, 0x000a0009,
383 0x9218, 0xffffffff, 0x000c000b,
384 0x921c, 0xffffffff, 0x0000000f,
385 0x9220, 0xffffffff, 0x000e000d,
386 0x9224, 0xffffffff, 0x00110010,
387 0x9228, 0xffffffff, 0x000b000a,
388 0x922c, 0xffffffff, 0x000d000c,
389 0x9230, 0xffffffff, 0x00000010,
390 0x9234, 0xffffffff, 0x000f000e,
391 0x9238, 0xffffffff, 0x00120011,
392 0x923c, 0xffffffff, 0x000c000b,
393 0x9240, 0xffffffff, 0x000e000d,
394 0x9244, 0xffffffff, 0x00000011,
395 0x9248, 0xffffffff, 0x0010000f,
396 0x924c, 0xffffffff, 0x00130012,
397 0x9250, 0xffffffff, 0x000d000c,
398 0x9254, 0xffffffff, 0x000f000e,
399 0x9258, 0xffffffff, 0x00100013,
400 0x925c, 0xffffffff, 0x00120011,
401 0x9260, 0xffffffff, 0x00150014,
402 0x9264, 0xffffffff, 0x000e000d,
403 0x9268, 0xffffffff, 0x0010000f,
404 0x926c, 0xffffffff, 0x00000013,
405 0x9270, 0xffffffff, 0x00120011,
406 0x9274, 0xffffffff, 0x00150014,
407 0x9278, 0xffffffff, 0x000f000e,
408 0x927c, 0xffffffff, 0x00110010,
409 0x9280, 0xffffffff, 0x00000014,
410 0x9284, 0xffffffff, 0x00130012,
411 0x9288, 0xffffffff, 0x00160015,
412 0x928c, 0xffffffff, 0x0010000f,
413 0x9290, 0xffffffff, 0x00120011,
414 0x9294, 0xffffffff, 0x00000015,
415 0x9298, 0xffffffff, 0x00140013,
416 0x929c, 0xffffffff, 0x00170016,
417 0x9150, 0xffffffff, 0x96940200,
418 0x8708, 0xffffffff, 0x00900100,
419 0xc478, 0xffffffff, 0x00000080,
420 0xc404, 0xffffffff, 0x0020003f,
421 0x30, 0xffffffff, 0x0000001c,
422 0x34, 0x000f0000, 0x000f0000,
423 0x160c, 0xffffffff, 0x00000100,
424 0x1024, 0xffffffff, 0x00000100,
425 0x102c, 0x00000101, 0x00000000,
426 0x20a8, 0xffffffff, 0x00000104,
427 0x264c, 0x000c0000, 0x000c0000,
428 0x2648, 0x000c0000, 0x000c0000,
429 0x55e4, 0xff000fff, 0x00000100,
430 0x55e8, 0x00000001, 0x00000001,
431 0x2f50, 0x00000001, 0x00000001,
432 0x30cc, 0xc0000fff, 0x00000104,
433 0xc1e4, 0x00000001, 0x00000001,
434 0xd0c0, 0xfffffff0, 0x00000100,
435 0xd8c0, 0xfffffff0, 0x00000100
436};
437
438static const u32 pitcairn_mgcg_cgcg_init[] =
439{
440 0xc400, 0xffffffff, 0xfffffffc,
441 0x802c, 0xffffffff, 0xe0000000,
442 0x9a60, 0xffffffff, 0x00000100,
443 0x92a4, 0xffffffff, 0x00000100,
444 0xc164, 0xffffffff, 0x00000100,
445 0x9774, 0xffffffff, 0x00000100,
446 0x8984, 0xffffffff, 0x06000100,
447 0x8a18, 0xffffffff, 0x00000100,
448 0x92a0, 0xffffffff, 0x00000100,
449 0xc380, 0xffffffff, 0x00000100,
450 0x8b28, 0xffffffff, 0x00000100,
451 0x9144, 0xffffffff, 0x00000100,
452 0x8d88, 0xffffffff, 0x00000100,
453 0x8d8c, 0xffffffff, 0x00000100,
454 0x9030, 0xffffffff, 0x00000100,
455 0x9034, 0xffffffff, 0x00000100,
456 0x9038, 0xffffffff, 0x00000100,
457 0x903c, 0xffffffff, 0x00000100,
458 0xad80, 0xffffffff, 0x00000100,
459 0xac54, 0xffffffff, 0x00000100,
460 0x897c, 0xffffffff, 0x06000100,
461 0x9868, 0xffffffff, 0x00000100,
462 0x9510, 0xffffffff, 0x00000100,
463 0xaf04, 0xffffffff, 0x00000100,
464 0xae04, 0xffffffff, 0x00000100,
465 0x949c, 0xffffffff, 0x00000100,
466 0x802c, 0xffffffff, 0xe0000000,
467 0x9160, 0xffffffff, 0x00010000,
468 0x9164, 0xffffffff, 0x00030002,
469 0x9168, 0xffffffff, 0x00040007,
470 0x916c, 0xffffffff, 0x00060005,
471 0x9170, 0xffffffff, 0x00090008,
472 0x9174, 0xffffffff, 0x00020001,
473 0x9178, 0xffffffff, 0x00040003,
474 0x917c, 0xffffffff, 0x00000007,
475 0x9180, 0xffffffff, 0x00060005,
476 0x9184, 0xffffffff, 0x00090008,
477 0x9188, 0xffffffff, 0x00030002,
478 0x918c, 0xffffffff, 0x00050004,
479 0x9190, 0xffffffff, 0x00000008,
480 0x9194, 0xffffffff, 0x00070006,
481 0x9198, 0xffffffff, 0x000a0009,
482 0x919c, 0xffffffff, 0x00040003,
483 0x91a0, 0xffffffff, 0x00060005,
484 0x91a4, 0xffffffff, 0x00000009,
485 0x91a8, 0xffffffff, 0x00080007,
486 0x91ac, 0xffffffff, 0x000b000a,
487 0x91b0, 0xffffffff, 0x00050004,
488 0x91b4, 0xffffffff, 0x00070006,
489 0x91b8, 0xffffffff, 0x0008000b,
490 0x91bc, 0xffffffff, 0x000a0009,
491 0x91c0, 0xffffffff, 0x000d000c,
492 0x9200, 0xffffffff, 0x00090008,
493 0x9204, 0xffffffff, 0x000b000a,
494 0x9208, 0xffffffff, 0x000c000f,
495 0x920c, 0xffffffff, 0x000e000d,
496 0x9210, 0xffffffff, 0x00110010,
497 0x9214, 0xffffffff, 0x000a0009,
498 0x9218, 0xffffffff, 0x000c000b,
499 0x921c, 0xffffffff, 0x0000000f,
500 0x9220, 0xffffffff, 0x000e000d,
501 0x9224, 0xffffffff, 0x00110010,
502 0x9228, 0xffffffff, 0x000b000a,
503 0x922c, 0xffffffff, 0x000d000c,
504 0x9230, 0xffffffff, 0x00000010,
505 0x9234, 0xffffffff, 0x000f000e,
506 0x9238, 0xffffffff, 0x00120011,
507 0x923c, 0xffffffff, 0x000c000b,
508 0x9240, 0xffffffff, 0x000e000d,
509 0x9244, 0xffffffff, 0x00000011,
510 0x9248, 0xffffffff, 0x0010000f,
511 0x924c, 0xffffffff, 0x00130012,
512 0x9250, 0xffffffff, 0x000d000c,
513 0x9254, 0xffffffff, 0x000f000e,
514 0x9258, 0xffffffff, 0x00100013,
515 0x925c, 0xffffffff, 0x00120011,
516 0x9260, 0xffffffff, 0x00150014,
517 0x9150, 0xffffffff, 0x96940200,
518 0x8708, 0xffffffff, 0x00900100,
519 0xc478, 0xffffffff, 0x00000080,
520 0xc404, 0xffffffff, 0x0020003f,
521 0x30, 0xffffffff, 0x0000001c,
522 0x34, 0x000f0000, 0x000f0000,
523 0x160c, 0xffffffff, 0x00000100,
524 0x1024, 0xffffffff, 0x00000100,
525 0x102c, 0x00000101, 0x00000000,
526 0x20a8, 0xffffffff, 0x00000104,
527 0x55e4, 0xff000fff, 0x00000100,
528 0x55e8, 0x00000001, 0x00000001,
529 0x2f50, 0x00000001, 0x00000001,
530 0x30cc, 0xc0000fff, 0x00000104,
531 0xc1e4, 0x00000001, 0x00000001,
532 0xd0c0, 0xfffffff0, 0x00000100,
533 0xd8c0, 0xfffffff0, 0x00000100
534};
535
536static const u32 verde_mgcg_cgcg_init[] =
537{
538 0xc400, 0xffffffff, 0xfffffffc,
539 0x802c, 0xffffffff, 0xe0000000,
540 0x9a60, 0xffffffff, 0x00000100,
541 0x92a4, 0xffffffff, 0x00000100,
542 0xc164, 0xffffffff, 0x00000100,
543 0x9774, 0xffffffff, 0x00000100,
544 0x8984, 0xffffffff, 0x06000100,
545 0x8a18, 0xffffffff, 0x00000100,
546 0x92a0, 0xffffffff, 0x00000100,
547 0xc380, 0xffffffff, 0x00000100,
548 0x8b28, 0xffffffff, 0x00000100,
549 0x9144, 0xffffffff, 0x00000100,
550 0x8d88, 0xffffffff, 0x00000100,
551 0x8d8c, 0xffffffff, 0x00000100,
552 0x9030, 0xffffffff, 0x00000100,
553 0x9034, 0xffffffff, 0x00000100,
554 0x9038, 0xffffffff, 0x00000100,
555 0x903c, 0xffffffff, 0x00000100,
556 0xad80, 0xffffffff, 0x00000100,
557 0xac54, 0xffffffff, 0x00000100,
558 0x897c, 0xffffffff, 0x06000100,
559 0x9868, 0xffffffff, 0x00000100,
560 0x9510, 0xffffffff, 0x00000100,
561 0xaf04, 0xffffffff, 0x00000100,
562 0xae04, 0xffffffff, 0x00000100,
563 0x949c, 0xffffffff, 0x00000100,
564 0x802c, 0xffffffff, 0xe0000000,
565 0x9160, 0xffffffff, 0x00010000,
566 0x9164, 0xffffffff, 0x00030002,
567 0x9168, 0xffffffff, 0x00040007,
568 0x916c, 0xffffffff, 0x00060005,
569 0x9170, 0xffffffff, 0x00090008,
570 0x9174, 0xffffffff, 0x00020001,
571 0x9178, 0xffffffff, 0x00040003,
572 0x917c, 0xffffffff, 0x00000007,
573 0x9180, 0xffffffff, 0x00060005,
574 0x9184, 0xffffffff, 0x00090008,
575 0x9188, 0xffffffff, 0x00030002,
576 0x918c, 0xffffffff, 0x00050004,
577 0x9190, 0xffffffff, 0x00000008,
578 0x9194, 0xffffffff, 0x00070006,
579 0x9198, 0xffffffff, 0x000a0009,
580 0x919c, 0xffffffff, 0x00040003,
581 0x91a0, 0xffffffff, 0x00060005,
582 0x91a4, 0xffffffff, 0x00000009,
583 0x91a8, 0xffffffff, 0x00080007,
584 0x91ac, 0xffffffff, 0x000b000a,
585 0x91b0, 0xffffffff, 0x00050004,
586 0x91b4, 0xffffffff, 0x00070006,
587 0x91b8, 0xffffffff, 0x0008000b,
588 0x91bc, 0xffffffff, 0x000a0009,
589 0x91c0, 0xffffffff, 0x000d000c,
590 0x9200, 0xffffffff, 0x00090008,
591 0x9204, 0xffffffff, 0x000b000a,
592 0x9208, 0xffffffff, 0x000c000f,
593 0x920c, 0xffffffff, 0x000e000d,
594 0x9210, 0xffffffff, 0x00110010,
595 0x9214, 0xffffffff, 0x000a0009,
596 0x9218, 0xffffffff, 0x000c000b,
597 0x921c, 0xffffffff, 0x0000000f,
598 0x9220, 0xffffffff, 0x000e000d,
599 0x9224, 0xffffffff, 0x00110010,
600 0x9228, 0xffffffff, 0x000b000a,
601 0x922c, 0xffffffff, 0x000d000c,
602 0x9230, 0xffffffff, 0x00000010,
603 0x9234, 0xffffffff, 0x000f000e,
604 0x9238, 0xffffffff, 0x00120011,
605 0x923c, 0xffffffff, 0x000c000b,
606 0x9240, 0xffffffff, 0x000e000d,
607 0x9244, 0xffffffff, 0x00000011,
608 0x9248, 0xffffffff, 0x0010000f,
609 0x924c, 0xffffffff, 0x00130012,
610 0x9250, 0xffffffff, 0x000d000c,
611 0x9254, 0xffffffff, 0x000f000e,
612 0x9258, 0xffffffff, 0x00100013,
613 0x925c, 0xffffffff, 0x00120011,
614 0x9260, 0xffffffff, 0x00150014,
615 0x9150, 0xffffffff, 0x96940200,
616 0x8708, 0xffffffff, 0x00900100,
617 0xc478, 0xffffffff, 0x00000080,
618 0xc404, 0xffffffff, 0x0020003f,
619 0x30, 0xffffffff, 0x0000001c,
620 0x34, 0x000f0000, 0x000f0000,
621 0x160c, 0xffffffff, 0x00000100,
622 0x1024, 0xffffffff, 0x00000100,
623 0x102c, 0x00000101, 0x00000000,
624 0x20a8, 0xffffffff, 0x00000104,
625 0x264c, 0x000c0000, 0x000c0000,
626 0x2648, 0x000c0000, 0x000c0000,
627 0x55e4, 0xff000fff, 0x00000100,
628 0x55e8, 0x00000001, 0x00000001,
629 0x2f50, 0x00000001, 0x00000001,
630 0x30cc, 0xc0000fff, 0x00000104,
631 0xc1e4, 0x00000001, 0x00000001,
632 0xd0c0, 0xfffffff0, 0x00000100,
633 0xd8c0, 0xfffffff0, 0x00000100
634};
635
636static const u32 oland_mgcg_cgcg_init[] =
637{
638 0xc400, 0xffffffff, 0xfffffffc,
639 0x802c, 0xffffffff, 0xe0000000,
640 0x9a60, 0xffffffff, 0x00000100,
641 0x92a4, 0xffffffff, 0x00000100,
642 0xc164, 0xffffffff, 0x00000100,
643 0x9774, 0xffffffff, 0x00000100,
644 0x8984, 0xffffffff, 0x06000100,
645 0x8a18, 0xffffffff, 0x00000100,
646 0x92a0, 0xffffffff, 0x00000100,
647 0xc380, 0xffffffff, 0x00000100,
648 0x8b28, 0xffffffff, 0x00000100,
649 0x9144, 0xffffffff, 0x00000100,
650 0x8d88, 0xffffffff, 0x00000100,
651 0x8d8c, 0xffffffff, 0x00000100,
652 0x9030, 0xffffffff, 0x00000100,
653 0x9034, 0xffffffff, 0x00000100,
654 0x9038, 0xffffffff, 0x00000100,
655 0x903c, 0xffffffff, 0x00000100,
656 0xad80, 0xffffffff, 0x00000100,
657 0xac54, 0xffffffff, 0x00000100,
658 0x897c, 0xffffffff, 0x06000100,
659 0x9868, 0xffffffff, 0x00000100,
660 0x9510, 0xffffffff, 0x00000100,
661 0xaf04, 0xffffffff, 0x00000100,
662 0xae04, 0xffffffff, 0x00000100,
663 0x949c, 0xffffffff, 0x00000100,
664 0x802c, 0xffffffff, 0xe0000000,
665 0x9160, 0xffffffff, 0x00010000,
666 0x9164, 0xffffffff, 0x00030002,
667 0x9168, 0xffffffff, 0x00040007,
668 0x916c, 0xffffffff, 0x00060005,
669 0x9170, 0xffffffff, 0x00090008,
670 0x9174, 0xffffffff, 0x00020001,
671 0x9178, 0xffffffff, 0x00040003,
672 0x917c, 0xffffffff, 0x00000007,
673 0x9180, 0xffffffff, 0x00060005,
674 0x9184, 0xffffffff, 0x00090008,
675 0x9188, 0xffffffff, 0x00030002,
676 0x918c, 0xffffffff, 0x00050004,
677 0x9190, 0xffffffff, 0x00000008,
678 0x9194, 0xffffffff, 0x00070006,
679 0x9198, 0xffffffff, 0x000a0009,
680 0x919c, 0xffffffff, 0x00040003,
681 0x91a0, 0xffffffff, 0x00060005,
682 0x91a4, 0xffffffff, 0x00000009,
683 0x91a8, 0xffffffff, 0x00080007,
684 0x91ac, 0xffffffff, 0x000b000a,
685 0x91b0, 0xffffffff, 0x00050004,
686 0x91b4, 0xffffffff, 0x00070006,
687 0x91b8, 0xffffffff, 0x0008000b,
688 0x91bc, 0xffffffff, 0x000a0009,
689 0x91c0, 0xffffffff, 0x000d000c,
690 0x91c4, 0xffffffff, 0x00060005,
691 0x91c8, 0xffffffff, 0x00080007,
692 0x91cc, 0xffffffff, 0x0000000b,
693 0x91d0, 0xffffffff, 0x000a0009,
694 0x91d4, 0xffffffff, 0x000d000c,
695 0x9150, 0xffffffff, 0x96940200,
696 0x8708, 0xffffffff, 0x00900100,
697 0xc478, 0xffffffff, 0x00000080,
698 0xc404, 0xffffffff, 0x0020003f,
699 0x30, 0xffffffff, 0x0000001c,
700 0x34, 0x000f0000, 0x000f0000,
701 0x160c, 0xffffffff, 0x00000100,
702 0x1024, 0xffffffff, 0x00000100,
703 0x102c, 0x00000101, 0x00000000,
704 0x20a8, 0xffffffff, 0x00000104,
705 0x264c, 0x000c0000, 0x000c0000,
706 0x2648, 0x000c0000, 0x000c0000,
707 0x55e4, 0xff000fff, 0x00000100,
708 0x55e8, 0x00000001, 0x00000001,
709 0x2f50, 0x00000001, 0x00000001,
710 0x30cc, 0xc0000fff, 0x00000104,
711 0xc1e4, 0x00000001, 0x00000001,
712 0xd0c0, 0xfffffff0, 0x00000100,
713 0xd8c0, 0xfffffff0, 0x00000100
714};
715
fffbdda4
AD		 716static const u32 hainan_mgcg_cgcg_init[] =
717{
718 0xc400, 0xffffffff, 0xfffffffc,
719 0x802c, 0xffffffff, 0xe0000000,
720 0x9a60, 0xffffffff, 0x00000100,
721 0x92a4, 0xffffffff, 0x00000100,
722 0xc164, 0xffffffff, 0x00000100,
723 0x9774, 0xffffffff, 0x00000100,
724 0x8984, 0xffffffff, 0x06000100,
725 0x8a18, 0xffffffff, 0x00000100,
726 0x92a0, 0xffffffff, 0x00000100,
727 0xc380, 0xffffffff, 0x00000100,
728 0x8b28, 0xffffffff, 0x00000100,
729 0x9144, 0xffffffff, 0x00000100,
730 0x8d88, 0xffffffff, 0x00000100,
731 0x8d8c, 0xffffffff, 0x00000100,
732 0x9030, 0xffffffff, 0x00000100,
733 0x9034, 0xffffffff, 0x00000100,
734 0x9038, 0xffffffff, 0x00000100,
735 0x903c, 0xffffffff, 0x00000100,
736 0xad80, 0xffffffff, 0x00000100,
737 0xac54, 0xffffffff, 0x00000100,
738 0x897c, 0xffffffff, 0x06000100,
739 0x9868, 0xffffffff, 0x00000100,
740 0x9510, 0xffffffff, 0x00000100,
741 0xaf04, 0xffffffff, 0x00000100,
742 0xae04, 0xffffffff, 0x00000100,
743 0x949c, 0xffffffff, 0x00000100,
744 0x802c, 0xffffffff, 0xe0000000,
745 0x9160, 0xffffffff, 0x00010000,
746 0x9164, 0xffffffff, 0x00030002,
747 0x9168, 0xffffffff, 0x00040007,
748 0x916c, 0xffffffff, 0x00060005,
749 0x9170, 0xffffffff, 0x00090008,
750 0x9174, 0xffffffff, 0x00020001,
751 0x9178, 0xffffffff, 0x00040003,
752 0x917c, 0xffffffff, 0x00000007,
753 0x9180, 0xffffffff, 0x00060005,
754 0x9184, 0xffffffff, 0x00090008,
755 0x9188, 0xffffffff, 0x00030002,
756 0x918c, 0xffffffff, 0x00050004,
757 0x9190, 0xffffffff, 0x00000008,
758 0x9194, 0xffffffff, 0x00070006,
759 0x9198, 0xffffffff, 0x000a0009,
760 0x919c, 0xffffffff, 0x00040003,
761 0x91a0, 0xffffffff, 0x00060005,
762 0x91a4, 0xffffffff, 0x00000009,
763 0x91a8, 0xffffffff, 0x00080007,
764 0x91ac, 0xffffffff, 0x000b000a,
765 0x91b0, 0xffffffff, 0x00050004,
766 0x91b4, 0xffffffff, 0x00070006,
767 0x91b8, 0xffffffff, 0x0008000b,
768 0x91bc, 0xffffffff, 0x000a0009,
769 0x91c0, 0xffffffff, 0x000d000c,
770 0x91c4, 0xffffffff, 0x00060005,
771 0x91c8, 0xffffffff, 0x00080007,
772 0x91cc, 0xffffffff, 0x0000000b,
773 0x91d0, 0xffffffff, 0x000a0009,
774 0x91d4, 0xffffffff, 0x000d000c,
775 0x9150, 0xffffffff, 0x96940200,
776 0x8708, 0xffffffff, 0x00900100,
777 0xc478, 0xffffffff, 0x00000080,
778 0xc404, 0xffffffff, 0x0020003f,
779 0x30, 0xffffffff, 0x0000001c,
780 0x34, 0x000f0000, 0x000f0000,
781 0x160c, 0xffffffff, 0x00000100,
782 0x1024, 0xffffffff, 0x00000100,
783 0x20a8, 0xffffffff, 0x00000104,
784 0x264c, 0x000c0000, 0x000c0000,
785 0x2648, 0x000c0000, 0x000c0000,
786 0x2f50, 0x00000001, 0x00000001,
787 0x30cc, 0xc0000fff, 0x00000104,
788 0xc1e4, 0x00000001, 0x00000001,
789 0xd0c0, 0xfffffff0, 0x00000100,
790 0xd8c0, 0xfffffff0, 0x00000100
791};
792
205996c0
AD		 793static u32 verde_pg_init[] =
794{
795 0x353c, 0xffffffff, 0x40000,
796 0x3538, 0xffffffff, 0x200010ff,
797 0x353c, 0xffffffff, 0x0,
798 0x353c, 0xffffffff, 0x0,
799 0x353c, 0xffffffff, 0x0,
800 0x353c, 0xffffffff, 0x0,
801 0x353c, 0xffffffff, 0x0,
802 0x353c, 0xffffffff, 0x7007,
803 0x3538, 0xffffffff, 0x300010ff,
804 0x353c, 0xffffffff, 0x0,
805 0x353c, 0xffffffff, 0x0,
806 0x353c, 0xffffffff, 0x0,
807 0x353c, 0xffffffff, 0x0,
808 0x353c, 0xffffffff, 0x0,
809 0x353c, 0xffffffff, 0x400000,
810 0x3538, 0xffffffff, 0x100010ff,
811 0x353c, 0xffffffff, 0x0,
812 0x353c, 0xffffffff, 0x0,
813 0x353c, 0xffffffff, 0x0,
814 0x353c, 0xffffffff, 0x0,
815 0x353c, 0xffffffff, 0x0,
816 0x353c, 0xffffffff, 0x120200,
817 0x3538, 0xffffffff, 0x500010ff,
818 0x353c, 0xffffffff, 0x0,
819 0x353c, 0xffffffff, 0x0,
820 0x353c, 0xffffffff, 0x0,
821 0x353c, 0xffffffff, 0x0,
822 0x353c, 0xffffffff, 0x0,
823 0x353c, 0xffffffff, 0x1e1e16,
824 0x3538, 0xffffffff, 0x600010ff,
825 0x353c, 0xffffffff, 0x0,
826 0x353c, 0xffffffff, 0x0,
827 0x353c, 0xffffffff, 0x0,
828 0x353c, 0xffffffff, 0x0,
829 0x353c, 0xffffffff, 0x0,
830 0x353c, 0xffffffff, 0x171f1e,
831 0x3538, 0xffffffff, 0x700010ff,
832 0x353c, 0xffffffff, 0x0,
833 0x353c, 0xffffffff, 0x0,
834 0x353c, 0xffffffff, 0x0,
835 0x353c, 0xffffffff, 0x0,
836 0x353c, 0xffffffff, 0x0,
837 0x353c, 0xffffffff, 0x0,
838 0x3538, 0xffffffff, 0x9ff,
839 0x3500, 0xffffffff, 0x0,
840 0x3504, 0xffffffff, 0x10000800,
841 0x3504, 0xffffffff, 0xf,
842 0x3504, 0xffffffff, 0xf,
843 0x3500, 0xffffffff, 0x4,
844 0x3504, 0xffffffff, 0x1000051e,
845 0x3504, 0xffffffff, 0xffff,
846 0x3504, 0xffffffff, 0xffff,
847 0x3500, 0xffffffff, 0x8,
848 0x3504, 0xffffffff, 0x80500,
849 0x3500, 0xffffffff, 0x12,
850 0x3504, 0xffffffff, 0x9050c,
851 0x3500, 0xffffffff, 0x1d,
852 0x3504, 0xffffffff, 0xb052c,
853 0x3500, 0xffffffff, 0x2a,
854 0x3504, 0xffffffff, 0x1053e,
855 0x3500, 0xffffffff, 0x2d,
856 0x3504, 0xffffffff, 0x10546,
857 0x3500, 0xffffffff, 0x30,
858 0x3504, 0xffffffff, 0xa054e,
859 0x3500, 0xffffffff, 0x3c,
860 0x3504, 0xffffffff, 0x1055f,
861 0x3500, 0xffffffff, 0x3f,
862 0x3504, 0xffffffff, 0x10567,
863 0x3500, 0xffffffff, 0x42,
864 0x3504, 0xffffffff, 0x1056f,
865 0x3500, 0xffffffff, 0x45,
866 0x3504, 0xffffffff, 0x10572,
867 0x3500, 0xffffffff, 0x48,
868 0x3504, 0xffffffff, 0x20575,
869 0x3500, 0xffffffff, 0x4c,
870 0x3504, 0xffffffff, 0x190801,
871 0x3500, 0xffffffff, 0x67,
872 0x3504, 0xffffffff, 0x1082a,
873 0x3500, 0xffffffff, 0x6a,
874 0x3504, 0xffffffff, 0x1b082d,
875 0x3500, 0xffffffff, 0x87,
876 0x3504, 0xffffffff, 0x310851,
877 0x3500, 0xffffffff, 0xba,
878 0x3504, 0xffffffff, 0x891,
879 0x3500, 0xffffffff, 0xbc,
880 0x3504, 0xffffffff, 0x893,
881 0x3500, 0xffffffff, 0xbe,
882 0x3504, 0xffffffff, 0x20895,
883 0x3500, 0xffffffff, 0xc2,
884 0x3504, 0xffffffff, 0x20899,
885 0x3500, 0xffffffff, 0xc6,
886 0x3504, 0xffffffff, 0x2089d,
887 0x3500, 0xffffffff, 0xca,
888 0x3504, 0xffffffff, 0x8a1,
889 0x3500, 0xffffffff, 0xcc,
890 0x3504, 0xffffffff, 0x8a3,
891 0x3500, 0xffffffff, 0xce,
892 0x3504, 0xffffffff, 0x308a5,
893 0x3500, 0xffffffff, 0xd3,
894 0x3504, 0xffffffff, 0x6d08cd,
895 0x3500, 0xffffffff, 0x142,
896 0x3504, 0xffffffff, 0x2000095a,
897 0x3504, 0xffffffff, 0x1,
898 0x3500, 0xffffffff, 0x144,
899 0x3504, 0xffffffff, 0x301f095b,
900 0x3500, 0xffffffff, 0x165,
901 0x3504, 0xffffffff, 0xc094d,
902 0x3500, 0xffffffff, 0x173,
903 0x3504, 0xffffffff, 0xf096d,
904 0x3500, 0xffffffff, 0x184,
905 0x3504, 0xffffffff, 0x15097f,
906 0x3500, 0xffffffff, 0x19b,
907 0x3504, 0xffffffff, 0xc0998,
908 0x3500, 0xffffffff, 0x1a9,
909 0x3504, 0xffffffff, 0x409a7,
910 0x3500, 0xffffffff, 0x1af,
911 0x3504, 0xffffffff, 0xcdc,
912 0x3500, 0xffffffff, 0x1b1,
913 0x3504, 0xffffffff, 0x800,
914 0x3508, 0xffffffff, 0x6c9b2000,
915 0x3510, 0xfc00, 0x2000,
916 0x3544, 0xffffffff, 0xfc0,
917 0x28d4, 0x00000100, 0x100
918};
919
920static void si_init_golden_registers(struct radeon_device *rdev)
921{
922 switch (rdev->family) {
923 case CHIP_TAHITI:
924 radeon_program_register_sequence(rdev,
925 tahiti_golden_registers,
926 (const u32)ARRAY_SIZE(tahiti_golden_registers));
927 radeon_program_register_sequence(rdev,
928 tahiti_golden_rlc_registers,
929 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
930 radeon_program_register_sequence(rdev,
931 tahiti_mgcg_cgcg_init,
932 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
933 radeon_program_register_sequence(rdev,
934 tahiti_golden_registers2,
935 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
936 break;
937 case CHIP_PITCAIRN:
938 radeon_program_register_sequence(rdev,
939 pitcairn_golden_registers,
940 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
941 radeon_program_register_sequence(rdev,
942 pitcairn_golden_rlc_registers,
943 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
944 radeon_program_register_sequence(rdev,
945 pitcairn_mgcg_cgcg_init,
946 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
947 break;
948 case CHIP_VERDE:
949 radeon_program_register_sequence(rdev,
950 verde_golden_registers,
951 (const u32)ARRAY_SIZE(verde_golden_registers));
952 radeon_program_register_sequence(rdev,
953 verde_golden_rlc_registers,
954 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
955 radeon_program_register_sequence(rdev,
956 verde_mgcg_cgcg_init,
957 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
958 radeon_program_register_sequence(rdev,
959 verde_pg_init,
960 (const u32)ARRAY_SIZE(verde_pg_init));
961 break;
962 case CHIP_OLAND:
963 radeon_program_register_sequence(rdev,
964 oland_golden_registers,
965 (const u32)ARRAY_SIZE(oland_golden_registers));
966 radeon_program_register_sequence(rdev,
967 oland_golden_rlc_registers,
968 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
969 radeon_program_register_sequence(rdev,
970 oland_mgcg_cgcg_init,
971 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
972 break;
fffbdda4
AD		 973 case CHIP_HAINAN:
974 radeon_program_register_sequence(rdev,
975 hainan_golden_registers,
976 (const u32)ARRAY_SIZE(hainan_golden_registers));
977 radeon_program_register_sequence(rdev,
978 hainan_golden_registers2,
979 (const u32)ARRAY_SIZE(hainan_golden_registers2));
980 radeon_program_register_sequence(rdev,
981 hainan_mgcg_cgcg_init,
982 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
983 break;
205996c0
AD		 984 default:
985 break;
986 }
987}
988
454d2e2a
AD		 989#define PCIE_BUS_CLK 10000
990#define TCLK (PCIE_BUS_CLK / 10)
991
992/**
993 * si_get_xclk - get the xclk
994 *
995 * @rdev: radeon_device pointer
996 *
997 * Returns the reference clock used by the gfx engine
998 * (SI).
999 */
1000u32 si_get_xclk(struct radeon_device *rdev)
1001{
1002 u32 reference_clock = rdev->clock.spll.reference_freq;
1003 u32 tmp;
1004
1005 tmp = RREG32(CG_CLKPIN_CNTL_2);
1006 if (tmp & MUX_TCLK_TO_XCLK)
1007 return TCLK;
1008
1009 tmp = RREG32(CG_CLKPIN_CNTL);
1010 if (tmp & XTALIN_DIVIDE)
1011 return reference_clock / 4;
1012
1013 return reference_clock;
1014}
1015
1bd47d2e
AD		 1016/* get temperature in millidegrees */
1017int si_get_temp(struct radeon_device *rdev)
1018{
1019 u32 temp;
1020 int actual_temp = 0;
1021
1022 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1023 CTF_TEMP_SHIFT;
1024
1025 if (temp & 0x200)
1026 actual_temp = 255;
1027 else
1028 actual_temp = temp & 0x1ff;
1029
1030 actual_temp = (actual_temp * 1000);
1031
1032 return actual_temp;
1033}
1034
8b074dd6
AD		 1035#define TAHITI_IO_MC_REGS_SIZE 36
1036
1037static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1038 {0x0000006f, 0x03044000},
1039 {0x00000070, 0x0480c018},
1040 {0x00000071, 0x00000040},
1041 {0x00000072, 0x01000000},
1042 {0x00000074, 0x000000ff},
1043 {0x00000075, 0x00143400},
1044 {0x00000076, 0x08ec0800},
1045 {0x00000077, 0x040000cc},
1046 {0x00000079, 0x00000000},
1047 {0x0000007a, 0x21000409},
1048 {0x0000007c, 0x00000000},
1049 {0x0000007d, 0xe8000000},
1050 {0x0000007e, 0x044408a8},
1051 {0x0000007f, 0x00000003},
1052 {0x00000080, 0x00000000},
1053 {0x00000081, 0x01000000},
1054 {0x00000082, 0x02000000},
1055 {0x00000083, 0x00000000},
1056 {0x00000084, 0xe3f3e4f4},
1057 {0x00000085, 0x00052024},
1058 {0x00000087, 0x00000000},
1059 {0x00000088, 0x66036603},
1060 {0x00000089, 0x01000000},
1061 {0x0000008b, 0x1c0a0000},
1062 {0x0000008c, 0xff010000},
1063 {0x0000008e, 0xffffefff},
1064 {0x0000008f, 0xfff3efff},
1065 {0x00000090, 0xfff3efbf},
1066 {0x00000094, 0x00101101},
1067 {0x00000095, 0x00000fff},
1068 {0x00000096, 0x00116fff},
1069 {0x00000097, 0x60010000},
1070 {0x00000098, 0x10010000},
1071 {0x00000099, 0x00006000},
1072 {0x0000009a, 0x00001000},
1073 {0x0000009f, 0x00a77400}
1074};
1075
1076static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1077 {0x0000006f, 0x03044000},
1078 {0x00000070, 0x0480c018},
1079 {0x00000071, 0x00000040},
1080 {0x00000072, 0x01000000},
1081 {0x00000074, 0x000000ff},
1082 {0x00000075, 0x00143400},
1083 {0x00000076, 0x08ec0800},
1084 {0x00000077, 0x040000cc},
1085 {0x00000079, 0x00000000},
1086 {0x0000007a, 0x21000409},
1087 {0x0000007c, 0x00000000},
1088 {0x0000007d, 0xe8000000},
1089 {0x0000007e, 0x044408a8},
1090 {0x0000007f, 0x00000003},
1091 {0x00000080, 0x00000000},
1092 {0x00000081, 0x01000000},
1093 {0x00000082, 0x02000000},
1094 {0x00000083, 0x00000000},
1095 {0x00000084, 0xe3f3e4f4},
1096 {0x00000085, 0x00052024},
1097 {0x00000087, 0x00000000},
1098 {0x00000088, 0x66036603},
1099 {0x00000089, 0x01000000},
1100 {0x0000008b, 0x1c0a0000},
1101 {0x0000008c, 0xff010000},
1102 {0x0000008e, 0xffffefff},
1103 {0x0000008f, 0xfff3efff},
1104 {0x00000090, 0xfff3efbf},
1105 {0x00000094, 0x00101101},
1106 {0x00000095, 0x00000fff},
1107 {0x00000096, 0x00116fff},
1108 {0x00000097, 0x60010000},
1109 {0x00000098, 0x10010000},
1110 {0x00000099, 0x00006000},
1111 {0x0000009a, 0x00001000},
1112 {0x0000009f, 0x00a47400}
1113};
1114
1115static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1116 {0x0000006f, 0x03044000},
1117 {0x00000070, 0x0480c018},
1118 {0x00000071, 0x00000040},
1119 {0x00000072, 0x01000000},
1120 {0x00000074, 0x000000ff},
1121 {0x00000075, 0x00143400},
1122 {0x00000076, 0x08ec0800},
1123 {0x00000077, 0x040000cc},
1124 {0x00000079, 0x00000000},
1125 {0x0000007a, 0x21000409},
1126 {0x0000007c, 0x00000000},
1127 {0x0000007d, 0xe8000000},
1128 {0x0000007e, 0x044408a8},
1129 {0x0000007f, 0x00000003},
1130 {0x00000080, 0x00000000},
1131 {0x00000081, 0x01000000},
1132 {0x00000082, 0x02000000},
1133 {0x00000083, 0x00000000},
1134 {0x00000084, 0xe3f3e4f4},
1135 {0x00000085, 0x00052024},
1136 {0x00000087, 0x00000000},
1137 {0x00000088, 0x66036603},
1138 {0x00000089, 0x01000000},
1139 {0x0000008b, 0x1c0a0000},
1140 {0x0000008c, 0xff010000},
1141 {0x0000008e, 0xffffefff},
1142 {0x0000008f, 0xfff3efff},
1143 {0x00000090, 0xfff3efbf},
1144 {0x00000094, 0x00101101},
1145 {0x00000095, 0x00000fff},
1146 {0x00000096, 0x00116fff},
1147 {0x00000097, 0x60010000},
1148 {0x00000098, 0x10010000},
1149 {0x00000099, 0x00006000},
1150 {0x0000009a, 0x00001000},
1151 {0x0000009f, 0x00a37400}
1152};
1153
bcc7f5d2
AD		 1154static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1155 {0x0000006f, 0x03044000},
1156 {0x00000070, 0x0480c018},
1157 {0x00000071, 0x00000040},
1158 {0x00000072, 0x01000000},
1159 {0x00000074, 0x000000ff},
1160 {0x00000075, 0x00143400},
1161 {0x00000076, 0x08ec0800},
1162 {0x00000077, 0x040000cc},
1163 {0x00000079, 0x00000000},
1164 {0x0000007a, 0x21000409},
1165 {0x0000007c, 0x00000000},
1166 {0x0000007d, 0xe8000000},
1167 {0x0000007e, 0x044408a8},
1168 {0x0000007f, 0x00000003},
1169 {0x00000080, 0x00000000},
1170 {0x00000081, 0x01000000},
1171 {0x00000082, 0x02000000},
1172 {0x00000083, 0x00000000},
1173 {0x00000084, 0xe3f3e4f4},
1174 {0x00000085, 0x00052024},
1175 {0x00000087, 0x00000000},
1176 {0x00000088, 0x66036603},
1177 {0x00000089, 0x01000000},
1178 {0x0000008b, 0x1c0a0000},
1179 {0x0000008c, 0xff010000},
1180 {0x0000008e, 0xffffefff},
1181 {0x0000008f, 0xfff3efff},
1182 {0x00000090, 0xfff3efbf},
1183 {0x00000094, 0x00101101},
1184 {0x00000095, 0x00000fff},
1185 {0x00000096, 0x00116fff},
1186 {0x00000097, 0x60010000},
1187 {0x00000098, 0x10010000},
1188 {0x00000099, 0x00006000},
1189 {0x0000009a, 0x00001000},
1190 {0x0000009f, 0x00a17730}
1191};
1192
c04c00b4
AD		 1193static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1194 {0x0000006f, 0x03044000},
1195 {0x00000070, 0x0480c018},
1196 {0x00000071, 0x00000040},
1197 {0x00000072, 0x01000000},
1198 {0x00000074, 0x000000ff},
1199 {0x00000075, 0x00143400},
1200 {0x00000076, 0x08ec0800},
1201 {0x00000077, 0x040000cc},
1202 {0x00000079, 0x00000000},
1203 {0x0000007a, 0x21000409},
1204 {0x0000007c, 0x00000000},
1205 {0x0000007d, 0xe8000000},
1206 {0x0000007e, 0x044408a8},
1207 {0x0000007f, 0x00000003},
1208 {0x00000080, 0x00000000},
1209 {0x00000081, 0x01000000},
1210 {0x00000082, 0x02000000},
1211 {0x00000083, 0x00000000},
1212 {0x00000084, 0xe3f3e4f4},
1213 {0x00000085, 0x00052024},
1214 {0x00000087, 0x00000000},
1215 {0x00000088, 0x66036603},
1216 {0x00000089, 0x01000000},
1217 {0x0000008b, 0x1c0a0000},
1218 {0x0000008c, 0xff010000},
1219 {0x0000008e, 0xffffefff},
1220 {0x0000008f, 0xfff3efff},
1221 {0x00000090, 0xfff3efbf},
1222 {0x00000094, 0x00101101},
1223 {0x00000095, 0x00000fff},
1224 {0x00000096, 0x00116fff},
1225 {0x00000097, 0x60010000},
1226 {0x00000098, 0x10010000},
1227 {0x00000099, 0x00006000},
1228 {0x0000009a, 0x00001000},
1229 {0x0000009f, 0x00a07730}
1230};
1231
8b074dd6
AD		 1232/* ucode loading */
1233static int si_mc_load_microcode(struct radeon_device *rdev)
1234{
1235 const __be32 *fw_data;
1236 u32 running, blackout = 0;
1237 u32 *io_mc_regs;
1238 int i, ucode_size, regs_size;
1239
1240 if (!rdev->mc_fw)
1241 return -EINVAL;
1242
1243 switch (rdev->family) {
1244 case CHIP_TAHITI:
1245 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1246 ucode_size = SI_MC_UCODE_SIZE;
1247 regs_size = TAHITI_IO_MC_REGS_SIZE;
1248 break;
1249 case CHIP_PITCAIRN:
1250 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1251 ucode_size = SI_MC_UCODE_SIZE;
1252 regs_size = TAHITI_IO_MC_REGS_SIZE;
1253 break;
1254 case CHIP_VERDE:
1255 default:
1256 io_mc_regs = (u32 *)&verde_io_mc_regs;
1257 ucode_size = SI_MC_UCODE_SIZE;
1258 regs_size = TAHITI_IO_MC_REGS_SIZE;
1259 break;
bcc7f5d2
AD		 1260 case CHIP_OLAND:
1261 io_mc_regs = (u32 *)&oland_io_mc_regs;
1262 ucode_size = OLAND_MC_UCODE_SIZE;
1263 regs_size = TAHITI_IO_MC_REGS_SIZE;
1264 break;
c04c00b4
AD		 1265 case CHIP_HAINAN:
1266 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1267 ucode_size = OLAND_MC_UCODE_SIZE;
1268 regs_size = TAHITI_IO_MC_REGS_SIZE;
1269 break;
8b074dd6
AD		 1270 }
1271
1272 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1273
1274 if (running == 0) {
1275 if (running) {
1276 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1277 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1278 }
1279
1280 /* reset the engine and set to writable */
1281 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1282 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1283
1284 /* load mc io regs */
1285 for (i = 0; i < regs_size; i++) {
1286 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1287 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1288 }
1289 /* load the MC ucode */
1290 fw_data = (const __be32 *)rdev->mc_fw->data;
1291 for (i = 0; i < ucode_size; i++)
1292 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1293
1294 /* put the engine back into the active state */
1295 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1296 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1297 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1298
1299 /* wait for training to complete */
1300 for (i = 0; i < rdev->usec_timeout; i++) {
1301 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1302 break;
1303 udelay(1);
1304 }
1305 for (i = 0; i < rdev->usec_timeout; i++) {
1306 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1307 break;
1308 udelay(1);
1309 }
1310
1311 if (running)
1312 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1313 }
1314
1315 return 0;
1316}
1317
0f0de06c
AD		 1318static int si_init_microcode(struct radeon_device *rdev)
1319{
1320 struct platform_device *pdev;
1321 const char *chip_name;
1322 const char *rlc_chip_name;
1323 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1324 char fw_name[30];
1325 int err;
1326
1327 DRM_DEBUG("\n");
1328
1329 pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1330 err = IS_ERR(pdev);
1331 if (err) {
1332 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1333 return -EINVAL;
1334 }
1335
1336 switch (rdev->family) {
1337 case CHIP_TAHITI:
1338 chip_name = "TAHITI";
1339 rlc_chip_name = "TAHITI";
1340 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1341 me_req_size = SI_PM4_UCODE_SIZE * 4;
1342 ce_req_size = SI_CE_UCODE_SIZE * 4;
1343 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1344 mc_req_size = SI_MC_UCODE_SIZE * 4;
1345 break;
1346 case CHIP_PITCAIRN:
1347 chip_name = "PITCAIRN";
1348 rlc_chip_name = "PITCAIRN";
1349 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1350 me_req_size = SI_PM4_UCODE_SIZE * 4;
1351 ce_req_size = SI_CE_UCODE_SIZE * 4;
1352 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1353 mc_req_size = SI_MC_UCODE_SIZE * 4;
1354 break;
1355 case CHIP_VERDE:
1356 chip_name = "VERDE";
1357 rlc_chip_name = "VERDE";
1358 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1359 me_req_size = SI_PM4_UCODE_SIZE * 4;
1360 ce_req_size = SI_CE_UCODE_SIZE * 4;
1361 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1362 mc_req_size = SI_MC_UCODE_SIZE * 4;
1363 break;
bcc7f5d2
AD		 1364 case CHIP_OLAND:
1365 chip_name = "OLAND";
1366 rlc_chip_name = "OLAND";
1367 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1368 me_req_size = SI_PM4_UCODE_SIZE * 4;
1369 ce_req_size = SI_CE_UCODE_SIZE * 4;
1370 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1371 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1372 break;
c04c00b4
AD		 1373 case CHIP_HAINAN:
1374 chip_name = "HAINAN";
1375 rlc_chip_name = "HAINAN";
1376 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1377 me_req_size = SI_PM4_UCODE_SIZE * 4;
1378 ce_req_size = SI_CE_UCODE_SIZE * 4;
1379 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1380 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1381 break;
0f0de06c
AD		 1382 default: BUG();
1383 }
1384
1385 DRM_INFO("Loading %s Microcode\n", chip_name);
1386
1387 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1388 err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1389 if (err)
1390 goto out;
1391 if (rdev->pfp_fw->size != pfp_req_size) {
1392 printk(KERN_ERR
1393 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1394 rdev->pfp_fw->size, fw_name);
1395 err = -EINVAL;
1396 goto out;
1397 }
1398
1399 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1400 err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1401 if (err)
1402 goto out;
1403 if (rdev->me_fw->size != me_req_size) {
1404 printk(KERN_ERR
1405 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1406 rdev->me_fw->size, fw_name);
1407 err = -EINVAL;
1408 }
1409
1410 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1411 err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
1412 if (err)
1413 goto out;
1414 if (rdev->ce_fw->size != ce_req_size) {
1415 printk(KERN_ERR
1416 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1417 rdev->ce_fw->size, fw_name);
1418 err = -EINVAL;
1419 }
1420
1421 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1422 err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1423 if (err)
1424 goto out;
1425 if (rdev->rlc_fw->size != rlc_req_size) {
1426 printk(KERN_ERR
1427 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1428 rdev->rlc_fw->size, fw_name);
1429 err = -EINVAL;
1430 }
1431
1432 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1433 err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
1434 if (err)
1435 goto out;
1436 if (rdev->mc_fw->size != mc_req_size) {
1437 printk(KERN_ERR
1438 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1439 rdev->mc_fw->size, fw_name);
1440 err = -EINVAL;
1441 }
1442
1443out:
1444 platform_device_unregister(pdev);
1445
1446 if (err) {
1447 if (err != -EINVAL)
1448 printk(KERN_ERR
1449 "si_cp: Failed to load firmware \"%s\"\n",
1450 fw_name);
1451 release_firmware(rdev->pfp_fw);
1452 rdev->pfp_fw = NULL;
1453 release_firmware(rdev->me_fw);
1454 rdev->me_fw = NULL;
1455 release_firmware(rdev->ce_fw);
1456 rdev->ce_fw = NULL;
1457 release_firmware(rdev->rlc_fw);
1458 rdev->rlc_fw = NULL;
1459 release_firmware(rdev->mc_fw);
1460 rdev->mc_fw = NULL;
1461 }
1462 return err;
1463}
1464
43b3cd99
AD		 1465/* watermark setup */
1466static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1467 struct radeon_crtc *radeon_crtc,
1468 struct drm_display_mode *mode,
1469 struct drm_display_mode *other_mode)
1470{
1471 u32 tmp;
1472 /*
1473 * Line Buffer Setup
1474 * There are 3 line buffers, each one shared by 2 display controllers.
1475 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1476 * the display controllers. The paritioning is done via one of four
1477 * preset allocations specified in bits 21:20:
1478 * 0 - half lb
1479 * 2 - whole lb, other crtc must be disabled
1480 */
1481 /* this can get tricky if we have two large displays on a paired group
1482 * of crtcs. Ideally for multiple large displays we'd assign them to
1483 * non-linked crtcs for maximum line buffer allocation.
1484 */
1485 if (radeon_crtc->base.enabled && mode) {
1486 if (other_mode)
1487 tmp = 0; /* 1/2 */
1488 else
1489 tmp = 2; /* whole */
1490 } else
1491 tmp = 0;
1492
1493 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1494 DC_LB_MEMORY_CONFIG(tmp));
1495
1496 if (radeon_crtc->base.enabled && mode) {
1497 switch (tmp) {
1498 case 0:
1499 default:
1500 return 4096 * 2;
1501 case 2:
1502 return 8192 * 2;
1503 }
1504 }
1505
1506 /* controller not enabled, so no lb used */
1507 return 0;
1508}
1509
ca7db22b 1510static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 1511{
1512 u32 tmp = RREG32(MC_SHARED_CHMAP);
1513
1514 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1515 case 0:
1516 default:
1517 return 1;
1518 case 1:
1519 return 2;
1520 case 2:
1521 return 4;
1522 case 3:
1523 return 8;
1524 case 4:
1525 return 3;
1526 case 5:
1527 return 6;
1528 case 6:
1529 return 10;
1530 case 7:
1531 return 12;
1532 case 8:
1533 return 16;
1534 }
1535}
1536
1537struct dce6_wm_params {
1538 u32 dram_channels; /* number of dram channels */
1539 u32 yclk; /* bandwidth per dram data pin in kHz */
1540 u32 sclk; /* engine clock in kHz */
1541 u32 disp_clk; /* display clock in kHz */
1542 u32 src_width; /* viewport width */
1543 u32 active_time; /* active display time in ns */
1544 u32 blank_time; /* blank time in ns */
1545 bool interlaced; /* mode is interlaced */
1546 fixed20_12 vsc; /* vertical scale ratio */
1547 u32 num_heads; /* number of active crtcs */
1548 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1549 u32 lb_size; /* line buffer allocated to pipe */
1550 u32 vtaps; /* vertical scaler taps */
1551};
1552
1553static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1554{
1555 /* Calculate raw DRAM Bandwidth */
1556 fixed20_12 dram_efficiency; /* 0.7 */
1557 fixed20_12 yclk, dram_channels, bandwidth;
1558 fixed20_12 a;
1559
1560 a.full = dfixed_const(1000);
1561 yclk.full = dfixed_const(wm->yclk);
1562 yclk.full = dfixed_div(yclk, a);
1563 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1564 a.full = dfixed_const(10);
1565 dram_efficiency.full = dfixed_const(7);
1566 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1567 bandwidth.full = dfixed_mul(dram_channels, yclk);
1568 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1569
1570 return dfixed_trunc(bandwidth);
1571}
1572
1573static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1574{
1575 /* Calculate DRAM Bandwidth and the part allocated to display. */
1576 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1577 fixed20_12 yclk, dram_channels, bandwidth;
1578 fixed20_12 a;
1579
1580 a.full = dfixed_const(1000);
1581 yclk.full = dfixed_const(wm->yclk);
1582 yclk.full = dfixed_div(yclk, a);
1583 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1584 a.full = dfixed_const(10);
1585 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1586 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1587 bandwidth.full = dfixed_mul(dram_channels, yclk);
1588 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1589
1590 return dfixed_trunc(bandwidth);
1591}
1592
1593static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1594{
1595 /* Calculate the display Data return Bandwidth */
1596 fixed20_12 return_efficiency; /* 0.8 */
1597 fixed20_12 sclk, bandwidth;
1598 fixed20_12 a;
1599
1600 a.full = dfixed_const(1000);
1601 sclk.full = dfixed_const(wm->sclk);
1602 sclk.full = dfixed_div(sclk, a);
1603 a.full = dfixed_const(10);
1604 return_efficiency.full = dfixed_const(8);
1605 return_efficiency.full = dfixed_div(return_efficiency, a);
1606 a.full = dfixed_const(32);
1607 bandwidth.full = dfixed_mul(a, sclk);
1608 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1609
1610 return dfixed_trunc(bandwidth);
1611}
1612
1613static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1614{
1615 return 32;
1616}
1617
1618static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1619{
1620 /* Calculate the DMIF Request Bandwidth */
1621 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1622 fixed20_12 disp_clk, sclk, bandwidth;
1623 fixed20_12 a, b1, b2;
1624 u32 min_bandwidth;
1625
1626 a.full = dfixed_const(1000);
1627 disp_clk.full = dfixed_const(wm->disp_clk);
1628 disp_clk.full = dfixed_div(disp_clk, a);
1629 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1630 b1.full = dfixed_mul(a, disp_clk);
1631
1632 a.full = dfixed_const(1000);
1633 sclk.full = dfixed_const(wm->sclk);
1634 sclk.full = dfixed_div(sclk, a);
1635 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1636 b2.full = dfixed_mul(a, sclk);
1637
1638 a.full = dfixed_const(10);
1639 disp_clk_request_efficiency.full = dfixed_const(8);
1640 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1641
1642 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1643
1644 a.full = dfixed_const(min_bandwidth);
1645 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1646
1647 return dfixed_trunc(bandwidth);
1648}
1649
1650static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1651{
1652 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1653 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1654 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1655 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1656
1657 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1658}
1659
1660static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1661{
1662 /* Calculate the display mode Average Bandwidth
1663 * DisplayMode should contain the source and destination dimensions,
1664 * timing, etc.
1665 */
1666 fixed20_12 bpp;
1667 fixed20_12 line_time;
1668 fixed20_12 src_width;
1669 fixed20_12 bandwidth;
1670 fixed20_12 a;
1671
1672 a.full = dfixed_const(1000);
1673 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1674 line_time.full = dfixed_div(line_time, a);
1675 bpp.full = dfixed_const(wm->bytes_per_pixel);
1676 src_width.full = dfixed_const(wm->src_width);
1677 bandwidth.full = dfixed_mul(src_width, bpp);
1678 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1679 bandwidth.full = dfixed_div(bandwidth, line_time);
1680
1681 return dfixed_trunc(bandwidth);
1682}
1683
1684static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1685{
1686 /* First calcualte the latency in ns */
1687 u32 mc_latency = 2000; /* 2000 ns. */
1688 u32 available_bandwidth = dce6_available_bandwidth(wm);
1689 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1690 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1691 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1692 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1693 (wm->num_heads * cursor_line_pair_return_time);
1694 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1695 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1696 u32 tmp, dmif_size = 12288;
1697 fixed20_12 a, b, c;
1698
1699 if (wm->num_heads == 0)
1700 return 0;
1701
1702 a.full = dfixed_const(2);
1703 b.full = dfixed_const(1);
1704 if ((wm->vsc.full > a.full) ||
1705 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1706 (wm->vtaps >= 5) ||
1707 ((wm->vsc.full >= a.full) && wm->interlaced))
1708 max_src_lines_per_dst_line = 4;
1709 else
1710 max_src_lines_per_dst_line = 2;
1711
1712 a.full = dfixed_const(available_bandwidth);
1713 b.full = dfixed_const(wm->num_heads);
1714 a.full = dfixed_div(a, b);
1715
1716 b.full = dfixed_const(mc_latency + 512);
1717 c.full = dfixed_const(wm->disp_clk);
1718 b.full = dfixed_div(b, c);
1719
1720 c.full = dfixed_const(dmif_size);
1721 b.full = dfixed_div(c, b);
1722
1723 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1724
1725 b.full = dfixed_const(1000);
1726 c.full = dfixed_const(wm->disp_clk);
1727 b.full = dfixed_div(c, b);
1728 c.full = dfixed_const(wm->bytes_per_pixel);
1729 b.full = dfixed_mul(b, c);
1730
1731 lb_fill_bw = min(tmp, dfixed_trunc(b));
1732
1733 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1734 b.full = dfixed_const(1000);
1735 c.full = dfixed_const(lb_fill_bw);
1736 b.full = dfixed_div(c, b);
1737 a.full = dfixed_div(a, b);
1738 line_fill_time = dfixed_trunc(a);
1739
1740 if (line_fill_time < wm->active_time)
1741 return latency;
1742 else
1743 return latency + (line_fill_time - wm->active_time);
1744
1745}
1746
1747static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1748{
1749 if (dce6_average_bandwidth(wm) <=
1750 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
1751 return true;
1752 else
1753 return false;
1754};
1755
1756static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
1757{
1758 if (dce6_average_bandwidth(wm) <=
1759 (dce6_available_bandwidth(wm) / wm->num_heads))
1760 return true;
1761 else
1762 return false;
1763};
1764
1765static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
1766{
1767 u32 lb_partitions = wm->lb_size / wm->src_width;
1768 u32 line_time = wm->active_time + wm->blank_time;
1769 u32 latency_tolerant_lines;
1770 u32 latency_hiding;
1771 fixed20_12 a;
1772
1773 a.full = dfixed_const(1);
1774 if (wm->vsc.full > a.full)
1775 latency_tolerant_lines = 1;
1776 else {
1777 if (lb_partitions <= (wm->vtaps + 1))
1778 latency_tolerant_lines = 1;
1779 else
1780 latency_tolerant_lines = 2;
1781 }
1782
1783 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
1784
1785 if (dce6_latency_watermark(wm) <= latency_hiding)
1786 return true;
1787 else
1788 return false;
1789}
1790
1791static void dce6_program_watermarks(struct radeon_device *rdev,
1792 struct radeon_crtc *radeon_crtc,
1793 u32 lb_size, u32 num_heads)
1794{
1795 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 1796 struct dce6_wm_params wm_low, wm_high;
1797 u32 dram_channels;
43b3cd99
AD		 1798 u32 pixel_period;
1799 u32 line_time = 0;
1800 u32 latency_watermark_a = 0, latency_watermark_b = 0;
1801 u32 priority_a_mark = 0, priority_b_mark = 0;
1802 u32 priority_a_cnt = PRIORITY_OFF;
1803 u32 priority_b_cnt = PRIORITY_OFF;
1804 u32 tmp, arb_control3;
1805 fixed20_12 a, b, c;
1806
1807 if (radeon_crtc->base.enabled && num_heads && mode) {
1808 pixel_period = 1000000 / (u32)mode->clock;
1809 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
1810 priority_a_cnt = 0;
1811 priority_b_cnt = 0;
1812
ca7db22b 1813 if (rdev->family == CHIP_ARUBA)
c696e53f 1814 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 1815 else
c696e53f
AD		 1816 dram_channels = si_get_number_of_dram_channels(rdev);
1817
1818 /* watermark for high clocks */
1819 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
1820 wm_high.yclk =
1821 radeon_dpm_get_mclk(rdev, false) * 10;
1822 wm_high.sclk =
1823 radeon_dpm_get_sclk(rdev, false) * 10;
1824 } else {
1825 wm_high.yclk = rdev->pm.current_mclk * 10;
1826 wm_high.sclk = rdev->pm.current_sclk * 10;
1827 }
1828
1829 wm_high.disp_clk = mode->clock;
1830 wm_high.src_width = mode->crtc_hdisplay;
1831 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
1832 wm_high.blank_time = line_time - wm_high.active_time;
1833 wm_high.interlaced = false;
1834 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
1835 wm_high.interlaced = true;
1836 wm_high.vsc = radeon_crtc->vsc;
1837 wm_high.vtaps = 1;
1838 if (radeon_crtc->rmx_type != RMX_OFF)
1839 wm_high.vtaps = 2;
1840 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
1841 wm_high.lb_size = lb_size;
1842 wm_high.dram_channels = dram_channels;
1843 wm_high.num_heads = num_heads;
1844
1845 /* watermark for low clocks */
1846 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
1847 wm_low.yclk =
1848 radeon_dpm_get_mclk(rdev, true) * 10;
1849 wm_low.sclk =
1850 radeon_dpm_get_sclk(rdev, true) * 10;
1851 } else {
1852 wm_low.yclk = rdev->pm.current_mclk * 10;
1853 wm_low.sclk = rdev->pm.current_sclk * 10;
1854 }
1855
1856 wm_low.disp_clk = mode->clock;
1857 wm_low.src_width = mode->crtc_hdisplay;
1858 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
1859 wm_low.blank_time = line_time - wm_low.active_time;
1860 wm_low.interlaced = false;
1861 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
1862 wm_low.interlaced = true;
1863 wm_low.vsc = radeon_crtc->vsc;
1864 wm_low.vtaps = 1;
1865 if (radeon_crtc->rmx_type != RMX_OFF)
1866 wm_low.vtaps = 2;
1867 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
1868 wm_low.lb_size = lb_size;
1869 wm_low.dram_channels = dram_channels;
1870 wm_low.num_heads = num_heads;
43b3cd99
AD		 1871
1872 /* set for high clocks */
c696e53f 1873 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 1874 /* set for low clocks */
c696e53f 1875 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 1876
1877 /* possibly force display priority to high */
1878 /* should really do this at mode validation time... */
c696e53f
AD		 1879 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
1880 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
1881 !dce6_check_latency_hiding(&wm_high) ||
1882 (rdev->disp_priority == 2)) {
1883 DRM_DEBUG_KMS("force priority to high\n");
1884 priority_a_cnt |= PRIORITY_ALWAYS_ON;
1885 priority_b_cnt |= PRIORITY_ALWAYS_ON;
1886 }
1887 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
1888 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
1889 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 1890 (rdev->disp_priority == 2)) {
1891 DRM_DEBUG_KMS("force priority to high\n");
1892 priority_a_cnt |= PRIORITY_ALWAYS_ON;
1893 priority_b_cnt |= PRIORITY_ALWAYS_ON;
1894 }
1895
1896 a.full = dfixed_const(1000);
1897 b.full = dfixed_const(mode->clock);
1898 b.full = dfixed_div(b, a);
1899 c.full = dfixed_const(latency_watermark_a);
1900 c.full = dfixed_mul(c, b);
1901 c.full = dfixed_mul(c, radeon_crtc->hsc);
1902 c.full = dfixed_div(c, a);
1903 a.full = dfixed_const(16);
1904 c.full = dfixed_div(c, a);
1905 priority_a_mark = dfixed_trunc(c);
1906 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
1907
1908 a.full = dfixed_const(1000);
1909 b.full = dfixed_const(mode->clock);
1910 b.full = dfixed_div(b, a);
1911 c.full = dfixed_const(latency_watermark_b);
1912 c.full = dfixed_mul(c, b);
1913 c.full = dfixed_mul(c, radeon_crtc->hsc);
1914 c.full = dfixed_div(c, a);
1915 a.full = dfixed_const(16);
1916 c.full = dfixed_div(c, a);
1917 priority_b_mark = dfixed_trunc(c);
1918 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
1919 }
1920
1921 /* select wm A */
1922 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
1923 tmp = arb_control3;
1924 tmp &= ~LATENCY_WATERMARK_MASK(3);
1925 tmp |= LATENCY_WATERMARK_MASK(1);
1926 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
1927 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
1928 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
1929 LATENCY_HIGH_WATERMARK(line_time)));
1930 /* select wm B */
1931 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
1932 tmp &= ~LATENCY_WATERMARK_MASK(3);
1933 tmp |= LATENCY_WATERMARK_MASK(2);
1934 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
1935 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
1936 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
1937 LATENCY_HIGH_WATERMARK(line_time)));
1938 /* restore original selection */
1939 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
1940
1941 /* write the priority marks */
1942 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
1943 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
1944
1945}
1946
1947void dce6_bandwidth_update(struct radeon_device *rdev)
1948{
1949 struct drm_display_mode *mode0 = NULL;
1950 struct drm_display_mode *mode1 = NULL;
1951 u32 num_heads = 0, lb_size;
1952 int i;
1953
1954 radeon_update_display_priority(rdev);
1955
1956 for (i = 0; i < rdev->num_crtc; i++) {
1957 if (rdev->mode_info.crtcs[i]->base.enabled)
1958 num_heads++;
1959 }
1960 for (i = 0; i < rdev->num_crtc; i += 2) {
1961 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
1962 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
1963 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
1964 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
1965 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
1966 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
1967 }
1968}
1969
0a96d72b
AD		 1970/*
1971 * Core functions
1972 */
0a96d72b
AD		 1973static void si_tiling_mode_table_init(struct radeon_device *rdev)
1974{
1975 const u32 num_tile_mode_states = 32;
1976 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1977
1978 switch (rdev->config.si.mem_row_size_in_kb) {
1979 case 1:
1980 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1981 break;
1982 case 2:
1983 default:
1984 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1985 break;
1986 case 4:
1987 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1988 break;
1989 }
1990
1991 if ((rdev->family == CHIP_TAHITI) ||
1992 (rdev->family == CHIP_PITCAIRN)) {
1993 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1994 switch (reg_offset) {
1995 case 0: /* non-AA compressed depth or any compressed stencil */
1996 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1997 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1998 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1999 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2000 NUM_BANKS(ADDR_SURF_16_BANK) |
2001 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2002 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2003 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2004 break;
2005 case 1: /* 2xAA/4xAA compressed depth only */
2006 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2007 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2008 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2009 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2010 NUM_BANKS(ADDR_SURF_16_BANK) |
2011 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2012 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2013 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2014 break;
2015 case 2: /* 8xAA compressed depth only */
2016 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2017 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2018 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2019 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2020 NUM_BANKS(ADDR_SURF_16_BANK) |
2021 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2022 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2023 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2024 break;
2025 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2026 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2027 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2028 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2029 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2030 NUM_BANKS(ADDR_SURF_16_BANK) |
2031 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2032 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2033 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2034 break;
2035 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2036 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2037 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2038 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2039 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2040 NUM_BANKS(ADDR_SURF_16_BANK) |
2041 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2042 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2043 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2044 break;
2045 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2046 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2047 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2048 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2049 TILE_SPLIT(split_equal_to_row_size) |
2050 NUM_BANKS(ADDR_SURF_16_BANK) |
2051 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2052 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2053 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2054 break;
2055 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2056 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2057 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2058 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2059 TILE_SPLIT(split_equal_to_row_size) |
2060 NUM_BANKS(ADDR_SURF_16_BANK) |
2061 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2062 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2063 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2064 break;
2065 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2066 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2067 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2068 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2069 TILE_SPLIT(split_equal_to_row_size) |
2070 NUM_BANKS(ADDR_SURF_16_BANK) |
2071 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2072 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2073 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2074 break;
2075 case 8: /* 1D and 1D Array Surfaces */
2076 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2077 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2078 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2079 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2080 NUM_BANKS(ADDR_SURF_16_BANK) |
2081 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2082 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2083 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2084 break;
2085 case 9: /* Displayable maps. */
2086 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2087 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2088 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2089 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2090 NUM_BANKS(ADDR_SURF_16_BANK) |
2091 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2092 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2093 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2094 break;
2095 case 10: /* Display 8bpp. */
2096 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2097 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2098 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2099 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2100 NUM_BANKS(ADDR_SURF_16_BANK) |
2101 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2102 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2103 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2104 break;
2105 case 11: /* Display 16bpp. */
2106 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2107 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2108 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2109 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2110 NUM_BANKS(ADDR_SURF_16_BANK) |
2111 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2112 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2113 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2114 break;
2115 case 12: /* Display 32bpp. */
2116 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2117 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2118 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2119 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2120 NUM_BANKS(ADDR_SURF_16_BANK) |
2121 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2122 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2123 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2124 break;
2125 case 13: /* Thin. */
2126 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2127 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2128 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2129 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2130 NUM_BANKS(ADDR_SURF_16_BANK) |
2131 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2132 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2133 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2134 break;
2135 case 14: /* Thin 8 bpp. */
2136 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2137 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2138 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2139 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2140 NUM_BANKS(ADDR_SURF_16_BANK) |
2141 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2142 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2143 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2144 break;
2145 case 15: /* Thin 16 bpp. */
2146 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2147 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2148 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2149 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2150 NUM_BANKS(ADDR_SURF_16_BANK) |
2151 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2152 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2153 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2154 break;
2155 case 16: /* Thin 32 bpp. */
2156 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2157 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2158 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2159 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2160 NUM_BANKS(ADDR_SURF_16_BANK) |
2161 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2162 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2163 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2164 break;
2165 case 17: /* Thin 64 bpp. */
2166 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2167 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2168 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2169 TILE_SPLIT(split_equal_to_row_size) |
2170 NUM_BANKS(ADDR_SURF_16_BANK) |
2171 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2172 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2173 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2174 break;
2175 case 21: /* 8 bpp PRT. */
2176 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2177 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2178 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2179 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2180 NUM_BANKS(ADDR_SURF_16_BANK) |
2181 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2182 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2183 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2184 break;
2185 case 22: /* 16 bpp PRT */
2186 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2187 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2188 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2189 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2190 NUM_BANKS(ADDR_SURF_16_BANK) |
2191 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2192 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2193 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2194 break;
2195 case 23: /* 32 bpp PRT */
2196 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2197 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2198 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2199 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2200 NUM_BANKS(ADDR_SURF_16_BANK) |
2201 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2202 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2203 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2204 break;
2205 case 24: /* 64 bpp PRT */
2206 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2207 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2208 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2209 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2210 NUM_BANKS(ADDR_SURF_16_BANK) |
2211 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2212 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2213 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2214 break;
2215 case 25: /* 128 bpp PRT */
2216 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2217 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2218 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2219 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2220 NUM_BANKS(ADDR_SURF_8_BANK) |
2221 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2222 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2223 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2224 break;
2225 default:
2226 gb_tile_moden = 0;
2227 break;
2228 }
64d7b8be 2229 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2230 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2231 }
d0ae7fcc 2232 } else if ((rdev->family == CHIP_VERDE) ||
8b02859d
AD		 2233 (rdev->family == CHIP_OLAND) ||
2234 (rdev->family == CHIP_HAINAN)) {
0a96d72b
AD		 2235 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2236 switch (reg_offset) {
2237 case 0: /* non-AA compressed depth or any compressed stencil */
2238 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2239 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2240 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2241 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2242 NUM_BANKS(ADDR_SURF_16_BANK) |
2243 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2244 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2245 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2246 break;
2247 case 1: /* 2xAA/4xAA compressed depth only */
2248 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2249 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2250 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2251 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2252 NUM_BANKS(ADDR_SURF_16_BANK) |
2253 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2254 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2255 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2256 break;
2257 case 2: /* 8xAA compressed depth only */
2258 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2259 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2260 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2261 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2262 NUM_BANKS(ADDR_SURF_16_BANK) |
2263 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2264 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2265 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2266 break;
2267 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2269 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2270 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2271 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2272 NUM_BANKS(ADDR_SURF_16_BANK) |
2273 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2274 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2275 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2276 break;
2277 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2278 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2279 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2280 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2281 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2282 NUM_BANKS(ADDR_SURF_16_BANK) |
2283 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2284 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2285 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2286 break;
2287 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2288 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2289 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2290 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2291 TILE_SPLIT(split_equal_to_row_size) |
2292 NUM_BANKS(ADDR_SURF_16_BANK) |
2293 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2294 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2295 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2296 break;
2297 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2298 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2299 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2300 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2301 TILE_SPLIT(split_equal_to_row_size) |
2302 NUM_BANKS(ADDR_SURF_16_BANK) |
2303 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2304 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2305 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2306 break;
2307 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2308 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2309 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2310 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2311 TILE_SPLIT(split_equal_to_row_size) |
2312 NUM_BANKS(ADDR_SURF_16_BANK) |
2313 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2314 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2315 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2316 break;
2317 case 8: /* 1D and 1D Array Surfaces */
2318 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2319 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2320 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2321 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2322 NUM_BANKS(ADDR_SURF_16_BANK) |
2323 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2324 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2325 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2326 break;
2327 case 9: /* Displayable maps. */
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2329 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2331 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2332 NUM_BANKS(ADDR_SURF_16_BANK) |
2333 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2334 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2335 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2336 break;
2337 case 10: /* Display 8bpp. */
2338 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2339 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2340 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2341 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2342 NUM_BANKS(ADDR_SURF_16_BANK) |
2343 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2346 break;
2347 case 11: /* Display 16bpp. */
2348 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2349 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2350 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2351 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2352 NUM_BANKS(ADDR_SURF_16_BANK) |
2353 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2354 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2355 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2356 break;
2357 case 12: /* Display 32bpp. */
2358 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2359 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2360 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2361 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2362 NUM_BANKS(ADDR_SURF_16_BANK) |
2363 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2364 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2365 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2366 break;
2367 case 13: /* Thin. */
2368 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2369 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2370 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2371 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2372 NUM_BANKS(ADDR_SURF_16_BANK) |
2373 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2376 break;
2377 case 14: /* Thin 8 bpp. */
2378 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2379 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2380 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2381 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2382 NUM_BANKS(ADDR_SURF_16_BANK) |
2383 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2384 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2385 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2386 break;
2387 case 15: /* Thin 16 bpp. */
2388 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2389 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2390 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2391 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2392 NUM_BANKS(ADDR_SURF_16_BANK) |
2393 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2394 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2395 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2396 break;
2397 case 16: /* Thin 32 bpp. */
2398 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2399 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2400 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2401 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2402 NUM_BANKS(ADDR_SURF_16_BANK) |
2403 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2406 break;
2407 case 17: /* Thin 64 bpp. */
2408 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2409 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2410 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2411 TILE_SPLIT(split_equal_to_row_size) |
2412 NUM_BANKS(ADDR_SURF_16_BANK) |
2413 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2414 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2415 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2416 break;
2417 case 21: /* 8 bpp PRT. */
2418 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2419 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2420 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2421 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2422 NUM_BANKS(ADDR_SURF_16_BANK) |
2423 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2424 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2425 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2426 break;
2427 case 22: /* 16 bpp PRT */
2428 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2429 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2430 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2431 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2432 NUM_BANKS(ADDR_SURF_16_BANK) |
2433 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2434 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2435 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2436 break;
2437 case 23: /* 32 bpp PRT */
2438 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2439 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2440 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2441 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2442 NUM_BANKS(ADDR_SURF_16_BANK) |
2443 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2444 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2445 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2446 break;
2447 case 24: /* 64 bpp PRT */
2448 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2449 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2450 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2451 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2452 NUM_BANKS(ADDR_SURF_16_BANK) |
2453 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2454 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2455 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2456 break;
2457 case 25: /* 128 bpp PRT */
2458 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2459 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2460 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2461 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2462 NUM_BANKS(ADDR_SURF_8_BANK) |
2463 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2466 break;
2467 default:
2468 gb_tile_moden = 0;
2469 break;
2470 }
64d7b8be 2471 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2472 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2473 }
2474 } else
2475 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2476}
2477
1a8ca750
AD		 2478static void si_select_se_sh(struct radeon_device *rdev,
2479 u32 se_num, u32 sh_num)
2480{
2481 u32 data = INSTANCE_BROADCAST_WRITES;
2482
2483 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2484 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2485 else if (se_num == 0xffffffff)
2486 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2487 else if (sh_num == 0xffffffff)
2488 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2489 else
2490 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2491 WREG32(GRBM_GFX_INDEX, data);
2492}
2493
2494static u32 si_create_bitmask(u32 bit_width)
2495{
2496 u32 i, mask = 0;
2497
2498 for (i = 0; i < bit_width; i++) {
2499 mask <<= 1;
2500 mask |= 1;
2501 }
2502 return mask;
2503}
2504
2505static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2506{
2507 u32 data, mask;
2508
2509 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2510 if (data & 1)
2511 data &= INACTIVE_CUS_MASK;
2512 else
2513 data = 0;
2514 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2515
2516 data >>= INACTIVE_CUS_SHIFT;
2517
2518 mask = si_create_bitmask(cu_per_sh);
2519
2520 return ~data & mask;
2521}
2522
2523static void si_setup_spi(struct radeon_device *rdev,
2524 u32 se_num, u32 sh_per_se,
2525 u32 cu_per_sh)
2526{
2527 int i, j, k;
2528 u32 data, mask, active_cu;
2529
2530 for (i = 0; i < se_num; i++) {
2531 for (j = 0; j < sh_per_se; j++) {
2532 si_select_se_sh(rdev, i, j);
2533 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2534 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2535
2536 mask = 1;
2537 for (k = 0; k < 16; k++) {
2538 mask <<= k;
2539 if (active_cu & mask) {
2540 data &= ~mask;
2541 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2542 break;
2543 }
2544 }
2545 }
2546 }
2547 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2548}
2549
2550static u32 si_get_rb_disabled(struct radeon_device *rdev,
2551 u32 max_rb_num, u32 se_num,
2552 u32 sh_per_se)
2553{
2554 u32 data, mask;
2555
2556 data = RREG32(CC_RB_BACKEND_DISABLE);
2557 if (data & 1)
2558 data &= BACKEND_DISABLE_MASK;
2559 else
2560 data = 0;
2561 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2562
2563 data >>= BACKEND_DISABLE_SHIFT;
2564
2565 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2566
2567 return data & mask;
2568}
2569
2570static void si_setup_rb(struct radeon_device *rdev,
2571 u32 se_num, u32 sh_per_se,
2572 u32 max_rb_num)
2573{
2574 int i, j;
2575 u32 data, mask;
2576 u32 disabled_rbs = 0;
2577 u32 enabled_rbs = 0;
2578
2579 for (i = 0; i < se_num; i++) {
2580 for (j = 0; j < sh_per_se; j++) {
2581 si_select_se_sh(rdev, i, j);
2582 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2583 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2584 }
2585 }
2586 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2587
2588 mask = 1;
2589 for (i = 0; i < max_rb_num; i++) {
2590 if (!(disabled_rbs & mask))
2591 enabled_rbs |= mask;
2592 mask <<= 1;
2593 }
2594
2595 for (i = 0; i < se_num; i++) {
2596 si_select_se_sh(rdev, i, 0xffffffff);
2597 data = 0;
2598 for (j = 0; j < sh_per_se; j++) {
2599 switch (enabled_rbs & 3) {
2600 case 1:
2601 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2602 break;
2603 case 2:
2604 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2605 break;
2606 case 3:
2607 default:
2608 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2609 break;
2610 }
2611 enabled_rbs >>= 2;
2612 }
2613 WREG32(PA_SC_RASTER_CONFIG, data);
2614 }
2615 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2616}
2617
0a96d72b
AD		 2618static void si_gpu_init(struct radeon_device *rdev)
2619{
0a96d72b
AD		 2620 u32 gb_addr_config = 0;
2621 u32 mc_shared_chmap, mc_arb_ramcfg;
0a96d72b 2622 u32 sx_debug_1;
0a96d72b
AD		 2623 u32 hdp_host_path_cntl;
2624 u32 tmp;
2625 int i, j;
2626
2627 switch (rdev->family) {
2628 case CHIP_TAHITI:
2629 rdev->config.si.max_shader_engines = 2;
0a96d72b 2630 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 2631 rdev->config.si.max_cu_per_sh = 8;
2632 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2633 rdev->config.si.max_backends_per_se = 4;
2634 rdev->config.si.max_texture_channel_caches = 12;
2635 rdev->config.si.max_gprs = 256;
2636 rdev->config.si.max_gs_threads = 32;
2637 rdev->config.si.max_hw_contexts = 8;
2638
2639 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2640 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2641 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2642 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2643 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2644 break;
2645 case CHIP_PITCAIRN:
2646 rdev->config.si.max_shader_engines = 2;
0a96d72b 2647 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 2648 rdev->config.si.max_cu_per_sh = 5;
2649 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2650 rdev->config.si.max_backends_per_se = 4;
2651 rdev->config.si.max_texture_channel_caches = 8;
2652 rdev->config.si.max_gprs = 256;
2653 rdev->config.si.max_gs_threads = 32;
2654 rdev->config.si.max_hw_contexts = 8;
2655
2656 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2657 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2658 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2659 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2660 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2661 break;
2662 case CHIP_VERDE:
2663 default:
2664 rdev->config.si.max_shader_engines = 1;
0a96d72b 2665 rdev->config.si.max_tile_pipes = 4;
468ef1a5 2666 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 2667 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2668 rdev->config.si.max_backends_per_se = 4;
2669 rdev->config.si.max_texture_channel_caches = 4;
2670 rdev->config.si.max_gprs = 256;
2671 rdev->config.si.max_gs_threads = 32;
2672 rdev->config.si.max_hw_contexts = 8;
2673
d0ae7fcc
AD		 2674 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2675 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2676 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2677 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2678 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2679 break;
2680 case CHIP_OLAND:
2681 rdev->config.si.max_shader_engines = 1;
2682 rdev->config.si.max_tile_pipes = 4;
2683 rdev->config.si.max_cu_per_sh = 6;
2684 rdev->config.si.max_sh_per_se = 1;
2685 rdev->config.si.max_backends_per_se = 2;
2686 rdev->config.si.max_texture_channel_caches = 4;
2687 rdev->config.si.max_gprs = 256;
2688 rdev->config.si.max_gs_threads = 16;
2689 rdev->config.si.max_hw_contexts = 8;
2690
0a96d72b
AD		 2691 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2692 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2693 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2694 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2695 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 2696 break;
8b02859d
AD		 2697 case CHIP_HAINAN:
2698 rdev->config.si.max_shader_engines = 1;
2699 rdev->config.si.max_tile_pipes = 4;
2700 rdev->config.si.max_cu_per_sh = 5;
2701 rdev->config.si.max_sh_per_se = 1;
2702 rdev->config.si.max_backends_per_se = 1;
2703 rdev->config.si.max_texture_channel_caches = 2;
2704 rdev->config.si.max_gprs = 256;
2705 rdev->config.si.max_gs_threads = 16;
2706 rdev->config.si.max_hw_contexts = 8;
2707
2708 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2709 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2710 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2711 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2712 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2713 break;
0a96d72b
AD		 2714 }
2715
2716 /* Initialize HDP */
2717 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2718 WREG32((0x2c14 + j), 0x00000000);
2719 WREG32((0x2c18 + j), 0x00000000);
2720 WREG32((0x2c1c + j), 0x00000000);
2721 WREG32((0x2c20 + j), 0x00000000);
2722 WREG32((0x2c24 + j), 0x00000000);
2723 }
2724
2725 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2726
2727 evergreen_fix_pci_max_read_req_size(rdev);
2728
2729 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2730
2731 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2732 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2733
0a96d72b 2734 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 2735 rdev->config.si.mem_max_burst_length_bytes = 256;
2736 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2737 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2738 if (rdev->config.si.mem_row_size_in_kb > 4)
2739 rdev->config.si.mem_row_size_in_kb = 4;
2740 /* XXX use MC settings? */
2741 rdev->config.si.shader_engine_tile_size = 32;
2742 rdev->config.si.num_gpus = 1;
2743 rdev->config.si.multi_gpu_tile_size = 64;
2744
1a8ca750
AD		 2745 /* fix up row size */
2746 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 2747 switch (rdev->config.si.mem_row_size_in_kb) {
2748 case 1:
2749 default:
2750 gb_addr_config |= ROW_SIZE(0);
2751 break;
2752 case 2:
2753 gb_addr_config |= ROW_SIZE(1);
2754 break;
2755 case 4:
2756 gb_addr_config |= ROW_SIZE(2);
2757 break;
2758 }
2759
0a96d72b
AD		 2760 /* setup tiling info dword. gb_addr_config is not adequate since it does
2761 * not have bank info, so create a custom tiling dword.
2762 * bits 3:0 num_pipes
2763 * bits 7:4 num_banks
2764 * bits 11:8 group_size
2765 * bits 15:12 row_size
2766 */
2767 rdev->config.si.tile_config = 0;
2768 switch (rdev->config.si.num_tile_pipes) {
2769 case 1:
2770 rdev->config.si.tile_config |= (0 << 0);
2771 break;
2772 case 2:
2773 rdev->config.si.tile_config |= (1 << 0);
2774 break;
2775 case 4:
2776 rdev->config.si.tile_config |= (2 << 0);
2777 break;
2778 case 8:
2779 default:
2780 /* XXX what about 12? */
2781 rdev->config.si.tile_config |= (3 << 0);
2782 break;
dca571a6
CK		 2783 }
2784 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
2785 case 0: /* four banks */
1a8ca750 2786 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 2787 break;
2788 case 1: /* eight banks */
2789 rdev->config.si.tile_config |= 1 << 4;
2790 break;
2791 case 2: /* sixteen banks */
2792 default:
2793 rdev->config.si.tile_config |= 2 << 4;
2794 break;
2795 }
0a96d72b
AD		 2796 rdev->config.si.tile_config |=
2797 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
2798 rdev->config.si.tile_config |=
2799 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
2800
0a96d72b
AD		 2801 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2802 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 2803 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 2804 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 2805 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
2806 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 2807 if (rdev->has_uvd) {
2808 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2809 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2810 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
2811 }
0a96d72b 2812
1a8ca750 2813 si_tiling_mode_table_init(rdev);
0a96d72b 2814
1a8ca750
AD		 2815 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
2816 rdev->config.si.max_sh_per_se,
2817 rdev->config.si.max_backends_per_se);
0a96d72b 2818
1a8ca750
AD		 2819 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
2820 rdev->config.si.max_sh_per_se,
2821 rdev->config.si.max_cu_per_sh);
0a96d72b 2822
0a96d72b
AD		 2823
2824 /* set HW defaults for 3D engine */
2825 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2826 ROQ_IB2_START(0x2b)));
2827 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
2828
2829 sx_debug_1 = RREG32(SX_DEBUG_1);
2830 WREG32(SX_DEBUG_1, sx_debug_1);
2831
2832 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2833
2834 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
2835 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
2836 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
2837 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
2838
2839 WREG32(VGT_NUM_INSTANCES, 1);
2840
2841 WREG32(CP_PERFMON_CNTL, 0);
2842
2843 WREG32(SQ_CONFIG, 0);
2844
2845 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2846 FORCE_EOV_MAX_REZ_CNT(255)));
2847
2848 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2849 AUTO_INVLD_EN(ES_AND_GS_AUTO));
2850
2851 WREG32(VGT_GS_VERTEX_REUSE, 16);
2852 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2853
2854 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
2855 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
2856 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
2857 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
2858 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
2859 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
2860 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
2861 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
2862
2863 tmp = RREG32(HDP_MISC_CNTL);
2864 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2865 WREG32(HDP_MISC_CNTL, tmp);
2866
2867 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2868 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2869
2870 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2871
2872 udelay(50);
2873}
c476dde2 2874
2ece2e8b
AD		 2875/*
2876 * GPU scratch registers helpers function.
2877 */
2878static void si_scratch_init(struct radeon_device *rdev)
2879{
2880 int i;
2881
2882 rdev->scratch.num_reg = 7;
2883 rdev->scratch.reg_base = SCRATCH_REG0;
2884 for (i = 0; i < rdev->scratch.num_reg; i++) {
2885 rdev->scratch.free[i] = true;
2886 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2887 }
2888}
2889
2890void si_fence_ring_emit(struct radeon_device *rdev,
2891 struct radeon_fence *fence)
2892{
2893 struct radeon_ring *ring = &rdev->ring[fence->ring];
2894 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2895
2896 /* flush read cache over gart */
2897 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2898 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
2899 radeon_ring_write(ring, 0);
2900 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2901 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
2902 PACKET3_TC_ACTION_ENA |
2903 PACKET3_SH_KCACHE_ACTION_ENA |
2904 PACKET3_SH_ICACHE_ACTION_ENA);
2905 radeon_ring_write(ring, 0xFFFFFFFF);
2906 radeon_ring_write(ring, 0);
2907 radeon_ring_write(ring, 10); /* poll interval */
2908 /* EVENT_WRITE_EOP - flush caches, send int */
2909 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2910 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
2911 radeon_ring_write(ring, addr & 0xffffffff);
2912 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2913 radeon_ring_write(ring, fence->seq);
2914 radeon_ring_write(ring, 0);
2915}
2916
2917/*
2918 * IB stuff
2919 */
2920void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2921{
876dc9f3 2922 struct radeon_ring *ring = &rdev->ring[ib->ring];
2ece2e8b
AD		 2923 u32 header;
2924
a85a7da4
AD		 2925 if (ib->is_const_ib) {
2926 /* set switch buffer packet before const IB */
2927 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2928 radeon_ring_write(ring, 0);
45df6803 2929
2ece2e8b 2930 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 2931 } else {
89d35807 2932 u32 next_rptr;
a85a7da4 2933 if (ring->rptr_save_reg) {
89d35807 2934 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 2935 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2936 radeon_ring_write(ring, ((ring->rptr_save_reg -
2937 PACKET3_SET_CONFIG_REG_START) >> 2));
2938 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2939 } else if (rdev->wb.enabled) {
2940 next_rptr = ring->wptr + 5 + 4 + 8;
2941 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2942 radeon_ring_write(ring, (1 << 8));
2943 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2944 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
2945 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 2946 }
2947
2ece2e8b 2948 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 2949 }
2ece2e8b
AD		 2950
2951 radeon_ring_write(ring, header);
2952 radeon_ring_write(ring,
2953#ifdef __BIG_ENDIAN
2954 (2 << 0) |
2955#endif
2956 (ib->gpu_addr & 0xFFFFFFFC));
2957 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4bf3dd92
CK		 2958 radeon_ring_write(ring, ib->length_dw |
2959 (ib->vm ? (ib->vm->id << 24) : 0));
2ece2e8b 2960
a85a7da4
AD		 2961 if (!ib->is_const_ib) {
2962 /* flush read cache over gart for this vmid */
2963 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2964 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 2965 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
a85a7da4
AD		 2966 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2967 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
2968 PACKET3_TC_ACTION_ENA |
2969 PACKET3_SH_KCACHE_ACTION_ENA |
2970 PACKET3_SH_ICACHE_ACTION_ENA);
2971 radeon_ring_write(ring, 0xFFFFFFFF);
2972 radeon_ring_write(ring, 0);
2973 radeon_ring_write(ring, 10); /* poll interval */
2974 }
2ece2e8b
AD		 2975}
2976
48c0c902
AD		 2977/*
2978 * CP.
2979 */
2980static void si_cp_enable(struct radeon_device *rdev, bool enable)
2981{
2982 if (enable)
2983 WREG32(CP_ME_CNTL, 0);
2984 else {
2985 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2986 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
2987 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 2988 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2989 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
2990 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 2991 }
2992 udelay(50);
2993}
2994
2995static int si_cp_load_microcode(struct radeon_device *rdev)
2996{
2997 const __be32 *fw_data;
2998 int i;
2999
3000 if (!rdev->me_fw || !rdev->pfp_fw)
3001 return -EINVAL;
3002
3003 si_cp_enable(rdev, false);
3004
3005 /* PFP */
3006 fw_data = (const __be32 *)rdev->pfp_fw->data;
3007 WREG32(CP_PFP_UCODE_ADDR, 0);
3008 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3009 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3010 WREG32(CP_PFP_UCODE_ADDR, 0);
3011
3012 /* CE */
3013 fw_data = (const __be32 *)rdev->ce_fw->data;
3014 WREG32(CP_CE_UCODE_ADDR, 0);
3015 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3016 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3017 WREG32(CP_CE_UCODE_ADDR, 0);
3018
3019 /* ME */
3020 fw_data = (const __be32 *)rdev->me_fw->data;
3021 WREG32(CP_ME_RAM_WADDR, 0);
3022 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3023 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3024 WREG32(CP_ME_RAM_WADDR, 0);
3025
3026 WREG32(CP_PFP_UCODE_ADDR, 0);
3027 WREG32(CP_CE_UCODE_ADDR, 0);
3028 WREG32(CP_ME_RAM_WADDR, 0);
3029 WREG32(CP_ME_RAM_RADDR, 0);
3030 return 0;
3031}
3032
3033static int si_cp_start(struct radeon_device *rdev)
3034{
3035 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3036 int r, i;
3037
3038 r = radeon_ring_lock(rdev, ring, 7 + 4);
3039 if (r) {
3040 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3041 return r;
3042 }
3043 /* init the CP */
3044 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3045 radeon_ring_write(ring, 0x1);
3046 radeon_ring_write(ring, 0x0);
3047 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3048 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3049 radeon_ring_write(ring, 0);
3050 radeon_ring_write(ring, 0);
3051
3052 /* init the CE partitions */
3053 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3054 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3055 radeon_ring_write(ring, 0xc000);
3056 radeon_ring_write(ring, 0xe000);
3057 radeon_ring_unlock_commit(rdev, ring);
3058
3059 si_cp_enable(rdev, true);
3060
3061 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3062 if (r) {
3063 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3064 return r;
3065 }
3066
3067 /* setup clear context state */
3068 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3069 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3070
3071 for (i = 0; i < si_default_size; i++)
3072 radeon_ring_write(ring, si_default_state[i]);
3073
3074 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3075 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3076
3077 /* set clear context state */
3078 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3079 radeon_ring_write(ring, 0);
3080
3081 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3082 radeon_ring_write(ring, 0x00000316);
3083 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3084 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3085
3086 radeon_ring_unlock_commit(rdev, ring);
3087
3088 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3089 ring = &rdev->ring[i];
3090 r = radeon_ring_lock(rdev, ring, 2);
3091
3092 /* clear the compute context state */
3093 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3094 radeon_ring_write(ring, 0);
3095
3096 radeon_ring_unlock_commit(rdev, ring);
3097 }
3098
3099 return 0;
3100}
3101
3102static void si_cp_fini(struct radeon_device *rdev)
3103{
45df6803 3104 struct radeon_ring *ring;
48c0c902 3105 si_cp_enable(rdev, false);
45df6803
CK		 3106
3107 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3108 radeon_ring_fini(rdev, ring);
3109 radeon_scratch_free(rdev, ring->rptr_save_reg);
3110
3111 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3112 radeon_ring_fini(rdev, ring);
3113 radeon_scratch_free(rdev, ring->rptr_save_reg);
3114
3115 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3116 radeon_ring_fini(rdev, ring);
3117 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3118}
3119
3120static int si_cp_resume(struct radeon_device *rdev)
3121{
3122 struct radeon_ring *ring;
3123 u32 tmp;
3124 u32 rb_bufsz;
3125 int r;
3126
3127 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3128 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3129 SOFT_RESET_PA |
3130 SOFT_RESET_VGT |
3131 SOFT_RESET_SPI |
3132 SOFT_RESET_SX));
3133 RREG32(GRBM_SOFT_RESET);
3134 mdelay(15);
3135 WREG32(GRBM_SOFT_RESET, 0);
3136 RREG32(GRBM_SOFT_RESET);
3137
3138 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3139 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3140
3141 /* Set the write pointer delay */
3142 WREG32(CP_RB_WPTR_DELAY, 0);
3143
3144 WREG32(CP_DEBUG, 0);
3145 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3146
3147 /* ring 0 - compute and gfx */
3148 /* Set ring buffer size */
3149 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3150 rb_bufsz = drm_order(ring->ring_size / 8);
3151 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3152#ifdef __BIG_ENDIAN
3153 tmp |= BUF_SWAP_32BIT;
3154#endif
3155 WREG32(CP_RB0_CNTL, tmp);
3156
3157 /* Initialize the ring buffer's read and write pointers */
3158 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3159 ring->wptr = 0;
3160 WREG32(CP_RB0_WPTR, ring->wptr);
3161
48fc7f7e 3162 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3163 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3164 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3165
3166 if (rdev->wb.enabled)
3167 WREG32(SCRATCH_UMSK, 0xff);
3168 else {
3169 tmp |= RB_NO_UPDATE;
3170 WREG32(SCRATCH_UMSK, 0);
3171 }
3172
3173 mdelay(1);
3174 WREG32(CP_RB0_CNTL, tmp);
3175
3176 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3177
3178 ring->rptr = RREG32(CP_RB0_RPTR);
3179
3180 /* ring1 - compute only */
3181 /* Set ring buffer size */
3182 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3183 rb_bufsz = drm_order(ring->ring_size / 8);
3184 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3185#ifdef __BIG_ENDIAN
3186 tmp |= BUF_SWAP_32BIT;
3187#endif
3188 WREG32(CP_RB1_CNTL, tmp);
3189
3190 /* Initialize the ring buffer's read and write pointers */
3191 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3192 ring->wptr = 0;
3193 WREG32(CP_RB1_WPTR, ring->wptr);
3194
48fc7f7e 3195 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3196 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3197 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3198
3199 mdelay(1);
3200 WREG32(CP_RB1_CNTL, tmp);
3201
3202 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3203
3204 ring->rptr = RREG32(CP_RB1_RPTR);
3205
3206 /* ring2 - compute only */
3207 /* Set ring buffer size */
3208 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3209 rb_bufsz = drm_order(ring->ring_size / 8);
3210 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3211#ifdef __BIG_ENDIAN
3212 tmp |= BUF_SWAP_32BIT;
3213#endif
3214 WREG32(CP_RB2_CNTL, tmp);
3215
3216 /* Initialize the ring buffer's read and write pointers */
3217 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3218 ring->wptr = 0;
3219 WREG32(CP_RB2_WPTR, ring->wptr);
3220
48fc7f7e 3221 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3222 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3223 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3224
3225 mdelay(1);
3226 WREG32(CP_RB2_CNTL, tmp);
3227
3228 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3229
3230 ring->rptr = RREG32(CP_RB2_RPTR);
3231
3232 /* start the rings */
3233 si_cp_start(rdev);
3234 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3235 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3236 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3237 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3238 if (r) {
3239 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3240 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3241 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3242 return r;
3243 }
3244 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3245 if (r) {
3246 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3247 }
3248 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3249 if (r) {
3250 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3251 }
3252
3253 return 0;
3254}
3255
014bb209 3256static u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3257{
014bb209 3258 u32 reset_mask = 0;
1c534671 3259 u32 tmp;
06bc6df0 3260
014bb209
AD		 3261 /* GRBM_STATUS */
3262 tmp = RREG32(GRBM_STATUS);
3263 if (tmp & (PA_BUSY | SC_BUSY |
3264 BCI_BUSY | SX_BUSY |
3265 TA_BUSY | VGT_BUSY |
3266 DB_BUSY | CB_BUSY |
3267 GDS_BUSY | SPI_BUSY |
3268 IA_BUSY | IA_BUSY_NO_DMA))
3269 reset_mask |= RADEON_RESET_GFX;
3270
3271 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3272 CP_BUSY | CP_COHERENCY_BUSY))
3273 reset_mask |= RADEON_RESET_CP;
3274
3275 if (tmp & GRBM_EE_BUSY)
3276 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3277
3278 /* GRBM_STATUS2 */
3279 tmp = RREG32(GRBM_STATUS2);
3280 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3281 reset_mask |= RADEON_RESET_RLC;
3282
3283 /* DMA_STATUS_REG 0 */
3284 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3285 if (!(tmp & DMA_IDLE))
3286 reset_mask |= RADEON_RESET_DMA;
3287
3288 /* DMA_STATUS_REG 1 */
3289 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3290 if (!(tmp & DMA_IDLE))
3291 reset_mask |= RADEON_RESET_DMA1;
3292
3293 /* SRBM_STATUS2 */
3294 tmp = RREG32(SRBM_STATUS2);
3295 if (tmp & DMA_BUSY)
3296 reset_mask |= RADEON_RESET_DMA;
3297
3298 if (tmp & DMA1_BUSY)
3299 reset_mask |= RADEON_RESET_DMA1;
3300
3301 /* SRBM_STATUS */
3302 tmp = RREG32(SRBM_STATUS);
3303
3304 if (tmp & IH_BUSY)
3305 reset_mask |= RADEON_RESET_IH;
3306
3307 if (tmp & SEM_BUSY)
3308 reset_mask |= RADEON_RESET_SEM;
3309
3310 if (tmp & GRBM_RQ_PENDING)
3311 reset_mask |= RADEON_RESET_GRBM;
3312
3313 if (tmp & VMC_BUSY)
3314 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3315
014bb209
AD		 3316 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3317 MCC_BUSY | MCD_BUSY))
3318 reset_mask |= RADEON_RESET_MC;
3319
3320 if (evergreen_is_display_hung(rdev))
3321 reset_mask |= RADEON_RESET_DISPLAY;
3322
3323 /* VM_L2_STATUS */
3324 tmp = RREG32(VM_L2_STATUS);
3325 if (tmp & L2_BUSY)
3326 reset_mask |= RADEON_RESET_VMC;
3327
d808fc88
AD		 3328 /* Skip MC reset as it's mostly likely not hung, just busy */
3329 if (reset_mask & RADEON_RESET_MC) {
3330 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3331 reset_mask &= ~RADEON_RESET_MC;
3332 }
3333
014bb209
AD		 3334 return reset_mask;
3335}
3336
3337static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3338{
3339 struct evergreen_mc_save save;
3340 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3341 u32 tmp;
19fc42ed 3342
06bc6df0 3343 if (reset_mask == 0)
014bb209 3344 return;
06bc6df0
AD		 3345
3346 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3347
1c534671 3348 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3349 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3350 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3351 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3352 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3353
1c534671
AD		 3354 /* Disable CP parsing/prefetching */
3355 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3356
3357 if (reset_mask & RADEON_RESET_DMA) {
3358 /* dma0 */
3359 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3360 tmp &= ~DMA_RB_ENABLE;
3361 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3362 }
3363 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3364 /* dma1 */
3365 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3366 tmp &= ~DMA_RB_ENABLE;
3367 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3368 }
3369
f770d78a
AD		 3370 udelay(50);
3371
3372 evergreen_mc_stop(rdev, &save);
3373 if (evergreen_mc_wait_for_idle(rdev)) {
3374 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3375 }
3376
1c534671
AD		 3377 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3378 grbm_soft_reset = SOFT_RESET_CB |
3379 SOFT_RESET_DB |
3380 SOFT_RESET_GDS |
3381 SOFT_RESET_PA |
3382 SOFT_RESET_SC |
3383 SOFT_RESET_BCI |
3384 SOFT_RESET_SPI |
3385 SOFT_RESET_SX |
3386 SOFT_RESET_TC |
3387 SOFT_RESET_TA |
3388 SOFT_RESET_VGT |
3389 SOFT_RESET_IA;
3390 }
3391
3392 if (reset_mask & RADEON_RESET_CP) {
3393 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3394
3395 srbm_soft_reset |= SOFT_RESET_GRBM;
3396 }
06bc6df0
AD		 3397
3398 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3399 srbm_soft_reset |= SOFT_RESET_DMA;
3400
3401 if (reset_mask & RADEON_RESET_DMA1)
3402 srbm_soft_reset |= SOFT_RESET_DMA1;
3403
3404 if (reset_mask & RADEON_RESET_DISPLAY)
3405 srbm_soft_reset |= SOFT_RESET_DC;
3406
3407 if (reset_mask & RADEON_RESET_RLC)
3408 grbm_soft_reset |= SOFT_RESET_RLC;
3409
3410 if (reset_mask & RADEON_RESET_SEM)
3411 srbm_soft_reset |= SOFT_RESET_SEM;
3412
3413 if (reset_mask & RADEON_RESET_IH)
3414 srbm_soft_reset |= SOFT_RESET_IH;
3415
3416 if (reset_mask & RADEON_RESET_GRBM)
3417 srbm_soft_reset |= SOFT_RESET_GRBM;
3418
3419 if (reset_mask & RADEON_RESET_VMC)
3420 srbm_soft_reset |= SOFT_RESET_VMC;
3421
3422 if (reset_mask & RADEON_RESET_MC)
3423 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3424
3425 if (grbm_soft_reset) {
3426 tmp = RREG32(GRBM_SOFT_RESET);
3427 tmp |= grbm_soft_reset;
3428 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3429 WREG32(GRBM_SOFT_RESET, tmp);
3430 tmp = RREG32(GRBM_SOFT_RESET);
3431
3432 udelay(50);
3433
3434 tmp &= ~grbm_soft_reset;
3435 WREG32(GRBM_SOFT_RESET, tmp);
3436 tmp = RREG32(GRBM_SOFT_RESET);
3437 }
3438
3439 if (srbm_soft_reset) {
3440 tmp = RREG32(SRBM_SOFT_RESET);
3441 tmp |= srbm_soft_reset;
3442 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3443 WREG32(SRBM_SOFT_RESET, tmp);
3444 tmp = RREG32(SRBM_SOFT_RESET);
3445
3446 udelay(50);
3447
3448 tmp &= ~srbm_soft_reset;
3449 WREG32(SRBM_SOFT_RESET, tmp);
3450 tmp = RREG32(SRBM_SOFT_RESET);
3451 }
06bc6df0
AD		 3452
3453 /* Wait a little for things to settle down */
3454 udelay(50);
3455
c476dde2 3456 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3457 udelay(50);
3458
1c534671 3459 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3460}
3461
3462int si_asic_reset(struct radeon_device *rdev)
3463{
014bb209
AD		 3464 u32 reset_mask;
3465
3466 reset_mask = si_gpu_check_soft_reset(rdev);
3467
3468 if (reset_mask)
3469 r600_set_bios_scratch_engine_hung(rdev, true);
3470
3471 si_gpu_soft_reset(rdev, reset_mask);
3472
3473 reset_mask = si_gpu_check_soft_reset(rdev);
3474
3475 if (!reset_mask)
3476 r600_set_bios_scratch_engine_hung(rdev, false);
3477
3478 return 0;
c476dde2
AD		 3479}
3480
123bc183
AD		 3481/**
3482 * si_gfx_is_lockup - Check if the GFX engine is locked up
3483 *
3484 * @rdev: radeon_device pointer
3485 * @ring: radeon_ring structure holding ring information
3486 *
3487 * Check if the GFX engine is locked up.
3488 * Returns true if the engine appears to be locked up, false if not.
3489 */
3490bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3491{
3492 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3493
3494 if (!(reset_mask & (RADEON_RESET_GFX |
3495 RADEON_RESET_COMPUTE |
3496 RADEON_RESET_CP))) {
3497 radeon_ring_lockup_update(ring);
3498 return false;
3499 }
3500 /* force CP activities */
3501 radeon_ring_force_activity(rdev, ring);
3502 return radeon_ring_test_lockup(rdev, ring);
3503}
3504
3505/**
3506 * si_dma_is_lockup - Check if the DMA engine is locked up
3507 *
3508 * @rdev: radeon_device pointer
3509 * @ring: radeon_ring structure holding ring information
3510 *
3511 * Check if the async DMA engine is locked up.
3512 * Returns true if the engine appears to be locked up, false if not.
3513 */
3514bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3515{
3516 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3517 u32 mask;
3518
3519 if (ring->idx == R600_RING_TYPE_DMA_INDEX)
3520 mask = RADEON_RESET_DMA;
3521 else
3522 mask = RADEON_RESET_DMA1;
3523
3524 if (!(reset_mask & mask)) {
3525 radeon_ring_lockup_update(ring);
3526 return false;
3527 }
3528 /* force ring activities */
3529 radeon_ring_force_activity(rdev, ring);
3530 return radeon_ring_test_lockup(rdev, ring);
3531}
3532
d2800ee5
AD		 3533/* MC */
3534static void si_mc_program(struct radeon_device *rdev)
3535{
3536 struct evergreen_mc_save save;
3537 u32 tmp;
3538 int i, j;
3539
3540 /* Initialize HDP */
3541 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3542 WREG32((0x2c14 + j), 0x00000000);
3543 WREG32((0x2c18 + j), 0x00000000);
3544 WREG32((0x2c1c + j), 0x00000000);
3545 WREG32((0x2c20 + j), 0x00000000);
3546 WREG32((0x2c24 + j), 0x00000000);
3547 }
3548 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3549
3550 evergreen_mc_stop(rdev, &save);
3551 if (radeon_mc_wait_for_idle(rdev)) {
3552 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3553 }
5153550a
AD		 3554 if (!ASIC_IS_NODCE(rdev))
3555 /* Lockout access through VGA aperture*/
3556 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 3557 /* Update configuration */
3558 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3559 rdev->mc.vram_start >> 12);
3560 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3561 rdev->mc.vram_end >> 12);
3562 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3563 rdev->vram_scratch.gpu_addr >> 12);
3564 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3565 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3566 WREG32(MC_VM_FB_LOCATION, tmp);
3567 /* XXX double check these! */
3568 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3569 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3570 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3571 WREG32(MC_VM_AGP_BASE, 0);
3572 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3573 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3574 if (radeon_mc_wait_for_idle(rdev)) {
3575 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3576 }
3577 evergreen_mc_resume(rdev, &save);
5153550a
AD		 3578 if (!ASIC_IS_NODCE(rdev)) {
3579 /* we need to own VRAM, so turn off the VGA renderer here
3580 * to stop it overwriting our objects */
3581 rv515_vga_render_disable(rdev);
3582 }
d2800ee5
AD		 3583}
3584
1c49165d
AD		 3585void si_vram_gtt_location(struct radeon_device *rdev,
3586 struct radeon_mc *mc)
d2800ee5
AD		 3587{
3588 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3589 /* leave room for at least 1024M GTT */
3590 dev_warn(rdev->dev, "limiting VRAM\n");
3591 mc->real_vram_size = 0xFFC0000000ULL;
3592 mc->mc_vram_size = 0xFFC0000000ULL;
3593 }
9ed8b1f9 3594 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 3595 rdev->mc.gtt_base_align = 0;
9ed8b1f9 3596 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 3597}
3598
3599static int si_mc_init(struct radeon_device *rdev)
3600{
3601 u32 tmp;
3602 int chansize, numchan;
3603
3604 /* Get VRAM informations */
3605 rdev->mc.vram_is_ddr = true;
3606 tmp = RREG32(MC_ARB_RAMCFG);
3607 if (tmp & CHANSIZE_OVERRIDE) {
3608 chansize = 16;
3609 } else if (tmp & CHANSIZE_MASK) {
3610 chansize = 64;
3611 } else {
3612 chansize = 32;
3613 }
3614 tmp = RREG32(MC_SHARED_CHMAP);
3615 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3616 case 0:
3617 default:
3618 numchan = 1;
3619 break;
3620 case 1:
3621 numchan = 2;
3622 break;
3623 case 2:
3624 numchan = 4;
3625 break;
3626 case 3:
3627 numchan = 8;
3628 break;
3629 case 4:
3630 numchan = 3;
3631 break;
3632 case 5:
3633 numchan = 6;
3634 break;
3635 case 6:
3636 numchan = 10;
3637 break;
3638 case 7:
3639 numchan = 12;
3640 break;
3641 case 8:
3642 numchan = 16;
3643 break;
3644 }
3645 rdev->mc.vram_width = numchan * chansize;
3646 /* Could aper size report 0 ? */
3647 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3648 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3649 /* size in MB on si */
fc986034
NOS		 3650 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3651 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
d2800ee5
AD		 3652 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3653 si_vram_gtt_location(rdev, &rdev->mc);
3654 radeon_update_bandwidth_info(rdev);
3655
3656 return 0;
3657}
3658
3659/*
3660 * GART
3661 */
3662void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3663{
3664 /* flush hdp cache */
3665 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3666
3667 /* bits 0-15 are the VM contexts0-15 */
3668 WREG32(VM_INVALIDATE_REQUEST, 1);
3669}
3670
1109ca09 3671static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 3672{
3673 int r, i;
3674
3675 if (rdev->gart.robj == NULL) {
3676 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3677 return -EINVAL;
3678 }
3679 r = radeon_gart_table_vram_pin(rdev);
3680 if (r)
3681 return r;
3682 radeon_gart_restore(rdev);
3683 /* Setup TLB control */
3684 WREG32(MC_VM_MX_L1_TLB_CNTL,
3685 (0xA << 7) |
3686 ENABLE_L1_TLB |
3687 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3688 ENABLE_ADVANCED_DRIVER_MODEL |
3689 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3690 /* Setup L2 cache */
3691 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3692 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3693 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3694 EFFECTIVE_L2_QUEUE_SIZE(7) |
3695 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3696 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3697 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3698 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3699 /* setup context0 */
3700 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3701 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3702 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3703 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3704 (u32)(rdev->dummy_page.addr >> 12));
3705 WREG32(VM_CONTEXT0_CNTL2, 0);
3706 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3707 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3708
3709 WREG32(0x15D4, 0);
3710 WREG32(0x15D8, 0);
3711 WREG32(0x15DC, 0);
3712
3713 /* empty context1-15 */
d2800ee5
AD		 3714 /* set vm size, must be a multiple of 4 */
3715 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
c21b328e 3716 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
23d4f1f2
AD		 3717 /* Assign the pt base to something valid for now; the pts used for
3718 * the VMs are determined by the application and setup and assigned
3719 * on the fly in the vm part of radeon_gart.c
3720 */
d2800ee5
AD		 3721 for (i = 1; i < 16; i++) {
3722 if (i < 8)
3723 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3724 rdev->gart.table_addr >> 12);
3725 else
3726 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3727 rdev->gart.table_addr >> 12);
3728 }
3729
3730 /* enable context1-15 */
3731 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3732 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 3733 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 3734 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
ae133a11
CK		 3735 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3736 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3737 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3738 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3739 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3740 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3741 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3742 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3743 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3744 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3745 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3746 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 3747
3748 si_pcie_gart_tlb_flush(rdev);
3749 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3750 (unsigned)(rdev->mc.gtt_size >> 20),
3751 (unsigned long long)rdev->gart.table_addr);
3752 rdev->gart.ready = true;
3753 return 0;
3754}
3755
1109ca09 3756static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5
AD		 3757{
3758 /* Disable all tables */
3759 WREG32(VM_CONTEXT0_CNTL, 0);
3760 WREG32(VM_CONTEXT1_CNTL, 0);
3761 /* Setup TLB control */
3762 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3763 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3764 /* Setup L2 cache */
3765 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3766 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3767 EFFECTIVE_L2_QUEUE_SIZE(7) |
3768 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3769 WREG32(VM_L2_CNTL2, 0);
3770 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3771 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3772 radeon_gart_table_vram_unpin(rdev);
3773}
3774
1109ca09 3775static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 3776{
3777 si_pcie_gart_disable(rdev);
3778 radeon_gart_table_vram_free(rdev);
3779 radeon_gart_fini(rdev);
3780}
3781
498dd8b3
AD		 3782/* vm parser */
3783static bool si_vm_reg_valid(u32 reg)
3784{
3785 /* context regs are fine */
3786 if (reg >= 0x28000)
3787 return true;
3788
3789 /* check config regs */
3790 switch (reg) {
3791 case GRBM_GFX_INDEX:
f418b88a 3792 case CP_STRMOUT_CNTL:
498dd8b3
AD		 3793 case VGT_VTX_VECT_EJECT_REG:
3794 case VGT_CACHE_INVALIDATION:
3795 case VGT_ESGS_RING_SIZE:
3796 case VGT_GSVS_RING_SIZE:
3797 case VGT_GS_VERTEX_REUSE:
3798 case VGT_PRIMITIVE_TYPE:
3799 case VGT_INDEX_TYPE:
3800 case VGT_NUM_INDICES:
3801 case VGT_NUM_INSTANCES:
3802 case VGT_TF_RING_SIZE:
3803 case VGT_HS_OFFCHIP_PARAM:
3804 case VGT_TF_MEMORY_BASE:
3805 case PA_CL_ENHANCE:
3806 case PA_SU_LINE_STIPPLE_VALUE:
3807 case PA_SC_LINE_STIPPLE_STATE:
3808 case PA_SC_ENHANCE:
3809 case SQC_CACHES:
3810 case SPI_STATIC_THREAD_MGMT_1:
3811 case SPI_STATIC_THREAD_MGMT_2:
3812 case SPI_STATIC_THREAD_MGMT_3:
3813 case SPI_PS_MAX_WAVE_ID:
3814 case SPI_CONFIG_CNTL:
3815 case SPI_CONFIG_CNTL_1:
3816 case TA_CNTL_AUX:
3817 return true;
3818 default:
3819 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
3820 return false;
3821 }
3822}
3823
3824static int si_vm_packet3_ce_check(struct radeon_device *rdev,
3825 u32 *ib, struct radeon_cs_packet *pkt)
3826{
3827 switch (pkt->opcode) {
3828 case PACKET3_NOP:
3829 case PACKET3_SET_BASE:
3830 case PACKET3_SET_CE_DE_COUNTERS:
3831 case PACKET3_LOAD_CONST_RAM:
3832 case PACKET3_WRITE_CONST_RAM:
3833 case PACKET3_WRITE_CONST_RAM_OFFSET:
3834 case PACKET3_DUMP_CONST_RAM:
3835 case PACKET3_INCREMENT_CE_COUNTER:
3836 case PACKET3_WAIT_ON_DE_COUNTER:
3837 case PACKET3_CE_WRITE:
3838 break;
3839 default:
3840 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
3841 return -EINVAL;
3842 }
3843 return 0;
3844}
3845
3846static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
3847 u32 *ib, struct radeon_cs_packet *pkt)
3848{
3849 u32 idx = pkt->idx + 1;
3850 u32 idx_value = ib[idx];
3851 u32 start_reg, end_reg, reg, i;
5aa709be 3852 u32 command, info;
498dd8b3
AD		 3853
3854 switch (pkt->opcode) {
3855 case PACKET3_NOP:
3856 case PACKET3_SET_BASE:
3857 case PACKET3_CLEAR_STATE:
3858 case PACKET3_INDEX_BUFFER_SIZE:
3859 case PACKET3_DISPATCH_DIRECT:
3860 case PACKET3_DISPATCH_INDIRECT:
3861 case PACKET3_ALLOC_GDS:
3862 case PACKET3_WRITE_GDS_RAM:
3863 case PACKET3_ATOMIC_GDS:
3864 case PACKET3_ATOMIC:
3865 case PACKET3_OCCLUSION_QUERY:
3866 case PACKET3_SET_PREDICATION:
3867 case PACKET3_COND_EXEC:
3868 case PACKET3_PRED_EXEC:
3869 case PACKET3_DRAW_INDIRECT:
3870 case PACKET3_DRAW_INDEX_INDIRECT:
3871 case PACKET3_INDEX_BASE:
3872 case PACKET3_DRAW_INDEX_2:
3873 case PACKET3_CONTEXT_CONTROL:
3874 case PACKET3_INDEX_TYPE:
3875 case PACKET3_DRAW_INDIRECT_MULTI:
3876 case PACKET3_DRAW_INDEX_AUTO:
3877 case PACKET3_DRAW_INDEX_IMMD:
3878 case PACKET3_NUM_INSTANCES:
3879 case PACKET3_DRAW_INDEX_MULTI_AUTO:
3880 case PACKET3_STRMOUT_BUFFER_UPDATE:
3881 case PACKET3_DRAW_INDEX_OFFSET_2:
3882 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
3883 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
3884 case PACKET3_MPEG_INDEX:
3885 case PACKET3_WAIT_REG_MEM:
3886 case PACKET3_MEM_WRITE:
3887 case PACKET3_PFP_SYNC_ME:
3888 case PACKET3_SURFACE_SYNC:
3889 case PACKET3_EVENT_WRITE:
3890 case PACKET3_EVENT_WRITE_EOP:
3891 case PACKET3_EVENT_WRITE_EOS:
3892 case PACKET3_SET_CONTEXT_REG:
3893 case PACKET3_SET_CONTEXT_REG_INDIRECT:
3894 case PACKET3_SET_SH_REG:
3895 case PACKET3_SET_SH_REG_OFFSET:
3896 case PACKET3_INCREMENT_DE_COUNTER:
3897 case PACKET3_WAIT_ON_CE_COUNTER:
3898 case PACKET3_WAIT_ON_AVAIL_BUFFER:
3899 case PACKET3_ME_WRITE:
3900 break;
3901 case PACKET3_COPY_DATA:
3902 if ((idx_value & 0xf00) == 0) {
3903 reg = ib[idx + 3] * 4;
3904 if (!si_vm_reg_valid(reg))
3905 return -EINVAL;
3906 }
3907 break;
3908 case PACKET3_WRITE_DATA:
3909 if ((idx_value & 0xf00) == 0) {
3910 start_reg = ib[idx + 1] * 4;
3911 if (idx_value & 0x10000) {
3912 if (!si_vm_reg_valid(start_reg))
3913 return -EINVAL;
3914 } else {
3915 for (i = 0; i < (pkt->count - 2); i++) {
3916 reg = start_reg + (4 * i);
3917 if (!si_vm_reg_valid(reg))
3918 return -EINVAL;
3919 }
3920 }
3921 }
3922 break;
3923 case PACKET3_COND_WRITE:
3924 if (idx_value & 0x100) {
3925 reg = ib[idx + 5] * 4;
3926 if (!si_vm_reg_valid(reg))
3927 return -EINVAL;
3928 }
3929 break;
3930 case PACKET3_COPY_DW:
3931 if (idx_value & 0x2) {
3932 reg = ib[idx + 3] * 4;
3933 if (!si_vm_reg_valid(reg))
3934 return -EINVAL;
3935 }
3936 break;
3937 case PACKET3_SET_CONFIG_REG:
3938 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
3939 end_reg = 4 * pkt->count + start_reg - 4;
3940 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
3941 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
3942 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
3943 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
3944 return -EINVAL;
3945 }
3946 for (i = 0; i < pkt->count; i++) {
3947 reg = start_reg + (4 * i);
3948 if (!si_vm_reg_valid(reg))
3949 return -EINVAL;
3950 }
3951 break;
5aa709be
AD		 3952 case PACKET3_CP_DMA:
3953 command = ib[idx + 4];
3954 info = ib[idx + 1];
3955 if (command & PACKET3_CP_DMA_CMD_SAS) {
3956 /* src address space is register */
3957 if (((info & 0x60000000) >> 29) == 0) {
3958 start_reg = idx_value << 2;
3959 if (command & PACKET3_CP_DMA_CMD_SAIC) {
3960 reg = start_reg;
3961 if (!si_vm_reg_valid(reg)) {
3962 DRM_ERROR("CP DMA Bad SRC register\n");
3963 return -EINVAL;
3964 }
3965 } else {
3966 for (i = 0; i < (command & 0x1fffff); i++) {
3967 reg = start_reg + (4 * i);
3968 if (!si_vm_reg_valid(reg)) {
3969 DRM_ERROR("CP DMA Bad SRC register\n");
3970 return -EINVAL;
3971 }
3972 }
3973 }
3974 }
3975 }
3976 if (command & PACKET3_CP_DMA_CMD_DAS) {
3977 /* dst address space is register */
3978 if (((info & 0x00300000) >> 20) == 0) {
3979 start_reg = ib[idx + 2];
3980 if (command & PACKET3_CP_DMA_CMD_DAIC) {
3981 reg = start_reg;
3982 if (!si_vm_reg_valid(reg)) {
3983 DRM_ERROR("CP DMA Bad DST register\n");
3984 return -EINVAL;
3985 }
3986 } else {
3987 for (i = 0; i < (command & 0x1fffff); i++) {
3988 reg = start_reg + (4 * i);
3989 if (!si_vm_reg_valid(reg)) {
3990 DRM_ERROR("CP DMA Bad DST register\n");
3991 return -EINVAL;
3992 }
3993 }
3994 }
3995 }
3996 }
3997 break;
498dd8b3
AD		 3998 default:
3999 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4000 return -EINVAL;
4001 }
4002 return 0;
4003}
4004
4005static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4006 u32 *ib, struct radeon_cs_packet *pkt)
4007{
4008 u32 idx = pkt->idx + 1;
4009 u32 idx_value = ib[idx];
4010 u32 start_reg, reg, i;
4011
4012 switch (pkt->opcode) {
4013 case PACKET3_NOP:
4014 case PACKET3_SET_BASE:
4015 case PACKET3_CLEAR_STATE:
4016 case PACKET3_DISPATCH_DIRECT:
4017 case PACKET3_DISPATCH_INDIRECT:
4018 case PACKET3_ALLOC_GDS:
4019 case PACKET3_WRITE_GDS_RAM:
4020 case PACKET3_ATOMIC_GDS:
4021 case PACKET3_ATOMIC:
4022 case PACKET3_OCCLUSION_QUERY:
4023 case PACKET3_SET_PREDICATION:
4024 case PACKET3_COND_EXEC:
4025 case PACKET3_PRED_EXEC:
4026 case PACKET3_CONTEXT_CONTROL:
4027 case PACKET3_STRMOUT_BUFFER_UPDATE:
4028 case PACKET3_WAIT_REG_MEM:
4029 case PACKET3_MEM_WRITE:
4030 case PACKET3_PFP_SYNC_ME:
4031 case PACKET3_SURFACE_SYNC:
4032 case PACKET3_EVENT_WRITE:
4033 case PACKET3_EVENT_WRITE_EOP:
4034 case PACKET3_EVENT_WRITE_EOS:
4035 case PACKET3_SET_CONTEXT_REG:
4036 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4037 case PACKET3_SET_SH_REG:
4038 case PACKET3_SET_SH_REG_OFFSET:
4039 case PACKET3_INCREMENT_DE_COUNTER:
4040 case PACKET3_WAIT_ON_CE_COUNTER:
4041 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4042 case PACKET3_ME_WRITE:
4043 break;
4044 case PACKET3_COPY_DATA:
4045 if ((idx_value & 0xf00) == 0) {
4046 reg = ib[idx + 3] * 4;
4047 if (!si_vm_reg_valid(reg))
4048 return -EINVAL;
4049 }
4050 break;
4051 case PACKET3_WRITE_DATA:
4052 if ((idx_value & 0xf00) == 0) {
4053 start_reg = ib[idx + 1] * 4;
4054 if (idx_value & 0x10000) {
4055 if (!si_vm_reg_valid(start_reg))
4056 return -EINVAL;
4057 } else {
4058 for (i = 0; i < (pkt->count - 2); i++) {
4059 reg = start_reg + (4 * i);
4060 if (!si_vm_reg_valid(reg))
4061 return -EINVAL;
4062 }
4063 }
4064 }
4065 break;
4066 case PACKET3_COND_WRITE:
4067 if (idx_value & 0x100) {
4068 reg = ib[idx + 5] * 4;
4069 if (!si_vm_reg_valid(reg))
4070 return -EINVAL;
4071 }
4072 break;
4073 case PACKET3_COPY_DW:
4074 if (idx_value & 0x2) {
4075 reg = ib[idx + 3] * 4;
4076 if (!si_vm_reg_valid(reg))
4077 return -EINVAL;
4078 }
4079 break;
4080 default:
4081 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4082 return -EINVAL;
4083 }
4084 return 0;
4085}
4086
4087int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4088{
4089 int ret = 0;
4090 u32 idx = 0;
4091 struct radeon_cs_packet pkt;
4092
4093 do {
4094 pkt.idx = idx;
4e872ae2
IH		 4095 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4096 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4097 pkt.one_reg_wr = 0;
4098 switch (pkt.type) {
4e872ae2 4099 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 4100 dev_err(rdev->dev, "Packet0 not allowed!\n");
4101 ret = -EINVAL;
4102 break;
4e872ae2 4103 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4104 idx += 1;
4105 break;
4e872ae2
IH		 4106 case RADEON_PACKET_TYPE3:
4107 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4108 if (ib->is_const_ib)
4109 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4110 else {
876dc9f3 4111 switch (ib->ring) {
498dd8b3
AD		 4112 case RADEON_RING_TYPE_GFX_INDEX:
4113 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4114 break;
4115 case CAYMAN_RING_TYPE_CP1_INDEX:
4116 case CAYMAN_RING_TYPE_CP2_INDEX:
4117 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4118 break;
4119 default:
876dc9f3 4120 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4121 ret = -EINVAL;
4122 break;
4123 }
4124 }
4125 idx += pkt.count + 2;
4126 break;
4127 default:
4128 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4129 ret = -EINVAL;
4130 break;
4131 }
4132 if (ret)
4133 break;
4134 } while (idx < ib->length_dw);
4135
4136 return ret;
4137}
4138
d2800ee5
AD		 4139/*
4140 * vm
4141 */
4142int si_vm_init(struct radeon_device *rdev)
4143{
4144 /* number of VMs */
4145 rdev->vm_manager.nvm = 16;
4146 /* base offset of vram pages */
4147 rdev->vm_manager.vram_base_offset = 0;
4148
4149 return 0;
4150}
4151
4152void si_vm_fini(struct radeon_device *rdev)
4153{
4154}
4155
82ffd92b
AD		 4156/**
4157 * si_vm_set_page - update the page tables using the CP
4158 *
4159 * @rdev: radeon_device pointer
43f1214a 4160 * @ib: indirect buffer to fill with commands
82ffd92b
AD		 4161 * @pe: addr of the page entry
4162 * @addr: dst addr to write into pe
4163 * @count: number of page entries to update
4164 * @incr: increase next addr by incr bytes
4165 * @flags: access flags
4166 *
43f1214a 4167 * Update the page tables using the CP (SI).
82ffd92b 4168 */
43f1214a
AD		 4169void si_vm_set_page(struct radeon_device *rdev,
4170 struct radeon_ib *ib,
4171 uint64_t pe,
82ffd92b
AD		 4172 uint64_t addr, unsigned count,
4173 uint32_t incr, uint32_t flags)
d2800ee5 4174{
82ffd92b 4175 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
deab48f1
AD		 4176 uint64_t value;
4177 unsigned ndw;
4178
4179 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4180 while (count) {
4181 ndw = 2 + count * 2;
4182 if (ndw > 0x3FFE)
4183 ndw = 0x3FFE;
4184
43f1214a
AD		 4185 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4186 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4187 WRITE_DATA_DST_SEL(1));
4188 ib->ptr[ib->length_dw++] = pe;
4189 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
deab48f1
AD		 4190 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4191 if (flags & RADEON_VM_PAGE_SYSTEM) {
4192 value = radeon_vm_map_gart(rdev, addr);
4193 value &= 0xFFFFFFFFFFFFF000ULL;
4194 } else if (flags & RADEON_VM_PAGE_VALID) {
4195 value = addr;
4196 } else {
4197 value = 0;
4198 }
4199 addr += incr;
4200 value |= r600_flags;
43f1214a
AD		 4201 ib->ptr[ib->length_dw++] = value;
4202 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4203 }
4204 }
4205 } else {
4206 /* DMA */
4207 if (flags & RADEON_VM_PAGE_SYSTEM) {
4208 while (count) {
4209 ndw = count * 2;
4210 if (ndw > 0xFFFFE)
4211 ndw = 0xFFFFE;
4212
4213 /* for non-physically contiguous pages (system) */
43f1214a
AD		 4214 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
4215 ib->ptr[ib->length_dw++] = pe;
4216 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
deab48f1
AD		 4217 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
4218 if (flags & RADEON_VM_PAGE_SYSTEM) {
4219 value = radeon_vm_map_gart(rdev, addr);
4220 value &= 0xFFFFFFFFFFFFF000ULL;
4221 } else if (flags & RADEON_VM_PAGE_VALID) {
4222 value = addr;
4223 } else {
4224 value = 0;
4225 }
4226 addr += incr;
4227 value |= r600_flags;
43f1214a
AD		 4228 ib->ptr[ib->length_dw++] = value;
4229 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4230 }
4231 }
4232 } else {
4233 while (count) {
4234 ndw = count * 2;
4235 if (ndw > 0xFFFFE)
4236 ndw = 0xFFFFE;
4237
4238 if (flags & RADEON_VM_PAGE_VALID)
4239 value = addr;
4240 else
4241 value = 0;
4242 /* for physically contiguous pages (vram) */
43f1214a
AD		 4243 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
4244 ib->ptr[ib->length_dw++] = pe; /* dst addr */
4245 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
4246 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
4247 ib->ptr[ib->length_dw++] = 0;
4248 ib->ptr[ib->length_dw++] = value; /* value */
4249 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4250 ib->ptr[ib->length_dw++] = incr; /* increment size */
4251 ib->ptr[ib->length_dw++] = 0;
deab48f1
AD		 4252 pe += ndw * 4;
4253 addr += (ndw / 2) * incr;
4254 count -= ndw / 2;
4255 }
d7025d89 4256 }
43f1214a
AD		 4257 while (ib->length_dw & 0x7)
4258 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
82ffd92b 4259 }
d2800ee5
AD		 4260}
4261
498522b4 4262void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
d2800ee5 4263{
498522b4 4264 struct radeon_ring *ring = &rdev->ring[ridx];
d2800ee5 4265
ee60e29f 4266 if (vm == NULL)
d2800ee5
AD		 4267 return;
4268
76c44f2c
AD		 4269 /* write new base address */
4270 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4271 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4272 WRITE_DATA_DST_SEL(0)));
4273
ee60e29f 4274 if (vm->id < 8) {
76c44f2c
AD		 4275 radeon_ring_write(ring,
4276 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
ee60e29f 4277 } else {
76c44f2c
AD		 4278 radeon_ring_write(ring,
4279 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
ee60e29f 4280 }
76c44f2c 4281 radeon_ring_write(ring, 0);
fa87e62d 4282 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 4283
d2800ee5 4284 /* flush hdp cache */
76c44f2c
AD		 4285 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4286 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4287 WRITE_DATA_DST_SEL(0)));
4288 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4289 radeon_ring_write(ring, 0);
ee60e29f
CK		 4290 radeon_ring_write(ring, 0x1);
4291
d2800ee5 4292 /* bits 0-15 are the VM contexts0-15 */
76c44f2c
AD		 4293 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4294 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4295 WRITE_DATA_DST_SEL(0)));
4296 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4297 radeon_ring_write(ring, 0);
498522b4 4298 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 4299
4300 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4301 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4302 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 4303}
4304
8c5fd7ef
AD		 4305void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4306{
4307 struct radeon_ring *ring = &rdev->ring[ridx];
4308
4309 if (vm == NULL)
4310 return;
4311
4312 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4313 if (vm->id < 8) {
4314 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
4315 } else {
4316 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2));
4317 }
4318 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4319
4320 /* flush hdp cache */
4321 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4322 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4323 radeon_ring_write(ring, 1);
4324
4325 /* bits 0-7 are the VM contexts0-7 */
4326 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4327 radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
4328 radeon_ring_write(ring, 1 << vm->id);
4329}
4330
347e7592
AD		 4331/*
4332 * RLC
4333 */
c420c745 4334void si_rlc_fini(struct radeon_device *rdev)
347e7592
AD		 4335{
4336 int r;
4337
4338 /* save restore block */
4339 if (rdev->rlc.save_restore_obj) {
4340 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4341 if (unlikely(r != 0))
4342 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4343 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4344 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4345
4346 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4347 rdev->rlc.save_restore_obj = NULL;
4348 }
4349
4350 /* clear state block */
4351 if (rdev->rlc.clear_state_obj) {
4352 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4353 if (unlikely(r != 0))
4354 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4355 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4356 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4357
4358 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4359 rdev->rlc.clear_state_obj = NULL;
4360 }
4361}
4362
c420c745 4363int si_rlc_init(struct radeon_device *rdev)
347e7592
AD		 4364{
4365 int r;
4366
4367 /* save restore block */
4368 if (rdev->rlc.save_restore_obj == NULL) {
4369 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
40f5cf99
AD		 4370 RADEON_GEM_DOMAIN_VRAM, NULL,
4371 &rdev->rlc.save_restore_obj);
347e7592
AD		 4372 if (r) {
4373 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4374 return r;
4375 }
4376 }
4377
4378 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4379 if (unlikely(r != 0)) {
4380 si_rlc_fini(rdev);
4381 return r;
4382 }
4383 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4384 &rdev->rlc.save_restore_gpu_addr);
5273db70 4385 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
347e7592 4386 if (r) {
347e7592
AD		 4387 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4388 si_rlc_fini(rdev);
4389 return r;
4390 }
4391
4392 /* clear state block */
4393 if (rdev->rlc.clear_state_obj == NULL) {
4394 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
40f5cf99
AD		 4395 RADEON_GEM_DOMAIN_VRAM, NULL,
4396 &rdev->rlc.clear_state_obj);
347e7592
AD		 4397 if (r) {
4398 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4399 si_rlc_fini(rdev);
4400 return r;
4401 }
4402 }
4403 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4404 if (unlikely(r != 0)) {
4405 si_rlc_fini(rdev);
4406 return r;
4407 }
4408 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4409 &rdev->rlc.clear_state_gpu_addr);
5273db70 4410 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
347e7592 4411 if (r) {
347e7592
AD		 4412 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4413 si_rlc_fini(rdev);
4414 return r;
4415 }
4416
4417 return 0;
4418}
4419
4420static void si_rlc_stop(struct radeon_device *rdev)
4421{
4422 WREG32(RLC_CNTL, 0);
4423}
4424
4425static void si_rlc_start(struct radeon_device *rdev)
4426{
4427 WREG32(RLC_CNTL, RLC_ENABLE);
4428}
4429
4430static int si_rlc_resume(struct radeon_device *rdev)
4431{
4432 u32 i;
4433 const __be32 *fw_data;
4434
4435 if (!rdev->rlc_fw)
4436 return -EINVAL;
4437
4438 si_rlc_stop(rdev);
4439
4440 WREG32(RLC_RL_BASE, 0);
4441 WREG32(RLC_RL_SIZE, 0);
4442 WREG32(RLC_LB_CNTL, 0);
4443 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
4444 WREG32(RLC_LB_CNTR_INIT, 0);
4445
4446 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4447 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4448
4449 WREG32(RLC_MC_CNTL, 0);
4450 WREG32(RLC_UCODE_CNTL, 0);
4451
4452 fw_data = (const __be32 *)rdev->rlc_fw->data;
4453 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
4454 WREG32(RLC_UCODE_ADDR, i);
4455 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4456 }
4457 WREG32(RLC_UCODE_ADDR, 0);
4458
4459 si_rlc_start(rdev);
4460
4461 return 0;
4462}
4463
25a857fb
AD		 4464static void si_enable_interrupts(struct radeon_device *rdev)
4465{
4466 u32 ih_cntl = RREG32(IH_CNTL);
4467 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4468
4469 ih_cntl |= ENABLE_INTR;
4470 ih_rb_cntl |= IH_RB_ENABLE;
4471 WREG32(IH_CNTL, ih_cntl);
4472 WREG32(IH_RB_CNTL, ih_rb_cntl);
4473 rdev->ih.enabled = true;
4474}
4475
4476static void si_disable_interrupts(struct radeon_device *rdev)
4477{
4478 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4479 u32 ih_cntl = RREG32(IH_CNTL);
4480
4481 ih_rb_cntl &= ~IH_RB_ENABLE;
4482 ih_cntl &= ~ENABLE_INTR;
4483 WREG32(IH_RB_CNTL, ih_rb_cntl);
4484 WREG32(IH_CNTL, ih_cntl);
4485 /* set rptr, wptr to 0 */
4486 WREG32(IH_RB_RPTR, 0);
4487 WREG32(IH_RB_WPTR, 0);
4488 rdev->ih.enabled = false;
25a857fb
AD		 4489 rdev->ih.rptr = 0;
4490}
4491
4492static void si_disable_interrupt_state(struct radeon_device *rdev)
4493{
4494 u32 tmp;
4495
4496 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4497 WREG32(CP_INT_CNTL_RING1, 0);
4498 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 4499 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4500 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
4501 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4502 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 4503 WREG32(GRBM_INT_CNTL, 0);
5153550a
AD		 4504 if (rdev->num_crtc >= 2) {
4505 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4506 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4507 }
25a857fb
AD		 4508 if (rdev->num_crtc >= 4) {
4509 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4510 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4511 }
4512 if (rdev->num_crtc >= 6) {
4513 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4514 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4515 }
4516
5153550a
AD		 4517 if (rdev->num_crtc >= 2) {
4518 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4519 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4520 }
25a857fb
AD		 4521 if (rdev->num_crtc >= 4) {
4522 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4523 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4524 }
4525 if (rdev->num_crtc >= 6) {
4526 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4527 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4528 }
4529
5153550a
AD		 4530 if (!ASIC_IS_NODCE(rdev)) {
4531 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4532
4533 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4534 WREG32(DC_HPD1_INT_CONTROL, tmp);
4535 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4536 WREG32(DC_HPD2_INT_CONTROL, tmp);
4537 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4538 WREG32(DC_HPD3_INT_CONTROL, tmp);
4539 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4540 WREG32(DC_HPD4_INT_CONTROL, tmp);
4541 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4542 WREG32(DC_HPD5_INT_CONTROL, tmp);
4543 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4544 WREG32(DC_HPD6_INT_CONTROL, tmp);
4545 }
25a857fb
AD		 4546}
4547
4548static int si_irq_init(struct radeon_device *rdev)
4549{
4550 int ret = 0;
4551 int rb_bufsz;
4552 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
4553
4554 /* allocate ring */
4555 ret = r600_ih_ring_alloc(rdev);
4556 if (ret)
4557 return ret;
4558
4559 /* disable irqs */
4560 si_disable_interrupts(rdev);
4561
4562 /* init rlc */
4563 ret = si_rlc_resume(rdev);
4564 if (ret) {
4565 r600_ih_ring_fini(rdev);
4566 return ret;
4567 }
4568
4569 /* setup interrupt control */
4570 /* set dummy read address to ring address */
4571 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
4572 interrupt_cntl = RREG32(INTERRUPT_CNTL);
4573 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
4574 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
4575 */
4576 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
4577 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
4578 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
4579 WREG32(INTERRUPT_CNTL, interrupt_cntl);
4580
4581 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
4582 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
4583
4584 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
4585 IH_WPTR_OVERFLOW_CLEAR |
4586 (rb_bufsz << 1));
4587
4588 if (rdev->wb.enabled)
4589 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
4590
4591 /* set the writeback address whether it's enabled or not */
4592 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
4593 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
4594
4595 WREG32(IH_RB_CNTL, ih_rb_cntl);
4596
4597 /* set rptr, wptr to 0 */
4598 WREG32(IH_RB_RPTR, 0);
4599 WREG32(IH_RB_WPTR, 0);
4600
4601 /* Default settings for IH_CNTL (disabled at first) */
4602 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
4603 /* RPTR_REARM only works if msi's are enabled */
4604 if (rdev->msi_enabled)
4605 ih_cntl |= RPTR_REARM;
4606 WREG32(IH_CNTL, ih_cntl);
4607
4608 /* force the active interrupt state to all disabled */
4609 si_disable_interrupt_state(rdev);
4610
2099810f
DA		 4611 pci_set_master(rdev->pdev);
4612
25a857fb
AD		 4613 /* enable irqs */
4614 si_enable_interrupts(rdev);
4615
4616 return ret;
4617}
4618
4619int si_irq_set(struct radeon_device *rdev)
4620{
4621 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4622 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4623 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5153550a 4624 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
25a857fb
AD		 4625 u32 grbm_int_cntl = 0;
4626 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
8c5fd7ef 4627 u32 dma_cntl, dma_cntl1;
25a857fb
AD		 4628
4629 if (!rdev->irq.installed) {
4630 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4631 return -EINVAL;
4632 }
4633 /* don't enable anything if the ih is disabled */
4634 if (!rdev->ih.enabled) {
4635 si_disable_interrupts(rdev);
4636 /* force the active interrupt state to all disabled */
4637 si_disable_interrupt_state(rdev);
4638 return 0;
4639 }
4640
5153550a
AD		 4641 if (!ASIC_IS_NODCE(rdev)) {
4642 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4643 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4644 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4645 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4646 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4647 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4648 }
25a857fb 4649
8c5fd7ef
AD		 4650 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4651 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4652
25a857fb 4653 /* enable CP interrupts on all rings */
736fc37f 4654 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 4655 DRM_DEBUG("si_irq_set: sw int gfx\n");
4656 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4657 }
736fc37f 4658 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 4659 DRM_DEBUG("si_irq_set: sw int cp1\n");
4660 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4661 }
736fc37f 4662 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 4663 DRM_DEBUG("si_irq_set: sw int cp2\n");
4664 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4665 }
8c5fd7ef
AD		 4666 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4667 DRM_DEBUG("si_irq_set: sw int dma\n");
4668 dma_cntl |= TRAP_ENABLE;
4669 }
4670
4671 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4672 DRM_DEBUG("si_irq_set: sw int dma1\n");
4673 dma_cntl1 |= TRAP_ENABLE;
4674 }
25a857fb 4675 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4676 atomic_read(&rdev->irq.pflip[0])) {
25a857fb
AD		 4677 DRM_DEBUG("si_irq_set: vblank 0\n");
4678 crtc1 |= VBLANK_INT_MASK;
4679 }
4680 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4681 atomic_read(&rdev->irq.pflip[1])) {
25a857fb
AD		 4682 DRM_DEBUG("si_irq_set: vblank 1\n");
4683 crtc2 |= VBLANK_INT_MASK;
4684 }
4685 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4686 atomic_read(&rdev->irq.pflip[2])) {
25a857fb
AD		 4687 DRM_DEBUG("si_irq_set: vblank 2\n");
4688 crtc3 |= VBLANK_INT_MASK;
4689 }
4690 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4691 atomic_read(&rdev->irq.pflip[3])) {
25a857fb
AD		 4692 DRM_DEBUG("si_irq_set: vblank 3\n");
4693 crtc4 |= VBLANK_INT_MASK;
4694 }
4695 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4696 atomic_read(&rdev->irq.pflip[4])) {
25a857fb
AD		 4697 DRM_DEBUG("si_irq_set: vblank 4\n");
4698 crtc5 |= VBLANK_INT_MASK;
4699 }
4700 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4701 atomic_read(&rdev->irq.pflip[5])) {
25a857fb
AD		 4702 DRM_DEBUG("si_irq_set: vblank 5\n");
4703 crtc6 |= VBLANK_INT_MASK;
4704 }
4705 if (rdev->irq.hpd[0]) {
4706 DRM_DEBUG("si_irq_set: hpd 1\n");
4707 hpd1 |= DC_HPDx_INT_EN;
4708 }
4709 if (rdev->irq.hpd[1]) {
4710 DRM_DEBUG("si_irq_set: hpd 2\n");
4711 hpd2 |= DC_HPDx_INT_EN;
4712 }
4713 if (rdev->irq.hpd[2]) {
4714 DRM_DEBUG("si_irq_set: hpd 3\n");
4715 hpd3 |= DC_HPDx_INT_EN;
4716 }
4717 if (rdev->irq.hpd[3]) {
4718 DRM_DEBUG("si_irq_set: hpd 4\n");
4719 hpd4 |= DC_HPDx_INT_EN;
4720 }
4721 if (rdev->irq.hpd[4]) {
4722 DRM_DEBUG("si_irq_set: hpd 5\n");
4723 hpd5 |= DC_HPDx_INT_EN;
4724 }
4725 if (rdev->irq.hpd[5]) {
4726 DRM_DEBUG("si_irq_set: hpd 6\n");
4727 hpd6 |= DC_HPDx_INT_EN;
4728 }
25a857fb
AD		 4729
4730 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
4731 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
4732 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
4733
8c5fd7ef
AD		 4734 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
4735 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
4736
25a857fb
AD		 4737 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4738
5153550a
AD		 4739 if (rdev->num_crtc >= 2) {
4740 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4741 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4742 }
25a857fb
AD		 4743 if (rdev->num_crtc >= 4) {
4744 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4745 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4746 }
4747 if (rdev->num_crtc >= 6) {
4748 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4749 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4750 }
4751
5153550a
AD		 4752 if (rdev->num_crtc >= 2) {
4753 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4754 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
4755 }
25a857fb
AD		 4756 if (rdev->num_crtc >= 4) {
4757 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4758 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4759 }
4760 if (rdev->num_crtc >= 6) {
4761 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4762 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4763 }
4764
5153550a
AD		 4765 if (!ASIC_IS_NODCE(rdev)) {
4766 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4767 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4768 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4769 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4770 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4771 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4772 }
25a857fb
AD		 4773
4774 return 0;
4775}
4776
4777static inline void si_irq_ack(struct radeon_device *rdev)
4778{
4779 u32 tmp;
4780
5153550a
AD		 4781 if (ASIC_IS_NODCE(rdev))
4782 return;
4783
25a857fb
AD		 4784 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4785 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4786 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4787 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4788 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4789 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4790 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4791 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4792 if (rdev->num_crtc >= 4) {
4793 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4794 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4795 }
4796 if (rdev->num_crtc >= 6) {
4797 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4798 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4799 }
4800
4801 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4802 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4803 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4804 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4805 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4806 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4807 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4808 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4809 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4810 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4811 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4812 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4813
4814 if (rdev->num_crtc >= 4) {
4815 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4816 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4817 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4818 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4819 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4820 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4821 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4822 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4823 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4824 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4825 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4826 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4827 }
4828
4829 if (rdev->num_crtc >= 6) {
4830 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4831 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4832 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4833 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4834 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4835 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4836 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4837 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4838 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4839 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4840 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4841 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4842 }
4843
4844 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4845 tmp = RREG32(DC_HPD1_INT_CONTROL);
4846 tmp |= DC_HPDx_INT_ACK;
4847 WREG32(DC_HPD1_INT_CONTROL, tmp);
4848 }
4849 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4850 tmp = RREG32(DC_HPD2_INT_CONTROL);
4851 tmp |= DC_HPDx_INT_ACK;
4852 WREG32(DC_HPD2_INT_CONTROL, tmp);
4853 }
4854 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4855 tmp = RREG32(DC_HPD3_INT_CONTROL);
4856 tmp |= DC_HPDx_INT_ACK;
4857 WREG32(DC_HPD3_INT_CONTROL, tmp);
4858 }
4859 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4860 tmp = RREG32(DC_HPD4_INT_CONTROL);
4861 tmp |= DC_HPDx_INT_ACK;
4862 WREG32(DC_HPD4_INT_CONTROL, tmp);
4863 }
4864 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4865 tmp = RREG32(DC_HPD5_INT_CONTROL);
4866 tmp |= DC_HPDx_INT_ACK;
4867 WREG32(DC_HPD5_INT_CONTROL, tmp);
4868 }
4869 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4870 tmp = RREG32(DC_HPD5_INT_CONTROL);
4871 tmp |= DC_HPDx_INT_ACK;
4872 WREG32(DC_HPD6_INT_CONTROL, tmp);
4873 }
4874}
4875
4876static void si_irq_disable(struct radeon_device *rdev)
4877{
4878 si_disable_interrupts(rdev);
4879 /* Wait and acknowledge irq */
4880 mdelay(1);
4881 si_irq_ack(rdev);
4882 si_disable_interrupt_state(rdev);
4883}
4884
4885static void si_irq_suspend(struct radeon_device *rdev)
4886{
4887 si_irq_disable(rdev);
4888 si_rlc_stop(rdev);
4889}
4890
9b136d51
AD		 4891static void si_irq_fini(struct radeon_device *rdev)
4892{
4893 si_irq_suspend(rdev);
4894 r600_ih_ring_fini(rdev);
4895}
4896
25a857fb
AD		 4897static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
4898{
4899 u32 wptr, tmp;
4900
4901 if (rdev->wb.enabled)
4902 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4903 else
4904 wptr = RREG32(IH_RB_WPTR);
4905
4906 if (wptr & RB_OVERFLOW) {
4907 /* When a ring buffer overflow happen start parsing interrupt
4908 * from the last not overwritten vector (wptr + 16). Hopefully
4909 * this should allow us to catchup.
4910 */
4911 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4912 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4913 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4914 tmp = RREG32(IH_RB_CNTL);
4915 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4916 WREG32(IH_RB_CNTL, tmp);
4917 }
4918 return (wptr & rdev->ih.ptr_mask);
4919}
4920
4921/* SI IV Ring
4922 * Each IV ring entry is 128 bits:
4923 * [7:0] - interrupt source id
4924 * [31:8] - reserved
4925 * [59:32] - interrupt source data
4926 * [63:60] - reserved
4927 * [71:64] - RINGID
4928 * [79:72] - VMID
4929 * [127:80] - reserved
4930 */
4931int si_irq_process(struct radeon_device *rdev)
4932{
4933 u32 wptr;
4934 u32 rptr;
4935 u32 src_id, src_data, ring_id;
4936 u32 ring_index;
25a857fb
AD		 4937 bool queue_hotplug = false;
4938
4939 if (!rdev->ih.enabled || rdev->shutdown)
4940 return IRQ_NONE;
4941
4942 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 4943
4944restart_ih:
4945 /* is somebody else already processing irqs? */
4946 if (atomic_xchg(&rdev->ih.lock, 1))
4947 return IRQ_NONE;
4948
25a857fb
AD		 4949 rptr = rdev->ih.rptr;
4950 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4951
25a857fb
AD		 4952 /* Order reading of wptr vs. reading of IH ring data */
4953 rmb();
4954
4955 /* display interrupts */
4956 si_irq_ack(rdev);
4957
25a857fb
AD		 4958 while (rptr != wptr) {
4959 /* wptr/rptr are in bytes! */
4960 ring_index = rptr / 4;
4961 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4962 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4963 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
4964
4965 switch (src_id) {
4966 case 1: /* D1 vblank/vline */
4967 switch (src_data) {
4968 case 0: /* D1 vblank */
4969 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4970 if (rdev->irq.crtc_vblank_int[0]) {
4971 drm_handle_vblank(rdev->ddev, 0);
4972 rdev->pm.vblank_sync = true;
4973 wake_up(&rdev->irq.vblank_queue);
4974 }
736fc37f 4975 if (atomic_read(&rdev->irq.pflip[0]))
25a857fb
AD		 4976 radeon_crtc_handle_flip(rdev, 0);
4977 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4978 DRM_DEBUG("IH: D1 vblank\n");
4979 }
4980 break;
4981 case 1: /* D1 vline */
4982 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4983 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4984 DRM_DEBUG("IH: D1 vline\n");
4985 }
4986 break;
4987 default:
4988 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4989 break;
4990 }
4991 break;
4992 case 2: /* D2 vblank/vline */
4993 switch (src_data) {
4994 case 0: /* D2 vblank */
4995 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4996 if (rdev->irq.crtc_vblank_int[1]) {
4997 drm_handle_vblank(rdev->ddev, 1);
4998 rdev->pm.vblank_sync = true;
4999 wake_up(&rdev->irq.vblank_queue);
5000 }
736fc37f 5001 if (atomic_read(&rdev->irq.pflip[1]))
25a857fb
AD		 5002 radeon_crtc_handle_flip(rdev, 1);
5003 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
5004 DRM_DEBUG("IH: D2 vblank\n");
5005 }
5006 break;
5007 case 1: /* D2 vline */
5008 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
5009 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
5010 DRM_DEBUG("IH: D2 vline\n");
5011 }
5012 break;
5013 default:
5014 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5015 break;
5016 }
5017 break;
5018 case 3: /* D3 vblank/vline */
5019 switch (src_data) {
5020 case 0: /* D3 vblank */
5021 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
5022 if (rdev->irq.crtc_vblank_int[2]) {
5023 drm_handle_vblank(rdev->ddev, 2);
5024 rdev->pm.vblank_sync = true;
5025 wake_up(&rdev->irq.vblank_queue);
5026 }
736fc37f 5027 if (atomic_read(&rdev->irq.pflip[2]))
25a857fb
AD		 5028 radeon_crtc_handle_flip(rdev, 2);
5029 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5030 DRM_DEBUG("IH: D3 vblank\n");
5031 }
5032 break;
5033 case 1: /* D3 vline */
5034 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
5035 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5036 DRM_DEBUG("IH: D3 vline\n");
5037 }
5038 break;
5039 default:
5040 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5041 break;
5042 }
5043 break;
5044 case 4: /* D4 vblank/vline */
5045 switch (src_data) {
5046 case 0: /* D4 vblank */
5047 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
5048 if (rdev->irq.crtc_vblank_int[3]) {
5049 drm_handle_vblank(rdev->ddev, 3);
5050 rdev->pm.vblank_sync = true;
5051 wake_up(&rdev->irq.vblank_queue);
5052 }
736fc37f 5053 if (atomic_read(&rdev->irq.pflip[3]))
25a857fb
AD		 5054 radeon_crtc_handle_flip(rdev, 3);
5055 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5056 DRM_DEBUG("IH: D4 vblank\n");
5057 }
5058 break;
5059 case 1: /* D4 vline */
5060 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
5061 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5062 DRM_DEBUG("IH: D4 vline\n");
5063 }
5064 break;
5065 default:
5066 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5067 break;
5068 }
5069 break;
5070 case 5: /* D5 vblank/vline */
5071 switch (src_data) {
5072 case 0: /* D5 vblank */
5073 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
5074 if (rdev->irq.crtc_vblank_int[4]) {
5075 drm_handle_vblank(rdev->ddev, 4);
5076 rdev->pm.vblank_sync = true;
5077 wake_up(&rdev->irq.vblank_queue);
5078 }
736fc37f 5079 if (atomic_read(&rdev->irq.pflip[4]))
25a857fb
AD		 5080 radeon_crtc_handle_flip(rdev, 4);
5081 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5082 DRM_DEBUG("IH: D5 vblank\n");
5083 }
5084 break;
5085 case 1: /* D5 vline */
5086 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
5087 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5088 DRM_DEBUG("IH: D5 vline\n");
5089 }
5090 break;
5091 default:
5092 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5093 break;
5094 }
5095 break;
5096 case 6: /* D6 vblank/vline */
5097 switch (src_data) {
5098 case 0: /* D6 vblank */
5099 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
5100 if (rdev->irq.crtc_vblank_int[5]) {
5101 drm_handle_vblank(rdev->ddev, 5);
5102 rdev->pm.vblank_sync = true;
5103 wake_up(&rdev->irq.vblank_queue);
5104 }
736fc37f 5105 if (atomic_read(&rdev->irq.pflip[5]))
25a857fb
AD		 5106 radeon_crtc_handle_flip(rdev, 5);
5107 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5108 DRM_DEBUG("IH: D6 vblank\n");
5109 }
5110 break;
5111 case 1: /* D6 vline */
5112 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
5113 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5114 DRM_DEBUG("IH: D6 vline\n");
5115 }
5116 break;
5117 default:
5118 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5119 break;
5120 }
5121 break;
5122 case 42: /* HPD hotplug */
5123 switch (src_data) {
5124 case 0:
5125 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5126 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5127 queue_hotplug = true;
5128 DRM_DEBUG("IH: HPD1\n");
5129 }
5130 break;
5131 case 1:
5132 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5133 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5134 queue_hotplug = true;
5135 DRM_DEBUG("IH: HPD2\n");
5136 }
5137 break;
5138 case 2:
5139 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5140 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5141 queue_hotplug = true;
5142 DRM_DEBUG("IH: HPD3\n");
5143 }
5144 break;
5145 case 3:
5146 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5147 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5148 queue_hotplug = true;
5149 DRM_DEBUG("IH: HPD4\n");
5150 }
5151 break;
5152 case 4:
5153 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5154 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5155 queue_hotplug = true;
5156 DRM_DEBUG("IH: HPD5\n");
5157 }
5158 break;
5159 case 5:
5160 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5161 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5162 queue_hotplug = true;
5163 DRM_DEBUG("IH: HPD6\n");
5164 }
5165 break;
5166 default:
5167 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5168 break;
5169 }
5170 break;
ae133a11
CK		 5171 case 146:
5172 case 147:
5173 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5174 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5175 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5176 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5177 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5178 /* reset addr and status */
5179 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5180 break;
25a857fb
AD		 5181 case 176: /* RINGID0 CP_INT */
5182 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5183 break;
5184 case 177: /* RINGID1 CP_INT */
5185 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5186 break;
5187 case 178: /* RINGID2 CP_INT */
5188 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5189 break;
5190 case 181: /* CP EOP event */
5191 DRM_DEBUG("IH: CP EOP\n");
5192 switch (ring_id) {
5193 case 0:
5194 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5195 break;
5196 case 1:
5197 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5198 break;
5199 case 2:
5200 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5201 break;
5202 }
5203 break;
8c5fd7ef
AD		 5204 case 224: /* DMA trap event */
5205 DRM_DEBUG("IH: DMA trap\n");
5206 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5207 break;
25a857fb
AD		 5208 case 233: /* GUI IDLE */
5209 DRM_DEBUG("IH: GUI idle\n");
25a857fb 5210 break;
8c5fd7ef
AD		 5211 case 244: /* DMA trap event */
5212 DRM_DEBUG("IH: DMA1 trap\n");
5213 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5214 break;
25a857fb
AD		 5215 default:
5216 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5217 break;
5218 }
5219
5220 /* wptr/rptr are in bytes! */
5221 rptr += 16;
5222 rptr &= rdev->ih.ptr_mask;
5223 }
25a857fb
AD		 5224 if (queue_hotplug)
5225 schedule_work(&rdev->hotplug_work);
5226 rdev->ih.rptr = rptr;
5227 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 5228 atomic_set(&rdev->ih.lock, 0);
5229
5230 /* make sure wptr hasn't changed while processing */
5231 wptr = si_get_ih_wptr(rdev);
5232 if (wptr != rptr)
5233 goto restart_ih;
5234
25a857fb
AD		 5235 return IRQ_HANDLED;
5236}
5237
8c5fd7ef
AD		 5238/**
5239 * si_copy_dma - copy pages using the DMA engine
5240 *
5241 * @rdev: radeon_device pointer
5242 * @src_offset: src GPU address
5243 * @dst_offset: dst GPU address
5244 * @num_gpu_pages: number of GPU pages to xfer
5245 * @fence: radeon fence object
5246 *
5247 * Copy GPU paging using the DMA engine (SI).
5248 * Used by the radeon ttm implementation to move pages if
5249 * registered as the asic copy callback.
5250 */
5251int si_copy_dma(struct radeon_device *rdev,
5252 uint64_t src_offset, uint64_t dst_offset,
5253 unsigned num_gpu_pages,
5254 struct radeon_fence **fence)
5255{
5256 struct radeon_semaphore *sem = NULL;
5257 int ring_index = rdev->asic->copy.dma_ring_index;
5258 struct radeon_ring *ring = &rdev->ring[ring_index];
5259 u32 size_in_bytes, cur_size_in_bytes;
5260 int i, num_loops;
5261 int r = 0;
5262
5263 r = radeon_semaphore_create(rdev, &sem);
5264 if (r) {
5265 DRM_ERROR("radeon: moving bo (%d).\n", r);
5266 return r;
5267 }
5268
5269 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
5270 num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
5271 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5272 if (r) {
5273 DRM_ERROR("radeon: moving bo (%d).\n", r);
5274 radeon_semaphore_free(rdev, &sem, NULL);
5275 return r;
5276 }
5277
5278 if (radeon_fence_need_sync(*fence, ring->idx)) {
5279 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5280 ring->idx);
5281 radeon_fence_note_sync(*fence, ring->idx);
5282 } else {
5283 radeon_semaphore_free(rdev, &sem, NULL);
5284 }
5285
5286 for (i = 0; i < num_loops; i++) {
5287 cur_size_in_bytes = size_in_bytes;
5288 if (cur_size_in_bytes > 0xFFFFF)
5289 cur_size_in_bytes = 0xFFFFF;
5290 size_in_bytes -= cur_size_in_bytes;
5291 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
5292 radeon_ring_write(ring, dst_offset & 0xffffffff);
5293 radeon_ring_write(ring, src_offset & 0xffffffff);
5294 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5295 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5296 src_offset += cur_size_in_bytes;
5297 dst_offset += cur_size_in_bytes;
5298 }
5299
5300 r = radeon_fence_emit(rdev, fence, ring->idx);
5301 if (r) {
5302 radeon_ring_unlock_undo(rdev, ring);
5303 return r;
5304 }
5305
5306 radeon_ring_unlock_commit(rdev, ring);
5307 radeon_semaphore_free(rdev, &sem, *fence);
5308
5309 return r;
5310}
5311
9b136d51
AD		 5312/*
5313 * startup/shutdown callbacks
5314 */
5315static int si_startup(struct radeon_device *rdev)
5316{
5317 struct radeon_ring *ring;
5318 int r;
5319
b9d305df
AD		 5320 /* enable pcie gen2/3 link */
5321 si_pcie_gen3_enable(rdev);
5322
9b136d51
AD		 5323 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
5324 !rdev->rlc_fw || !rdev->mc_fw) {
5325 r = si_init_microcode(rdev);
5326 if (r) {
5327 DRM_ERROR("Failed to load firmware!\n");
5328 return r;
5329 }
5330 }
5331
5332 r = si_mc_load_microcode(rdev);
5333 if (r) {
5334 DRM_ERROR("Failed to load MC firmware!\n");
5335 return r;
5336 }
5337
5338 r = r600_vram_scratch_init(rdev);
5339 if (r)
5340 return r;
5341
5342 si_mc_program(rdev);
5343 r = si_pcie_gart_enable(rdev);
5344 if (r)
5345 return r;
5346 si_gpu_init(rdev);
5347
9b136d51
AD		 5348 /* allocate rlc buffers */
5349 r = si_rlc_init(rdev);
5350 if (r) {
5351 DRM_ERROR("Failed to init rlc BOs!\n");
5352 return r;
5353 }
5354
5355 /* allocate wb buffer */
5356 r = radeon_wb_init(rdev);
5357 if (r)
5358 return r;
5359
5360 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5361 if (r) {
5362 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5363 return r;
5364 }
5365
5366 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5367 if (r) {
5368 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5369 return r;
5370 }
5371
5372 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5373 if (r) {
5374 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5375 return r;
5376 }
5377
8c5fd7ef
AD		 5378 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5379 if (r) {
5380 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5381 return r;
5382 }
5383
5384 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5385 if (r) {
5386 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5387 return r;
5388 }
5389
1df0d523
AD		 5390 if (rdev->has_uvd) {
5391 r = rv770_uvd_resume(rdev);
5392 if (!r) {
5393 r = radeon_fence_driver_start_ring(rdev,
5394 R600_RING_TYPE_UVD_INDEX);
5395 if (r)
5396 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5397 }
f2ba57b5 5398 if (r)
1df0d523 5399 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
f2ba57b5 5400 }
f2ba57b5 5401
9b136d51 5402 /* Enable IRQ */
e49f3959
AH		 5403 if (!rdev->irq.installed) {
5404 r = radeon_irq_kms_init(rdev);
5405 if (r)
5406 return r;
5407 }
5408
9b136d51
AD		 5409 r = si_irq_init(rdev);
5410 if (r) {
5411 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5412 radeon_irq_kms_fini(rdev);
5413 return r;
5414 }
5415 si_irq_set(rdev);
5416
5417 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5418 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5419 CP_RB0_RPTR, CP_RB0_WPTR,
5420 0, 0xfffff, RADEON_CP_PACKET2);
5421 if (r)
5422 return r;
5423
5424 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5425 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
5426 CP_RB1_RPTR, CP_RB1_WPTR,
5427 0, 0xfffff, RADEON_CP_PACKET2);
5428 if (r)
5429 return r;
5430
5431 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5432 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
5433 CP_RB2_RPTR, CP_RB2_WPTR,
5434 0, 0xfffff, RADEON_CP_PACKET2);
5435 if (r)
5436 return r;
5437
8c5fd7ef
AD		 5438 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5439 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5440 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
5441 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
5442 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5443 if (r)
5444 return r;
5445
5446 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5447 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
5448 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
5449 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
5450 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5451 if (r)
5452 return r;
5453
9b136d51
AD		 5454 r = si_cp_load_microcode(rdev);
5455 if (r)
5456 return r;
5457 r = si_cp_resume(rdev);
5458 if (r)
5459 return r;
5460
8c5fd7ef
AD		 5461 r = cayman_dma_resume(rdev);
5462 if (r)
5463 return r;
5464
1df0d523
AD		 5465 if (rdev->has_uvd) {
5466 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5467 if (ring->ring_size) {
5468 r = radeon_ring_init(rdev, ring, ring->ring_size,
5469 R600_WB_UVD_RPTR_OFFSET,
5470 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5471 0, 0xfffff, RADEON_CP_PACKET2);
5472 if (!r)
5473 r = r600_uvd_init(rdev);
5474 if (r)
5475 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
5476 }
f2ba57b5
CK		 5477 }
5478
2898c348
CK		 5479 r = radeon_ib_pool_init(rdev);
5480 if (r) {
5481 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 5482 return r;
2898c348 5483 }
9b136d51 5484
c6105f24
CK		 5485 r = radeon_vm_manager_init(rdev);
5486 if (r) {
5487 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 5488 return r;
c6105f24 5489 }
9b136d51
AD		 5490
5491 return 0;
5492}
5493
5494int si_resume(struct radeon_device *rdev)
5495{
5496 int r;
5497
5498 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5499 * posting will perform necessary task to bring back GPU into good
5500 * shape.
5501 */
5502 /* post card */
5503 atom_asic_init(rdev->mode_info.atom_context);
5504
205996c0
AD		 5505 /* init golden registers */
5506 si_init_golden_registers(rdev);
5507
9b136d51
AD		 5508 rdev->accel_working = true;
5509 r = si_startup(rdev);
5510 if (r) {
5511 DRM_ERROR("si startup failed on resume\n");
5512 rdev->accel_working = false;
5513 return r;
5514 }
5515
5516 return r;
5517
5518}
5519
5520int si_suspend(struct radeon_device *rdev)
5521{
fa3daf9a 5522 radeon_vm_manager_fini(rdev);
9b136d51 5523 si_cp_enable(rdev, false);
8c5fd7ef 5524 cayman_dma_stop(rdev);
1df0d523
AD		 5525 if (rdev->has_uvd) {
5526 r600_uvd_rbc_stop(rdev);
5527 radeon_uvd_suspend(rdev);
5528 }
9b136d51
AD		 5529 si_irq_suspend(rdev);
5530 radeon_wb_disable(rdev);
5531 si_pcie_gart_disable(rdev);
5532 return 0;
5533}
5534
5535/* Plan is to move initialization in that function and use
5536 * helper function so that radeon_device_init pretty much
5537 * do nothing more than calling asic specific function. This
5538 * should also allow to remove a bunch of callback function
5539 * like vram_info.
5540 */
5541int si_init(struct radeon_device *rdev)
5542{
5543 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5544 int r;
5545
9b136d51
AD		 5546 /* Read BIOS */
5547 if (!radeon_get_bios(rdev)) {
5548 if (ASIC_IS_AVIVO(rdev))
5549 return -EINVAL;
5550 }
5551 /* Must be an ATOMBIOS */
5552 if (!rdev->is_atom_bios) {
5553 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
5554 return -EINVAL;
5555 }
5556 r = radeon_atombios_init(rdev);
5557 if (r)
5558 return r;
5559
5560 /* Post card if necessary */
5561 if (!radeon_card_posted(rdev)) {
5562 if (!rdev->bios) {
5563 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5564 return -EINVAL;
5565 }
5566 DRM_INFO("GPU not posted. posting now...\n");
5567 atom_asic_init(rdev->mode_info.atom_context);
5568 }
205996c0
AD		 5569 /* init golden registers */
5570 si_init_golden_registers(rdev);
9b136d51
AD		 5571 /* Initialize scratch registers */
5572 si_scratch_init(rdev);
5573 /* Initialize surface registers */
5574 radeon_surface_init(rdev);
5575 /* Initialize clocks */
5576 radeon_get_clock_info(rdev->ddev);
5577
5578 /* Fence driver */
5579 r = radeon_fence_driver_init(rdev);
5580 if (r)
5581 return r;
5582
5583 /* initialize memory controller */
5584 r = si_mc_init(rdev);
5585 if (r)
5586 return r;
5587 /* Memory manager */
5588 r = radeon_bo_init(rdev);
5589 if (r)
5590 return r;
5591
9b136d51
AD		 5592 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5593 ring->ring_obj = NULL;
5594 r600_ring_init(rdev, ring, 1024 * 1024);
5595
5596 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5597 ring->ring_obj = NULL;
5598 r600_ring_init(rdev, ring, 1024 * 1024);
5599
5600 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5601 ring->ring_obj = NULL;
5602 r600_ring_init(rdev, ring, 1024 * 1024);
5603
8c5fd7ef
AD		 5604 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5605 ring->ring_obj = NULL;
5606 r600_ring_init(rdev, ring, 64 * 1024);
5607
5608 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5609 ring->ring_obj = NULL;
5610 r600_ring_init(rdev, ring, 64 * 1024);
5611
1df0d523
AD		 5612 if (rdev->has_uvd) {
5613 r = radeon_uvd_init(rdev);
5614 if (!r) {
5615 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5616 ring->ring_obj = NULL;
5617 r600_ring_init(rdev, ring, 4096);
5618 }
f2ba57b5
CK		 5619 }
5620
9b136d51
AD		 5621 rdev->ih.ring_obj = NULL;
5622 r600_ih_ring_init(rdev, 64 * 1024);
5623
5624 r = r600_pcie_gart_init(rdev);
5625 if (r)
5626 return r;
5627
9b136d51 5628 rdev->accel_working = true;
9b136d51
AD		 5629 r = si_startup(rdev);
5630 if (r) {
5631 dev_err(rdev->dev, "disabling GPU acceleration\n");
5632 si_cp_fini(rdev);
8c5fd7ef 5633 cayman_dma_fini(rdev);
9b136d51
AD		 5634 si_irq_fini(rdev);
5635 si_rlc_fini(rdev);
5636 radeon_wb_fini(rdev);
2898c348 5637 radeon_ib_pool_fini(rdev);
9b136d51
AD		 5638 radeon_vm_manager_fini(rdev);
5639 radeon_irq_kms_fini(rdev);
5640 si_pcie_gart_fini(rdev);
5641 rdev->accel_working = false;
5642 }
5643
5644 /* Don't start up if the MC ucode is missing.
5645 * The default clocks and voltages before the MC ucode
5646 * is loaded are not suffient for advanced operations.
5647 */
5648 if (!rdev->mc_fw) {
5649 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5650 return -EINVAL;
5651 }
5652
5653 return 0;
5654}
5655
5656void si_fini(struct radeon_device *rdev)
5657{
9b136d51 5658 si_cp_fini(rdev);
8c5fd7ef 5659 cayman_dma_fini(rdev);
9b136d51
AD		 5660 si_irq_fini(rdev);
5661 si_rlc_fini(rdev);
5662 radeon_wb_fini(rdev);
5663 radeon_vm_manager_fini(rdev);
2898c348 5664 radeon_ib_pool_fini(rdev);
9b136d51 5665 radeon_irq_kms_fini(rdev);
1df0d523
AD		 5666 if (rdev->has_uvd)
5667 radeon_uvd_fini(rdev);
9b136d51
AD		 5668 si_pcie_gart_fini(rdev);
5669 r600_vram_scratch_fini(rdev);
5670 radeon_gem_fini(rdev);
9b136d51
AD		 5671 radeon_fence_driver_fini(rdev);
5672 radeon_bo_fini(rdev);
5673 radeon_atombios_fini(rdev);
5674 kfree(rdev->bios);
5675 rdev->bios = NULL;
5676}
5677
6759a0a7 5678/**
d0418894 5679 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 5680 *
5681 * @rdev: radeon_device pointer
5682 *
5683 * Fetches a GPU clock counter snapshot (SI).
5684 * Returns the 64 bit clock counter snapshot.
5685 */
d0418894 5686uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 5687{
5688 uint64_t clock;
5689
5690 mutex_lock(&rdev->gpu_clock_mutex);
5691 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
5692 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
5693 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
5694 mutex_unlock(&rdev->gpu_clock_mutex);
5695 return clock;
5696}
2539eb02 5697
2539eb02
CK		 5698int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
5699{
facd112d 5700 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 5701 int r;
5702
4ed10835
CK		 5703 /* bypass vclk and dclk with bclk */
5704 WREG32_P(CG_UPLL_FUNC_CNTL_2,
5705 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
5706 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
5707
5708 /* put PLL in bypass mode */
5709 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
5710
5711 if (!vclk || !dclk) {
5712 /* keep the Bypass mode, put PLL to sleep */
5713 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
5714 return 0;
5715 }
5716
facd112d
CK		 5717 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
5718 16384, 0x03FFFFFF, 0, 128, 5,
5719 &fb_div, &vclk_div, &dclk_div);
5720 if (r)
5721 return r;
2539eb02
CK		 5722
5723 /* set RESET_ANTI_MUX to 0 */
5724 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
5725
5726 /* set VCO_MODE to 1 */
5727 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
5728
5729 /* toggle UPLL_SLEEP to 1 then back to 0 */
5730 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
5731 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
5732
5733 /* deassert UPLL_RESET */
5734 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
5735
5736 mdelay(1);
5737
facd112d 5738 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 5739 if (r)
5740 return r;
5741
5742 /* assert UPLL_RESET again */
5743 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
5744
5745 /* disable spread spectrum. */
5746 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
5747
5748 /* set feedback divider */
facd112d 5749 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 5750
5751 /* set ref divider to 0 */
5752 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
5753
facd112d 5754 if (fb_div < 307200)
2539eb02
CK		 5755 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
5756 else
5757 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
5758
5759 /* set PDIV_A and PDIV_B */
5760 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 5761 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 5762 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
5763
5764 /* give the PLL some time to settle */
5765 mdelay(15);
5766
5767 /* deassert PLL_RESET */
5768 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
5769
5770 mdelay(15);
5771
5772 /* switch from bypass mode to normal mode */
5773 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
5774
facd112d 5775 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 5776 if (r)
5777 return r;
5778
5779 /* switch VCLK and DCLK selection */
5780 WREG32_P(CG_UPLL_FUNC_CNTL_2,
5781 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
5782 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
5783
5784 mdelay(100);
5785
5786 return 0;
5787}
b9d305df
AD		 5788
5789static void si_pcie_gen3_enable(struct radeon_device *rdev)
5790{
5791 struct pci_dev *root = rdev->pdev->bus->self;
5792 int bridge_pos, gpu_pos;
5793 u32 speed_cntl, mask, current_data_rate;
5794 int ret, i;
5795 u16 tmp16;
5796
5797 if (radeon_pcie_gen2 == 0)
5798 return;
5799
5800 if (rdev->flags & RADEON_IS_IGP)
5801 return;
5802
5803 if (!(rdev->flags & RADEON_IS_PCIE))
5804 return;
5805
5806 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
5807 if (ret != 0)
5808 return;
5809
5810 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
5811 return;
5812
5813 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5814 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
5815 LC_CURRENT_DATA_RATE_SHIFT;
5816 if (mask & DRM_PCIE_SPEED_80) {
5817 if (current_data_rate == 2) {
5818 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
5819 return;
5820 }
5821 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
5822 } else if (mask & DRM_PCIE_SPEED_50) {
5823 if (current_data_rate == 1) {
5824 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5825 return;
5826 }
5827 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5828 }
5829
5830 bridge_pos = pci_pcie_cap(root);
5831 if (!bridge_pos)
5832 return;
5833
5834 gpu_pos = pci_pcie_cap(rdev->pdev);
5835 if (!gpu_pos)
5836 return;
5837
5838 if (mask & DRM_PCIE_SPEED_80) {
5839 /* re-try equalization if gen3 is not already enabled */
5840 if (current_data_rate != 2) {
5841 u16 bridge_cfg, gpu_cfg;
5842 u16 bridge_cfg2, gpu_cfg2;
5843 u32 max_lw, current_lw, tmp;
5844
5845 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
5846 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
5847
5848 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
5849 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
5850
5851 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
5852 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
5853
5854 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
5855 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
5856 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
5857
5858 if (current_lw < max_lw) {
5859 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5860 if (tmp & LC_RENEGOTIATION_SUPPORT) {
5861 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
5862 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
5863 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
5864 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
5865 }
5866 }
5867
5868 for (i = 0; i < 10; i++) {
5869 /* check status */
5870 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
5871 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
5872 break;
5873
5874 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
5875 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
5876
5877 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
5878 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
5879
5880 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
5881 tmp |= LC_SET_QUIESCE;
5882 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
5883
5884 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
5885 tmp |= LC_REDO_EQ;
5886 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
5887
5888 mdelay(100);
5889
5890 /* linkctl */
5891 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
5892 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
5893 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
5894 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
5895
5896 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
5897 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
5898 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
5899 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
5900
5901 /* linkctl2 */
5902 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
5903 tmp16 &= ~((1 << 4) | (7 << 9));
5904 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
5905 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
5906
5907 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
5908 tmp16 &= ~((1 << 4) | (7 << 9));
5909 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
5910 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
5911
5912 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
5913 tmp &= ~LC_SET_QUIESCE;
5914 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
5915 }
5916 }
5917 }
5918
5919 /* set the link speed */
5920 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
5921 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
5922 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5923
5924 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
5925 tmp16 &= ~0xf;
5926 if (mask & DRM_PCIE_SPEED_80)
5927 tmp16 |= 3; /* gen3 */
5928 else if (mask & DRM_PCIE_SPEED_50)
5929 tmp16 |= 2; /* gen2 */
5930 else
5931 tmp16 |= 1; /* gen1 */
5932 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
5933
5934 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5935 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
5936 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5937
5938 for (i = 0; i < rdev->usec_timeout; i++) {
5939 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5940 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
5941 break;
5942 udelay(1);
5943 }
5944}
5945
Linux 6.x block layer and io_uring tree(s)