projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: update radeon_atom_get_voltage_table() for SI
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
0f0de06c
AD		 24#include <linux/firmware.h>
25#include <linux/platform_device.h>
26#include <linux/slab.h>
27#include <linux/module.h>
760285e7 28#include <drm/drmP.h>
43b3cd99
AD		 29#include "radeon.h"
30#include "radeon_asic.h"
760285e7 31#include <drm/radeon_drm.h>
43b3cd99
AD		 32#include "sid.h"
33#include "atom.h"
48c0c902 34#include "si_blit_shaders.h"
43b3cd99 35
0f0de06c
AD		 36#define SI_PFP_UCODE_SIZE 2144
37#define SI_PM4_UCODE_SIZE 2144
38#define SI_CE_UCODE_SIZE 2144
39#define SI_RLC_UCODE_SIZE 2048
40#define SI_MC_UCODE_SIZE 7769
bcc7f5d2 41#define OLAND_MC_UCODE_SIZE 7863
0f0de06c
AD		 42
43MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
44MODULE_FIRMWARE("radeon/TAHITI_me.bin");
45MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
46MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
47MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
48MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
49MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
50MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
51MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
52MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
53MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
54MODULE_FIRMWARE("radeon/VERDE_me.bin");
55MODULE_FIRMWARE("radeon/VERDE_ce.bin");
56MODULE_FIRMWARE("radeon/VERDE_mc.bin");
57MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
bcc7f5d2
AD		 58MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
59MODULE_FIRMWARE("radeon/OLAND_me.bin");
60MODULE_FIRMWARE("radeon/OLAND_ce.bin");
61MODULE_FIRMWARE("radeon/OLAND_mc.bin");
62MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
c04c00b4
AD		 63MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
64MODULE_FIRMWARE("radeon/HAINAN_me.bin");
65MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
66MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
67MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
0f0de06c 68
25a857fb
AD		 69extern int r600_ih_ring_alloc(struct radeon_device *rdev);
70extern void r600_ih_ring_fini(struct radeon_device *rdev);
0a96d72b 71extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
c476dde2
AD		 72extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
73extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
ca7db22b 74extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
1c534671 75extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
014bb209 76extern bool evergreen_is_display_hung(struct radeon_device *rdev);
0a96d72b 77
205996c0
AD		 78static const u32 tahiti_golden_rlc_registers[] =
79{
80 0xc424, 0xffffffff, 0x00601005,
81 0xc47c, 0xffffffff, 0x10104040,
82 0xc488, 0xffffffff, 0x0100000a,
83 0xc314, 0xffffffff, 0x00000800,
84 0xc30c, 0xffffffff, 0x800000f4,
85 0xf4a8, 0xffffffff, 0x00000000
86};
87
88static const u32 tahiti_golden_registers[] =
89{
90 0x9a10, 0x00010000, 0x00018208,
91 0x9830, 0xffffffff, 0x00000000,
92 0x9834, 0xf00fffff, 0x00000400,
93 0x9838, 0x0002021c, 0x00020200,
94 0xc78, 0x00000080, 0x00000000,
95 0xd030, 0x000300c0, 0x00800040,
96 0xd830, 0x000300c0, 0x00800040,
97 0x5bb0, 0x000000f0, 0x00000070,
98 0x5bc0, 0x00200000, 0x50100000,
99 0x7030, 0x31000311, 0x00000011,
100 0x277c, 0x00000003, 0x000007ff,
101 0x240c, 0x000007ff, 0x00000000,
102 0x8a14, 0xf000001f, 0x00000007,
103 0x8b24, 0xffffffff, 0x00ffffff,
104 0x8b10, 0x0000ff0f, 0x00000000,
105 0x28a4c, 0x07ffffff, 0x4e000000,
106 0x28350, 0x3f3f3fff, 0x2a00126a,
107 0x30, 0x000000ff, 0x0040,
108 0x34, 0x00000040, 0x00004040,
109 0x9100, 0x07ffffff, 0x03000000,
110 0x8e88, 0x01ff1f3f, 0x00000000,
111 0x8e84, 0x01ff1f3f, 0x00000000,
112 0x9060, 0x0000007f, 0x00000020,
113 0x9508, 0x00010000, 0x00010000,
114 0xac14, 0x00000200, 0x000002fb,
115 0xac10, 0xffffffff, 0x0000543b,
116 0xac0c, 0xffffffff, 0xa9210876,
117 0x88d0, 0xffffffff, 0x000fff40,
118 0x88d4, 0x0000001f, 0x00000010,
119 0x1410, 0x20000000, 0x20fffed8,
120 0x15c0, 0x000c0fc0, 0x000c0400
121};
122
123static const u32 tahiti_golden_registers2[] =
124{
125 0xc64, 0x00000001, 0x00000001
126};
127
128static const u32 pitcairn_golden_rlc_registers[] =
129{
130 0xc424, 0xffffffff, 0x00601004,
131 0xc47c, 0xffffffff, 0x10102020,
132 0xc488, 0xffffffff, 0x01000020,
133 0xc314, 0xffffffff, 0x00000800,
134 0xc30c, 0xffffffff, 0x800000a4
135};
136
137static const u32 pitcairn_golden_registers[] =
138{
139 0x9a10, 0x00010000, 0x00018208,
140 0x9830, 0xffffffff, 0x00000000,
141 0x9834, 0xf00fffff, 0x00000400,
142 0x9838, 0x0002021c, 0x00020200,
143 0xc78, 0x00000080, 0x00000000,
144 0xd030, 0x000300c0, 0x00800040,
145 0xd830, 0x000300c0, 0x00800040,
146 0x5bb0, 0x000000f0, 0x00000070,
147 0x5bc0, 0x00200000, 0x50100000,
148 0x7030, 0x31000311, 0x00000011,
149 0x2ae4, 0x00073ffe, 0x000022a2,
150 0x240c, 0x000007ff, 0x00000000,
151 0x8a14, 0xf000001f, 0x00000007,
152 0x8b24, 0xffffffff, 0x00ffffff,
153 0x8b10, 0x0000ff0f, 0x00000000,
154 0x28a4c, 0x07ffffff, 0x4e000000,
155 0x28350, 0x3f3f3fff, 0x2a00126a,
156 0x30, 0x000000ff, 0x0040,
157 0x34, 0x00000040, 0x00004040,
158 0x9100, 0x07ffffff, 0x03000000,
159 0x9060, 0x0000007f, 0x00000020,
160 0x9508, 0x00010000, 0x00010000,
161 0xac14, 0x000003ff, 0x000000f7,
162 0xac10, 0xffffffff, 0x00000000,
163 0xac0c, 0xffffffff, 0x32761054,
164 0x88d4, 0x0000001f, 0x00000010,
165 0x15c0, 0x000c0fc0, 0x000c0400
166};
167
168static const u32 verde_golden_rlc_registers[] =
169{
170 0xc424, 0xffffffff, 0x033f1005,
171 0xc47c, 0xffffffff, 0x10808020,
172 0xc488, 0xffffffff, 0x00800008,
173 0xc314, 0xffffffff, 0x00001000,
174 0xc30c, 0xffffffff, 0x80010014
175};
176
177static const u32 verde_golden_registers[] =
178{
179 0x9a10, 0x00010000, 0x00018208,
180 0x9830, 0xffffffff, 0x00000000,
181 0x9834, 0xf00fffff, 0x00000400,
182 0x9838, 0x0002021c, 0x00020200,
183 0xc78, 0x00000080, 0x00000000,
184 0xd030, 0x000300c0, 0x00800040,
185 0xd030, 0x000300c0, 0x00800040,
186 0xd830, 0x000300c0, 0x00800040,
187 0xd830, 0x000300c0, 0x00800040,
188 0x5bb0, 0x000000f0, 0x00000070,
189 0x5bc0, 0x00200000, 0x50100000,
190 0x7030, 0x31000311, 0x00000011,
191 0x2ae4, 0x00073ffe, 0x000022a2,
192 0x2ae4, 0x00073ffe, 0x000022a2,
193 0x2ae4, 0x00073ffe, 0x000022a2,
194 0x240c, 0x000007ff, 0x00000000,
195 0x240c, 0x000007ff, 0x00000000,
196 0x240c, 0x000007ff, 0x00000000,
197 0x8a14, 0xf000001f, 0x00000007,
198 0x8a14, 0xf000001f, 0x00000007,
199 0x8a14, 0xf000001f, 0x00000007,
200 0x8b24, 0xffffffff, 0x00ffffff,
201 0x8b10, 0x0000ff0f, 0x00000000,
202 0x28a4c, 0x07ffffff, 0x4e000000,
203 0x28350, 0x3f3f3fff, 0x0000124a,
204 0x28350, 0x3f3f3fff, 0x0000124a,
205 0x28350, 0x3f3f3fff, 0x0000124a,
206 0x30, 0x000000ff, 0x0040,
207 0x34, 0x00000040, 0x00004040,
208 0x9100, 0x07ffffff, 0x03000000,
209 0x9100, 0x07ffffff, 0x03000000,
210 0x8e88, 0x01ff1f3f, 0x00000000,
211 0x8e88, 0x01ff1f3f, 0x00000000,
212 0x8e88, 0x01ff1f3f, 0x00000000,
213 0x8e84, 0x01ff1f3f, 0x00000000,
214 0x8e84, 0x01ff1f3f, 0x00000000,
215 0x8e84, 0x01ff1f3f, 0x00000000,
216 0x9060, 0x0000007f, 0x00000020,
217 0x9508, 0x00010000, 0x00010000,
218 0xac14, 0x000003ff, 0x00000003,
219 0xac14, 0x000003ff, 0x00000003,
220 0xac14, 0x000003ff, 0x00000003,
221 0xac10, 0xffffffff, 0x00000000,
222 0xac10, 0xffffffff, 0x00000000,
223 0xac10, 0xffffffff, 0x00000000,
224 0xac0c, 0xffffffff, 0x00001032,
225 0xac0c, 0xffffffff, 0x00001032,
226 0xac0c, 0xffffffff, 0x00001032,
227 0x88d4, 0x0000001f, 0x00000010,
228 0x88d4, 0x0000001f, 0x00000010,
229 0x88d4, 0x0000001f, 0x00000010,
230 0x15c0, 0x000c0fc0, 0x000c0400
231};
232
233static const u32 oland_golden_rlc_registers[] =
234{
235 0xc424, 0xffffffff, 0x00601005,
236 0xc47c, 0xffffffff, 0x10104040,
237 0xc488, 0xffffffff, 0x0100000a,
238 0xc314, 0xffffffff, 0x00000800,
239 0xc30c, 0xffffffff, 0x800000f4
240};
241
242static const u32 oland_golden_registers[] =
243{
244 0x9a10, 0x00010000, 0x00018208,
245 0x9830, 0xffffffff, 0x00000000,
246 0x9834, 0xf00fffff, 0x00000400,
247 0x9838, 0x0002021c, 0x00020200,
248 0xc78, 0x00000080, 0x00000000,
249 0xd030, 0x000300c0, 0x00800040,
250 0xd830, 0x000300c0, 0x00800040,
251 0x5bb0, 0x000000f0, 0x00000070,
252 0x5bc0, 0x00200000, 0x50100000,
253 0x7030, 0x31000311, 0x00000011,
254 0x2ae4, 0x00073ffe, 0x000022a2,
255 0x240c, 0x000007ff, 0x00000000,
256 0x8a14, 0xf000001f, 0x00000007,
257 0x8b24, 0xffffffff, 0x00ffffff,
258 0x8b10, 0x0000ff0f, 0x00000000,
259 0x28a4c, 0x07ffffff, 0x4e000000,
260 0x28350, 0x3f3f3fff, 0x00000082,
261 0x30, 0x000000ff, 0x0040,
262 0x34, 0x00000040, 0x00004040,
263 0x9100, 0x07ffffff, 0x03000000,
264 0x9060, 0x0000007f, 0x00000020,
265 0x9508, 0x00010000, 0x00010000,
266 0xac14, 0x000003ff, 0x000000f3,
267 0xac10, 0xffffffff, 0x00000000,
268 0xac0c, 0xffffffff, 0x00003210,
269 0x88d4, 0x0000001f, 0x00000010,
270 0x15c0, 0x000c0fc0, 0x000c0400
271};
272
fffbdda4
AD		 273static const u32 hainan_golden_registers[] =
274{
275 0x9a10, 0x00010000, 0x00018208,
276 0x9830, 0xffffffff, 0x00000000,
277 0x9834, 0xf00fffff, 0x00000400,
278 0x9838, 0x0002021c, 0x00020200,
279 0xd0c0, 0xff000fff, 0x00000100,
280 0xd030, 0x000300c0, 0x00800040,
281 0xd8c0, 0xff000fff, 0x00000100,
282 0xd830, 0x000300c0, 0x00800040,
283 0x2ae4, 0x00073ffe, 0x000022a2,
284 0x240c, 0x000007ff, 0x00000000,
285 0x8a14, 0xf000001f, 0x00000007,
286 0x8b24, 0xffffffff, 0x00ffffff,
287 0x8b10, 0x0000ff0f, 0x00000000,
288 0x28a4c, 0x07ffffff, 0x4e000000,
289 0x28350, 0x3f3f3fff, 0x00000000,
290 0x30, 0x000000ff, 0x0040,
291 0x34, 0x00000040, 0x00004040,
292 0x9100, 0x03e00000, 0x03600000,
293 0x9060, 0x0000007f, 0x00000020,
294 0x9508, 0x00010000, 0x00010000,
295 0xac14, 0x000003ff, 0x000000f1,
296 0xac10, 0xffffffff, 0x00000000,
297 0xac0c, 0xffffffff, 0x00003210,
298 0x88d4, 0x0000001f, 0x00000010,
299 0x15c0, 0x000c0fc0, 0x000c0400
300};
301
302static const u32 hainan_golden_registers2[] =
303{
304 0x98f8, 0xffffffff, 0x02010001
305};
306
205996c0
AD		 307static const u32 tahiti_mgcg_cgcg_init[] =
308{
309 0xc400, 0xffffffff, 0xfffffffc,
310 0x802c, 0xffffffff, 0xe0000000,
311 0x9a60, 0xffffffff, 0x00000100,
312 0x92a4, 0xffffffff, 0x00000100,
313 0xc164, 0xffffffff, 0x00000100,
314 0x9774, 0xffffffff, 0x00000100,
315 0x8984, 0xffffffff, 0x06000100,
316 0x8a18, 0xffffffff, 0x00000100,
317 0x92a0, 0xffffffff, 0x00000100,
318 0xc380, 0xffffffff, 0x00000100,
319 0x8b28, 0xffffffff, 0x00000100,
320 0x9144, 0xffffffff, 0x00000100,
321 0x8d88, 0xffffffff, 0x00000100,
322 0x8d8c, 0xffffffff, 0x00000100,
323 0x9030, 0xffffffff, 0x00000100,
324 0x9034, 0xffffffff, 0x00000100,
325 0x9038, 0xffffffff, 0x00000100,
326 0x903c, 0xffffffff, 0x00000100,
327 0xad80, 0xffffffff, 0x00000100,
328 0xac54, 0xffffffff, 0x00000100,
329 0x897c, 0xffffffff, 0x06000100,
330 0x9868, 0xffffffff, 0x00000100,
331 0x9510, 0xffffffff, 0x00000100,
332 0xaf04, 0xffffffff, 0x00000100,
333 0xae04, 0xffffffff, 0x00000100,
334 0x949c, 0xffffffff, 0x00000100,
335 0x802c, 0xffffffff, 0xe0000000,
336 0x9160, 0xffffffff, 0x00010000,
337 0x9164, 0xffffffff, 0x00030002,
338 0x9168, 0xffffffff, 0x00040007,
339 0x916c, 0xffffffff, 0x00060005,
340 0x9170, 0xffffffff, 0x00090008,
341 0x9174, 0xffffffff, 0x00020001,
342 0x9178, 0xffffffff, 0x00040003,
343 0x917c, 0xffffffff, 0x00000007,
344 0x9180, 0xffffffff, 0x00060005,
345 0x9184, 0xffffffff, 0x00090008,
346 0x9188, 0xffffffff, 0x00030002,
347 0x918c, 0xffffffff, 0x00050004,
348 0x9190, 0xffffffff, 0x00000008,
349 0x9194, 0xffffffff, 0x00070006,
350 0x9198, 0xffffffff, 0x000a0009,
351 0x919c, 0xffffffff, 0x00040003,
352 0x91a0, 0xffffffff, 0x00060005,
353 0x91a4, 0xffffffff, 0x00000009,
354 0x91a8, 0xffffffff, 0x00080007,
355 0x91ac, 0xffffffff, 0x000b000a,
356 0x91b0, 0xffffffff, 0x00050004,
357 0x91b4, 0xffffffff, 0x00070006,
358 0x91b8, 0xffffffff, 0x0008000b,
359 0x91bc, 0xffffffff, 0x000a0009,
360 0x91c0, 0xffffffff, 0x000d000c,
361 0x91c4, 0xffffffff, 0x00060005,
362 0x91c8, 0xffffffff, 0x00080007,
363 0x91cc, 0xffffffff, 0x0000000b,
364 0x91d0, 0xffffffff, 0x000a0009,
365 0x91d4, 0xffffffff, 0x000d000c,
366 0x91d8, 0xffffffff, 0x00070006,
367 0x91dc, 0xffffffff, 0x00090008,
368 0x91e0, 0xffffffff, 0x0000000c,
369 0x91e4, 0xffffffff, 0x000b000a,
370 0x91e8, 0xffffffff, 0x000e000d,
371 0x91ec, 0xffffffff, 0x00080007,
372 0x91f0, 0xffffffff, 0x000a0009,
373 0x91f4, 0xffffffff, 0x0000000d,
374 0x91f8, 0xffffffff, 0x000c000b,
375 0x91fc, 0xffffffff, 0x000f000e,
376 0x9200, 0xffffffff, 0x00090008,
377 0x9204, 0xffffffff, 0x000b000a,
378 0x9208, 0xffffffff, 0x000c000f,
379 0x920c, 0xffffffff, 0x000e000d,
380 0x9210, 0xffffffff, 0x00110010,
381 0x9214, 0xffffffff, 0x000a0009,
382 0x9218, 0xffffffff, 0x000c000b,
383 0x921c, 0xffffffff, 0x0000000f,
384 0x9220, 0xffffffff, 0x000e000d,
385 0x9224, 0xffffffff, 0x00110010,
386 0x9228, 0xffffffff, 0x000b000a,
387 0x922c, 0xffffffff, 0x000d000c,
388 0x9230, 0xffffffff, 0x00000010,
389 0x9234, 0xffffffff, 0x000f000e,
390 0x9238, 0xffffffff, 0x00120011,
391 0x923c, 0xffffffff, 0x000c000b,
392 0x9240, 0xffffffff, 0x000e000d,
393 0x9244, 0xffffffff, 0x00000011,
394 0x9248, 0xffffffff, 0x0010000f,
395 0x924c, 0xffffffff, 0x00130012,
396 0x9250, 0xffffffff, 0x000d000c,
397 0x9254, 0xffffffff, 0x000f000e,
398 0x9258, 0xffffffff, 0x00100013,
399 0x925c, 0xffffffff, 0x00120011,
400 0x9260, 0xffffffff, 0x00150014,
401 0x9264, 0xffffffff, 0x000e000d,
402 0x9268, 0xffffffff, 0x0010000f,
403 0x926c, 0xffffffff, 0x00000013,
404 0x9270, 0xffffffff, 0x00120011,
405 0x9274, 0xffffffff, 0x00150014,
406 0x9278, 0xffffffff, 0x000f000e,
407 0x927c, 0xffffffff, 0x00110010,
408 0x9280, 0xffffffff, 0x00000014,
409 0x9284, 0xffffffff, 0x00130012,
410 0x9288, 0xffffffff, 0x00160015,
411 0x928c, 0xffffffff, 0x0010000f,
412 0x9290, 0xffffffff, 0x00120011,
413 0x9294, 0xffffffff, 0x00000015,
414 0x9298, 0xffffffff, 0x00140013,
415 0x929c, 0xffffffff, 0x00170016,
416 0x9150, 0xffffffff, 0x96940200,
417 0x8708, 0xffffffff, 0x00900100,
418 0xc478, 0xffffffff, 0x00000080,
419 0xc404, 0xffffffff, 0x0020003f,
420 0x30, 0xffffffff, 0x0000001c,
421 0x34, 0x000f0000, 0x000f0000,
422 0x160c, 0xffffffff, 0x00000100,
423 0x1024, 0xffffffff, 0x00000100,
424 0x102c, 0x00000101, 0x00000000,
425 0x20a8, 0xffffffff, 0x00000104,
426 0x264c, 0x000c0000, 0x000c0000,
427 0x2648, 0x000c0000, 0x000c0000,
428 0x55e4, 0xff000fff, 0x00000100,
429 0x55e8, 0x00000001, 0x00000001,
430 0x2f50, 0x00000001, 0x00000001,
431 0x30cc, 0xc0000fff, 0x00000104,
432 0xc1e4, 0x00000001, 0x00000001,
433 0xd0c0, 0xfffffff0, 0x00000100,
434 0xd8c0, 0xfffffff0, 0x00000100
435};
436
437static const u32 pitcairn_mgcg_cgcg_init[] =
438{
439 0xc400, 0xffffffff, 0xfffffffc,
440 0x802c, 0xffffffff, 0xe0000000,
441 0x9a60, 0xffffffff, 0x00000100,
442 0x92a4, 0xffffffff, 0x00000100,
443 0xc164, 0xffffffff, 0x00000100,
444 0x9774, 0xffffffff, 0x00000100,
445 0x8984, 0xffffffff, 0x06000100,
446 0x8a18, 0xffffffff, 0x00000100,
447 0x92a0, 0xffffffff, 0x00000100,
448 0xc380, 0xffffffff, 0x00000100,
449 0x8b28, 0xffffffff, 0x00000100,
450 0x9144, 0xffffffff, 0x00000100,
451 0x8d88, 0xffffffff, 0x00000100,
452 0x8d8c, 0xffffffff, 0x00000100,
453 0x9030, 0xffffffff, 0x00000100,
454 0x9034, 0xffffffff, 0x00000100,
455 0x9038, 0xffffffff, 0x00000100,
456 0x903c, 0xffffffff, 0x00000100,
457 0xad80, 0xffffffff, 0x00000100,
458 0xac54, 0xffffffff, 0x00000100,
459 0x897c, 0xffffffff, 0x06000100,
460 0x9868, 0xffffffff, 0x00000100,
461 0x9510, 0xffffffff, 0x00000100,
462 0xaf04, 0xffffffff, 0x00000100,
463 0xae04, 0xffffffff, 0x00000100,
464 0x949c, 0xffffffff, 0x00000100,
465 0x802c, 0xffffffff, 0xe0000000,
466 0x9160, 0xffffffff, 0x00010000,
467 0x9164, 0xffffffff, 0x00030002,
468 0x9168, 0xffffffff, 0x00040007,
469 0x916c, 0xffffffff, 0x00060005,
470 0x9170, 0xffffffff, 0x00090008,
471 0x9174, 0xffffffff, 0x00020001,
472 0x9178, 0xffffffff, 0x00040003,
473 0x917c, 0xffffffff, 0x00000007,
474 0x9180, 0xffffffff, 0x00060005,
475 0x9184, 0xffffffff, 0x00090008,
476 0x9188, 0xffffffff, 0x00030002,
477 0x918c, 0xffffffff, 0x00050004,
478 0x9190, 0xffffffff, 0x00000008,
479 0x9194, 0xffffffff, 0x00070006,
480 0x9198, 0xffffffff, 0x000a0009,
481 0x919c, 0xffffffff, 0x00040003,
482 0x91a0, 0xffffffff, 0x00060005,
483 0x91a4, 0xffffffff, 0x00000009,
484 0x91a8, 0xffffffff, 0x00080007,
485 0x91ac, 0xffffffff, 0x000b000a,
486 0x91b0, 0xffffffff, 0x00050004,
487 0x91b4, 0xffffffff, 0x00070006,
488 0x91b8, 0xffffffff, 0x0008000b,
489 0x91bc, 0xffffffff, 0x000a0009,
490 0x91c0, 0xffffffff, 0x000d000c,
491 0x9200, 0xffffffff, 0x00090008,
492 0x9204, 0xffffffff, 0x000b000a,
493 0x9208, 0xffffffff, 0x000c000f,
494 0x920c, 0xffffffff, 0x000e000d,
495 0x9210, 0xffffffff, 0x00110010,
496 0x9214, 0xffffffff, 0x000a0009,
497 0x9218, 0xffffffff, 0x000c000b,
498 0x921c, 0xffffffff, 0x0000000f,
499 0x9220, 0xffffffff, 0x000e000d,
500 0x9224, 0xffffffff, 0x00110010,
501 0x9228, 0xffffffff, 0x000b000a,
502 0x922c, 0xffffffff, 0x000d000c,
503 0x9230, 0xffffffff, 0x00000010,
504 0x9234, 0xffffffff, 0x000f000e,
505 0x9238, 0xffffffff, 0x00120011,
506 0x923c, 0xffffffff, 0x000c000b,
507 0x9240, 0xffffffff, 0x000e000d,
508 0x9244, 0xffffffff, 0x00000011,
509 0x9248, 0xffffffff, 0x0010000f,
510 0x924c, 0xffffffff, 0x00130012,
511 0x9250, 0xffffffff, 0x000d000c,
512 0x9254, 0xffffffff, 0x000f000e,
513 0x9258, 0xffffffff, 0x00100013,
514 0x925c, 0xffffffff, 0x00120011,
515 0x9260, 0xffffffff, 0x00150014,
516 0x9150, 0xffffffff, 0x96940200,
517 0x8708, 0xffffffff, 0x00900100,
518 0xc478, 0xffffffff, 0x00000080,
519 0xc404, 0xffffffff, 0x0020003f,
520 0x30, 0xffffffff, 0x0000001c,
521 0x34, 0x000f0000, 0x000f0000,
522 0x160c, 0xffffffff, 0x00000100,
523 0x1024, 0xffffffff, 0x00000100,
524 0x102c, 0x00000101, 0x00000000,
525 0x20a8, 0xffffffff, 0x00000104,
526 0x55e4, 0xff000fff, 0x00000100,
527 0x55e8, 0x00000001, 0x00000001,
528 0x2f50, 0x00000001, 0x00000001,
529 0x30cc, 0xc0000fff, 0x00000104,
530 0xc1e4, 0x00000001, 0x00000001,
531 0xd0c0, 0xfffffff0, 0x00000100,
532 0xd8c0, 0xfffffff0, 0x00000100
533};
534
535static const u32 verde_mgcg_cgcg_init[] =
536{
537 0xc400, 0xffffffff, 0xfffffffc,
538 0x802c, 0xffffffff, 0xe0000000,
539 0x9a60, 0xffffffff, 0x00000100,
540 0x92a4, 0xffffffff, 0x00000100,
541 0xc164, 0xffffffff, 0x00000100,
542 0x9774, 0xffffffff, 0x00000100,
543 0x8984, 0xffffffff, 0x06000100,
544 0x8a18, 0xffffffff, 0x00000100,
545 0x92a0, 0xffffffff, 0x00000100,
546 0xc380, 0xffffffff, 0x00000100,
547 0x8b28, 0xffffffff, 0x00000100,
548 0x9144, 0xffffffff, 0x00000100,
549 0x8d88, 0xffffffff, 0x00000100,
550 0x8d8c, 0xffffffff, 0x00000100,
551 0x9030, 0xffffffff, 0x00000100,
552 0x9034, 0xffffffff, 0x00000100,
553 0x9038, 0xffffffff, 0x00000100,
554 0x903c, 0xffffffff, 0x00000100,
555 0xad80, 0xffffffff, 0x00000100,
556 0xac54, 0xffffffff, 0x00000100,
557 0x897c, 0xffffffff, 0x06000100,
558 0x9868, 0xffffffff, 0x00000100,
559 0x9510, 0xffffffff, 0x00000100,
560 0xaf04, 0xffffffff, 0x00000100,
561 0xae04, 0xffffffff, 0x00000100,
562 0x949c, 0xffffffff, 0x00000100,
563 0x802c, 0xffffffff, 0xe0000000,
564 0x9160, 0xffffffff, 0x00010000,
565 0x9164, 0xffffffff, 0x00030002,
566 0x9168, 0xffffffff, 0x00040007,
567 0x916c, 0xffffffff, 0x00060005,
568 0x9170, 0xffffffff, 0x00090008,
569 0x9174, 0xffffffff, 0x00020001,
570 0x9178, 0xffffffff, 0x00040003,
571 0x917c, 0xffffffff, 0x00000007,
572 0x9180, 0xffffffff, 0x00060005,
573 0x9184, 0xffffffff, 0x00090008,
574 0x9188, 0xffffffff, 0x00030002,
575 0x918c, 0xffffffff, 0x00050004,
576 0x9190, 0xffffffff, 0x00000008,
577 0x9194, 0xffffffff, 0x00070006,
578 0x9198, 0xffffffff, 0x000a0009,
579 0x919c, 0xffffffff, 0x00040003,
580 0x91a0, 0xffffffff, 0x00060005,
581 0x91a4, 0xffffffff, 0x00000009,
582 0x91a8, 0xffffffff, 0x00080007,
583 0x91ac, 0xffffffff, 0x000b000a,
584 0x91b0, 0xffffffff, 0x00050004,
585 0x91b4, 0xffffffff, 0x00070006,
586 0x91b8, 0xffffffff, 0x0008000b,
587 0x91bc, 0xffffffff, 0x000a0009,
588 0x91c0, 0xffffffff, 0x000d000c,
589 0x9200, 0xffffffff, 0x00090008,
590 0x9204, 0xffffffff, 0x000b000a,
591 0x9208, 0xffffffff, 0x000c000f,
592 0x920c, 0xffffffff, 0x000e000d,
593 0x9210, 0xffffffff, 0x00110010,
594 0x9214, 0xffffffff, 0x000a0009,
595 0x9218, 0xffffffff, 0x000c000b,
596 0x921c, 0xffffffff, 0x0000000f,
597 0x9220, 0xffffffff, 0x000e000d,
598 0x9224, 0xffffffff, 0x00110010,
599 0x9228, 0xffffffff, 0x000b000a,
600 0x922c, 0xffffffff, 0x000d000c,
601 0x9230, 0xffffffff, 0x00000010,
602 0x9234, 0xffffffff, 0x000f000e,
603 0x9238, 0xffffffff, 0x00120011,
604 0x923c, 0xffffffff, 0x000c000b,
605 0x9240, 0xffffffff, 0x000e000d,
606 0x9244, 0xffffffff, 0x00000011,
607 0x9248, 0xffffffff, 0x0010000f,
608 0x924c, 0xffffffff, 0x00130012,
609 0x9250, 0xffffffff, 0x000d000c,
610 0x9254, 0xffffffff, 0x000f000e,
611 0x9258, 0xffffffff, 0x00100013,
612 0x925c, 0xffffffff, 0x00120011,
613 0x9260, 0xffffffff, 0x00150014,
614 0x9150, 0xffffffff, 0x96940200,
615 0x8708, 0xffffffff, 0x00900100,
616 0xc478, 0xffffffff, 0x00000080,
617 0xc404, 0xffffffff, 0x0020003f,
618 0x30, 0xffffffff, 0x0000001c,
619 0x34, 0x000f0000, 0x000f0000,
620 0x160c, 0xffffffff, 0x00000100,
621 0x1024, 0xffffffff, 0x00000100,
622 0x102c, 0x00000101, 0x00000000,
623 0x20a8, 0xffffffff, 0x00000104,
624 0x264c, 0x000c0000, 0x000c0000,
625 0x2648, 0x000c0000, 0x000c0000,
626 0x55e4, 0xff000fff, 0x00000100,
627 0x55e8, 0x00000001, 0x00000001,
628 0x2f50, 0x00000001, 0x00000001,
629 0x30cc, 0xc0000fff, 0x00000104,
630 0xc1e4, 0x00000001, 0x00000001,
631 0xd0c0, 0xfffffff0, 0x00000100,
632 0xd8c0, 0xfffffff0, 0x00000100
633};
634
635static const u32 oland_mgcg_cgcg_init[] =
636{
637 0xc400, 0xffffffff, 0xfffffffc,
638 0x802c, 0xffffffff, 0xe0000000,
639 0x9a60, 0xffffffff, 0x00000100,
640 0x92a4, 0xffffffff, 0x00000100,
641 0xc164, 0xffffffff, 0x00000100,
642 0x9774, 0xffffffff, 0x00000100,
643 0x8984, 0xffffffff, 0x06000100,
644 0x8a18, 0xffffffff, 0x00000100,
645 0x92a0, 0xffffffff, 0x00000100,
646 0xc380, 0xffffffff, 0x00000100,
647 0x8b28, 0xffffffff, 0x00000100,
648 0x9144, 0xffffffff, 0x00000100,
649 0x8d88, 0xffffffff, 0x00000100,
650 0x8d8c, 0xffffffff, 0x00000100,
651 0x9030, 0xffffffff, 0x00000100,
652 0x9034, 0xffffffff, 0x00000100,
653 0x9038, 0xffffffff, 0x00000100,
654 0x903c, 0xffffffff, 0x00000100,
655 0xad80, 0xffffffff, 0x00000100,
656 0xac54, 0xffffffff, 0x00000100,
657 0x897c, 0xffffffff, 0x06000100,
658 0x9868, 0xffffffff, 0x00000100,
659 0x9510, 0xffffffff, 0x00000100,
660 0xaf04, 0xffffffff, 0x00000100,
661 0xae04, 0xffffffff, 0x00000100,
662 0x949c, 0xffffffff, 0x00000100,
663 0x802c, 0xffffffff, 0xe0000000,
664 0x9160, 0xffffffff, 0x00010000,
665 0x9164, 0xffffffff, 0x00030002,
666 0x9168, 0xffffffff, 0x00040007,
667 0x916c, 0xffffffff, 0x00060005,
668 0x9170, 0xffffffff, 0x00090008,
669 0x9174, 0xffffffff, 0x00020001,
670 0x9178, 0xffffffff, 0x00040003,
671 0x917c, 0xffffffff, 0x00000007,
672 0x9180, 0xffffffff, 0x00060005,
673 0x9184, 0xffffffff, 0x00090008,
674 0x9188, 0xffffffff, 0x00030002,
675 0x918c, 0xffffffff, 0x00050004,
676 0x9190, 0xffffffff, 0x00000008,
677 0x9194, 0xffffffff, 0x00070006,
678 0x9198, 0xffffffff, 0x000a0009,
679 0x919c, 0xffffffff, 0x00040003,
680 0x91a0, 0xffffffff, 0x00060005,
681 0x91a4, 0xffffffff, 0x00000009,
682 0x91a8, 0xffffffff, 0x00080007,
683 0x91ac, 0xffffffff, 0x000b000a,
684 0x91b0, 0xffffffff, 0x00050004,
685 0x91b4, 0xffffffff, 0x00070006,
686 0x91b8, 0xffffffff, 0x0008000b,
687 0x91bc, 0xffffffff, 0x000a0009,
688 0x91c0, 0xffffffff, 0x000d000c,
689 0x91c4, 0xffffffff, 0x00060005,
690 0x91c8, 0xffffffff, 0x00080007,
691 0x91cc, 0xffffffff, 0x0000000b,
692 0x91d0, 0xffffffff, 0x000a0009,
693 0x91d4, 0xffffffff, 0x000d000c,
694 0x9150, 0xffffffff, 0x96940200,
695 0x8708, 0xffffffff, 0x00900100,
696 0xc478, 0xffffffff, 0x00000080,
697 0xc404, 0xffffffff, 0x0020003f,
698 0x30, 0xffffffff, 0x0000001c,
699 0x34, 0x000f0000, 0x000f0000,
700 0x160c, 0xffffffff, 0x00000100,
701 0x1024, 0xffffffff, 0x00000100,
702 0x102c, 0x00000101, 0x00000000,
703 0x20a8, 0xffffffff, 0x00000104,
704 0x264c, 0x000c0000, 0x000c0000,
705 0x2648, 0x000c0000, 0x000c0000,
706 0x55e4, 0xff000fff, 0x00000100,
707 0x55e8, 0x00000001, 0x00000001,
708 0x2f50, 0x00000001, 0x00000001,
709 0x30cc, 0xc0000fff, 0x00000104,
710 0xc1e4, 0x00000001, 0x00000001,
711 0xd0c0, 0xfffffff0, 0x00000100,
712 0xd8c0, 0xfffffff0, 0x00000100
713};
714
fffbdda4
AD		 715static const u32 hainan_mgcg_cgcg_init[] =
716{
717 0xc400, 0xffffffff, 0xfffffffc,
718 0x802c, 0xffffffff, 0xe0000000,
719 0x9a60, 0xffffffff, 0x00000100,
720 0x92a4, 0xffffffff, 0x00000100,
721 0xc164, 0xffffffff, 0x00000100,
722 0x9774, 0xffffffff, 0x00000100,
723 0x8984, 0xffffffff, 0x06000100,
724 0x8a18, 0xffffffff, 0x00000100,
725 0x92a0, 0xffffffff, 0x00000100,
726 0xc380, 0xffffffff, 0x00000100,
727 0x8b28, 0xffffffff, 0x00000100,
728 0x9144, 0xffffffff, 0x00000100,
729 0x8d88, 0xffffffff, 0x00000100,
730 0x8d8c, 0xffffffff, 0x00000100,
731 0x9030, 0xffffffff, 0x00000100,
732 0x9034, 0xffffffff, 0x00000100,
733 0x9038, 0xffffffff, 0x00000100,
734 0x903c, 0xffffffff, 0x00000100,
735 0xad80, 0xffffffff, 0x00000100,
736 0xac54, 0xffffffff, 0x00000100,
737 0x897c, 0xffffffff, 0x06000100,
738 0x9868, 0xffffffff, 0x00000100,
739 0x9510, 0xffffffff, 0x00000100,
740 0xaf04, 0xffffffff, 0x00000100,
741 0xae04, 0xffffffff, 0x00000100,
742 0x949c, 0xffffffff, 0x00000100,
743 0x802c, 0xffffffff, 0xe0000000,
744 0x9160, 0xffffffff, 0x00010000,
745 0x9164, 0xffffffff, 0x00030002,
746 0x9168, 0xffffffff, 0x00040007,
747 0x916c, 0xffffffff, 0x00060005,
748 0x9170, 0xffffffff, 0x00090008,
749 0x9174, 0xffffffff, 0x00020001,
750 0x9178, 0xffffffff, 0x00040003,
751 0x917c, 0xffffffff, 0x00000007,
752 0x9180, 0xffffffff, 0x00060005,
753 0x9184, 0xffffffff, 0x00090008,
754 0x9188, 0xffffffff, 0x00030002,
755 0x918c, 0xffffffff, 0x00050004,
756 0x9190, 0xffffffff, 0x00000008,
757 0x9194, 0xffffffff, 0x00070006,
758 0x9198, 0xffffffff, 0x000a0009,
759 0x919c, 0xffffffff, 0x00040003,
760 0x91a0, 0xffffffff, 0x00060005,
761 0x91a4, 0xffffffff, 0x00000009,
762 0x91a8, 0xffffffff, 0x00080007,
763 0x91ac, 0xffffffff, 0x000b000a,
764 0x91b0, 0xffffffff, 0x00050004,
765 0x91b4, 0xffffffff, 0x00070006,
766 0x91b8, 0xffffffff, 0x0008000b,
767 0x91bc, 0xffffffff, 0x000a0009,
768 0x91c0, 0xffffffff, 0x000d000c,
769 0x91c4, 0xffffffff, 0x00060005,
770 0x91c8, 0xffffffff, 0x00080007,
771 0x91cc, 0xffffffff, 0x0000000b,
772 0x91d0, 0xffffffff, 0x000a0009,
773 0x91d4, 0xffffffff, 0x000d000c,
774 0x9150, 0xffffffff, 0x96940200,
775 0x8708, 0xffffffff, 0x00900100,
776 0xc478, 0xffffffff, 0x00000080,
777 0xc404, 0xffffffff, 0x0020003f,
778 0x30, 0xffffffff, 0x0000001c,
779 0x34, 0x000f0000, 0x000f0000,
780 0x160c, 0xffffffff, 0x00000100,
781 0x1024, 0xffffffff, 0x00000100,
782 0x20a8, 0xffffffff, 0x00000104,
783 0x264c, 0x000c0000, 0x000c0000,
784 0x2648, 0x000c0000, 0x000c0000,
785 0x2f50, 0x00000001, 0x00000001,
786 0x30cc, 0xc0000fff, 0x00000104,
787 0xc1e4, 0x00000001, 0x00000001,
788 0xd0c0, 0xfffffff0, 0x00000100,
789 0xd8c0, 0xfffffff0, 0x00000100
790};
791
205996c0
AD		 792static u32 verde_pg_init[] =
793{
794 0x353c, 0xffffffff, 0x40000,
795 0x3538, 0xffffffff, 0x200010ff,
796 0x353c, 0xffffffff, 0x0,
797 0x353c, 0xffffffff, 0x0,
798 0x353c, 0xffffffff, 0x0,
799 0x353c, 0xffffffff, 0x0,
800 0x353c, 0xffffffff, 0x0,
801 0x353c, 0xffffffff, 0x7007,
802 0x3538, 0xffffffff, 0x300010ff,
803 0x353c, 0xffffffff, 0x0,
804 0x353c, 0xffffffff, 0x0,
805 0x353c, 0xffffffff, 0x0,
806 0x353c, 0xffffffff, 0x0,
807 0x353c, 0xffffffff, 0x0,
808 0x353c, 0xffffffff, 0x400000,
809 0x3538, 0xffffffff, 0x100010ff,
810 0x353c, 0xffffffff, 0x0,
811 0x353c, 0xffffffff, 0x0,
812 0x353c, 0xffffffff, 0x0,
813 0x353c, 0xffffffff, 0x0,
814 0x353c, 0xffffffff, 0x0,
815 0x353c, 0xffffffff, 0x120200,
816 0x3538, 0xffffffff, 0x500010ff,
817 0x353c, 0xffffffff, 0x0,
818 0x353c, 0xffffffff, 0x0,
819 0x353c, 0xffffffff, 0x0,
820 0x353c, 0xffffffff, 0x0,
821 0x353c, 0xffffffff, 0x0,
822 0x353c, 0xffffffff, 0x1e1e16,
823 0x3538, 0xffffffff, 0x600010ff,
824 0x353c, 0xffffffff, 0x0,
825 0x353c, 0xffffffff, 0x0,
826 0x353c, 0xffffffff, 0x0,
827 0x353c, 0xffffffff, 0x0,
828 0x353c, 0xffffffff, 0x0,
829 0x353c, 0xffffffff, 0x171f1e,
830 0x3538, 0xffffffff, 0x700010ff,
831 0x353c, 0xffffffff, 0x0,
832 0x353c, 0xffffffff, 0x0,
833 0x353c, 0xffffffff, 0x0,
834 0x353c, 0xffffffff, 0x0,
835 0x353c, 0xffffffff, 0x0,
836 0x353c, 0xffffffff, 0x0,
837 0x3538, 0xffffffff, 0x9ff,
838 0x3500, 0xffffffff, 0x0,
839 0x3504, 0xffffffff, 0x10000800,
840 0x3504, 0xffffffff, 0xf,
841 0x3504, 0xffffffff, 0xf,
842 0x3500, 0xffffffff, 0x4,
843 0x3504, 0xffffffff, 0x1000051e,
844 0x3504, 0xffffffff, 0xffff,
845 0x3504, 0xffffffff, 0xffff,
846 0x3500, 0xffffffff, 0x8,
847 0x3504, 0xffffffff, 0x80500,
848 0x3500, 0xffffffff, 0x12,
849 0x3504, 0xffffffff, 0x9050c,
850 0x3500, 0xffffffff, 0x1d,
851 0x3504, 0xffffffff, 0xb052c,
852 0x3500, 0xffffffff, 0x2a,
853 0x3504, 0xffffffff, 0x1053e,
854 0x3500, 0xffffffff, 0x2d,
855 0x3504, 0xffffffff, 0x10546,
856 0x3500, 0xffffffff, 0x30,
857 0x3504, 0xffffffff, 0xa054e,
858 0x3500, 0xffffffff, 0x3c,
859 0x3504, 0xffffffff, 0x1055f,
860 0x3500, 0xffffffff, 0x3f,
861 0x3504, 0xffffffff, 0x10567,
862 0x3500, 0xffffffff, 0x42,
863 0x3504, 0xffffffff, 0x1056f,
864 0x3500, 0xffffffff, 0x45,
865 0x3504, 0xffffffff, 0x10572,
866 0x3500, 0xffffffff, 0x48,
867 0x3504, 0xffffffff, 0x20575,
868 0x3500, 0xffffffff, 0x4c,
869 0x3504, 0xffffffff, 0x190801,
870 0x3500, 0xffffffff, 0x67,
871 0x3504, 0xffffffff, 0x1082a,
872 0x3500, 0xffffffff, 0x6a,
873 0x3504, 0xffffffff, 0x1b082d,
874 0x3500, 0xffffffff, 0x87,
875 0x3504, 0xffffffff, 0x310851,
876 0x3500, 0xffffffff, 0xba,
877 0x3504, 0xffffffff, 0x891,
878 0x3500, 0xffffffff, 0xbc,
879 0x3504, 0xffffffff, 0x893,
880 0x3500, 0xffffffff, 0xbe,
881 0x3504, 0xffffffff, 0x20895,
882 0x3500, 0xffffffff, 0xc2,
883 0x3504, 0xffffffff, 0x20899,
884 0x3500, 0xffffffff, 0xc6,
885 0x3504, 0xffffffff, 0x2089d,
886 0x3500, 0xffffffff, 0xca,
887 0x3504, 0xffffffff, 0x8a1,
888 0x3500, 0xffffffff, 0xcc,
889 0x3504, 0xffffffff, 0x8a3,
890 0x3500, 0xffffffff, 0xce,
891 0x3504, 0xffffffff, 0x308a5,
892 0x3500, 0xffffffff, 0xd3,
893 0x3504, 0xffffffff, 0x6d08cd,
894 0x3500, 0xffffffff, 0x142,
895 0x3504, 0xffffffff, 0x2000095a,
896 0x3504, 0xffffffff, 0x1,
897 0x3500, 0xffffffff, 0x144,
898 0x3504, 0xffffffff, 0x301f095b,
899 0x3500, 0xffffffff, 0x165,
900 0x3504, 0xffffffff, 0xc094d,
901 0x3500, 0xffffffff, 0x173,
902 0x3504, 0xffffffff, 0xf096d,
903 0x3500, 0xffffffff, 0x184,
904 0x3504, 0xffffffff, 0x15097f,
905 0x3500, 0xffffffff, 0x19b,
906 0x3504, 0xffffffff, 0xc0998,
907 0x3500, 0xffffffff, 0x1a9,
908 0x3504, 0xffffffff, 0x409a7,
909 0x3500, 0xffffffff, 0x1af,
910 0x3504, 0xffffffff, 0xcdc,
911 0x3500, 0xffffffff, 0x1b1,
912 0x3504, 0xffffffff, 0x800,
913 0x3508, 0xffffffff, 0x6c9b2000,
914 0x3510, 0xfc00, 0x2000,
915 0x3544, 0xffffffff, 0xfc0,
916 0x28d4, 0x00000100, 0x100
917};
918
919static void si_init_golden_registers(struct radeon_device *rdev)
920{
921 switch (rdev->family) {
922 case CHIP_TAHITI:
923 radeon_program_register_sequence(rdev,
924 tahiti_golden_registers,
925 (const u32)ARRAY_SIZE(tahiti_golden_registers));
926 radeon_program_register_sequence(rdev,
927 tahiti_golden_rlc_registers,
928 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
929 radeon_program_register_sequence(rdev,
930 tahiti_mgcg_cgcg_init,
931 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
932 radeon_program_register_sequence(rdev,
933 tahiti_golden_registers2,
934 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
935 break;
936 case CHIP_PITCAIRN:
937 radeon_program_register_sequence(rdev,
938 pitcairn_golden_registers,
939 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
940 radeon_program_register_sequence(rdev,
941 pitcairn_golden_rlc_registers,
942 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
943 radeon_program_register_sequence(rdev,
944 pitcairn_mgcg_cgcg_init,
945 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
946 break;
947 case CHIP_VERDE:
948 radeon_program_register_sequence(rdev,
949 verde_golden_registers,
950 (const u32)ARRAY_SIZE(verde_golden_registers));
951 radeon_program_register_sequence(rdev,
952 verde_golden_rlc_registers,
953 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
954 radeon_program_register_sequence(rdev,
955 verde_mgcg_cgcg_init,
956 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
957 radeon_program_register_sequence(rdev,
958 verde_pg_init,
959 (const u32)ARRAY_SIZE(verde_pg_init));
960 break;
961 case CHIP_OLAND:
962 radeon_program_register_sequence(rdev,
963 oland_golden_registers,
964 (const u32)ARRAY_SIZE(oland_golden_registers));
965 radeon_program_register_sequence(rdev,
966 oland_golden_rlc_registers,
967 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
968 radeon_program_register_sequence(rdev,
969 oland_mgcg_cgcg_init,
970 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
971 break;
fffbdda4
AD		 972 case CHIP_HAINAN:
973 radeon_program_register_sequence(rdev,
974 hainan_golden_registers,
975 (const u32)ARRAY_SIZE(hainan_golden_registers));
976 radeon_program_register_sequence(rdev,
977 hainan_golden_registers2,
978 (const u32)ARRAY_SIZE(hainan_golden_registers2));
979 radeon_program_register_sequence(rdev,
980 hainan_mgcg_cgcg_init,
981 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
982 break;
205996c0
AD		 983 default:
984 break;
985 }
986}
987
454d2e2a
AD		 988#define PCIE_BUS_CLK 10000
989#define TCLK (PCIE_BUS_CLK / 10)
990
991/**
992 * si_get_xclk - get the xclk
993 *
994 * @rdev: radeon_device pointer
995 *
996 * Returns the reference clock used by the gfx engine
997 * (SI).
998 */
999u32 si_get_xclk(struct radeon_device *rdev)
1000{
1001 u32 reference_clock = rdev->clock.spll.reference_freq;
1002 u32 tmp;
1003
1004 tmp = RREG32(CG_CLKPIN_CNTL_2);
1005 if (tmp & MUX_TCLK_TO_XCLK)
1006 return TCLK;
1007
1008 tmp = RREG32(CG_CLKPIN_CNTL);
1009 if (tmp & XTALIN_DIVIDE)
1010 return reference_clock / 4;
1011
1012 return reference_clock;
1013}
1014
1bd47d2e
AD		 1015/* get temperature in millidegrees */
1016int si_get_temp(struct radeon_device *rdev)
1017{
1018 u32 temp;
1019 int actual_temp = 0;
1020
1021 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1022 CTF_TEMP_SHIFT;
1023
1024 if (temp & 0x200)
1025 actual_temp = 255;
1026 else
1027 actual_temp = temp & 0x1ff;
1028
1029 actual_temp = (actual_temp * 1000);
1030
1031 return actual_temp;
1032}
1033
8b074dd6
AD		 1034#define TAHITI_IO_MC_REGS_SIZE 36
1035
1036static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1037 {0x0000006f, 0x03044000},
1038 {0x00000070, 0x0480c018},
1039 {0x00000071, 0x00000040},
1040 {0x00000072, 0x01000000},
1041 {0x00000074, 0x000000ff},
1042 {0x00000075, 0x00143400},
1043 {0x00000076, 0x08ec0800},
1044 {0x00000077, 0x040000cc},
1045 {0x00000079, 0x00000000},
1046 {0x0000007a, 0x21000409},
1047 {0x0000007c, 0x00000000},
1048 {0x0000007d, 0xe8000000},
1049 {0x0000007e, 0x044408a8},
1050 {0x0000007f, 0x00000003},
1051 {0x00000080, 0x00000000},
1052 {0x00000081, 0x01000000},
1053 {0x00000082, 0x02000000},
1054 {0x00000083, 0x00000000},
1055 {0x00000084, 0xe3f3e4f4},
1056 {0x00000085, 0x00052024},
1057 {0x00000087, 0x00000000},
1058 {0x00000088, 0x66036603},
1059 {0x00000089, 0x01000000},
1060 {0x0000008b, 0x1c0a0000},
1061 {0x0000008c, 0xff010000},
1062 {0x0000008e, 0xffffefff},
1063 {0x0000008f, 0xfff3efff},
1064 {0x00000090, 0xfff3efbf},
1065 {0x00000094, 0x00101101},
1066 {0x00000095, 0x00000fff},
1067 {0x00000096, 0x00116fff},
1068 {0x00000097, 0x60010000},
1069 {0x00000098, 0x10010000},
1070 {0x00000099, 0x00006000},
1071 {0x0000009a, 0x00001000},
1072 {0x0000009f, 0x00a77400}
1073};
1074
1075static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1076 {0x0000006f, 0x03044000},
1077 {0x00000070, 0x0480c018},
1078 {0x00000071, 0x00000040},
1079 {0x00000072, 0x01000000},
1080 {0x00000074, 0x000000ff},
1081 {0x00000075, 0x00143400},
1082 {0x00000076, 0x08ec0800},
1083 {0x00000077, 0x040000cc},
1084 {0x00000079, 0x00000000},
1085 {0x0000007a, 0x21000409},
1086 {0x0000007c, 0x00000000},
1087 {0x0000007d, 0xe8000000},
1088 {0x0000007e, 0x044408a8},
1089 {0x0000007f, 0x00000003},
1090 {0x00000080, 0x00000000},
1091 {0x00000081, 0x01000000},
1092 {0x00000082, 0x02000000},
1093 {0x00000083, 0x00000000},
1094 {0x00000084, 0xe3f3e4f4},
1095 {0x00000085, 0x00052024},
1096 {0x00000087, 0x00000000},
1097 {0x00000088, 0x66036603},
1098 {0x00000089, 0x01000000},
1099 {0x0000008b, 0x1c0a0000},
1100 {0x0000008c, 0xff010000},
1101 {0x0000008e, 0xffffefff},
1102 {0x0000008f, 0xfff3efff},
1103 {0x00000090, 0xfff3efbf},
1104 {0x00000094, 0x00101101},
1105 {0x00000095, 0x00000fff},
1106 {0x00000096, 0x00116fff},
1107 {0x00000097, 0x60010000},
1108 {0x00000098, 0x10010000},
1109 {0x00000099, 0x00006000},
1110 {0x0000009a, 0x00001000},
1111 {0x0000009f, 0x00a47400}
1112};
1113
1114static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1115 {0x0000006f, 0x03044000},
1116 {0x00000070, 0x0480c018},
1117 {0x00000071, 0x00000040},
1118 {0x00000072, 0x01000000},
1119 {0x00000074, 0x000000ff},
1120 {0x00000075, 0x00143400},
1121 {0x00000076, 0x08ec0800},
1122 {0x00000077, 0x040000cc},
1123 {0x00000079, 0x00000000},
1124 {0x0000007a, 0x21000409},
1125 {0x0000007c, 0x00000000},
1126 {0x0000007d, 0xe8000000},
1127 {0x0000007e, 0x044408a8},
1128 {0x0000007f, 0x00000003},
1129 {0x00000080, 0x00000000},
1130 {0x00000081, 0x01000000},
1131 {0x00000082, 0x02000000},
1132 {0x00000083, 0x00000000},
1133 {0x00000084, 0xe3f3e4f4},
1134 {0x00000085, 0x00052024},
1135 {0x00000087, 0x00000000},
1136 {0x00000088, 0x66036603},
1137 {0x00000089, 0x01000000},
1138 {0x0000008b, 0x1c0a0000},
1139 {0x0000008c, 0xff010000},
1140 {0x0000008e, 0xffffefff},
1141 {0x0000008f, 0xfff3efff},
1142 {0x00000090, 0xfff3efbf},
1143 {0x00000094, 0x00101101},
1144 {0x00000095, 0x00000fff},
1145 {0x00000096, 0x00116fff},
1146 {0x00000097, 0x60010000},
1147 {0x00000098, 0x10010000},
1148 {0x00000099, 0x00006000},
1149 {0x0000009a, 0x00001000},
1150 {0x0000009f, 0x00a37400}
1151};
1152
bcc7f5d2
AD		 1153static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1154 {0x0000006f, 0x03044000},
1155 {0x00000070, 0x0480c018},
1156 {0x00000071, 0x00000040},
1157 {0x00000072, 0x01000000},
1158 {0x00000074, 0x000000ff},
1159 {0x00000075, 0x00143400},
1160 {0x00000076, 0x08ec0800},
1161 {0x00000077, 0x040000cc},
1162 {0x00000079, 0x00000000},
1163 {0x0000007a, 0x21000409},
1164 {0x0000007c, 0x00000000},
1165 {0x0000007d, 0xe8000000},
1166 {0x0000007e, 0x044408a8},
1167 {0x0000007f, 0x00000003},
1168 {0x00000080, 0x00000000},
1169 {0x00000081, 0x01000000},
1170 {0x00000082, 0x02000000},
1171 {0x00000083, 0x00000000},
1172 {0x00000084, 0xe3f3e4f4},
1173 {0x00000085, 0x00052024},
1174 {0x00000087, 0x00000000},
1175 {0x00000088, 0x66036603},
1176 {0x00000089, 0x01000000},
1177 {0x0000008b, 0x1c0a0000},
1178 {0x0000008c, 0xff010000},
1179 {0x0000008e, 0xffffefff},
1180 {0x0000008f, 0xfff3efff},
1181 {0x00000090, 0xfff3efbf},
1182 {0x00000094, 0x00101101},
1183 {0x00000095, 0x00000fff},
1184 {0x00000096, 0x00116fff},
1185 {0x00000097, 0x60010000},
1186 {0x00000098, 0x10010000},
1187 {0x00000099, 0x00006000},
1188 {0x0000009a, 0x00001000},
1189 {0x0000009f, 0x00a17730}
1190};
1191
c04c00b4
AD		 1192static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1193 {0x0000006f, 0x03044000},
1194 {0x00000070, 0x0480c018},
1195 {0x00000071, 0x00000040},
1196 {0x00000072, 0x01000000},
1197 {0x00000074, 0x000000ff},
1198 {0x00000075, 0x00143400},
1199 {0x00000076, 0x08ec0800},
1200 {0x00000077, 0x040000cc},
1201 {0x00000079, 0x00000000},
1202 {0x0000007a, 0x21000409},
1203 {0x0000007c, 0x00000000},
1204 {0x0000007d, 0xe8000000},
1205 {0x0000007e, 0x044408a8},
1206 {0x0000007f, 0x00000003},
1207 {0x00000080, 0x00000000},
1208 {0x00000081, 0x01000000},
1209 {0x00000082, 0x02000000},
1210 {0x00000083, 0x00000000},
1211 {0x00000084, 0xe3f3e4f4},
1212 {0x00000085, 0x00052024},
1213 {0x00000087, 0x00000000},
1214 {0x00000088, 0x66036603},
1215 {0x00000089, 0x01000000},
1216 {0x0000008b, 0x1c0a0000},
1217 {0x0000008c, 0xff010000},
1218 {0x0000008e, 0xffffefff},
1219 {0x0000008f, 0xfff3efff},
1220 {0x00000090, 0xfff3efbf},
1221 {0x00000094, 0x00101101},
1222 {0x00000095, 0x00000fff},
1223 {0x00000096, 0x00116fff},
1224 {0x00000097, 0x60010000},
1225 {0x00000098, 0x10010000},
1226 {0x00000099, 0x00006000},
1227 {0x0000009a, 0x00001000},
1228 {0x0000009f, 0x00a07730}
1229};
1230
8b074dd6
AD		 1231/* ucode loading */
1232static int si_mc_load_microcode(struct radeon_device *rdev)
1233{
1234 const __be32 *fw_data;
1235 u32 running, blackout = 0;
1236 u32 *io_mc_regs;
1237 int i, ucode_size, regs_size;
1238
1239 if (!rdev->mc_fw)
1240 return -EINVAL;
1241
1242 switch (rdev->family) {
1243 case CHIP_TAHITI:
1244 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1245 ucode_size = SI_MC_UCODE_SIZE;
1246 regs_size = TAHITI_IO_MC_REGS_SIZE;
1247 break;
1248 case CHIP_PITCAIRN:
1249 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1250 ucode_size = SI_MC_UCODE_SIZE;
1251 regs_size = TAHITI_IO_MC_REGS_SIZE;
1252 break;
1253 case CHIP_VERDE:
1254 default:
1255 io_mc_regs = (u32 *)&verde_io_mc_regs;
1256 ucode_size = SI_MC_UCODE_SIZE;
1257 regs_size = TAHITI_IO_MC_REGS_SIZE;
1258 break;
bcc7f5d2
AD		 1259 case CHIP_OLAND:
1260 io_mc_regs = (u32 *)&oland_io_mc_regs;
1261 ucode_size = OLAND_MC_UCODE_SIZE;
1262 regs_size = TAHITI_IO_MC_REGS_SIZE;
1263 break;
c04c00b4
AD		 1264 case CHIP_HAINAN:
1265 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1266 ucode_size = OLAND_MC_UCODE_SIZE;
1267 regs_size = TAHITI_IO_MC_REGS_SIZE;
1268 break;
8b074dd6
AD		 1269 }
1270
1271 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1272
1273 if (running == 0) {
1274 if (running) {
1275 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1276 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1277 }
1278
1279 /* reset the engine and set to writable */
1280 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1281 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1282
1283 /* load mc io regs */
1284 for (i = 0; i < regs_size; i++) {
1285 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1286 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1287 }
1288 /* load the MC ucode */
1289 fw_data = (const __be32 *)rdev->mc_fw->data;
1290 for (i = 0; i < ucode_size; i++)
1291 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1292
1293 /* put the engine back into the active state */
1294 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1295 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1296 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1297
1298 /* wait for training to complete */
1299 for (i = 0; i < rdev->usec_timeout; i++) {
1300 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1301 break;
1302 udelay(1);
1303 }
1304 for (i = 0; i < rdev->usec_timeout; i++) {
1305 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1306 break;
1307 udelay(1);
1308 }
1309
1310 if (running)
1311 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1312 }
1313
1314 return 0;
1315}
1316
0f0de06c
AD		 1317static int si_init_microcode(struct radeon_device *rdev)
1318{
1319 struct platform_device *pdev;
1320 const char *chip_name;
1321 const char *rlc_chip_name;
1322 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1323 char fw_name[30];
1324 int err;
1325
1326 DRM_DEBUG("\n");
1327
1328 pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1329 err = IS_ERR(pdev);
1330 if (err) {
1331 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1332 return -EINVAL;
1333 }
1334
1335 switch (rdev->family) {
1336 case CHIP_TAHITI:
1337 chip_name = "TAHITI";
1338 rlc_chip_name = "TAHITI";
1339 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1340 me_req_size = SI_PM4_UCODE_SIZE * 4;
1341 ce_req_size = SI_CE_UCODE_SIZE * 4;
1342 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1343 mc_req_size = SI_MC_UCODE_SIZE * 4;
1344 break;
1345 case CHIP_PITCAIRN:
1346 chip_name = "PITCAIRN";
1347 rlc_chip_name = "PITCAIRN";
1348 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1349 me_req_size = SI_PM4_UCODE_SIZE * 4;
1350 ce_req_size = SI_CE_UCODE_SIZE * 4;
1351 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1352 mc_req_size = SI_MC_UCODE_SIZE * 4;
1353 break;
1354 case CHIP_VERDE:
1355 chip_name = "VERDE";
1356 rlc_chip_name = "VERDE";
1357 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1358 me_req_size = SI_PM4_UCODE_SIZE * 4;
1359 ce_req_size = SI_CE_UCODE_SIZE * 4;
1360 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1361 mc_req_size = SI_MC_UCODE_SIZE * 4;
1362 break;
bcc7f5d2
AD		 1363 case CHIP_OLAND:
1364 chip_name = "OLAND";
1365 rlc_chip_name = "OLAND";
1366 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1367 me_req_size = SI_PM4_UCODE_SIZE * 4;
1368 ce_req_size = SI_CE_UCODE_SIZE * 4;
1369 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1370 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1371 break;
c04c00b4
AD		 1372 case CHIP_HAINAN:
1373 chip_name = "HAINAN";
1374 rlc_chip_name = "HAINAN";
1375 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1376 me_req_size = SI_PM4_UCODE_SIZE * 4;
1377 ce_req_size = SI_CE_UCODE_SIZE * 4;
1378 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1379 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1380 break;
0f0de06c
AD		 1381 default: BUG();
1382 }
1383
1384 DRM_INFO("Loading %s Microcode\n", chip_name);
1385
1386 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1387 err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1388 if (err)
1389 goto out;
1390 if (rdev->pfp_fw->size != pfp_req_size) {
1391 printk(KERN_ERR
1392 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1393 rdev->pfp_fw->size, fw_name);
1394 err = -EINVAL;
1395 goto out;
1396 }
1397
1398 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1399 err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1400 if (err)
1401 goto out;
1402 if (rdev->me_fw->size != me_req_size) {
1403 printk(KERN_ERR
1404 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1405 rdev->me_fw->size, fw_name);
1406 err = -EINVAL;
1407 }
1408
1409 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1410 err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
1411 if (err)
1412 goto out;
1413 if (rdev->ce_fw->size != ce_req_size) {
1414 printk(KERN_ERR
1415 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1416 rdev->ce_fw->size, fw_name);
1417 err = -EINVAL;
1418 }
1419
1420 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1421 err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1422 if (err)
1423 goto out;
1424 if (rdev->rlc_fw->size != rlc_req_size) {
1425 printk(KERN_ERR
1426 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1427 rdev->rlc_fw->size, fw_name);
1428 err = -EINVAL;
1429 }
1430
1431 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1432 err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
1433 if (err)
1434 goto out;
1435 if (rdev->mc_fw->size != mc_req_size) {
1436 printk(KERN_ERR
1437 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1438 rdev->mc_fw->size, fw_name);
1439 err = -EINVAL;
1440 }
1441
1442out:
1443 platform_device_unregister(pdev);
1444
1445 if (err) {
1446 if (err != -EINVAL)
1447 printk(KERN_ERR
1448 "si_cp: Failed to load firmware \"%s\"\n",
1449 fw_name);
1450 release_firmware(rdev->pfp_fw);
1451 rdev->pfp_fw = NULL;
1452 release_firmware(rdev->me_fw);
1453 rdev->me_fw = NULL;
1454 release_firmware(rdev->ce_fw);
1455 rdev->ce_fw = NULL;
1456 release_firmware(rdev->rlc_fw);
1457 rdev->rlc_fw = NULL;
1458 release_firmware(rdev->mc_fw);
1459 rdev->mc_fw = NULL;
1460 }
1461 return err;
1462}
1463
43b3cd99
AD		 1464/* watermark setup */
1465static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1466 struct radeon_crtc *radeon_crtc,
1467 struct drm_display_mode *mode,
1468 struct drm_display_mode *other_mode)
1469{
1470 u32 tmp;
1471 /*
1472 * Line Buffer Setup
1473 * There are 3 line buffers, each one shared by 2 display controllers.
1474 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1475 * the display controllers. The paritioning is done via one of four
1476 * preset allocations specified in bits 21:20:
1477 * 0 - half lb
1478 * 2 - whole lb, other crtc must be disabled
1479 */
1480 /* this can get tricky if we have two large displays on a paired group
1481 * of crtcs. Ideally for multiple large displays we'd assign them to
1482 * non-linked crtcs for maximum line buffer allocation.
1483 */
1484 if (radeon_crtc->base.enabled && mode) {
1485 if (other_mode)
1486 tmp = 0; /* 1/2 */
1487 else
1488 tmp = 2; /* whole */
1489 } else
1490 tmp = 0;
1491
1492 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1493 DC_LB_MEMORY_CONFIG(tmp));
1494
1495 if (radeon_crtc->base.enabled && mode) {
1496 switch (tmp) {
1497 case 0:
1498 default:
1499 return 4096 * 2;
1500 case 2:
1501 return 8192 * 2;
1502 }
1503 }
1504
1505 /* controller not enabled, so no lb used */
1506 return 0;
1507}
1508
ca7db22b 1509static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 1510{
1511 u32 tmp = RREG32(MC_SHARED_CHMAP);
1512
1513 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1514 case 0:
1515 default:
1516 return 1;
1517 case 1:
1518 return 2;
1519 case 2:
1520 return 4;
1521 case 3:
1522 return 8;
1523 case 4:
1524 return 3;
1525 case 5:
1526 return 6;
1527 case 6:
1528 return 10;
1529 case 7:
1530 return 12;
1531 case 8:
1532 return 16;
1533 }
1534}
1535
1536struct dce6_wm_params {
1537 u32 dram_channels; /* number of dram channels */
1538 u32 yclk; /* bandwidth per dram data pin in kHz */
1539 u32 sclk; /* engine clock in kHz */
1540 u32 disp_clk; /* display clock in kHz */
1541 u32 src_width; /* viewport width */
1542 u32 active_time; /* active display time in ns */
1543 u32 blank_time; /* blank time in ns */
1544 bool interlaced; /* mode is interlaced */
1545 fixed20_12 vsc; /* vertical scale ratio */
1546 u32 num_heads; /* number of active crtcs */
1547 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1548 u32 lb_size; /* line buffer allocated to pipe */
1549 u32 vtaps; /* vertical scaler taps */
1550};
1551
1552static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1553{
1554 /* Calculate raw DRAM Bandwidth */
1555 fixed20_12 dram_efficiency; /* 0.7 */
1556 fixed20_12 yclk, dram_channels, bandwidth;
1557 fixed20_12 a;
1558
1559 a.full = dfixed_const(1000);
1560 yclk.full = dfixed_const(wm->yclk);
1561 yclk.full = dfixed_div(yclk, a);
1562 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1563 a.full = dfixed_const(10);
1564 dram_efficiency.full = dfixed_const(7);
1565 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1566 bandwidth.full = dfixed_mul(dram_channels, yclk);
1567 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1568
1569 return dfixed_trunc(bandwidth);
1570}
1571
1572static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1573{
1574 /* Calculate DRAM Bandwidth and the part allocated to display. */
1575 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1576 fixed20_12 yclk, dram_channels, bandwidth;
1577 fixed20_12 a;
1578
1579 a.full = dfixed_const(1000);
1580 yclk.full = dfixed_const(wm->yclk);
1581 yclk.full = dfixed_div(yclk, a);
1582 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1583 a.full = dfixed_const(10);
1584 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1585 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1586 bandwidth.full = dfixed_mul(dram_channels, yclk);
1587 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1588
1589 return dfixed_trunc(bandwidth);
1590}
1591
1592static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1593{
1594 /* Calculate the display Data return Bandwidth */
1595 fixed20_12 return_efficiency; /* 0.8 */
1596 fixed20_12 sclk, bandwidth;
1597 fixed20_12 a;
1598
1599 a.full = dfixed_const(1000);
1600 sclk.full = dfixed_const(wm->sclk);
1601 sclk.full = dfixed_div(sclk, a);
1602 a.full = dfixed_const(10);
1603 return_efficiency.full = dfixed_const(8);
1604 return_efficiency.full = dfixed_div(return_efficiency, a);
1605 a.full = dfixed_const(32);
1606 bandwidth.full = dfixed_mul(a, sclk);
1607 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1608
1609 return dfixed_trunc(bandwidth);
1610}
1611
1612static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1613{
1614 return 32;
1615}
1616
1617static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1618{
1619 /* Calculate the DMIF Request Bandwidth */
1620 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1621 fixed20_12 disp_clk, sclk, bandwidth;
1622 fixed20_12 a, b1, b2;
1623 u32 min_bandwidth;
1624
1625 a.full = dfixed_const(1000);
1626 disp_clk.full = dfixed_const(wm->disp_clk);
1627 disp_clk.full = dfixed_div(disp_clk, a);
1628 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1629 b1.full = dfixed_mul(a, disp_clk);
1630
1631 a.full = dfixed_const(1000);
1632 sclk.full = dfixed_const(wm->sclk);
1633 sclk.full = dfixed_div(sclk, a);
1634 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1635 b2.full = dfixed_mul(a, sclk);
1636
1637 a.full = dfixed_const(10);
1638 disp_clk_request_efficiency.full = dfixed_const(8);
1639 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1640
1641 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1642
1643 a.full = dfixed_const(min_bandwidth);
1644 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1645
1646 return dfixed_trunc(bandwidth);
1647}
1648
1649static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1650{
1651 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1652 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1653 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1654 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1655
1656 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1657}
1658
1659static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1660{
1661 /* Calculate the display mode Average Bandwidth
1662 * DisplayMode should contain the source and destination dimensions,
1663 * timing, etc.
1664 */
1665 fixed20_12 bpp;
1666 fixed20_12 line_time;
1667 fixed20_12 src_width;
1668 fixed20_12 bandwidth;
1669 fixed20_12 a;
1670
1671 a.full = dfixed_const(1000);
1672 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1673 line_time.full = dfixed_div(line_time, a);
1674 bpp.full = dfixed_const(wm->bytes_per_pixel);
1675 src_width.full = dfixed_const(wm->src_width);
1676 bandwidth.full = dfixed_mul(src_width, bpp);
1677 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1678 bandwidth.full = dfixed_div(bandwidth, line_time);
1679
1680 return dfixed_trunc(bandwidth);
1681}
1682
1683static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1684{
1685 /* First calcualte the latency in ns */
1686 u32 mc_latency = 2000; /* 2000 ns. */
1687 u32 available_bandwidth = dce6_available_bandwidth(wm);
1688 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1689 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1690 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1691 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1692 (wm->num_heads * cursor_line_pair_return_time);
1693 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1694 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1695 u32 tmp, dmif_size = 12288;
1696 fixed20_12 a, b, c;
1697
1698 if (wm->num_heads == 0)
1699 return 0;
1700
1701 a.full = dfixed_const(2);
1702 b.full = dfixed_const(1);
1703 if ((wm->vsc.full > a.full) ||
1704 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1705 (wm->vtaps >= 5) ||
1706 ((wm->vsc.full >= a.full) && wm->interlaced))
1707 max_src_lines_per_dst_line = 4;
1708 else
1709 max_src_lines_per_dst_line = 2;
1710
1711 a.full = dfixed_const(available_bandwidth);
1712 b.full = dfixed_const(wm->num_heads);
1713 a.full = dfixed_div(a, b);
1714
1715 b.full = dfixed_const(mc_latency + 512);
1716 c.full = dfixed_const(wm->disp_clk);
1717 b.full = dfixed_div(b, c);
1718
1719 c.full = dfixed_const(dmif_size);
1720 b.full = dfixed_div(c, b);
1721
1722 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1723
1724 b.full = dfixed_const(1000);
1725 c.full = dfixed_const(wm->disp_clk);
1726 b.full = dfixed_div(c, b);
1727 c.full = dfixed_const(wm->bytes_per_pixel);
1728 b.full = dfixed_mul(b, c);
1729
1730 lb_fill_bw = min(tmp, dfixed_trunc(b));
1731
1732 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1733 b.full = dfixed_const(1000);
1734 c.full = dfixed_const(lb_fill_bw);
1735 b.full = dfixed_div(c, b);
1736 a.full = dfixed_div(a, b);
1737 line_fill_time = dfixed_trunc(a);
1738
1739 if (line_fill_time < wm->active_time)
1740 return latency;
1741 else
1742 return latency + (line_fill_time - wm->active_time);
1743
1744}
1745
1746static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1747{
1748 if (dce6_average_bandwidth(wm) <=
1749 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
1750 return true;
1751 else
1752 return false;
1753};
1754
1755static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
1756{
1757 if (dce6_average_bandwidth(wm) <=
1758 (dce6_available_bandwidth(wm) / wm->num_heads))
1759 return true;
1760 else
1761 return false;
1762};
1763
1764static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
1765{
1766 u32 lb_partitions = wm->lb_size / wm->src_width;
1767 u32 line_time = wm->active_time + wm->blank_time;
1768 u32 latency_tolerant_lines;
1769 u32 latency_hiding;
1770 fixed20_12 a;
1771
1772 a.full = dfixed_const(1);
1773 if (wm->vsc.full > a.full)
1774 latency_tolerant_lines = 1;
1775 else {
1776 if (lb_partitions <= (wm->vtaps + 1))
1777 latency_tolerant_lines = 1;
1778 else
1779 latency_tolerant_lines = 2;
1780 }
1781
1782 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
1783
1784 if (dce6_latency_watermark(wm) <= latency_hiding)
1785 return true;
1786 else
1787 return false;
1788}
1789
1790static void dce6_program_watermarks(struct radeon_device *rdev,
1791 struct radeon_crtc *radeon_crtc,
1792 u32 lb_size, u32 num_heads)
1793{
1794 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 1795 struct dce6_wm_params wm_low, wm_high;
1796 u32 dram_channels;
43b3cd99
AD		 1797 u32 pixel_period;
1798 u32 line_time = 0;
1799 u32 latency_watermark_a = 0, latency_watermark_b = 0;
1800 u32 priority_a_mark = 0, priority_b_mark = 0;
1801 u32 priority_a_cnt = PRIORITY_OFF;
1802 u32 priority_b_cnt = PRIORITY_OFF;
1803 u32 tmp, arb_control3;
1804 fixed20_12 a, b, c;
1805
1806 if (radeon_crtc->base.enabled && num_heads && mode) {
1807 pixel_period = 1000000 / (u32)mode->clock;
1808 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
1809 priority_a_cnt = 0;
1810 priority_b_cnt = 0;
1811
ca7db22b 1812 if (rdev->family == CHIP_ARUBA)
c696e53f 1813 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 1814 else
c696e53f
AD		 1815 dram_channels = si_get_number_of_dram_channels(rdev);
1816
1817 /* watermark for high clocks */
1818 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
1819 wm_high.yclk =
1820 radeon_dpm_get_mclk(rdev, false) * 10;
1821 wm_high.sclk =
1822 radeon_dpm_get_sclk(rdev, false) * 10;
1823 } else {
1824 wm_high.yclk = rdev->pm.current_mclk * 10;
1825 wm_high.sclk = rdev->pm.current_sclk * 10;
1826 }
1827
1828 wm_high.disp_clk = mode->clock;
1829 wm_high.src_width = mode->crtc_hdisplay;
1830 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
1831 wm_high.blank_time = line_time - wm_high.active_time;
1832 wm_high.interlaced = false;
1833 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
1834 wm_high.interlaced = true;
1835 wm_high.vsc = radeon_crtc->vsc;
1836 wm_high.vtaps = 1;
1837 if (radeon_crtc->rmx_type != RMX_OFF)
1838 wm_high.vtaps = 2;
1839 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
1840 wm_high.lb_size = lb_size;
1841 wm_high.dram_channels = dram_channels;
1842 wm_high.num_heads = num_heads;
1843
1844 /* watermark for low clocks */
1845 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
1846 wm_low.yclk =
1847 radeon_dpm_get_mclk(rdev, true) * 10;
1848 wm_low.sclk =
1849 radeon_dpm_get_sclk(rdev, true) * 10;
1850 } else {
1851 wm_low.yclk = rdev->pm.current_mclk * 10;
1852 wm_low.sclk = rdev->pm.current_sclk * 10;
1853 }
1854
1855 wm_low.disp_clk = mode->clock;
1856 wm_low.src_width = mode->crtc_hdisplay;
1857 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
1858 wm_low.blank_time = line_time - wm_low.active_time;
1859 wm_low.interlaced = false;
1860 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
1861 wm_low.interlaced = true;
1862 wm_low.vsc = radeon_crtc->vsc;
1863 wm_low.vtaps = 1;
1864 if (radeon_crtc->rmx_type != RMX_OFF)
1865 wm_low.vtaps = 2;
1866 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
1867 wm_low.lb_size = lb_size;
1868 wm_low.dram_channels = dram_channels;
1869 wm_low.num_heads = num_heads;
43b3cd99
AD		 1870
1871 /* set for high clocks */
c696e53f 1872 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 1873 /* set for low clocks */
c696e53f 1874 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 1875
1876 /* possibly force display priority to high */
1877 /* should really do this at mode validation time... */
c696e53f
AD		 1878 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
1879 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
1880 !dce6_check_latency_hiding(&wm_high) ||
1881 (rdev->disp_priority == 2)) {
1882 DRM_DEBUG_KMS("force priority to high\n");
1883 priority_a_cnt |= PRIORITY_ALWAYS_ON;
1884 priority_b_cnt |= PRIORITY_ALWAYS_ON;
1885 }
1886 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
1887 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
1888 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 1889 (rdev->disp_priority == 2)) {
1890 DRM_DEBUG_KMS("force priority to high\n");
1891 priority_a_cnt |= PRIORITY_ALWAYS_ON;
1892 priority_b_cnt |= PRIORITY_ALWAYS_ON;
1893 }
1894
1895 a.full = dfixed_const(1000);
1896 b.full = dfixed_const(mode->clock);
1897 b.full = dfixed_div(b, a);
1898 c.full = dfixed_const(latency_watermark_a);
1899 c.full = dfixed_mul(c, b);
1900 c.full = dfixed_mul(c, radeon_crtc->hsc);
1901 c.full = dfixed_div(c, a);
1902 a.full = dfixed_const(16);
1903 c.full = dfixed_div(c, a);
1904 priority_a_mark = dfixed_trunc(c);
1905 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
1906
1907 a.full = dfixed_const(1000);
1908 b.full = dfixed_const(mode->clock);
1909 b.full = dfixed_div(b, a);
1910 c.full = dfixed_const(latency_watermark_b);
1911 c.full = dfixed_mul(c, b);
1912 c.full = dfixed_mul(c, radeon_crtc->hsc);
1913 c.full = dfixed_div(c, a);
1914 a.full = dfixed_const(16);
1915 c.full = dfixed_div(c, a);
1916 priority_b_mark = dfixed_trunc(c);
1917 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
1918 }
1919
1920 /* select wm A */
1921 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
1922 tmp = arb_control3;
1923 tmp &= ~LATENCY_WATERMARK_MASK(3);
1924 tmp |= LATENCY_WATERMARK_MASK(1);
1925 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
1926 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
1927 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
1928 LATENCY_HIGH_WATERMARK(line_time)));
1929 /* select wm B */
1930 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
1931 tmp &= ~LATENCY_WATERMARK_MASK(3);
1932 tmp |= LATENCY_WATERMARK_MASK(2);
1933 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
1934 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
1935 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
1936 LATENCY_HIGH_WATERMARK(line_time)));
1937 /* restore original selection */
1938 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
1939
1940 /* write the priority marks */
1941 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
1942 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
1943
1944}
1945
1946void dce6_bandwidth_update(struct radeon_device *rdev)
1947{
1948 struct drm_display_mode *mode0 = NULL;
1949 struct drm_display_mode *mode1 = NULL;
1950 u32 num_heads = 0, lb_size;
1951 int i;
1952
1953 radeon_update_display_priority(rdev);
1954
1955 for (i = 0; i < rdev->num_crtc; i++) {
1956 if (rdev->mode_info.crtcs[i]->base.enabled)
1957 num_heads++;
1958 }
1959 for (i = 0; i < rdev->num_crtc; i += 2) {
1960 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
1961 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
1962 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
1963 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
1964 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
1965 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
1966 }
1967}
1968
0a96d72b
AD		 1969/*
1970 * Core functions
1971 */
0a96d72b
AD		 1972static void si_tiling_mode_table_init(struct radeon_device *rdev)
1973{
1974 const u32 num_tile_mode_states = 32;
1975 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1976
1977 switch (rdev->config.si.mem_row_size_in_kb) {
1978 case 1:
1979 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1980 break;
1981 case 2:
1982 default:
1983 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1984 break;
1985 case 4:
1986 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1987 break;
1988 }
1989
1990 if ((rdev->family == CHIP_TAHITI) ||
1991 (rdev->family == CHIP_PITCAIRN)) {
1992 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1993 switch (reg_offset) {
1994 case 0: /* non-AA compressed depth or any compressed stencil */
1995 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1996 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1997 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1998 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1999 NUM_BANKS(ADDR_SURF_16_BANK) |
2000 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2001 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2002 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2003 break;
2004 case 1: /* 2xAA/4xAA compressed depth only */
2005 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2006 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2007 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2008 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2009 NUM_BANKS(ADDR_SURF_16_BANK) |
2010 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2011 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2012 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2013 break;
2014 case 2: /* 8xAA compressed depth only */
2015 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2016 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2017 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2018 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2019 NUM_BANKS(ADDR_SURF_16_BANK) |
2020 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2021 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2022 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2023 break;
2024 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2025 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2026 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2027 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2028 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2029 NUM_BANKS(ADDR_SURF_16_BANK) |
2030 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2031 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2032 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2033 break;
2034 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2035 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2036 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2037 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2038 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2039 NUM_BANKS(ADDR_SURF_16_BANK) |
2040 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2041 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2042 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2043 break;
2044 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2045 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2046 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2047 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2048 TILE_SPLIT(split_equal_to_row_size) |
2049 NUM_BANKS(ADDR_SURF_16_BANK) |
2050 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2051 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2052 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2053 break;
2054 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2055 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2056 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2057 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2058 TILE_SPLIT(split_equal_to_row_size) |
2059 NUM_BANKS(ADDR_SURF_16_BANK) |
2060 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2061 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2062 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2063 break;
2064 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2065 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2066 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2067 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2068 TILE_SPLIT(split_equal_to_row_size) |
2069 NUM_BANKS(ADDR_SURF_16_BANK) |
2070 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2071 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2072 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2073 break;
2074 case 8: /* 1D and 1D Array Surfaces */
2075 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2076 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2077 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2078 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2079 NUM_BANKS(ADDR_SURF_16_BANK) |
2080 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2081 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2082 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2083 break;
2084 case 9: /* Displayable maps. */
2085 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2086 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2087 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2088 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2089 NUM_BANKS(ADDR_SURF_16_BANK) |
2090 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2091 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2092 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2093 break;
2094 case 10: /* Display 8bpp. */
2095 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2096 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2097 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2098 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2099 NUM_BANKS(ADDR_SURF_16_BANK) |
2100 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2101 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2102 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2103 break;
2104 case 11: /* Display 16bpp. */
2105 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2106 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2107 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2108 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2109 NUM_BANKS(ADDR_SURF_16_BANK) |
2110 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2111 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2112 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2113 break;
2114 case 12: /* Display 32bpp. */
2115 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2116 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2117 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2118 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2119 NUM_BANKS(ADDR_SURF_16_BANK) |
2120 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2121 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2122 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2123 break;
2124 case 13: /* Thin. */
2125 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2126 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2127 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2128 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2129 NUM_BANKS(ADDR_SURF_16_BANK) |
2130 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2131 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2132 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2133 break;
2134 case 14: /* Thin 8 bpp. */
2135 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2136 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2137 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2138 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2139 NUM_BANKS(ADDR_SURF_16_BANK) |
2140 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2141 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2142 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2143 break;
2144 case 15: /* Thin 16 bpp. */
2145 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2146 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2147 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2148 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2149 NUM_BANKS(ADDR_SURF_16_BANK) |
2150 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2151 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2152 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2153 break;
2154 case 16: /* Thin 32 bpp. */
2155 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2156 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2157 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2158 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2159 NUM_BANKS(ADDR_SURF_16_BANK) |
2160 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2161 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2162 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2163 break;
2164 case 17: /* Thin 64 bpp. */
2165 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2166 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2167 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2168 TILE_SPLIT(split_equal_to_row_size) |
2169 NUM_BANKS(ADDR_SURF_16_BANK) |
2170 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2171 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2172 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2173 break;
2174 case 21: /* 8 bpp PRT. */
2175 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2176 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2177 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2178 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2179 NUM_BANKS(ADDR_SURF_16_BANK) |
2180 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2181 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2182 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2183 break;
2184 case 22: /* 16 bpp PRT */
2185 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2186 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2187 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2188 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2189 NUM_BANKS(ADDR_SURF_16_BANK) |
2190 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2191 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2192 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2193 break;
2194 case 23: /* 32 bpp PRT */
2195 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2196 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2197 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2198 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2199 NUM_BANKS(ADDR_SURF_16_BANK) |
2200 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2201 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2202 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2203 break;
2204 case 24: /* 64 bpp PRT */
2205 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2206 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2207 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2208 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2209 NUM_BANKS(ADDR_SURF_16_BANK) |
2210 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2211 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2212 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2213 break;
2214 case 25: /* 128 bpp PRT */
2215 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2216 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2217 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2218 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2219 NUM_BANKS(ADDR_SURF_8_BANK) |
2220 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2221 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2222 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2223 break;
2224 default:
2225 gb_tile_moden = 0;
2226 break;
2227 }
64d7b8be 2228 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2229 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2230 }
d0ae7fcc 2231 } else if ((rdev->family == CHIP_VERDE) ||
8b02859d
AD		 2232 (rdev->family == CHIP_OLAND) ||
2233 (rdev->family == CHIP_HAINAN)) {
0a96d72b
AD		 2234 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2235 switch (reg_offset) {
2236 case 0: /* non-AA compressed depth or any compressed stencil */
2237 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2238 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2239 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2240 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2241 NUM_BANKS(ADDR_SURF_16_BANK) |
2242 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2243 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2244 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2245 break;
2246 case 1: /* 2xAA/4xAA compressed depth only */
2247 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2248 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2249 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2250 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2251 NUM_BANKS(ADDR_SURF_16_BANK) |
2252 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2253 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2254 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2255 break;
2256 case 2: /* 8xAA compressed depth only */
2257 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2258 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2259 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2260 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2261 NUM_BANKS(ADDR_SURF_16_BANK) |
2262 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2263 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2264 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2265 break;
2266 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2267 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2268 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2269 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2270 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2271 NUM_BANKS(ADDR_SURF_16_BANK) |
2272 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2273 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2274 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2275 break;
2276 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2277 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2278 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2279 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2280 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2281 NUM_BANKS(ADDR_SURF_16_BANK) |
2282 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2283 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2284 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2285 break;
2286 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2287 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2288 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2289 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2290 TILE_SPLIT(split_equal_to_row_size) |
2291 NUM_BANKS(ADDR_SURF_16_BANK) |
2292 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2293 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2294 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2295 break;
2296 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2297 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2298 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2299 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2300 TILE_SPLIT(split_equal_to_row_size) |
2301 NUM_BANKS(ADDR_SURF_16_BANK) |
2302 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2303 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2304 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2305 break;
2306 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2307 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2308 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2309 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2310 TILE_SPLIT(split_equal_to_row_size) |
2311 NUM_BANKS(ADDR_SURF_16_BANK) |
2312 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2313 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2314 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2315 break;
2316 case 8: /* 1D and 1D Array Surfaces */
2317 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2318 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2319 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2320 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2321 NUM_BANKS(ADDR_SURF_16_BANK) |
2322 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2323 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2324 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2325 break;
2326 case 9: /* Displayable maps. */
2327 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2328 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2329 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2330 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2331 NUM_BANKS(ADDR_SURF_16_BANK) |
2332 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2333 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2334 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2335 break;
2336 case 10: /* Display 8bpp. */
2337 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2338 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2339 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2340 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2341 NUM_BANKS(ADDR_SURF_16_BANK) |
2342 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2343 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2344 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2345 break;
2346 case 11: /* Display 16bpp. */
2347 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2348 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2349 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2350 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2351 NUM_BANKS(ADDR_SURF_16_BANK) |
2352 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2353 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2354 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2355 break;
2356 case 12: /* Display 32bpp. */
2357 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2358 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2359 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2360 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2361 NUM_BANKS(ADDR_SURF_16_BANK) |
2362 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2363 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2364 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2365 break;
2366 case 13: /* Thin. */
2367 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2368 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2369 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2370 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2371 NUM_BANKS(ADDR_SURF_16_BANK) |
2372 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2373 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2374 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2375 break;
2376 case 14: /* Thin 8 bpp. */
2377 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2378 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2379 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2380 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2381 NUM_BANKS(ADDR_SURF_16_BANK) |
2382 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2383 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2384 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2385 break;
2386 case 15: /* Thin 16 bpp. */
2387 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2388 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2389 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2390 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2391 NUM_BANKS(ADDR_SURF_16_BANK) |
2392 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2393 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2394 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2395 break;
2396 case 16: /* Thin 32 bpp. */
2397 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2398 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2399 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2400 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2401 NUM_BANKS(ADDR_SURF_16_BANK) |
2402 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2403 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2404 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2405 break;
2406 case 17: /* Thin 64 bpp. */
2407 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2408 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2409 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2410 TILE_SPLIT(split_equal_to_row_size) |
2411 NUM_BANKS(ADDR_SURF_16_BANK) |
2412 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2413 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2414 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2415 break;
2416 case 21: /* 8 bpp PRT. */
2417 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2418 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2419 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2420 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2421 NUM_BANKS(ADDR_SURF_16_BANK) |
2422 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2423 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2424 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2425 break;
2426 case 22: /* 16 bpp PRT */
2427 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2428 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2429 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2430 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2431 NUM_BANKS(ADDR_SURF_16_BANK) |
2432 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2433 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2434 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2435 break;
2436 case 23: /* 32 bpp PRT */
2437 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2438 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2439 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2440 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2441 NUM_BANKS(ADDR_SURF_16_BANK) |
2442 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2443 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2444 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2445 break;
2446 case 24: /* 64 bpp PRT */
2447 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2448 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2449 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2450 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2451 NUM_BANKS(ADDR_SURF_16_BANK) |
2452 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2453 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2454 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2455 break;
2456 case 25: /* 128 bpp PRT */
2457 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2458 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2459 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2460 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2461 NUM_BANKS(ADDR_SURF_8_BANK) |
2462 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2463 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2464 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2465 break;
2466 default:
2467 gb_tile_moden = 0;
2468 break;
2469 }
64d7b8be 2470 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2471 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2472 }
2473 } else
2474 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2475}
2476
1a8ca750
AD		 2477static void si_select_se_sh(struct radeon_device *rdev,
2478 u32 se_num, u32 sh_num)
2479{
2480 u32 data = INSTANCE_BROADCAST_WRITES;
2481
2482 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2483 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2484 else if (se_num == 0xffffffff)
2485 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2486 else if (sh_num == 0xffffffff)
2487 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2488 else
2489 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2490 WREG32(GRBM_GFX_INDEX, data);
2491}
2492
2493static u32 si_create_bitmask(u32 bit_width)
2494{
2495 u32 i, mask = 0;
2496
2497 for (i = 0; i < bit_width; i++) {
2498 mask <<= 1;
2499 mask |= 1;
2500 }
2501 return mask;
2502}
2503
2504static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2505{
2506 u32 data, mask;
2507
2508 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2509 if (data & 1)
2510 data &= INACTIVE_CUS_MASK;
2511 else
2512 data = 0;
2513 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2514
2515 data >>= INACTIVE_CUS_SHIFT;
2516
2517 mask = si_create_bitmask(cu_per_sh);
2518
2519 return ~data & mask;
2520}
2521
2522static void si_setup_spi(struct radeon_device *rdev,
2523 u32 se_num, u32 sh_per_se,
2524 u32 cu_per_sh)
2525{
2526 int i, j, k;
2527 u32 data, mask, active_cu;
2528
2529 for (i = 0; i < se_num; i++) {
2530 for (j = 0; j < sh_per_se; j++) {
2531 si_select_se_sh(rdev, i, j);
2532 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2533 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2534
2535 mask = 1;
2536 for (k = 0; k < 16; k++) {
2537 mask <<= k;
2538 if (active_cu & mask) {
2539 data &= ~mask;
2540 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2541 break;
2542 }
2543 }
2544 }
2545 }
2546 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2547}
2548
2549static u32 si_get_rb_disabled(struct radeon_device *rdev,
2550 u32 max_rb_num, u32 se_num,
2551 u32 sh_per_se)
2552{
2553 u32 data, mask;
2554
2555 data = RREG32(CC_RB_BACKEND_DISABLE);
2556 if (data & 1)
2557 data &= BACKEND_DISABLE_MASK;
2558 else
2559 data = 0;
2560 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2561
2562 data >>= BACKEND_DISABLE_SHIFT;
2563
2564 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2565
2566 return data & mask;
2567}
2568
2569static void si_setup_rb(struct radeon_device *rdev,
2570 u32 se_num, u32 sh_per_se,
2571 u32 max_rb_num)
2572{
2573 int i, j;
2574 u32 data, mask;
2575 u32 disabled_rbs = 0;
2576 u32 enabled_rbs = 0;
2577
2578 for (i = 0; i < se_num; i++) {
2579 for (j = 0; j < sh_per_se; j++) {
2580 si_select_se_sh(rdev, i, j);
2581 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2582 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2583 }
2584 }
2585 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2586
2587 mask = 1;
2588 for (i = 0; i < max_rb_num; i++) {
2589 if (!(disabled_rbs & mask))
2590 enabled_rbs |= mask;
2591 mask <<= 1;
2592 }
2593
2594 for (i = 0; i < se_num; i++) {
2595 si_select_se_sh(rdev, i, 0xffffffff);
2596 data = 0;
2597 for (j = 0; j < sh_per_se; j++) {
2598 switch (enabled_rbs & 3) {
2599 case 1:
2600 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2601 break;
2602 case 2:
2603 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2604 break;
2605 case 3:
2606 default:
2607 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2608 break;
2609 }
2610 enabled_rbs >>= 2;
2611 }
2612 WREG32(PA_SC_RASTER_CONFIG, data);
2613 }
2614 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2615}
2616
0a96d72b
AD		 2617static void si_gpu_init(struct radeon_device *rdev)
2618{
0a96d72b
AD		 2619 u32 gb_addr_config = 0;
2620 u32 mc_shared_chmap, mc_arb_ramcfg;
0a96d72b 2621 u32 sx_debug_1;
0a96d72b
AD		 2622 u32 hdp_host_path_cntl;
2623 u32 tmp;
2624 int i, j;
2625
2626 switch (rdev->family) {
2627 case CHIP_TAHITI:
2628 rdev->config.si.max_shader_engines = 2;
0a96d72b 2629 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 2630 rdev->config.si.max_cu_per_sh = 8;
2631 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2632 rdev->config.si.max_backends_per_se = 4;
2633 rdev->config.si.max_texture_channel_caches = 12;
2634 rdev->config.si.max_gprs = 256;
2635 rdev->config.si.max_gs_threads = 32;
2636 rdev->config.si.max_hw_contexts = 8;
2637
2638 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2639 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2640 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2641 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2642 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2643 break;
2644 case CHIP_PITCAIRN:
2645 rdev->config.si.max_shader_engines = 2;
0a96d72b 2646 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 2647 rdev->config.si.max_cu_per_sh = 5;
2648 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2649 rdev->config.si.max_backends_per_se = 4;
2650 rdev->config.si.max_texture_channel_caches = 8;
2651 rdev->config.si.max_gprs = 256;
2652 rdev->config.si.max_gs_threads = 32;
2653 rdev->config.si.max_hw_contexts = 8;
2654
2655 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2656 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2657 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2658 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2659 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2660 break;
2661 case CHIP_VERDE:
2662 default:
2663 rdev->config.si.max_shader_engines = 1;
0a96d72b 2664 rdev->config.si.max_tile_pipes = 4;
468ef1a5 2665 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 2666 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2667 rdev->config.si.max_backends_per_se = 4;
2668 rdev->config.si.max_texture_channel_caches = 4;
2669 rdev->config.si.max_gprs = 256;
2670 rdev->config.si.max_gs_threads = 32;
2671 rdev->config.si.max_hw_contexts = 8;
2672
d0ae7fcc
AD		 2673 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2674 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2675 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2676 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2677 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2678 break;
2679 case CHIP_OLAND:
2680 rdev->config.si.max_shader_engines = 1;
2681 rdev->config.si.max_tile_pipes = 4;
2682 rdev->config.si.max_cu_per_sh = 6;
2683 rdev->config.si.max_sh_per_se = 1;
2684 rdev->config.si.max_backends_per_se = 2;
2685 rdev->config.si.max_texture_channel_caches = 4;
2686 rdev->config.si.max_gprs = 256;
2687 rdev->config.si.max_gs_threads = 16;
2688 rdev->config.si.max_hw_contexts = 8;
2689
0a96d72b
AD		 2690 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2691 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2692 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2693 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2694 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 2695 break;
8b02859d
AD		 2696 case CHIP_HAINAN:
2697 rdev->config.si.max_shader_engines = 1;
2698 rdev->config.si.max_tile_pipes = 4;
2699 rdev->config.si.max_cu_per_sh = 5;
2700 rdev->config.si.max_sh_per_se = 1;
2701 rdev->config.si.max_backends_per_se = 1;
2702 rdev->config.si.max_texture_channel_caches = 2;
2703 rdev->config.si.max_gprs = 256;
2704 rdev->config.si.max_gs_threads = 16;
2705 rdev->config.si.max_hw_contexts = 8;
2706
2707 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2708 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2709 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2710 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2711 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2712 break;
0a96d72b
AD		 2713 }
2714
2715 /* Initialize HDP */
2716 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2717 WREG32((0x2c14 + j), 0x00000000);
2718 WREG32((0x2c18 + j), 0x00000000);
2719 WREG32((0x2c1c + j), 0x00000000);
2720 WREG32((0x2c20 + j), 0x00000000);
2721 WREG32((0x2c24 + j), 0x00000000);
2722 }
2723
2724 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2725
2726 evergreen_fix_pci_max_read_req_size(rdev);
2727
2728 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2729
2730 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2731 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2732
0a96d72b 2733 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 2734 rdev->config.si.mem_max_burst_length_bytes = 256;
2735 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2736 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2737 if (rdev->config.si.mem_row_size_in_kb > 4)
2738 rdev->config.si.mem_row_size_in_kb = 4;
2739 /* XXX use MC settings? */
2740 rdev->config.si.shader_engine_tile_size = 32;
2741 rdev->config.si.num_gpus = 1;
2742 rdev->config.si.multi_gpu_tile_size = 64;
2743
1a8ca750
AD		 2744 /* fix up row size */
2745 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 2746 switch (rdev->config.si.mem_row_size_in_kb) {
2747 case 1:
2748 default:
2749 gb_addr_config |= ROW_SIZE(0);
2750 break;
2751 case 2:
2752 gb_addr_config |= ROW_SIZE(1);
2753 break;
2754 case 4:
2755 gb_addr_config |= ROW_SIZE(2);
2756 break;
2757 }
2758
0a96d72b
AD		 2759 /* setup tiling info dword. gb_addr_config is not adequate since it does
2760 * not have bank info, so create a custom tiling dword.
2761 * bits 3:0 num_pipes
2762 * bits 7:4 num_banks
2763 * bits 11:8 group_size
2764 * bits 15:12 row_size
2765 */
2766 rdev->config.si.tile_config = 0;
2767 switch (rdev->config.si.num_tile_pipes) {
2768 case 1:
2769 rdev->config.si.tile_config |= (0 << 0);
2770 break;
2771 case 2:
2772 rdev->config.si.tile_config |= (1 << 0);
2773 break;
2774 case 4:
2775 rdev->config.si.tile_config |= (2 << 0);
2776 break;
2777 case 8:
2778 default:
2779 /* XXX what about 12? */
2780 rdev->config.si.tile_config |= (3 << 0);
2781 break;
dca571a6
CK		 2782 }
2783 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
2784 case 0: /* four banks */
1a8ca750 2785 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 2786 break;
2787 case 1: /* eight banks */
2788 rdev->config.si.tile_config |= 1 << 4;
2789 break;
2790 case 2: /* sixteen banks */
2791 default:
2792 rdev->config.si.tile_config |= 2 << 4;
2793 break;
2794 }
0a96d72b
AD		 2795 rdev->config.si.tile_config |=
2796 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
2797 rdev->config.si.tile_config |=
2798 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
2799
0a96d72b
AD		 2800 WREG32(GB_ADDR_CONFIG, gb_addr_config);
2801 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 2802 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 2803 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 2804 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
2805 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 2806 if (rdev->has_uvd) {
2807 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2808 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2809 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
2810 }
0a96d72b 2811
1a8ca750 2812 si_tiling_mode_table_init(rdev);
0a96d72b 2813
1a8ca750
AD		 2814 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
2815 rdev->config.si.max_sh_per_se,
2816 rdev->config.si.max_backends_per_se);
0a96d72b 2817
1a8ca750
AD		 2818 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
2819 rdev->config.si.max_sh_per_se,
2820 rdev->config.si.max_cu_per_sh);
0a96d72b 2821
0a96d72b
AD		 2822
2823 /* set HW defaults for 3D engine */
2824 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2825 ROQ_IB2_START(0x2b)));
2826 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
2827
2828 sx_debug_1 = RREG32(SX_DEBUG_1);
2829 WREG32(SX_DEBUG_1, sx_debug_1);
2830
2831 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2832
2833 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
2834 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
2835 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
2836 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
2837
2838 WREG32(VGT_NUM_INSTANCES, 1);
2839
2840 WREG32(CP_PERFMON_CNTL, 0);
2841
2842 WREG32(SQ_CONFIG, 0);
2843
2844 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2845 FORCE_EOV_MAX_REZ_CNT(255)));
2846
2847 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2848 AUTO_INVLD_EN(ES_AND_GS_AUTO));
2849
2850 WREG32(VGT_GS_VERTEX_REUSE, 16);
2851 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2852
2853 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
2854 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
2855 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
2856 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
2857 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
2858 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
2859 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
2860 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
2861
2862 tmp = RREG32(HDP_MISC_CNTL);
2863 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2864 WREG32(HDP_MISC_CNTL, tmp);
2865
2866 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2867 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2868
2869 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2870
2871 udelay(50);
2872}
c476dde2 2873
2ece2e8b
AD		 2874/*
2875 * GPU scratch registers helpers function.
2876 */
2877static void si_scratch_init(struct radeon_device *rdev)
2878{
2879 int i;
2880
2881 rdev->scratch.num_reg = 7;
2882 rdev->scratch.reg_base = SCRATCH_REG0;
2883 for (i = 0; i < rdev->scratch.num_reg; i++) {
2884 rdev->scratch.free[i] = true;
2885 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2886 }
2887}
2888
2889void si_fence_ring_emit(struct radeon_device *rdev,
2890 struct radeon_fence *fence)
2891{
2892 struct radeon_ring *ring = &rdev->ring[fence->ring];
2893 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2894
2895 /* flush read cache over gart */
2896 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2897 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
2898 radeon_ring_write(ring, 0);
2899 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2900 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
2901 PACKET3_TC_ACTION_ENA |
2902 PACKET3_SH_KCACHE_ACTION_ENA |
2903 PACKET3_SH_ICACHE_ACTION_ENA);
2904 radeon_ring_write(ring, 0xFFFFFFFF);
2905 radeon_ring_write(ring, 0);
2906 radeon_ring_write(ring, 10); /* poll interval */
2907 /* EVENT_WRITE_EOP - flush caches, send int */
2908 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2909 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
2910 radeon_ring_write(ring, addr & 0xffffffff);
2911 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2912 radeon_ring_write(ring, fence->seq);
2913 radeon_ring_write(ring, 0);
2914}
2915
2916/*
2917 * IB stuff
2918 */
2919void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2920{
876dc9f3 2921 struct radeon_ring *ring = &rdev->ring[ib->ring];
2ece2e8b
AD		 2922 u32 header;
2923
a85a7da4
AD		 2924 if (ib->is_const_ib) {
2925 /* set switch buffer packet before const IB */
2926 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2927 radeon_ring_write(ring, 0);
45df6803 2928
2ece2e8b 2929 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 2930 } else {
89d35807 2931 u32 next_rptr;
a85a7da4 2932 if (ring->rptr_save_reg) {
89d35807 2933 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 2934 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2935 radeon_ring_write(ring, ((ring->rptr_save_reg -
2936 PACKET3_SET_CONFIG_REG_START) >> 2));
2937 radeon_ring_write(ring, next_rptr);
89d35807
AD		 2938 } else if (rdev->wb.enabled) {
2939 next_rptr = ring->wptr + 5 + 4 + 8;
2940 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2941 radeon_ring_write(ring, (1 << 8));
2942 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2943 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
2944 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 2945 }
2946
2ece2e8b 2947 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 2948 }
2ece2e8b
AD		 2949
2950 radeon_ring_write(ring, header);
2951 radeon_ring_write(ring,
2952#ifdef __BIG_ENDIAN
2953 (2 << 0) |
2954#endif
2955 (ib->gpu_addr & 0xFFFFFFFC));
2956 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4bf3dd92
CK		 2957 radeon_ring_write(ring, ib->length_dw |
2958 (ib->vm ? (ib->vm->id << 24) : 0));
2ece2e8b 2959
a85a7da4
AD		 2960 if (!ib->is_const_ib) {
2961 /* flush read cache over gart for this vmid */
2962 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2963 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 2964 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
a85a7da4
AD		 2965 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2966 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
2967 PACKET3_TC_ACTION_ENA |
2968 PACKET3_SH_KCACHE_ACTION_ENA |
2969 PACKET3_SH_ICACHE_ACTION_ENA);
2970 radeon_ring_write(ring, 0xFFFFFFFF);
2971 radeon_ring_write(ring, 0);
2972 radeon_ring_write(ring, 10); /* poll interval */
2973 }
2ece2e8b
AD		 2974}
2975
48c0c902
AD		 2976/*
2977 * CP.
2978 */
2979static void si_cp_enable(struct radeon_device *rdev, bool enable)
2980{
2981 if (enable)
2982 WREG32(CP_ME_CNTL, 0);
2983 else {
2984 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2985 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
2986 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 2987 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2988 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
2989 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 2990 }
2991 udelay(50);
2992}
2993
2994static int si_cp_load_microcode(struct radeon_device *rdev)
2995{
2996 const __be32 *fw_data;
2997 int i;
2998
2999 if (!rdev->me_fw || !rdev->pfp_fw)
3000 return -EINVAL;
3001
3002 si_cp_enable(rdev, false);
3003
3004 /* PFP */
3005 fw_data = (const __be32 *)rdev->pfp_fw->data;
3006 WREG32(CP_PFP_UCODE_ADDR, 0);
3007 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3008 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3009 WREG32(CP_PFP_UCODE_ADDR, 0);
3010
3011 /* CE */
3012 fw_data = (const __be32 *)rdev->ce_fw->data;
3013 WREG32(CP_CE_UCODE_ADDR, 0);
3014 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3015 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3016 WREG32(CP_CE_UCODE_ADDR, 0);
3017
3018 /* ME */
3019 fw_data = (const __be32 *)rdev->me_fw->data;
3020 WREG32(CP_ME_RAM_WADDR, 0);
3021 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3022 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3023 WREG32(CP_ME_RAM_WADDR, 0);
3024
3025 WREG32(CP_PFP_UCODE_ADDR, 0);
3026 WREG32(CP_CE_UCODE_ADDR, 0);
3027 WREG32(CP_ME_RAM_WADDR, 0);
3028 WREG32(CP_ME_RAM_RADDR, 0);
3029 return 0;
3030}
3031
3032static int si_cp_start(struct radeon_device *rdev)
3033{
3034 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3035 int r, i;
3036
3037 r = radeon_ring_lock(rdev, ring, 7 + 4);
3038 if (r) {
3039 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3040 return r;
3041 }
3042 /* init the CP */
3043 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3044 radeon_ring_write(ring, 0x1);
3045 radeon_ring_write(ring, 0x0);
3046 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3047 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3048 radeon_ring_write(ring, 0);
3049 radeon_ring_write(ring, 0);
3050
3051 /* init the CE partitions */
3052 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3053 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3054 radeon_ring_write(ring, 0xc000);
3055 radeon_ring_write(ring, 0xe000);
3056 radeon_ring_unlock_commit(rdev, ring);
3057
3058 si_cp_enable(rdev, true);
3059
3060 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3061 if (r) {
3062 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3063 return r;
3064 }
3065
3066 /* setup clear context state */
3067 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3068 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3069
3070 for (i = 0; i < si_default_size; i++)
3071 radeon_ring_write(ring, si_default_state[i]);
3072
3073 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3074 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3075
3076 /* set clear context state */
3077 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3078 radeon_ring_write(ring, 0);
3079
3080 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3081 radeon_ring_write(ring, 0x00000316);
3082 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3083 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3084
3085 radeon_ring_unlock_commit(rdev, ring);
3086
3087 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3088 ring = &rdev->ring[i];
3089 r = radeon_ring_lock(rdev, ring, 2);
3090
3091 /* clear the compute context state */
3092 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3093 radeon_ring_write(ring, 0);
3094
3095 radeon_ring_unlock_commit(rdev, ring);
3096 }
3097
3098 return 0;
3099}
3100
3101static void si_cp_fini(struct radeon_device *rdev)
3102{
45df6803 3103 struct radeon_ring *ring;
48c0c902 3104 si_cp_enable(rdev, false);
45df6803
CK		 3105
3106 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3107 radeon_ring_fini(rdev, ring);
3108 radeon_scratch_free(rdev, ring->rptr_save_reg);
3109
3110 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3111 radeon_ring_fini(rdev, ring);
3112 radeon_scratch_free(rdev, ring->rptr_save_reg);
3113
3114 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3115 radeon_ring_fini(rdev, ring);
3116 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3117}
3118
3119static int si_cp_resume(struct radeon_device *rdev)
3120{
3121 struct radeon_ring *ring;
3122 u32 tmp;
3123 u32 rb_bufsz;
3124 int r;
3125
3126 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3127 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3128 SOFT_RESET_PA |
3129 SOFT_RESET_VGT |
3130 SOFT_RESET_SPI |
3131 SOFT_RESET_SX));
3132 RREG32(GRBM_SOFT_RESET);
3133 mdelay(15);
3134 WREG32(GRBM_SOFT_RESET, 0);
3135 RREG32(GRBM_SOFT_RESET);
3136
3137 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3138 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3139
3140 /* Set the write pointer delay */
3141 WREG32(CP_RB_WPTR_DELAY, 0);
3142
3143 WREG32(CP_DEBUG, 0);
3144 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3145
3146 /* ring 0 - compute and gfx */
3147 /* Set ring buffer size */
3148 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3149 rb_bufsz = drm_order(ring->ring_size / 8);
3150 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3151#ifdef __BIG_ENDIAN
3152 tmp |= BUF_SWAP_32BIT;
3153#endif
3154 WREG32(CP_RB0_CNTL, tmp);
3155
3156 /* Initialize the ring buffer's read and write pointers */
3157 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3158 ring->wptr = 0;
3159 WREG32(CP_RB0_WPTR, ring->wptr);
3160
48fc7f7e 3161 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3162 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3163 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3164
3165 if (rdev->wb.enabled)
3166 WREG32(SCRATCH_UMSK, 0xff);
3167 else {
3168 tmp |= RB_NO_UPDATE;
3169 WREG32(SCRATCH_UMSK, 0);
3170 }
3171
3172 mdelay(1);
3173 WREG32(CP_RB0_CNTL, tmp);
3174
3175 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3176
3177 ring->rptr = RREG32(CP_RB0_RPTR);
3178
3179 /* ring1 - compute only */
3180 /* Set ring buffer size */
3181 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3182 rb_bufsz = drm_order(ring->ring_size / 8);
3183 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3184#ifdef __BIG_ENDIAN
3185 tmp |= BUF_SWAP_32BIT;
3186#endif
3187 WREG32(CP_RB1_CNTL, tmp);
3188
3189 /* Initialize the ring buffer's read and write pointers */
3190 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3191 ring->wptr = 0;
3192 WREG32(CP_RB1_WPTR, ring->wptr);
3193
48fc7f7e 3194 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3195 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3196 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3197
3198 mdelay(1);
3199 WREG32(CP_RB1_CNTL, tmp);
3200
3201 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3202
3203 ring->rptr = RREG32(CP_RB1_RPTR);
3204
3205 /* ring2 - compute only */
3206 /* Set ring buffer size */
3207 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3208 rb_bufsz = drm_order(ring->ring_size / 8);
3209 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3210#ifdef __BIG_ENDIAN
3211 tmp |= BUF_SWAP_32BIT;
3212#endif
3213 WREG32(CP_RB2_CNTL, tmp);
3214
3215 /* Initialize the ring buffer's read and write pointers */
3216 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3217 ring->wptr = 0;
3218 WREG32(CP_RB2_WPTR, ring->wptr);
3219
48fc7f7e 3220 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3221 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3222 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3223
3224 mdelay(1);
3225 WREG32(CP_RB2_CNTL, tmp);
3226
3227 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3228
3229 ring->rptr = RREG32(CP_RB2_RPTR);
3230
3231 /* start the rings */
3232 si_cp_start(rdev);
3233 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3234 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3235 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3236 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3237 if (r) {
3238 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3239 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3240 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3241 return r;
3242 }
3243 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3244 if (r) {
3245 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3246 }
3247 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3248 if (r) {
3249 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3250 }
3251
3252 return 0;
3253}
3254
014bb209 3255static u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3256{
014bb209 3257 u32 reset_mask = 0;
1c534671 3258 u32 tmp;
06bc6df0 3259
014bb209
AD		 3260 /* GRBM_STATUS */
3261 tmp = RREG32(GRBM_STATUS);
3262 if (tmp & (PA_BUSY | SC_BUSY |
3263 BCI_BUSY | SX_BUSY |
3264 TA_BUSY | VGT_BUSY |
3265 DB_BUSY | CB_BUSY |
3266 GDS_BUSY | SPI_BUSY |
3267 IA_BUSY | IA_BUSY_NO_DMA))
3268 reset_mask |= RADEON_RESET_GFX;
3269
3270 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3271 CP_BUSY | CP_COHERENCY_BUSY))
3272 reset_mask |= RADEON_RESET_CP;
3273
3274 if (tmp & GRBM_EE_BUSY)
3275 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3276
3277 /* GRBM_STATUS2 */
3278 tmp = RREG32(GRBM_STATUS2);
3279 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3280 reset_mask |= RADEON_RESET_RLC;
3281
3282 /* DMA_STATUS_REG 0 */
3283 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3284 if (!(tmp & DMA_IDLE))
3285 reset_mask |= RADEON_RESET_DMA;
3286
3287 /* DMA_STATUS_REG 1 */
3288 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3289 if (!(tmp & DMA_IDLE))
3290 reset_mask |= RADEON_RESET_DMA1;
3291
3292 /* SRBM_STATUS2 */
3293 tmp = RREG32(SRBM_STATUS2);
3294 if (tmp & DMA_BUSY)
3295 reset_mask |= RADEON_RESET_DMA;
3296
3297 if (tmp & DMA1_BUSY)
3298 reset_mask |= RADEON_RESET_DMA1;
3299
3300 /* SRBM_STATUS */
3301 tmp = RREG32(SRBM_STATUS);
3302
3303 if (tmp & IH_BUSY)
3304 reset_mask |= RADEON_RESET_IH;
3305
3306 if (tmp & SEM_BUSY)
3307 reset_mask |= RADEON_RESET_SEM;
3308
3309 if (tmp & GRBM_RQ_PENDING)
3310 reset_mask |= RADEON_RESET_GRBM;
3311
3312 if (tmp & VMC_BUSY)
3313 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3314
014bb209
AD		 3315 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3316 MCC_BUSY | MCD_BUSY))
3317 reset_mask |= RADEON_RESET_MC;
3318
3319 if (evergreen_is_display_hung(rdev))
3320 reset_mask |= RADEON_RESET_DISPLAY;
3321
3322 /* VM_L2_STATUS */
3323 tmp = RREG32(VM_L2_STATUS);
3324 if (tmp & L2_BUSY)
3325 reset_mask |= RADEON_RESET_VMC;
3326
d808fc88
AD		 3327 /* Skip MC reset as it's mostly likely not hung, just busy */
3328 if (reset_mask & RADEON_RESET_MC) {
3329 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3330 reset_mask &= ~RADEON_RESET_MC;
3331 }
3332
014bb209
AD		 3333 return reset_mask;
3334}
3335
3336static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3337{
3338 struct evergreen_mc_save save;
3339 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3340 u32 tmp;
19fc42ed 3341
06bc6df0 3342 if (reset_mask == 0)
014bb209 3343 return;
06bc6df0
AD		 3344
3345 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3346
1c534671 3347 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3348 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3349 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3350 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3351 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3352
1c534671
AD		 3353 /* Disable CP parsing/prefetching */
3354 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3355
3356 if (reset_mask & RADEON_RESET_DMA) {
3357 /* dma0 */
3358 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3359 tmp &= ~DMA_RB_ENABLE;
3360 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3361 }
3362 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3363 /* dma1 */
3364 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3365 tmp &= ~DMA_RB_ENABLE;
3366 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3367 }
3368
f770d78a
AD		 3369 udelay(50);
3370
3371 evergreen_mc_stop(rdev, &save);
3372 if (evergreen_mc_wait_for_idle(rdev)) {
3373 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3374 }
3375
1c534671
AD		 3376 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3377 grbm_soft_reset = SOFT_RESET_CB |
3378 SOFT_RESET_DB |
3379 SOFT_RESET_GDS |
3380 SOFT_RESET_PA |
3381 SOFT_RESET_SC |
3382 SOFT_RESET_BCI |
3383 SOFT_RESET_SPI |
3384 SOFT_RESET_SX |
3385 SOFT_RESET_TC |
3386 SOFT_RESET_TA |
3387 SOFT_RESET_VGT |
3388 SOFT_RESET_IA;
3389 }
3390
3391 if (reset_mask & RADEON_RESET_CP) {
3392 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3393
3394 srbm_soft_reset |= SOFT_RESET_GRBM;
3395 }
06bc6df0
AD		 3396
3397 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3398 srbm_soft_reset |= SOFT_RESET_DMA;
3399
3400 if (reset_mask & RADEON_RESET_DMA1)
3401 srbm_soft_reset |= SOFT_RESET_DMA1;
3402
3403 if (reset_mask & RADEON_RESET_DISPLAY)
3404 srbm_soft_reset |= SOFT_RESET_DC;
3405
3406 if (reset_mask & RADEON_RESET_RLC)
3407 grbm_soft_reset |= SOFT_RESET_RLC;
3408
3409 if (reset_mask & RADEON_RESET_SEM)
3410 srbm_soft_reset |= SOFT_RESET_SEM;
3411
3412 if (reset_mask & RADEON_RESET_IH)
3413 srbm_soft_reset |= SOFT_RESET_IH;
3414
3415 if (reset_mask & RADEON_RESET_GRBM)
3416 srbm_soft_reset |= SOFT_RESET_GRBM;
3417
3418 if (reset_mask & RADEON_RESET_VMC)
3419 srbm_soft_reset |= SOFT_RESET_VMC;
3420
3421 if (reset_mask & RADEON_RESET_MC)
3422 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3423
3424 if (grbm_soft_reset) {
3425 tmp = RREG32(GRBM_SOFT_RESET);
3426 tmp |= grbm_soft_reset;
3427 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3428 WREG32(GRBM_SOFT_RESET, tmp);
3429 tmp = RREG32(GRBM_SOFT_RESET);
3430
3431 udelay(50);
3432
3433 tmp &= ~grbm_soft_reset;
3434 WREG32(GRBM_SOFT_RESET, tmp);
3435 tmp = RREG32(GRBM_SOFT_RESET);
3436 }
3437
3438 if (srbm_soft_reset) {
3439 tmp = RREG32(SRBM_SOFT_RESET);
3440 tmp |= srbm_soft_reset;
3441 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3442 WREG32(SRBM_SOFT_RESET, tmp);
3443 tmp = RREG32(SRBM_SOFT_RESET);
3444
3445 udelay(50);
3446
3447 tmp &= ~srbm_soft_reset;
3448 WREG32(SRBM_SOFT_RESET, tmp);
3449 tmp = RREG32(SRBM_SOFT_RESET);
3450 }
06bc6df0
AD		 3451
3452 /* Wait a little for things to settle down */
3453 udelay(50);
3454
c476dde2 3455 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3456 udelay(50);
3457
1c534671 3458 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3459}
3460
3461int si_asic_reset(struct radeon_device *rdev)
3462{
014bb209
AD		 3463 u32 reset_mask;
3464
3465 reset_mask = si_gpu_check_soft_reset(rdev);
3466
3467 if (reset_mask)
3468 r600_set_bios_scratch_engine_hung(rdev, true);
3469
3470 si_gpu_soft_reset(rdev, reset_mask);
3471
3472 reset_mask = si_gpu_check_soft_reset(rdev);
3473
3474 if (!reset_mask)
3475 r600_set_bios_scratch_engine_hung(rdev, false);
3476
3477 return 0;
c476dde2
AD		 3478}
3479
123bc183
AD		 3480/**
3481 * si_gfx_is_lockup - Check if the GFX engine is locked up
3482 *
3483 * @rdev: radeon_device pointer
3484 * @ring: radeon_ring structure holding ring information
3485 *
3486 * Check if the GFX engine is locked up.
3487 * Returns true if the engine appears to be locked up, false if not.
3488 */
3489bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3490{
3491 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3492
3493 if (!(reset_mask & (RADEON_RESET_GFX |
3494 RADEON_RESET_COMPUTE |
3495 RADEON_RESET_CP))) {
3496 radeon_ring_lockup_update(ring);
3497 return false;
3498 }
3499 /* force CP activities */
3500 radeon_ring_force_activity(rdev, ring);
3501 return radeon_ring_test_lockup(rdev, ring);
3502}
3503
3504/**
3505 * si_dma_is_lockup - Check if the DMA engine is locked up
3506 *
3507 * @rdev: radeon_device pointer
3508 * @ring: radeon_ring structure holding ring information
3509 *
3510 * Check if the async DMA engine is locked up.
3511 * Returns true if the engine appears to be locked up, false if not.
3512 */
3513bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3514{
3515 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3516 u32 mask;
3517
3518 if (ring->idx == R600_RING_TYPE_DMA_INDEX)
3519 mask = RADEON_RESET_DMA;
3520 else
3521 mask = RADEON_RESET_DMA1;
3522
3523 if (!(reset_mask & mask)) {
3524 radeon_ring_lockup_update(ring);
3525 return false;
3526 }
3527 /* force ring activities */
3528 radeon_ring_force_activity(rdev, ring);
3529 return radeon_ring_test_lockup(rdev, ring);
3530}
3531
d2800ee5
AD		 3532/* MC */
3533static void si_mc_program(struct radeon_device *rdev)
3534{
3535 struct evergreen_mc_save save;
3536 u32 tmp;
3537 int i, j;
3538
3539 /* Initialize HDP */
3540 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3541 WREG32((0x2c14 + j), 0x00000000);
3542 WREG32((0x2c18 + j), 0x00000000);
3543 WREG32((0x2c1c + j), 0x00000000);
3544 WREG32((0x2c20 + j), 0x00000000);
3545 WREG32((0x2c24 + j), 0x00000000);
3546 }
3547 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3548
3549 evergreen_mc_stop(rdev, &save);
3550 if (radeon_mc_wait_for_idle(rdev)) {
3551 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3552 }
5153550a
AD		 3553 if (!ASIC_IS_NODCE(rdev))
3554 /* Lockout access through VGA aperture*/
3555 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 3556 /* Update configuration */
3557 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3558 rdev->mc.vram_start >> 12);
3559 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3560 rdev->mc.vram_end >> 12);
3561 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3562 rdev->vram_scratch.gpu_addr >> 12);
3563 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3564 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3565 WREG32(MC_VM_FB_LOCATION, tmp);
3566 /* XXX double check these! */
3567 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3568 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3569 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3570 WREG32(MC_VM_AGP_BASE, 0);
3571 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3572 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3573 if (radeon_mc_wait_for_idle(rdev)) {
3574 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3575 }
3576 evergreen_mc_resume(rdev, &save);
5153550a
AD		 3577 if (!ASIC_IS_NODCE(rdev)) {
3578 /* we need to own VRAM, so turn off the VGA renderer here
3579 * to stop it overwriting our objects */
3580 rv515_vga_render_disable(rdev);
3581 }
d2800ee5
AD		 3582}
3583
1c49165d
AD		 3584void si_vram_gtt_location(struct radeon_device *rdev,
3585 struct radeon_mc *mc)
d2800ee5
AD		 3586{
3587 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3588 /* leave room for at least 1024M GTT */
3589 dev_warn(rdev->dev, "limiting VRAM\n");
3590 mc->real_vram_size = 0xFFC0000000ULL;
3591 mc->mc_vram_size = 0xFFC0000000ULL;
3592 }
9ed8b1f9 3593 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 3594 rdev->mc.gtt_base_align = 0;
9ed8b1f9 3595 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 3596}
3597
3598static int si_mc_init(struct radeon_device *rdev)
3599{
3600 u32 tmp;
3601 int chansize, numchan;
3602
3603 /* Get VRAM informations */
3604 rdev->mc.vram_is_ddr = true;
3605 tmp = RREG32(MC_ARB_RAMCFG);
3606 if (tmp & CHANSIZE_OVERRIDE) {
3607 chansize = 16;
3608 } else if (tmp & CHANSIZE_MASK) {
3609 chansize = 64;
3610 } else {
3611 chansize = 32;
3612 }
3613 tmp = RREG32(MC_SHARED_CHMAP);
3614 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3615 case 0:
3616 default:
3617 numchan = 1;
3618 break;
3619 case 1:
3620 numchan = 2;
3621 break;
3622 case 2:
3623 numchan = 4;
3624 break;
3625 case 3:
3626 numchan = 8;
3627 break;
3628 case 4:
3629 numchan = 3;
3630 break;
3631 case 5:
3632 numchan = 6;
3633 break;
3634 case 6:
3635 numchan = 10;
3636 break;
3637 case 7:
3638 numchan = 12;
3639 break;
3640 case 8:
3641 numchan = 16;
3642 break;
3643 }
3644 rdev->mc.vram_width = numchan * chansize;
3645 /* Could aper size report 0 ? */
3646 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3647 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3648 /* size in MB on si */
fc986034
NOS		 3649 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3650 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
d2800ee5
AD		 3651 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3652 si_vram_gtt_location(rdev, &rdev->mc);
3653 radeon_update_bandwidth_info(rdev);
3654
3655 return 0;
3656}
3657
3658/*
3659 * GART
3660 */
3661void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3662{
3663 /* flush hdp cache */
3664 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3665
3666 /* bits 0-15 are the VM contexts0-15 */
3667 WREG32(VM_INVALIDATE_REQUEST, 1);
3668}
3669
1109ca09 3670static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 3671{
3672 int r, i;
3673
3674 if (rdev->gart.robj == NULL) {
3675 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3676 return -EINVAL;
3677 }
3678 r = radeon_gart_table_vram_pin(rdev);
3679 if (r)
3680 return r;
3681 radeon_gart_restore(rdev);
3682 /* Setup TLB control */
3683 WREG32(MC_VM_MX_L1_TLB_CNTL,
3684 (0xA << 7) |
3685 ENABLE_L1_TLB |
3686 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3687 ENABLE_ADVANCED_DRIVER_MODEL |
3688 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3689 /* Setup L2 cache */
3690 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3691 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3692 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3693 EFFECTIVE_L2_QUEUE_SIZE(7) |
3694 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3695 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3696 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3697 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3698 /* setup context0 */
3699 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3700 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3701 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3702 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3703 (u32)(rdev->dummy_page.addr >> 12));
3704 WREG32(VM_CONTEXT0_CNTL2, 0);
3705 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3706 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3707
3708 WREG32(0x15D4, 0);
3709 WREG32(0x15D8, 0);
3710 WREG32(0x15DC, 0);
3711
3712 /* empty context1-15 */
d2800ee5
AD		 3713 /* set vm size, must be a multiple of 4 */
3714 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
c21b328e 3715 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
23d4f1f2
AD		 3716 /* Assign the pt base to something valid for now; the pts used for
3717 * the VMs are determined by the application and setup and assigned
3718 * on the fly in the vm part of radeon_gart.c
3719 */
d2800ee5
AD		 3720 for (i = 1; i < 16; i++) {
3721 if (i < 8)
3722 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3723 rdev->gart.table_addr >> 12);
3724 else
3725 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3726 rdev->gart.table_addr >> 12);
3727 }
3728
3729 /* enable context1-15 */
3730 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3731 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 3732 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 3733 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
ae133a11
CK		 3734 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3735 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3736 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3737 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3738 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3739 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3740 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3741 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3742 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3743 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3744 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3745 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 3746
3747 si_pcie_gart_tlb_flush(rdev);
3748 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3749 (unsigned)(rdev->mc.gtt_size >> 20),
3750 (unsigned long long)rdev->gart.table_addr);
3751 rdev->gart.ready = true;
3752 return 0;
3753}
3754
1109ca09 3755static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5
AD		 3756{
3757 /* Disable all tables */
3758 WREG32(VM_CONTEXT0_CNTL, 0);
3759 WREG32(VM_CONTEXT1_CNTL, 0);
3760 /* Setup TLB control */
3761 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3762 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3763 /* Setup L2 cache */
3764 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3765 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3766 EFFECTIVE_L2_QUEUE_SIZE(7) |
3767 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3768 WREG32(VM_L2_CNTL2, 0);
3769 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3770 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3771 radeon_gart_table_vram_unpin(rdev);
3772}
3773
1109ca09 3774static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 3775{
3776 si_pcie_gart_disable(rdev);
3777 radeon_gart_table_vram_free(rdev);
3778 radeon_gart_fini(rdev);
3779}
3780
498dd8b3
AD		 3781/* vm parser */
3782static bool si_vm_reg_valid(u32 reg)
3783{
3784 /* context regs are fine */
3785 if (reg >= 0x28000)
3786 return true;
3787
3788 /* check config regs */
3789 switch (reg) {
3790 case GRBM_GFX_INDEX:
f418b88a 3791 case CP_STRMOUT_CNTL:
498dd8b3
AD		 3792 case VGT_VTX_VECT_EJECT_REG:
3793 case VGT_CACHE_INVALIDATION:
3794 case VGT_ESGS_RING_SIZE:
3795 case VGT_GSVS_RING_SIZE:
3796 case VGT_GS_VERTEX_REUSE:
3797 case VGT_PRIMITIVE_TYPE:
3798 case VGT_INDEX_TYPE:
3799 case VGT_NUM_INDICES:
3800 case VGT_NUM_INSTANCES:
3801 case VGT_TF_RING_SIZE:
3802 case VGT_HS_OFFCHIP_PARAM:
3803 case VGT_TF_MEMORY_BASE:
3804 case PA_CL_ENHANCE:
3805 case PA_SU_LINE_STIPPLE_VALUE:
3806 case PA_SC_LINE_STIPPLE_STATE:
3807 case PA_SC_ENHANCE:
3808 case SQC_CACHES:
3809 case SPI_STATIC_THREAD_MGMT_1:
3810 case SPI_STATIC_THREAD_MGMT_2:
3811 case SPI_STATIC_THREAD_MGMT_3:
3812 case SPI_PS_MAX_WAVE_ID:
3813 case SPI_CONFIG_CNTL:
3814 case SPI_CONFIG_CNTL_1:
3815 case TA_CNTL_AUX:
3816 return true;
3817 default:
3818 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
3819 return false;
3820 }
3821}
3822
3823static int si_vm_packet3_ce_check(struct radeon_device *rdev,
3824 u32 *ib, struct radeon_cs_packet *pkt)
3825{
3826 switch (pkt->opcode) {
3827 case PACKET3_NOP:
3828 case PACKET3_SET_BASE:
3829 case PACKET3_SET_CE_DE_COUNTERS:
3830 case PACKET3_LOAD_CONST_RAM:
3831 case PACKET3_WRITE_CONST_RAM:
3832 case PACKET3_WRITE_CONST_RAM_OFFSET:
3833 case PACKET3_DUMP_CONST_RAM:
3834 case PACKET3_INCREMENT_CE_COUNTER:
3835 case PACKET3_WAIT_ON_DE_COUNTER:
3836 case PACKET3_CE_WRITE:
3837 break;
3838 default:
3839 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
3840 return -EINVAL;
3841 }
3842 return 0;
3843}
3844
3845static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
3846 u32 *ib, struct radeon_cs_packet *pkt)
3847{
3848 u32 idx = pkt->idx + 1;
3849 u32 idx_value = ib[idx];
3850 u32 start_reg, end_reg, reg, i;
5aa709be 3851 u32 command, info;
498dd8b3
AD		 3852
3853 switch (pkt->opcode) {
3854 case PACKET3_NOP:
3855 case PACKET3_SET_BASE:
3856 case PACKET3_CLEAR_STATE:
3857 case PACKET3_INDEX_BUFFER_SIZE:
3858 case PACKET3_DISPATCH_DIRECT:
3859 case PACKET3_DISPATCH_INDIRECT:
3860 case PACKET3_ALLOC_GDS:
3861 case PACKET3_WRITE_GDS_RAM:
3862 case PACKET3_ATOMIC_GDS:
3863 case PACKET3_ATOMIC:
3864 case PACKET3_OCCLUSION_QUERY:
3865 case PACKET3_SET_PREDICATION:
3866 case PACKET3_COND_EXEC:
3867 case PACKET3_PRED_EXEC:
3868 case PACKET3_DRAW_INDIRECT:
3869 case PACKET3_DRAW_INDEX_INDIRECT:
3870 case PACKET3_INDEX_BASE:
3871 case PACKET3_DRAW_INDEX_2:
3872 case PACKET3_CONTEXT_CONTROL:
3873 case PACKET3_INDEX_TYPE:
3874 case PACKET3_DRAW_INDIRECT_MULTI:
3875 case PACKET3_DRAW_INDEX_AUTO:
3876 case PACKET3_DRAW_INDEX_IMMD:
3877 case PACKET3_NUM_INSTANCES:
3878 case PACKET3_DRAW_INDEX_MULTI_AUTO:
3879 case PACKET3_STRMOUT_BUFFER_UPDATE:
3880 case PACKET3_DRAW_INDEX_OFFSET_2:
3881 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
3882 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
3883 case PACKET3_MPEG_INDEX:
3884 case PACKET3_WAIT_REG_MEM:
3885 case PACKET3_MEM_WRITE:
3886 case PACKET3_PFP_SYNC_ME:
3887 case PACKET3_SURFACE_SYNC:
3888 case PACKET3_EVENT_WRITE:
3889 case PACKET3_EVENT_WRITE_EOP:
3890 case PACKET3_EVENT_WRITE_EOS:
3891 case PACKET3_SET_CONTEXT_REG:
3892 case PACKET3_SET_CONTEXT_REG_INDIRECT:
3893 case PACKET3_SET_SH_REG:
3894 case PACKET3_SET_SH_REG_OFFSET:
3895 case PACKET3_INCREMENT_DE_COUNTER:
3896 case PACKET3_WAIT_ON_CE_COUNTER:
3897 case PACKET3_WAIT_ON_AVAIL_BUFFER:
3898 case PACKET3_ME_WRITE:
3899 break;
3900 case PACKET3_COPY_DATA:
3901 if ((idx_value & 0xf00) == 0) {
3902 reg = ib[idx + 3] * 4;
3903 if (!si_vm_reg_valid(reg))
3904 return -EINVAL;
3905 }
3906 break;
3907 case PACKET3_WRITE_DATA:
3908 if ((idx_value & 0xf00) == 0) {
3909 start_reg = ib[idx + 1] * 4;
3910 if (idx_value & 0x10000) {
3911 if (!si_vm_reg_valid(start_reg))
3912 return -EINVAL;
3913 } else {
3914 for (i = 0; i < (pkt->count - 2); i++) {
3915 reg = start_reg + (4 * i);
3916 if (!si_vm_reg_valid(reg))
3917 return -EINVAL;
3918 }
3919 }
3920 }
3921 break;
3922 case PACKET3_COND_WRITE:
3923 if (idx_value & 0x100) {
3924 reg = ib[idx + 5] * 4;
3925 if (!si_vm_reg_valid(reg))
3926 return -EINVAL;
3927 }
3928 break;
3929 case PACKET3_COPY_DW:
3930 if (idx_value & 0x2) {
3931 reg = ib[idx + 3] * 4;
3932 if (!si_vm_reg_valid(reg))
3933 return -EINVAL;
3934 }
3935 break;
3936 case PACKET3_SET_CONFIG_REG:
3937 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
3938 end_reg = 4 * pkt->count + start_reg - 4;
3939 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
3940 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
3941 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
3942 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
3943 return -EINVAL;
3944 }
3945 for (i = 0; i < pkt->count; i++) {
3946 reg = start_reg + (4 * i);
3947 if (!si_vm_reg_valid(reg))
3948 return -EINVAL;
3949 }
3950 break;
5aa709be
AD		 3951 case PACKET3_CP_DMA:
3952 command = ib[idx + 4];
3953 info = ib[idx + 1];
3954 if (command & PACKET3_CP_DMA_CMD_SAS) {
3955 /* src address space is register */
3956 if (((info & 0x60000000) >> 29) == 0) {
3957 start_reg = idx_value << 2;
3958 if (command & PACKET3_CP_DMA_CMD_SAIC) {
3959 reg = start_reg;
3960 if (!si_vm_reg_valid(reg)) {
3961 DRM_ERROR("CP DMA Bad SRC register\n");
3962 return -EINVAL;
3963 }
3964 } else {
3965 for (i = 0; i < (command & 0x1fffff); i++) {
3966 reg = start_reg + (4 * i);
3967 if (!si_vm_reg_valid(reg)) {
3968 DRM_ERROR("CP DMA Bad SRC register\n");
3969 return -EINVAL;
3970 }
3971 }
3972 }
3973 }
3974 }
3975 if (command & PACKET3_CP_DMA_CMD_DAS) {
3976 /* dst address space is register */
3977 if (((info & 0x00300000) >> 20) == 0) {
3978 start_reg = ib[idx + 2];
3979 if (command & PACKET3_CP_DMA_CMD_DAIC) {
3980 reg = start_reg;
3981 if (!si_vm_reg_valid(reg)) {
3982 DRM_ERROR("CP DMA Bad DST register\n");
3983 return -EINVAL;
3984 }
3985 } else {
3986 for (i = 0; i < (command & 0x1fffff); i++) {
3987 reg = start_reg + (4 * i);
3988 if (!si_vm_reg_valid(reg)) {
3989 DRM_ERROR("CP DMA Bad DST register\n");
3990 return -EINVAL;
3991 }
3992 }
3993 }
3994 }
3995 }
3996 break;
498dd8b3
AD		 3997 default:
3998 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
3999 return -EINVAL;
4000 }
4001 return 0;
4002}
4003
4004static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4005 u32 *ib, struct radeon_cs_packet *pkt)
4006{
4007 u32 idx = pkt->idx + 1;
4008 u32 idx_value = ib[idx];
4009 u32 start_reg, reg, i;
4010
4011 switch (pkt->opcode) {
4012 case PACKET3_NOP:
4013 case PACKET3_SET_BASE:
4014 case PACKET3_CLEAR_STATE:
4015 case PACKET3_DISPATCH_DIRECT:
4016 case PACKET3_DISPATCH_INDIRECT:
4017 case PACKET3_ALLOC_GDS:
4018 case PACKET3_WRITE_GDS_RAM:
4019 case PACKET3_ATOMIC_GDS:
4020 case PACKET3_ATOMIC:
4021 case PACKET3_OCCLUSION_QUERY:
4022 case PACKET3_SET_PREDICATION:
4023 case PACKET3_COND_EXEC:
4024 case PACKET3_PRED_EXEC:
4025 case PACKET3_CONTEXT_CONTROL:
4026 case PACKET3_STRMOUT_BUFFER_UPDATE:
4027 case PACKET3_WAIT_REG_MEM:
4028 case PACKET3_MEM_WRITE:
4029 case PACKET3_PFP_SYNC_ME:
4030 case PACKET3_SURFACE_SYNC:
4031 case PACKET3_EVENT_WRITE:
4032 case PACKET3_EVENT_WRITE_EOP:
4033 case PACKET3_EVENT_WRITE_EOS:
4034 case PACKET3_SET_CONTEXT_REG:
4035 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4036 case PACKET3_SET_SH_REG:
4037 case PACKET3_SET_SH_REG_OFFSET:
4038 case PACKET3_INCREMENT_DE_COUNTER:
4039 case PACKET3_WAIT_ON_CE_COUNTER:
4040 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4041 case PACKET3_ME_WRITE:
4042 break;
4043 case PACKET3_COPY_DATA:
4044 if ((idx_value & 0xf00) == 0) {
4045 reg = ib[idx + 3] * 4;
4046 if (!si_vm_reg_valid(reg))
4047 return -EINVAL;
4048 }
4049 break;
4050 case PACKET3_WRITE_DATA:
4051 if ((idx_value & 0xf00) == 0) {
4052 start_reg = ib[idx + 1] * 4;
4053 if (idx_value & 0x10000) {
4054 if (!si_vm_reg_valid(start_reg))
4055 return -EINVAL;
4056 } else {
4057 for (i = 0; i < (pkt->count - 2); i++) {
4058 reg = start_reg + (4 * i);
4059 if (!si_vm_reg_valid(reg))
4060 return -EINVAL;
4061 }
4062 }
4063 }
4064 break;
4065 case PACKET3_COND_WRITE:
4066 if (idx_value & 0x100) {
4067 reg = ib[idx + 5] * 4;
4068 if (!si_vm_reg_valid(reg))
4069 return -EINVAL;
4070 }
4071 break;
4072 case PACKET3_COPY_DW:
4073 if (idx_value & 0x2) {
4074 reg = ib[idx + 3] * 4;
4075 if (!si_vm_reg_valid(reg))
4076 return -EINVAL;
4077 }
4078 break;
4079 default:
4080 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4081 return -EINVAL;
4082 }
4083 return 0;
4084}
4085
4086int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4087{
4088 int ret = 0;
4089 u32 idx = 0;
4090 struct radeon_cs_packet pkt;
4091
4092 do {
4093 pkt.idx = idx;
4e872ae2
IH		 4094 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4095 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4096 pkt.one_reg_wr = 0;
4097 switch (pkt.type) {
4e872ae2 4098 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 4099 dev_err(rdev->dev, "Packet0 not allowed!\n");
4100 ret = -EINVAL;
4101 break;
4e872ae2 4102 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4103 idx += 1;
4104 break;
4e872ae2
IH		 4105 case RADEON_PACKET_TYPE3:
4106 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4107 if (ib->is_const_ib)
4108 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4109 else {
876dc9f3 4110 switch (ib->ring) {
498dd8b3
AD		 4111 case RADEON_RING_TYPE_GFX_INDEX:
4112 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4113 break;
4114 case CAYMAN_RING_TYPE_CP1_INDEX:
4115 case CAYMAN_RING_TYPE_CP2_INDEX:
4116 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4117 break;
4118 default:
876dc9f3 4119 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4120 ret = -EINVAL;
4121 break;
4122 }
4123 }
4124 idx += pkt.count + 2;
4125 break;
4126 default:
4127 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4128 ret = -EINVAL;
4129 break;
4130 }
4131 if (ret)
4132 break;
4133 } while (idx < ib->length_dw);
4134
4135 return ret;
4136}
4137
d2800ee5
AD		 4138/*
4139 * vm
4140 */
4141int si_vm_init(struct radeon_device *rdev)
4142{
4143 /* number of VMs */
4144 rdev->vm_manager.nvm = 16;
4145 /* base offset of vram pages */
4146 rdev->vm_manager.vram_base_offset = 0;
4147
4148 return 0;
4149}
4150
4151void si_vm_fini(struct radeon_device *rdev)
4152{
4153}
4154
82ffd92b
AD		 4155/**
4156 * si_vm_set_page - update the page tables using the CP
4157 *
4158 * @rdev: radeon_device pointer
43f1214a 4159 * @ib: indirect buffer to fill with commands
82ffd92b
AD		 4160 * @pe: addr of the page entry
4161 * @addr: dst addr to write into pe
4162 * @count: number of page entries to update
4163 * @incr: increase next addr by incr bytes
4164 * @flags: access flags
4165 *
43f1214a 4166 * Update the page tables using the CP (SI).
82ffd92b 4167 */
43f1214a
AD		 4168void si_vm_set_page(struct radeon_device *rdev,
4169 struct radeon_ib *ib,
4170 uint64_t pe,
82ffd92b
AD		 4171 uint64_t addr, unsigned count,
4172 uint32_t incr, uint32_t flags)
d2800ee5 4173{
82ffd92b 4174 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
deab48f1
AD		 4175 uint64_t value;
4176 unsigned ndw;
4177
4178 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4179 while (count) {
4180 ndw = 2 + count * 2;
4181 if (ndw > 0x3FFE)
4182 ndw = 0x3FFE;
4183
43f1214a
AD		 4184 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4185 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4186 WRITE_DATA_DST_SEL(1));
4187 ib->ptr[ib->length_dw++] = pe;
4188 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
deab48f1
AD		 4189 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4190 if (flags & RADEON_VM_PAGE_SYSTEM) {
4191 value = radeon_vm_map_gart(rdev, addr);
4192 value &= 0xFFFFFFFFFFFFF000ULL;
4193 } else if (flags & RADEON_VM_PAGE_VALID) {
4194 value = addr;
4195 } else {
4196 value = 0;
4197 }
4198 addr += incr;
4199 value |= r600_flags;
43f1214a
AD		 4200 ib->ptr[ib->length_dw++] = value;
4201 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4202 }
4203 }
4204 } else {
4205 /* DMA */
4206 if (flags & RADEON_VM_PAGE_SYSTEM) {
4207 while (count) {
4208 ndw = count * 2;
4209 if (ndw > 0xFFFFE)
4210 ndw = 0xFFFFE;
4211
4212 /* for non-physically contiguous pages (system) */
43f1214a
AD		 4213 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
4214 ib->ptr[ib->length_dw++] = pe;
4215 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
deab48f1
AD		 4216 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
4217 if (flags & RADEON_VM_PAGE_SYSTEM) {
4218 value = radeon_vm_map_gart(rdev, addr);
4219 value &= 0xFFFFFFFFFFFFF000ULL;
4220 } else if (flags & RADEON_VM_PAGE_VALID) {
4221 value = addr;
4222 } else {
4223 value = 0;
4224 }
4225 addr += incr;
4226 value |= r600_flags;
43f1214a
AD		 4227 ib->ptr[ib->length_dw++] = value;
4228 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4229 }
4230 }
4231 } else {
4232 while (count) {
4233 ndw = count * 2;
4234 if (ndw > 0xFFFFE)
4235 ndw = 0xFFFFE;
4236
4237 if (flags & RADEON_VM_PAGE_VALID)
4238 value = addr;
4239 else
4240 value = 0;
4241 /* for physically contiguous pages (vram) */
43f1214a
AD		 4242 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
4243 ib->ptr[ib->length_dw++] = pe; /* dst addr */
4244 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
4245 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
4246 ib->ptr[ib->length_dw++] = 0;
4247 ib->ptr[ib->length_dw++] = value; /* value */
4248 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4249 ib->ptr[ib->length_dw++] = incr; /* increment size */
4250 ib->ptr[ib->length_dw++] = 0;
deab48f1
AD		 4251 pe += ndw * 4;
4252 addr += (ndw / 2) * incr;
4253 count -= ndw / 2;
4254 }
d7025d89 4255 }
43f1214a
AD		 4256 while (ib->length_dw & 0x7)
4257 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
82ffd92b 4258 }
d2800ee5
AD		 4259}
4260
498522b4 4261void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
d2800ee5 4262{
498522b4 4263 struct radeon_ring *ring = &rdev->ring[ridx];
d2800ee5 4264
ee60e29f 4265 if (vm == NULL)
d2800ee5
AD		 4266 return;
4267
76c44f2c
AD		 4268 /* write new base address */
4269 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4270 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4271 WRITE_DATA_DST_SEL(0)));
4272
ee60e29f 4273 if (vm->id < 8) {
76c44f2c
AD		 4274 radeon_ring_write(ring,
4275 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
ee60e29f 4276 } else {
76c44f2c
AD		 4277 radeon_ring_write(ring,
4278 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
ee60e29f 4279 }
76c44f2c 4280 radeon_ring_write(ring, 0);
fa87e62d 4281 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 4282
d2800ee5 4283 /* flush hdp cache */
76c44f2c
AD		 4284 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4285 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4286 WRITE_DATA_DST_SEL(0)));
4287 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4288 radeon_ring_write(ring, 0);
ee60e29f
CK		 4289 radeon_ring_write(ring, 0x1);
4290
d2800ee5 4291 /* bits 0-15 are the VM contexts0-15 */
76c44f2c
AD		 4292 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4293 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4294 WRITE_DATA_DST_SEL(0)));
4295 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4296 radeon_ring_write(ring, 0);
498522b4 4297 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 4298
4299 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4300 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4301 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 4302}
4303
8c5fd7ef
AD		 4304void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4305{
4306 struct radeon_ring *ring = &rdev->ring[ridx];
4307
4308 if (vm == NULL)
4309 return;
4310
4311 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4312 if (vm->id < 8) {
4313 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
4314 } else {
4315 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2));
4316 }
4317 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4318
4319 /* flush hdp cache */
4320 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4321 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4322 radeon_ring_write(ring, 1);
4323
4324 /* bits 0-7 are the VM contexts0-7 */
4325 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4326 radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
4327 radeon_ring_write(ring, 1 << vm->id);
4328}
4329
347e7592
AD		 4330/*
4331 * RLC
4332 */
c420c745 4333void si_rlc_fini(struct radeon_device *rdev)
347e7592
AD		 4334{
4335 int r;
4336
4337 /* save restore block */
4338 if (rdev->rlc.save_restore_obj) {
4339 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4340 if (unlikely(r != 0))
4341 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4342 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4343 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4344
4345 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4346 rdev->rlc.save_restore_obj = NULL;
4347 }
4348
4349 /* clear state block */
4350 if (rdev->rlc.clear_state_obj) {
4351 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4352 if (unlikely(r != 0))
4353 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4354 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4355 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4356
4357 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4358 rdev->rlc.clear_state_obj = NULL;
4359 }
4360}
4361
c420c745 4362int si_rlc_init(struct radeon_device *rdev)
347e7592
AD		 4363{
4364 int r;
4365
4366 /* save restore block */
4367 if (rdev->rlc.save_restore_obj == NULL) {
4368 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
40f5cf99
AD		 4369 RADEON_GEM_DOMAIN_VRAM, NULL,
4370 &rdev->rlc.save_restore_obj);
347e7592
AD		 4371 if (r) {
4372 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4373 return r;
4374 }
4375 }
4376
4377 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4378 if (unlikely(r != 0)) {
4379 si_rlc_fini(rdev);
4380 return r;
4381 }
4382 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4383 &rdev->rlc.save_restore_gpu_addr);
5273db70 4384 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
347e7592 4385 if (r) {
347e7592
AD		 4386 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4387 si_rlc_fini(rdev);
4388 return r;
4389 }
4390
4391 /* clear state block */
4392 if (rdev->rlc.clear_state_obj == NULL) {
4393 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
40f5cf99
AD		 4394 RADEON_GEM_DOMAIN_VRAM, NULL,
4395 &rdev->rlc.clear_state_obj);
347e7592
AD		 4396 if (r) {
4397 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4398 si_rlc_fini(rdev);
4399 return r;
4400 }
4401 }
4402 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4403 if (unlikely(r != 0)) {
4404 si_rlc_fini(rdev);
4405 return r;
4406 }
4407 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4408 &rdev->rlc.clear_state_gpu_addr);
5273db70 4409 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
347e7592 4410 if (r) {
347e7592
AD		 4411 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4412 si_rlc_fini(rdev);
4413 return r;
4414 }
4415
4416 return 0;
4417}
4418
4419static void si_rlc_stop(struct radeon_device *rdev)
4420{
4421 WREG32(RLC_CNTL, 0);
4422}
4423
4424static void si_rlc_start(struct radeon_device *rdev)
4425{
4426 WREG32(RLC_CNTL, RLC_ENABLE);
4427}
4428
4429static int si_rlc_resume(struct radeon_device *rdev)
4430{
4431 u32 i;
4432 const __be32 *fw_data;
4433
4434 if (!rdev->rlc_fw)
4435 return -EINVAL;
4436
4437 si_rlc_stop(rdev);
4438
4439 WREG32(RLC_RL_BASE, 0);
4440 WREG32(RLC_RL_SIZE, 0);
4441 WREG32(RLC_LB_CNTL, 0);
4442 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
4443 WREG32(RLC_LB_CNTR_INIT, 0);
4444
4445 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4446 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4447
4448 WREG32(RLC_MC_CNTL, 0);
4449 WREG32(RLC_UCODE_CNTL, 0);
4450
4451 fw_data = (const __be32 *)rdev->rlc_fw->data;
4452 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
4453 WREG32(RLC_UCODE_ADDR, i);
4454 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4455 }
4456 WREG32(RLC_UCODE_ADDR, 0);
4457
4458 si_rlc_start(rdev);
4459
4460 return 0;
4461}
4462
25a857fb
AD		 4463static void si_enable_interrupts(struct radeon_device *rdev)
4464{
4465 u32 ih_cntl = RREG32(IH_CNTL);
4466 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4467
4468 ih_cntl |= ENABLE_INTR;
4469 ih_rb_cntl |= IH_RB_ENABLE;
4470 WREG32(IH_CNTL, ih_cntl);
4471 WREG32(IH_RB_CNTL, ih_rb_cntl);
4472 rdev->ih.enabled = true;
4473}
4474
4475static void si_disable_interrupts(struct radeon_device *rdev)
4476{
4477 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4478 u32 ih_cntl = RREG32(IH_CNTL);
4479
4480 ih_rb_cntl &= ~IH_RB_ENABLE;
4481 ih_cntl &= ~ENABLE_INTR;
4482 WREG32(IH_RB_CNTL, ih_rb_cntl);
4483 WREG32(IH_CNTL, ih_cntl);
4484 /* set rptr, wptr to 0 */
4485 WREG32(IH_RB_RPTR, 0);
4486 WREG32(IH_RB_WPTR, 0);
4487 rdev->ih.enabled = false;
25a857fb
AD		 4488 rdev->ih.rptr = 0;
4489}
4490
4491static void si_disable_interrupt_state(struct radeon_device *rdev)
4492{
4493 u32 tmp;
4494
4495 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4496 WREG32(CP_INT_CNTL_RING1, 0);
4497 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 4498 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4499 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
4500 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4501 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 4502 WREG32(GRBM_INT_CNTL, 0);
5153550a
AD		 4503 if (rdev->num_crtc >= 2) {
4504 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4505 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4506 }
25a857fb
AD		 4507 if (rdev->num_crtc >= 4) {
4508 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4509 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4510 }
4511 if (rdev->num_crtc >= 6) {
4512 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4513 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4514 }
4515
5153550a
AD		 4516 if (rdev->num_crtc >= 2) {
4517 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4518 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4519 }
25a857fb
AD		 4520 if (rdev->num_crtc >= 4) {
4521 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4522 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4523 }
4524 if (rdev->num_crtc >= 6) {
4525 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4526 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4527 }
4528
5153550a
AD		 4529 if (!ASIC_IS_NODCE(rdev)) {
4530 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4531
4532 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4533 WREG32(DC_HPD1_INT_CONTROL, tmp);
4534 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4535 WREG32(DC_HPD2_INT_CONTROL, tmp);
4536 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4537 WREG32(DC_HPD3_INT_CONTROL, tmp);
4538 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4539 WREG32(DC_HPD4_INT_CONTROL, tmp);
4540 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4541 WREG32(DC_HPD5_INT_CONTROL, tmp);
4542 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4543 WREG32(DC_HPD6_INT_CONTROL, tmp);
4544 }
25a857fb
AD		 4545}
4546
4547static int si_irq_init(struct radeon_device *rdev)
4548{
4549 int ret = 0;
4550 int rb_bufsz;
4551 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
4552
4553 /* allocate ring */
4554 ret = r600_ih_ring_alloc(rdev);
4555 if (ret)
4556 return ret;
4557
4558 /* disable irqs */
4559 si_disable_interrupts(rdev);
4560
4561 /* init rlc */
4562 ret = si_rlc_resume(rdev);
4563 if (ret) {
4564 r600_ih_ring_fini(rdev);
4565 return ret;
4566 }
4567
4568 /* setup interrupt control */
4569 /* set dummy read address to ring address */
4570 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
4571 interrupt_cntl = RREG32(INTERRUPT_CNTL);
4572 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
4573 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
4574 */
4575 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
4576 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
4577 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
4578 WREG32(INTERRUPT_CNTL, interrupt_cntl);
4579
4580 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
4581 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
4582
4583 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
4584 IH_WPTR_OVERFLOW_CLEAR |
4585 (rb_bufsz << 1));
4586
4587 if (rdev->wb.enabled)
4588 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
4589
4590 /* set the writeback address whether it's enabled or not */
4591 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
4592 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
4593
4594 WREG32(IH_RB_CNTL, ih_rb_cntl);
4595
4596 /* set rptr, wptr to 0 */
4597 WREG32(IH_RB_RPTR, 0);
4598 WREG32(IH_RB_WPTR, 0);
4599
4600 /* Default settings for IH_CNTL (disabled at first) */
4601 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
4602 /* RPTR_REARM only works if msi's are enabled */
4603 if (rdev->msi_enabled)
4604 ih_cntl |= RPTR_REARM;
4605 WREG32(IH_CNTL, ih_cntl);
4606
4607 /* force the active interrupt state to all disabled */
4608 si_disable_interrupt_state(rdev);
4609
2099810f
DA		 4610 pci_set_master(rdev->pdev);
4611
25a857fb
AD		 4612 /* enable irqs */
4613 si_enable_interrupts(rdev);
4614
4615 return ret;
4616}
4617
4618int si_irq_set(struct radeon_device *rdev)
4619{
4620 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4621 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4622 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5153550a 4623 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
25a857fb
AD		 4624 u32 grbm_int_cntl = 0;
4625 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
8c5fd7ef 4626 u32 dma_cntl, dma_cntl1;
25a857fb
AD		 4627
4628 if (!rdev->irq.installed) {
4629 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4630 return -EINVAL;
4631 }
4632 /* don't enable anything if the ih is disabled */
4633 if (!rdev->ih.enabled) {
4634 si_disable_interrupts(rdev);
4635 /* force the active interrupt state to all disabled */
4636 si_disable_interrupt_state(rdev);
4637 return 0;
4638 }
4639
5153550a
AD		 4640 if (!ASIC_IS_NODCE(rdev)) {
4641 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4642 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4643 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4644 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4645 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4646 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4647 }
25a857fb 4648
8c5fd7ef
AD		 4649 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4650 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4651
25a857fb 4652 /* enable CP interrupts on all rings */
736fc37f 4653 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 4654 DRM_DEBUG("si_irq_set: sw int gfx\n");
4655 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4656 }
736fc37f 4657 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 4658 DRM_DEBUG("si_irq_set: sw int cp1\n");
4659 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4660 }
736fc37f 4661 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 4662 DRM_DEBUG("si_irq_set: sw int cp2\n");
4663 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4664 }
8c5fd7ef
AD		 4665 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4666 DRM_DEBUG("si_irq_set: sw int dma\n");
4667 dma_cntl |= TRAP_ENABLE;
4668 }
4669
4670 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4671 DRM_DEBUG("si_irq_set: sw int dma1\n");
4672 dma_cntl1 |= TRAP_ENABLE;
4673 }
25a857fb 4674 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 4675 atomic_read(&rdev->irq.pflip[0])) {
25a857fb
AD		 4676 DRM_DEBUG("si_irq_set: vblank 0\n");
4677 crtc1 |= VBLANK_INT_MASK;
4678 }
4679 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 4680 atomic_read(&rdev->irq.pflip[1])) {
25a857fb
AD		 4681 DRM_DEBUG("si_irq_set: vblank 1\n");
4682 crtc2 |= VBLANK_INT_MASK;
4683 }
4684 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 4685 atomic_read(&rdev->irq.pflip[2])) {
25a857fb
AD		 4686 DRM_DEBUG("si_irq_set: vblank 2\n");
4687 crtc3 |= VBLANK_INT_MASK;
4688 }
4689 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 4690 atomic_read(&rdev->irq.pflip[3])) {
25a857fb
AD		 4691 DRM_DEBUG("si_irq_set: vblank 3\n");
4692 crtc4 |= VBLANK_INT_MASK;
4693 }
4694 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 4695 atomic_read(&rdev->irq.pflip[4])) {
25a857fb
AD		 4696 DRM_DEBUG("si_irq_set: vblank 4\n");
4697 crtc5 |= VBLANK_INT_MASK;
4698 }
4699 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 4700 atomic_read(&rdev->irq.pflip[5])) {
25a857fb
AD		 4701 DRM_DEBUG("si_irq_set: vblank 5\n");
4702 crtc6 |= VBLANK_INT_MASK;
4703 }
4704 if (rdev->irq.hpd[0]) {
4705 DRM_DEBUG("si_irq_set: hpd 1\n");
4706 hpd1 |= DC_HPDx_INT_EN;
4707 }
4708 if (rdev->irq.hpd[1]) {
4709 DRM_DEBUG("si_irq_set: hpd 2\n");
4710 hpd2 |= DC_HPDx_INT_EN;
4711 }
4712 if (rdev->irq.hpd[2]) {
4713 DRM_DEBUG("si_irq_set: hpd 3\n");
4714 hpd3 |= DC_HPDx_INT_EN;
4715 }
4716 if (rdev->irq.hpd[3]) {
4717 DRM_DEBUG("si_irq_set: hpd 4\n");
4718 hpd4 |= DC_HPDx_INT_EN;
4719 }
4720 if (rdev->irq.hpd[4]) {
4721 DRM_DEBUG("si_irq_set: hpd 5\n");
4722 hpd5 |= DC_HPDx_INT_EN;
4723 }
4724 if (rdev->irq.hpd[5]) {
4725 DRM_DEBUG("si_irq_set: hpd 6\n");
4726 hpd6 |= DC_HPDx_INT_EN;
4727 }
25a857fb
AD		 4728
4729 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
4730 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
4731 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
4732
8c5fd7ef
AD		 4733 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
4734 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
4735
25a857fb
AD		 4736 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4737
5153550a
AD		 4738 if (rdev->num_crtc >= 2) {
4739 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4740 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4741 }
25a857fb
AD		 4742 if (rdev->num_crtc >= 4) {
4743 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4744 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4745 }
4746 if (rdev->num_crtc >= 6) {
4747 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4748 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4749 }
4750
5153550a
AD		 4751 if (rdev->num_crtc >= 2) {
4752 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4753 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
4754 }
25a857fb
AD		 4755 if (rdev->num_crtc >= 4) {
4756 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4757 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4758 }
4759 if (rdev->num_crtc >= 6) {
4760 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4761 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4762 }
4763
5153550a
AD		 4764 if (!ASIC_IS_NODCE(rdev)) {
4765 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4766 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4767 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4768 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4769 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4770 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4771 }
25a857fb
AD		 4772
4773 return 0;
4774}
4775
4776static inline void si_irq_ack(struct radeon_device *rdev)
4777{
4778 u32 tmp;
4779
5153550a
AD		 4780 if (ASIC_IS_NODCE(rdev))
4781 return;
4782
25a857fb
AD		 4783 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4784 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4785 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4786 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4787 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4788 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4789 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4790 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4791 if (rdev->num_crtc >= 4) {
4792 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4793 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4794 }
4795 if (rdev->num_crtc >= 6) {
4796 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4797 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4798 }
4799
4800 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4801 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4802 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4803 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4804 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4805 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4806 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4807 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4808 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4809 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4810 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4811 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4812
4813 if (rdev->num_crtc >= 4) {
4814 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4815 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4816 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4817 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4818 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4819 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4820 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4821 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4822 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4823 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4824 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4825 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4826 }
4827
4828 if (rdev->num_crtc >= 6) {
4829 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4830 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4831 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4832 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4833 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4834 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4835 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4836 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4837 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4838 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4839 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4840 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4841 }
4842
4843 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4844 tmp = RREG32(DC_HPD1_INT_CONTROL);
4845 tmp |= DC_HPDx_INT_ACK;
4846 WREG32(DC_HPD1_INT_CONTROL, tmp);
4847 }
4848 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4849 tmp = RREG32(DC_HPD2_INT_CONTROL);
4850 tmp |= DC_HPDx_INT_ACK;
4851 WREG32(DC_HPD2_INT_CONTROL, tmp);
4852 }
4853 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4854 tmp = RREG32(DC_HPD3_INT_CONTROL);
4855 tmp |= DC_HPDx_INT_ACK;
4856 WREG32(DC_HPD3_INT_CONTROL, tmp);
4857 }
4858 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4859 tmp = RREG32(DC_HPD4_INT_CONTROL);
4860 tmp |= DC_HPDx_INT_ACK;
4861 WREG32(DC_HPD4_INT_CONTROL, tmp);
4862 }
4863 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4864 tmp = RREG32(DC_HPD5_INT_CONTROL);
4865 tmp |= DC_HPDx_INT_ACK;
4866 WREG32(DC_HPD5_INT_CONTROL, tmp);
4867 }
4868 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4869 tmp = RREG32(DC_HPD5_INT_CONTROL);
4870 tmp |= DC_HPDx_INT_ACK;
4871 WREG32(DC_HPD6_INT_CONTROL, tmp);
4872 }
4873}
4874
4875static void si_irq_disable(struct radeon_device *rdev)
4876{
4877 si_disable_interrupts(rdev);
4878 /* Wait and acknowledge irq */
4879 mdelay(1);
4880 si_irq_ack(rdev);
4881 si_disable_interrupt_state(rdev);
4882}
4883
4884static void si_irq_suspend(struct radeon_device *rdev)
4885{
4886 si_irq_disable(rdev);
4887 si_rlc_stop(rdev);
4888}
4889
9b136d51
AD		 4890static void si_irq_fini(struct radeon_device *rdev)
4891{
4892 si_irq_suspend(rdev);
4893 r600_ih_ring_fini(rdev);
4894}
4895
25a857fb
AD		 4896static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
4897{
4898 u32 wptr, tmp;
4899
4900 if (rdev->wb.enabled)
4901 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4902 else
4903 wptr = RREG32(IH_RB_WPTR);
4904
4905 if (wptr & RB_OVERFLOW) {
4906 /* When a ring buffer overflow happen start parsing interrupt
4907 * from the last not overwritten vector (wptr + 16). Hopefully
4908 * this should allow us to catchup.
4909 */
4910 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4911 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4912 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4913 tmp = RREG32(IH_RB_CNTL);
4914 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4915 WREG32(IH_RB_CNTL, tmp);
4916 }
4917 return (wptr & rdev->ih.ptr_mask);
4918}
4919
4920/* SI IV Ring
4921 * Each IV ring entry is 128 bits:
4922 * [7:0] - interrupt source id
4923 * [31:8] - reserved
4924 * [59:32] - interrupt source data
4925 * [63:60] - reserved
4926 * [71:64] - RINGID
4927 * [79:72] - VMID
4928 * [127:80] - reserved
4929 */
4930int si_irq_process(struct radeon_device *rdev)
4931{
4932 u32 wptr;
4933 u32 rptr;
4934 u32 src_id, src_data, ring_id;
4935 u32 ring_index;
25a857fb
AD		 4936 bool queue_hotplug = false;
4937
4938 if (!rdev->ih.enabled || rdev->shutdown)
4939 return IRQ_NONE;
4940
4941 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 4942
4943restart_ih:
4944 /* is somebody else already processing irqs? */
4945 if (atomic_xchg(&rdev->ih.lock, 1))
4946 return IRQ_NONE;
4947
25a857fb
AD		 4948 rptr = rdev->ih.rptr;
4949 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4950
25a857fb
AD		 4951 /* Order reading of wptr vs. reading of IH ring data */
4952 rmb();
4953
4954 /* display interrupts */
4955 si_irq_ack(rdev);
4956
25a857fb
AD		 4957 while (rptr != wptr) {
4958 /* wptr/rptr are in bytes! */
4959 ring_index = rptr / 4;
4960 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4961 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4962 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
4963
4964 switch (src_id) {
4965 case 1: /* D1 vblank/vline */
4966 switch (src_data) {
4967 case 0: /* D1 vblank */
4968 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4969 if (rdev->irq.crtc_vblank_int[0]) {
4970 drm_handle_vblank(rdev->ddev, 0);
4971 rdev->pm.vblank_sync = true;
4972 wake_up(&rdev->irq.vblank_queue);
4973 }
736fc37f 4974 if (atomic_read(&rdev->irq.pflip[0]))
25a857fb
AD		 4975 radeon_crtc_handle_flip(rdev, 0);
4976 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4977 DRM_DEBUG("IH: D1 vblank\n");
4978 }
4979 break;
4980 case 1: /* D1 vline */
4981 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4982 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4983 DRM_DEBUG("IH: D1 vline\n");
4984 }
4985 break;
4986 default:
4987 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4988 break;
4989 }
4990 break;
4991 case 2: /* D2 vblank/vline */
4992 switch (src_data) {
4993 case 0: /* D2 vblank */
4994 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4995 if (rdev->irq.crtc_vblank_int[1]) {
4996 drm_handle_vblank(rdev->ddev, 1);
4997 rdev->pm.vblank_sync = true;
4998 wake_up(&rdev->irq.vblank_queue);
4999 }
736fc37f 5000 if (atomic_read(&rdev->irq.pflip[1]))
25a857fb
AD		 5001 radeon_crtc_handle_flip(rdev, 1);
5002 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
5003 DRM_DEBUG("IH: D2 vblank\n");
5004 }
5005 break;
5006 case 1: /* D2 vline */
5007 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
5008 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
5009 DRM_DEBUG("IH: D2 vline\n");
5010 }
5011 break;
5012 default:
5013 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5014 break;
5015 }
5016 break;
5017 case 3: /* D3 vblank/vline */
5018 switch (src_data) {
5019 case 0: /* D3 vblank */
5020 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
5021 if (rdev->irq.crtc_vblank_int[2]) {
5022 drm_handle_vblank(rdev->ddev, 2);
5023 rdev->pm.vblank_sync = true;
5024 wake_up(&rdev->irq.vblank_queue);
5025 }
736fc37f 5026 if (atomic_read(&rdev->irq.pflip[2]))
25a857fb
AD		 5027 radeon_crtc_handle_flip(rdev, 2);
5028 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5029 DRM_DEBUG("IH: D3 vblank\n");
5030 }
5031 break;
5032 case 1: /* D3 vline */
5033 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
5034 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5035 DRM_DEBUG("IH: D3 vline\n");
5036 }
5037 break;
5038 default:
5039 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5040 break;
5041 }
5042 break;
5043 case 4: /* D4 vblank/vline */
5044 switch (src_data) {
5045 case 0: /* D4 vblank */
5046 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
5047 if (rdev->irq.crtc_vblank_int[3]) {
5048 drm_handle_vblank(rdev->ddev, 3);
5049 rdev->pm.vblank_sync = true;
5050 wake_up(&rdev->irq.vblank_queue);
5051 }
736fc37f 5052 if (atomic_read(&rdev->irq.pflip[3]))
25a857fb
AD		 5053 radeon_crtc_handle_flip(rdev, 3);
5054 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5055 DRM_DEBUG("IH: D4 vblank\n");
5056 }
5057 break;
5058 case 1: /* D4 vline */
5059 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
5060 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5061 DRM_DEBUG("IH: D4 vline\n");
5062 }
5063 break;
5064 default:
5065 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5066 break;
5067 }
5068 break;
5069 case 5: /* D5 vblank/vline */
5070 switch (src_data) {
5071 case 0: /* D5 vblank */
5072 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
5073 if (rdev->irq.crtc_vblank_int[4]) {
5074 drm_handle_vblank(rdev->ddev, 4);
5075 rdev->pm.vblank_sync = true;
5076 wake_up(&rdev->irq.vblank_queue);
5077 }
736fc37f 5078 if (atomic_read(&rdev->irq.pflip[4]))
25a857fb
AD		 5079 radeon_crtc_handle_flip(rdev, 4);
5080 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5081 DRM_DEBUG("IH: D5 vblank\n");
5082 }
5083 break;
5084 case 1: /* D5 vline */
5085 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
5086 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5087 DRM_DEBUG("IH: D5 vline\n");
5088 }
5089 break;
5090 default:
5091 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5092 break;
5093 }
5094 break;
5095 case 6: /* D6 vblank/vline */
5096 switch (src_data) {
5097 case 0: /* D6 vblank */
5098 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
5099 if (rdev->irq.crtc_vblank_int[5]) {
5100 drm_handle_vblank(rdev->ddev, 5);
5101 rdev->pm.vblank_sync = true;
5102 wake_up(&rdev->irq.vblank_queue);
5103 }
736fc37f 5104 if (atomic_read(&rdev->irq.pflip[5]))
25a857fb
AD		 5105 radeon_crtc_handle_flip(rdev, 5);
5106 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5107 DRM_DEBUG("IH: D6 vblank\n");
5108 }
5109 break;
5110 case 1: /* D6 vline */
5111 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
5112 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5113 DRM_DEBUG("IH: D6 vline\n");
5114 }
5115 break;
5116 default:
5117 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5118 break;
5119 }
5120 break;
5121 case 42: /* HPD hotplug */
5122 switch (src_data) {
5123 case 0:
5124 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5125 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5126 queue_hotplug = true;
5127 DRM_DEBUG("IH: HPD1\n");
5128 }
5129 break;
5130 case 1:
5131 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5132 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5133 queue_hotplug = true;
5134 DRM_DEBUG("IH: HPD2\n");
5135 }
5136 break;
5137 case 2:
5138 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5139 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5140 queue_hotplug = true;
5141 DRM_DEBUG("IH: HPD3\n");
5142 }
5143 break;
5144 case 3:
5145 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5146 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5147 queue_hotplug = true;
5148 DRM_DEBUG("IH: HPD4\n");
5149 }
5150 break;
5151 case 4:
5152 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5153 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5154 queue_hotplug = true;
5155 DRM_DEBUG("IH: HPD5\n");
5156 }
5157 break;
5158 case 5:
5159 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5160 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5161 queue_hotplug = true;
5162 DRM_DEBUG("IH: HPD6\n");
5163 }
5164 break;
5165 default:
5166 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5167 break;
5168 }
5169 break;
ae133a11
CK		 5170 case 146:
5171 case 147:
5172 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5173 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5174 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5175 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5176 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5177 /* reset addr and status */
5178 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5179 break;
25a857fb
AD		 5180 case 176: /* RINGID0 CP_INT */
5181 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5182 break;
5183 case 177: /* RINGID1 CP_INT */
5184 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5185 break;
5186 case 178: /* RINGID2 CP_INT */
5187 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5188 break;
5189 case 181: /* CP EOP event */
5190 DRM_DEBUG("IH: CP EOP\n");
5191 switch (ring_id) {
5192 case 0:
5193 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5194 break;
5195 case 1:
5196 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5197 break;
5198 case 2:
5199 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5200 break;
5201 }
5202 break;
8c5fd7ef
AD		 5203 case 224: /* DMA trap event */
5204 DRM_DEBUG("IH: DMA trap\n");
5205 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5206 break;
25a857fb
AD		 5207 case 233: /* GUI IDLE */
5208 DRM_DEBUG("IH: GUI idle\n");
25a857fb 5209 break;
8c5fd7ef
AD		 5210 case 244: /* DMA trap event */
5211 DRM_DEBUG("IH: DMA1 trap\n");
5212 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5213 break;
25a857fb
AD		 5214 default:
5215 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5216 break;
5217 }
5218
5219 /* wptr/rptr are in bytes! */
5220 rptr += 16;
5221 rptr &= rdev->ih.ptr_mask;
5222 }
25a857fb
AD		 5223 if (queue_hotplug)
5224 schedule_work(&rdev->hotplug_work);
5225 rdev->ih.rptr = rptr;
5226 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 5227 atomic_set(&rdev->ih.lock, 0);
5228
5229 /* make sure wptr hasn't changed while processing */
5230 wptr = si_get_ih_wptr(rdev);
5231 if (wptr != rptr)
5232 goto restart_ih;
5233
25a857fb
AD		 5234 return IRQ_HANDLED;
5235}
5236
8c5fd7ef
AD		 5237/**
5238 * si_copy_dma - copy pages using the DMA engine
5239 *
5240 * @rdev: radeon_device pointer
5241 * @src_offset: src GPU address
5242 * @dst_offset: dst GPU address
5243 * @num_gpu_pages: number of GPU pages to xfer
5244 * @fence: radeon fence object
5245 *
5246 * Copy GPU paging using the DMA engine (SI).
5247 * Used by the radeon ttm implementation to move pages if
5248 * registered as the asic copy callback.
5249 */
5250int si_copy_dma(struct radeon_device *rdev,
5251 uint64_t src_offset, uint64_t dst_offset,
5252 unsigned num_gpu_pages,
5253 struct radeon_fence **fence)
5254{
5255 struct radeon_semaphore *sem = NULL;
5256 int ring_index = rdev->asic->copy.dma_ring_index;
5257 struct radeon_ring *ring = &rdev->ring[ring_index];
5258 u32 size_in_bytes, cur_size_in_bytes;
5259 int i, num_loops;
5260 int r = 0;
5261
5262 r = radeon_semaphore_create(rdev, &sem);
5263 if (r) {
5264 DRM_ERROR("radeon: moving bo (%d).\n", r);
5265 return r;
5266 }
5267
5268 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
5269 num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
5270 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5271 if (r) {
5272 DRM_ERROR("radeon: moving bo (%d).\n", r);
5273 radeon_semaphore_free(rdev, &sem, NULL);
5274 return r;
5275 }
5276
5277 if (radeon_fence_need_sync(*fence, ring->idx)) {
5278 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5279 ring->idx);
5280 radeon_fence_note_sync(*fence, ring->idx);
5281 } else {
5282 radeon_semaphore_free(rdev, &sem, NULL);
5283 }
5284
5285 for (i = 0; i < num_loops; i++) {
5286 cur_size_in_bytes = size_in_bytes;
5287 if (cur_size_in_bytes > 0xFFFFF)
5288 cur_size_in_bytes = 0xFFFFF;
5289 size_in_bytes -= cur_size_in_bytes;
5290 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
5291 radeon_ring_write(ring, dst_offset & 0xffffffff);
5292 radeon_ring_write(ring, src_offset & 0xffffffff);
5293 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5294 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5295 src_offset += cur_size_in_bytes;
5296 dst_offset += cur_size_in_bytes;
5297 }
5298
5299 r = radeon_fence_emit(rdev, fence, ring->idx);
5300 if (r) {
5301 radeon_ring_unlock_undo(rdev, ring);
5302 return r;
5303 }
5304
5305 radeon_ring_unlock_commit(rdev, ring);
5306 radeon_semaphore_free(rdev, &sem, *fence);
5307
5308 return r;
5309}
5310
9b136d51
AD		 5311/*
5312 * startup/shutdown callbacks
5313 */
5314static int si_startup(struct radeon_device *rdev)
5315{
5316 struct radeon_ring *ring;
5317 int r;
5318
5319 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
5320 !rdev->rlc_fw || !rdev->mc_fw) {
5321 r = si_init_microcode(rdev);
5322 if (r) {
5323 DRM_ERROR("Failed to load firmware!\n");
5324 return r;
5325 }
5326 }
5327
5328 r = si_mc_load_microcode(rdev);
5329 if (r) {
5330 DRM_ERROR("Failed to load MC firmware!\n");
5331 return r;
5332 }
5333
5334 r = r600_vram_scratch_init(rdev);
5335 if (r)
5336 return r;
5337
5338 si_mc_program(rdev);
5339 r = si_pcie_gart_enable(rdev);
5340 if (r)
5341 return r;
5342 si_gpu_init(rdev);
5343
9b136d51
AD		 5344 /* allocate rlc buffers */
5345 r = si_rlc_init(rdev);
5346 if (r) {
5347 DRM_ERROR("Failed to init rlc BOs!\n");
5348 return r;
5349 }
5350
5351 /* allocate wb buffer */
5352 r = radeon_wb_init(rdev);
5353 if (r)
5354 return r;
5355
5356 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5357 if (r) {
5358 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5359 return r;
5360 }
5361
5362 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5363 if (r) {
5364 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5365 return r;
5366 }
5367
5368 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5369 if (r) {
5370 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5371 return r;
5372 }
5373
8c5fd7ef
AD		 5374 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5375 if (r) {
5376 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5377 return r;
5378 }
5379
5380 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5381 if (r) {
5382 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5383 return r;
5384 }
5385
1df0d523
AD		 5386 if (rdev->has_uvd) {
5387 r = rv770_uvd_resume(rdev);
5388 if (!r) {
5389 r = radeon_fence_driver_start_ring(rdev,
5390 R600_RING_TYPE_UVD_INDEX);
5391 if (r)
5392 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5393 }
f2ba57b5 5394 if (r)
1df0d523 5395 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
f2ba57b5 5396 }
f2ba57b5 5397
9b136d51 5398 /* Enable IRQ */
e49f3959
AH		 5399 if (!rdev->irq.installed) {
5400 r = radeon_irq_kms_init(rdev);
5401 if (r)
5402 return r;
5403 }
5404
9b136d51
AD		 5405 r = si_irq_init(rdev);
5406 if (r) {
5407 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5408 radeon_irq_kms_fini(rdev);
5409 return r;
5410 }
5411 si_irq_set(rdev);
5412
5413 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5414 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5415 CP_RB0_RPTR, CP_RB0_WPTR,
5416 0, 0xfffff, RADEON_CP_PACKET2);
5417 if (r)
5418 return r;
5419
5420 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5421 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
5422 CP_RB1_RPTR, CP_RB1_WPTR,
5423 0, 0xfffff, RADEON_CP_PACKET2);
5424 if (r)
5425 return r;
5426
5427 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5428 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
5429 CP_RB2_RPTR, CP_RB2_WPTR,
5430 0, 0xfffff, RADEON_CP_PACKET2);
5431 if (r)
5432 return r;
5433
8c5fd7ef
AD		 5434 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5435 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5436 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
5437 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
5438 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5439 if (r)
5440 return r;
5441
5442 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5443 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
5444 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
5445 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
5446 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5447 if (r)
5448 return r;
5449
9b136d51
AD		 5450 r = si_cp_load_microcode(rdev);
5451 if (r)
5452 return r;
5453 r = si_cp_resume(rdev);
5454 if (r)
5455 return r;
5456
8c5fd7ef
AD		 5457 r = cayman_dma_resume(rdev);
5458 if (r)
5459 return r;
5460
1df0d523
AD		 5461 if (rdev->has_uvd) {
5462 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5463 if (ring->ring_size) {
5464 r = radeon_ring_init(rdev, ring, ring->ring_size,
5465 R600_WB_UVD_RPTR_OFFSET,
5466 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5467 0, 0xfffff, RADEON_CP_PACKET2);
5468 if (!r)
5469 r = r600_uvd_init(rdev);
5470 if (r)
5471 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
5472 }
f2ba57b5
CK		 5473 }
5474
2898c348
CK		 5475 r = radeon_ib_pool_init(rdev);
5476 if (r) {
5477 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 5478 return r;
2898c348 5479 }
9b136d51 5480
c6105f24
CK		 5481 r = radeon_vm_manager_init(rdev);
5482 if (r) {
5483 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 5484 return r;
c6105f24 5485 }
9b136d51
AD		 5486
5487 return 0;
5488}
5489
5490int si_resume(struct radeon_device *rdev)
5491{
5492 int r;
5493
5494 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5495 * posting will perform necessary task to bring back GPU into good
5496 * shape.
5497 */
5498 /* post card */
5499 atom_asic_init(rdev->mode_info.atom_context);
5500
205996c0
AD		 5501 /* init golden registers */
5502 si_init_golden_registers(rdev);
5503
9b136d51
AD		 5504 rdev->accel_working = true;
5505 r = si_startup(rdev);
5506 if (r) {
5507 DRM_ERROR("si startup failed on resume\n");
5508 rdev->accel_working = false;
5509 return r;
5510 }
5511
5512 return r;
5513
5514}
5515
5516int si_suspend(struct radeon_device *rdev)
5517{
fa3daf9a 5518 radeon_vm_manager_fini(rdev);
9b136d51 5519 si_cp_enable(rdev, false);
8c5fd7ef 5520 cayman_dma_stop(rdev);
1df0d523
AD		 5521 if (rdev->has_uvd) {
5522 r600_uvd_rbc_stop(rdev);
5523 radeon_uvd_suspend(rdev);
5524 }
9b136d51
AD		 5525 si_irq_suspend(rdev);
5526 radeon_wb_disable(rdev);
5527 si_pcie_gart_disable(rdev);
5528 return 0;
5529}
5530
5531/* Plan is to move initialization in that function and use
5532 * helper function so that radeon_device_init pretty much
5533 * do nothing more than calling asic specific function. This
5534 * should also allow to remove a bunch of callback function
5535 * like vram_info.
5536 */
5537int si_init(struct radeon_device *rdev)
5538{
5539 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5540 int r;
5541
9b136d51
AD		 5542 /* Read BIOS */
5543 if (!radeon_get_bios(rdev)) {
5544 if (ASIC_IS_AVIVO(rdev))
5545 return -EINVAL;
5546 }
5547 /* Must be an ATOMBIOS */
5548 if (!rdev->is_atom_bios) {
5549 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
5550 return -EINVAL;
5551 }
5552 r = radeon_atombios_init(rdev);
5553 if (r)
5554 return r;
5555
5556 /* Post card if necessary */
5557 if (!radeon_card_posted(rdev)) {
5558 if (!rdev->bios) {
5559 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5560 return -EINVAL;
5561 }
5562 DRM_INFO("GPU not posted. posting now...\n");
5563 atom_asic_init(rdev->mode_info.atom_context);
5564 }
205996c0
AD		 5565 /* init golden registers */
5566 si_init_golden_registers(rdev);
9b136d51
AD		 5567 /* Initialize scratch registers */
5568 si_scratch_init(rdev);
5569 /* Initialize surface registers */
5570 radeon_surface_init(rdev);
5571 /* Initialize clocks */
5572 radeon_get_clock_info(rdev->ddev);
5573
5574 /* Fence driver */
5575 r = radeon_fence_driver_init(rdev);
5576 if (r)
5577 return r;
5578
5579 /* initialize memory controller */
5580 r = si_mc_init(rdev);
5581 if (r)
5582 return r;
5583 /* Memory manager */
5584 r = radeon_bo_init(rdev);
5585 if (r)
5586 return r;
5587
9b136d51
AD		 5588 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5589 ring->ring_obj = NULL;
5590 r600_ring_init(rdev, ring, 1024 * 1024);
5591
5592 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5593 ring->ring_obj = NULL;
5594 r600_ring_init(rdev, ring, 1024 * 1024);
5595
5596 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5597 ring->ring_obj = NULL;
5598 r600_ring_init(rdev, ring, 1024 * 1024);
5599
8c5fd7ef
AD		 5600 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5601 ring->ring_obj = NULL;
5602 r600_ring_init(rdev, ring, 64 * 1024);
5603
5604 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5605 ring->ring_obj = NULL;
5606 r600_ring_init(rdev, ring, 64 * 1024);
5607
1df0d523
AD		 5608 if (rdev->has_uvd) {
5609 r = radeon_uvd_init(rdev);
5610 if (!r) {
5611 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5612 ring->ring_obj = NULL;
5613 r600_ring_init(rdev, ring, 4096);
5614 }
f2ba57b5
CK		 5615 }
5616
9b136d51
AD		 5617 rdev->ih.ring_obj = NULL;
5618 r600_ih_ring_init(rdev, 64 * 1024);
5619
5620 r = r600_pcie_gart_init(rdev);
5621 if (r)
5622 return r;
5623
9b136d51 5624 rdev->accel_working = true;
9b136d51
AD		 5625 r = si_startup(rdev);
5626 if (r) {
5627 dev_err(rdev->dev, "disabling GPU acceleration\n");
5628 si_cp_fini(rdev);
8c5fd7ef 5629 cayman_dma_fini(rdev);
9b136d51
AD		 5630 si_irq_fini(rdev);
5631 si_rlc_fini(rdev);
5632 radeon_wb_fini(rdev);
2898c348 5633 radeon_ib_pool_fini(rdev);
9b136d51
AD		 5634 radeon_vm_manager_fini(rdev);
5635 radeon_irq_kms_fini(rdev);
5636 si_pcie_gart_fini(rdev);
5637 rdev->accel_working = false;
5638 }
5639
5640 /* Don't start up if the MC ucode is missing.
5641 * The default clocks and voltages before the MC ucode
5642 * is loaded are not suffient for advanced operations.
5643 */
5644 if (!rdev->mc_fw) {
5645 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5646 return -EINVAL;
5647 }
5648
5649 return 0;
5650}
5651
5652void si_fini(struct radeon_device *rdev)
5653{
9b136d51 5654 si_cp_fini(rdev);
8c5fd7ef 5655 cayman_dma_fini(rdev);
9b136d51
AD		 5656 si_irq_fini(rdev);
5657 si_rlc_fini(rdev);
5658 radeon_wb_fini(rdev);
5659 radeon_vm_manager_fini(rdev);
2898c348 5660 radeon_ib_pool_fini(rdev);
9b136d51 5661 radeon_irq_kms_fini(rdev);
1df0d523
AD		 5662 if (rdev->has_uvd)
5663 radeon_uvd_fini(rdev);
9b136d51
AD		 5664 si_pcie_gart_fini(rdev);
5665 r600_vram_scratch_fini(rdev);
5666 radeon_gem_fini(rdev);
9b136d51
AD		 5667 radeon_fence_driver_fini(rdev);
5668 radeon_bo_fini(rdev);
5669 radeon_atombios_fini(rdev);
5670 kfree(rdev->bios);
5671 rdev->bios = NULL;
5672}
5673
6759a0a7 5674/**
d0418894 5675 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 5676 *
5677 * @rdev: radeon_device pointer
5678 *
5679 * Fetches a GPU clock counter snapshot (SI).
5680 * Returns the 64 bit clock counter snapshot.
5681 */
d0418894 5682uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 5683{
5684 uint64_t clock;
5685
5686 mutex_lock(&rdev->gpu_clock_mutex);
5687 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
5688 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
5689 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
5690 mutex_unlock(&rdev->gpu_clock_mutex);
5691 return clock;
5692}
2539eb02 5693
2539eb02
CK		 5694int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
5695{
facd112d 5696 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 5697 int r;
5698
4ed10835
CK		 5699 /* bypass vclk and dclk with bclk */
5700 WREG32_P(CG_UPLL_FUNC_CNTL_2,
5701 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
5702 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
5703
5704 /* put PLL in bypass mode */
5705 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
5706
5707 if (!vclk || !dclk) {
5708 /* keep the Bypass mode, put PLL to sleep */
5709 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
5710 return 0;
5711 }
5712
facd112d
CK		 5713 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
5714 16384, 0x03FFFFFF, 0, 128, 5,
5715 &fb_div, &vclk_div, &dclk_div);
5716 if (r)
5717 return r;
2539eb02
CK		 5718
5719 /* set RESET_ANTI_MUX to 0 */
5720 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
5721
5722 /* set VCO_MODE to 1 */
5723 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
5724
5725 /* toggle UPLL_SLEEP to 1 then back to 0 */
5726 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
5727 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
5728
5729 /* deassert UPLL_RESET */
5730 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
5731
5732 mdelay(1);
5733
facd112d 5734 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 5735 if (r)
5736 return r;
5737
5738 /* assert UPLL_RESET again */
5739 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
5740
5741 /* disable spread spectrum. */
5742 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
5743
5744 /* set feedback divider */
facd112d 5745 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 5746
5747 /* set ref divider to 0 */
5748 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
5749
facd112d 5750 if (fb_div < 307200)
2539eb02
CK		 5751 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
5752 else
5753 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
5754
5755 /* set PDIV_A and PDIV_B */
5756 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 5757 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 5758 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
5759
5760 /* give the PLL some time to settle */
5761 mdelay(15);
5762
5763 /* deassert PLL_RESET */
5764 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
5765
5766 mdelay(15);
5767
5768 /* switch from bypass mode to normal mode */
5769 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
5770
facd112d 5771 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 5772 if (r)
5773 return r;
5774
5775 /* switch VCLK and DCLK selection */
5776 WREG32_P(CG_UPLL_FUNC_CNTL_2,
5777 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
5778 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
5779
5780 mdelay(100);
5781
5782 return 0;
5783}
Linux 6.x block layer and io_uring tree(s)