projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: add some additional berlin pci ids
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
0f0de06c 24#include <linux/firmware.h>
0f0de06c
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
760285e7 27#include <drm/drmP.h>
43b3cd99
AD		 28#include "radeon.h"
29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
43b3cd99
AD		 31#include "sid.h"
32#include "atom.h"
48c0c902 33#include "si_blit_shaders.h"
bd8cd539 34#include "clearstate_si.h"
a0ceada6 35#include "radeon_ucode.h"
43b3cd99 36
0f0de06c
AD		 37
38MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
42MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
a9e61410 43MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
0f0de06c
AD		 44MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
45MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
46MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
47MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
48MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
a9e61410 49MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
0f0de06c
AD		 50MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
51MODULE_FIRMWARE("radeon/VERDE_me.bin");
52MODULE_FIRMWARE("radeon/VERDE_ce.bin");
53MODULE_FIRMWARE("radeon/VERDE_mc.bin");
54MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
a9e61410 55MODULE_FIRMWARE("radeon/VERDE_smc.bin");
bcc7f5d2
AD		 56MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
57MODULE_FIRMWARE("radeon/OLAND_me.bin");
58MODULE_FIRMWARE("radeon/OLAND_ce.bin");
59MODULE_FIRMWARE("radeon/OLAND_mc.bin");
60MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
a9e61410 61MODULE_FIRMWARE("radeon/OLAND_smc.bin");
c04c00b4
AD		 62MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
63MODULE_FIRMWARE("radeon/HAINAN_me.bin");
64MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
65MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
66MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
a9e61410 67MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
0f0de06c 68
b9d305df 69static void si_pcie_gen3_enable(struct radeon_device *rdev);
e0bcf165 70static void si_program_aspm(struct radeon_device *rdev);
1fd11777
AD		 71extern void sumo_rlc_fini(struct radeon_device *rdev);
72extern int sumo_rlc_init(struct radeon_device *rdev);
25a857fb
AD		 73extern int r600_ih_ring_alloc(struct radeon_device *rdev);
74extern void r600_ih_ring_fini(struct radeon_device *rdev);
0a96d72b 75extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
c476dde2
AD		 76extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
77extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
ca7db22b 78extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
1c534671 79extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
014bb209 80extern bool evergreen_is_display_hung(struct radeon_device *rdev);
2483b4ea
CK		 81extern void si_dma_vm_set_page(struct radeon_device *rdev,
82 struct radeon_ib *ib,
83 uint64_t pe,
84 uint64_t addr, unsigned count,
85 uint32_t incr, uint32_t flags);
811e4d58
AD		 86static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
87 bool enable);
0a96d72b 88
6d8cf000
AD		 89static const u32 verde_rlc_save_restore_register_list[] =
90{
91 (0x8000 << 16) | (0x98f4 >> 2),
92 0x00000000,
93 (0x8040 << 16) | (0x98f4 >> 2),
94 0x00000000,
95 (0x8000 << 16) | (0xe80 >> 2),
96 0x00000000,
97 (0x8040 << 16) | (0xe80 >> 2),
98 0x00000000,
99 (0x8000 << 16) | (0x89bc >> 2),
100 0x00000000,
101 (0x8040 << 16) | (0x89bc >> 2),
102 0x00000000,
103 (0x8000 << 16) | (0x8c1c >> 2),
104 0x00000000,
105 (0x8040 << 16) | (0x8c1c >> 2),
106 0x00000000,
107 (0x9c00 << 16) | (0x98f0 >> 2),
108 0x00000000,
109 (0x9c00 << 16) | (0xe7c >> 2),
110 0x00000000,
111 (0x8000 << 16) | (0x9148 >> 2),
112 0x00000000,
113 (0x8040 << 16) | (0x9148 >> 2),
114 0x00000000,
115 (0x9c00 << 16) | (0x9150 >> 2),
116 0x00000000,
117 (0x9c00 << 16) | (0x897c >> 2),
118 0x00000000,
119 (0x9c00 << 16) | (0x8d8c >> 2),
120 0x00000000,
121 (0x9c00 << 16) | (0xac54 >> 2),
122 0X00000000,
123 0x3,
124 (0x9c00 << 16) | (0x98f8 >> 2),
125 0x00000000,
126 (0x9c00 << 16) | (0x9910 >> 2),
127 0x00000000,
128 (0x9c00 << 16) | (0x9914 >> 2),
129 0x00000000,
130 (0x9c00 << 16) | (0x9918 >> 2),
131 0x00000000,
132 (0x9c00 << 16) | (0x991c >> 2),
133 0x00000000,
134 (0x9c00 << 16) | (0x9920 >> 2),
135 0x00000000,
136 (0x9c00 << 16) | (0x9924 >> 2),
137 0x00000000,
138 (0x9c00 << 16) | (0x9928 >> 2),
139 0x00000000,
140 (0x9c00 << 16) | (0x992c >> 2),
141 0x00000000,
142 (0x9c00 << 16) | (0x9930 >> 2),
143 0x00000000,
144 (0x9c00 << 16) | (0x9934 >> 2),
145 0x00000000,
146 (0x9c00 << 16) | (0x9938 >> 2),
147 0x00000000,
148 (0x9c00 << 16) | (0x993c >> 2),
149 0x00000000,
150 (0x9c00 << 16) | (0x9940 >> 2),
151 0x00000000,
152 (0x9c00 << 16) | (0x9944 >> 2),
153 0x00000000,
154 (0x9c00 << 16) | (0x9948 >> 2),
155 0x00000000,
156 (0x9c00 << 16) | (0x994c >> 2),
157 0x00000000,
158 (0x9c00 << 16) | (0x9950 >> 2),
159 0x00000000,
160 (0x9c00 << 16) | (0x9954 >> 2),
161 0x00000000,
162 (0x9c00 << 16) | (0x9958 >> 2),
163 0x00000000,
164 (0x9c00 << 16) | (0x995c >> 2),
165 0x00000000,
166 (0x9c00 << 16) | (0x9960 >> 2),
167 0x00000000,
168 (0x9c00 << 16) | (0x9964 >> 2),
169 0x00000000,
170 (0x9c00 << 16) | (0x9968 >> 2),
171 0x00000000,
172 (0x9c00 << 16) | (0x996c >> 2),
173 0x00000000,
174 (0x9c00 << 16) | (0x9970 >> 2),
175 0x00000000,
176 (0x9c00 << 16) | (0x9974 >> 2),
177 0x00000000,
178 (0x9c00 << 16) | (0x9978 >> 2),
179 0x00000000,
180 (0x9c00 << 16) | (0x997c >> 2),
181 0x00000000,
182 (0x9c00 << 16) | (0x9980 >> 2),
183 0x00000000,
184 (0x9c00 << 16) | (0x9984 >> 2),
185 0x00000000,
186 (0x9c00 << 16) | (0x9988 >> 2),
187 0x00000000,
188 (0x9c00 << 16) | (0x998c >> 2),
189 0x00000000,
190 (0x9c00 << 16) | (0x8c00 >> 2),
191 0x00000000,
192 (0x9c00 << 16) | (0x8c14 >> 2),
193 0x00000000,
194 (0x9c00 << 16) | (0x8c04 >> 2),
195 0x00000000,
196 (0x9c00 << 16) | (0x8c08 >> 2),
197 0x00000000,
198 (0x8000 << 16) | (0x9b7c >> 2),
199 0x00000000,
200 (0x8040 << 16) | (0x9b7c >> 2),
201 0x00000000,
202 (0x8000 << 16) | (0xe84 >> 2),
203 0x00000000,
204 (0x8040 << 16) | (0xe84 >> 2),
205 0x00000000,
206 (0x8000 << 16) | (0x89c0 >> 2),
207 0x00000000,
208 (0x8040 << 16) | (0x89c0 >> 2),
209 0x00000000,
210 (0x8000 << 16) | (0x914c >> 2),
211 0x00000000,
212 (0x8040 << 16) | (0x914c >> 2),
213 0x00000000,
214 (0x8000 << 16) | (0x8c20 >> 2),
215 0x00000000,
216 (0x8040 << 16) | (0x8c20 >> 2),
217 0x00000000,
218 (0x8000 << 16) | (0x9354 >> 2),
219 0x00000000,
220 (0x8040 << 16) | (0x9354 >> 2),
221 0x00000000,
222 (0x9c00 << 16) | (0x9060 >> 2),
223 0x00000000,
224 (0x9c00 << 16) | (0x9364 >> 2),
225 0x00000000,
226 (0x9c00 << 16) | (0x9100 >> 2),
227 0x00000000,
228 (0x9c00 << 16) | (0x913c >> 2),
229 0x00000000,
230 (0x8000 << 16) | (0x90e0 >> 2),
231 0x00000000,
232 (0x8000 << 16) | (0x90e4 >> 2),
233 0x00000000,
234 (0x8000 << 16) | (0x90e8 >> 2),
235 0x00000000,
236 (0x8040 << 16) | (0x90e0 >> 2),
237 0x00000000,
238 (0x8040 << 16) | (0x90e4 >> 2),
239 0x00000000,
240 (0x8040 << 16) | (0x90e8 >> 2),
241 0x00000000,
242 (0x9c00 << 16) | (0x8bcc >> 2),
243 0x00000000,
244 (0x9c00 << 16) | (0x8b24 >> 2),
245 0x00000000,
246 (0x9c00 << 16) | (0x88c4 >> 2),
247 0x00000000,
248 (0x9c00 << 16) | (0x8e50 >> 2),
249 0x00000000,
250 (0x9c00 << 16) | (0x8c0c >> 2),
251 0x00000000,
252 (0x9c00 << 16) | (0x8e58 >> 2),
253 0x00000000,
254 (0x9c00 << 16) | (0x8e5c >> 2),
255 0x00000000,
256 (0x9c00 << 16) | (0x9508 >> 2),
257 0x00000000,
258 (0x9c00 << 16) | (0x950c >> 2),
259 0x00000000,
260 (0x9c00 << 16) | (0x9494 >> 2),
261 0x00000000,
262 (0x9c00 << 16) | (0xac0c >> 2),
263 0x00000000,
264 (0x9c00 << 16) | (0xac10 >> 2),
265 0x00000000,
266 (0x9c00 << 16) | (0xac14 >> 2),
267 0x00000000,
268 (0x9c00 << 16) | (0xae00 >> 2),
269 0x00000000,
270 (0x9c00 << 16) | (0xac08 >> 2),
271 0x00000000,
272 (0x9c00 << 16) | (0x88d4 >> 2),
273 0x00000000,
274 (0x9c00 << 16) | (0x88c8 >> 2),
275 0x00000000,
276 (0x9c00 << 16) | (0x88cc >> 2),
277 0x00000000,
278 (0x9c00 << 16) | (0x89b0 >> 2),
279 0x00000000,
280 (0x9c00 << 16) | (0x8b10 >> 2),
281 0x00000000,
282 (0x9c00 << 16) | (0x8a14 >> 2),
283 0x00000000,
284 (0x9c00 << 16) | (0x9830 >> 2),
285 0x00000000,
286 (0x9c00 << 16) | (0x9834 >> 2),
287 0x00000000,
288 (0x9c00 << 16) | (0x9838 >> 2),
289 0x00000000,
290 (0x9c00 << 16) | (0x9a10 >> 2),
291 0x00000000,
292 (0x8000 << 16) | (0x9870 >> 2),
293 0x00000000,
294 (0x8000 << 16) | (0x9874 >> 2),
295 0x00000000,
296 (0x8001 << 16) | (0x9870 >> 2),
297 0x00000000,
298 (0x8001 << 16) | (0x9874 >> 2),
299 0x00000000,
300 (0x8040 << 16) | (0x9870 >> 2),
301 0x00000000,
302 (0x8040 << 16) | (0x9874 >> 2),
303 0x00000000,
304 (0x8041 << 16) | (0x9870 >> 2),
305 0x00000000,
306 (0x8041 << 16) | (0x9874 >> 2),
307 0x00000000,
308 0x00000000
309};
310
205996c0
AD		 311static const u32 tahiti_golden_rlc_registers[] =
312{
313 0xc424, 0xffffffff, 0x00601005,
314 0xc47c, 0xffffffff, 0x10104040,
315 0xc488, 0xffffffff, 0x0100000a,
316 0xc314, 0xffffffff, 0x00000800,
317 0xc30c, 0xffffffff, 0x800000f4,
318 0xf4a8, 0xffffffff, 0x00000000
319};
320
321static const u32 tahiti_golden_registers[] =
322{
323 0x9a10, 0x00010000, 0x00018208,
324 0x9830, 0xffffffff, 0x00000000,
325 0x9834, 0xf00fffff, 0x00000400,
326 0x9838, 0x0002021c, 0x00020200,
327 0xc78, 0x00000080, 0x00000000,
328 0xd030, 0x000300c0, 0x00800040,
329 0xd830, 0x000300c0, 0x00800040,
330 0x5bb0, 0x000000f0, 0x00000070,
331 0x5bc0, 0x00200000, 0x50100000,
332 0x7030, 0x31000311, 0x00000011,
333 0x277c, 0x00000003, 0x000007ff,
334 0x240c, 0x000007ff, 0x00000000,
335 0x8a14, 0xf000001f, 0x00000007,
336 0x8b24, 0xffffffff, 0x00ffffff,
337 0x8b10, 0x0000ff0f, 0x00000000,
338 0x28a4c, 0x07ffffff, 0x4e000000,
339 0x28350, 0x3f3f3fff, 0x2a00126a,
340 0x30, 0x000000ff, 0x0040,
341 0x34, 0x00000040, 0x00004040,
342 0x9100, 0x07ffffff, 0x03000000,
343 0x8e88, 0x01ff1f3f, 0x00000000,
344 0x8e84, 0x01ff1f3f, 0x00000000,
345 0x9060, 0x0000007f, 0x00000020,
346 0x9508, 0x00010000, 0x00010000,
347 0xac14, 0x00000200, 0x000002fb,
348 0xac10, 0xffffffff, 0x0000543b,
349 0xac0c, 0xffffffff, 0xa9210876,
350 0x88d0, 0xffffffff, 0x000fff40,
351 0x88d4, 0x0000001f, 0x00000010,
352 0x1410, 0x20000000, 0x20fffed8,
353 0x15c0, 0x000c0fc0, 0x000c0400
354};
355
356static const u32 tahiti_golden_registers2[] =
357{
358 0xc64, 0x00000001, 0x00000001
359};
360
361static const u32 pitcairn_golden_rlc_registers[] =
362{
363 0xc424, 0xffffffff, 0x00601004,
364 0xc47c, 0xffffffff, 0x10102020,
365 0xc488, 0xffffffff, 0x01000020,
366 0xc314, 0xffffffff, 0x00000800,
367 0xc30c, 0xffffffff, 0x800000a4
368};
369
370static const u32 pitcairn_golden_registers[] =
371{
372 0x9a10, 0x00010000, 0x00018208,
373 0x9830, 0xffffffff, 0x00000000,
374 0x9834, 0xf00fffff, 0x00000400,
375 0x9838, 0x0002021c, 0x00020200,
376 0xc78, 0x00000080, 0x00000000,
377 0xd030, 0x000300c0, 0x00800040,
378 0xd830, 0x000300c0, 0x00800040,
379 0x5bb0, 0x000000f0, 0x00000070,
380 0x5bc0, 0x00200000, 0x50100000,
381 0x7030, 0x31000311, 0x00000011,
382 0x2ae4, 0x00073ffe, 0x000022a2,
383 0x240c, 0x000007ff, 0x00000000,
384 0x8a14, 0xf000001f, 0x00000007,
385 0x8b24, 0xffffffff, 0x00ffffff,
386 0x8b10, 0x0000ff0f, 0x00000000,
387 0x28a4c, 0x07ffffff, 0x4e000000,
388 0x28350, 0x3f3f3fff, 0x2a00126a,
389 0x30, 0x000000ff, 0x0040,
390 0x34, 0x00000040, 0x00004040,
391 0x9100, 0x07ffffff, 0x03000000,
392 0x9060, 0x0000007f, 0x00000020,
393 0x9508, 0x00010000, 0x00010000,
394 0xac14, 0x000003ff, 0x000000f7,
395 0xac10, 0xffffffff, 0x00000000,
396 0xac0c, 0xffffffff, 0x32761054,
397 0x88d4, 0x0000001f, 0x00000010,
398 0x15c0, 0x000c0fc0, 0x000c0400
399};
400
401static const u32 verde_golden_rlc_registers[] =
402{
403 0xc424, 0xffffffff, 0x033f1005,
404 0xc47c, 0xffffffff, 0x10808020,
405 0xc488, 0xffffffff, 0x00800008,
406 0xc314, 0xffffffff, 0x00001000,
407 0xc30c, 0xffffffff, 0x80010014
408};
409
410static const u32 verde_golden_registers[] =
411{
412 0x9a10, 0x00010000, 0x00018208,
413 0x9830, 0xffffffff, 0x00000000,
414 0x9834, 0xf00fffff, 0x00000400,
415 0x9838, 0x0002021c, 0x00020200,
416 0xc78, 0x00000080, 0x00000000,
417 0xd030, 0x000300c0, 0x00800040,
418 0xd030, 0x000300c0, 0x00800040,
419 0xd830, 0x000300c0, 0x00800040,
420 0xd830, 0x000300c0, 0x00800040,
421 0x5bb0, 0x000000f0, 0x00000070,
422 0x5bc0, 0x00200000, 0x50100000,
423 0x7030, 0x31000311, 0x00000011,
424 0x2ae4, 0x00073ffe, 0x000022a2,
425 0x2ae4, 0x00073ffe, 0x000022a2,
426 0x2ae4, 0x00073ffe, 0x000022a2,
427 0x240c, 0x000007ff, 0x00000000,
428 0x240c, 0x000007ff, 0x00000000,
429 0x240c, 0x000007ff, 0x00000000,
430 0x8a14, 0xf000001f, 0x00000007,
431 0x8a14, 0xf000001f, 0x00000007,
432 0x8a14, 0xf000001f, 0x00000007,
433 0x8b24, 0xffffffff, 0x00ffffff,
434 0x8b10, 0x0000ff0f, 0x00000000,
435 0x28a4c, 0x07ffffff, 0x4e000000,
436 0x28350, 0x3f3f3fff, 0x0000124a,
437 0x28350, 0x3f3f3fff, 0x0000124a,
438 0x28350, 0x3f3f3fff, 0x0000124a,
439 0x30, 0x000000ff, 0x0040,
440 0x34, 0x00000040, 0x00004040,
441 0x9100, 0x07ffffff, 0x03000000,
442 0x9100, 0x07ffffff, 0x03000000,
443 0x8e88, 0x01ff1f3f, 0x00000000,
444 0x8e88, 0x01ff1f3f, 0x00000000,
445 0x8e88, 0x01ff1f3f, 0x00000000,
446 0x8e84, 0x01ff1f3f, 0x00000000,
447 0x8e84, 0x01ff1f3f, 0x00000000,
448 0x8e84, 0x01ff1f3f, 0x00000000,
449 0x9060, 0x0000007f, 0x00000020,
450 0x9508, 0x00010000, 0x00010000,
451 0xac14, 0x000003ff, 0x00000003,
452 0xac14, 0x000003ff, 0x00000003,
453 0xac14, 0x000003ff, 0x00000003,
454 0xac10, 0xffffffff, 0x00000000,
455 0xac10, 0xffffffff, 0x00000000,
456 0xac10, 0xffffffff, 0x00000000,
457 0xac0c, 0xffffffff, 0x00001032,
458 0xac0c, 0xffffffff, 0x00001032,
459 0xac0c, 0xffffffff, 0x00001032,
460 0x88d4, 0x0000001f, 0x00000010,
461 0x88d4, 0x0000001f, 0x00000010,
462 0x88d4, 0x0000001f, 0x00000010,
463 0x15c0, 0x000c0fc0, 0x000c0400
464};
465
466static const u32 oland_golden_rlc_registers[] =
467{
468 0xc424, 0xffffffff, 0x00601005,
469 0xc47c, 0xffffffff, 0x10104040,
470 0xc488, 0xffffffff, 0x0100000a,
471 0xc314, 0xffffffff, 0x00000800,
472 0xc30c, 0xffffffff, 0x800000f4
473};
474
475static const u32 oland_golden_registers[] =
476{
477 0x9a10, 0x00010000, 0x00018208,
478 0x9830, 0xffffffff, 0x00000000,
479 0x9834, 0xf00fffff, 0x00000400,
480 0x9838, 0x0002021c, 0x00020200,
481 0xc78, 0x00000080, 0x00000000,
482 0xd030, 0x000300c0, 0x00800040,
483 0xd830, 0x000300c0, 0x00800040,
484 0x5bb0, 0x000000f0, 0x00000070,
485 0x5bc0, 0x00200000, 0x50100000,
486 0x7030, 0x31000311, 0x00000011,
487 0x2ae4, 0x00073ffe, 0x000022a2,
488 0x240c, 0x000007ff, 0x00000000,
489 0x8a14, 0xf000001f, 0x00000007,
490 0x8b24, 0xffffffff, 0x00ffffff,
491 0x8b10, 0x0000ff0f, 0x00000000,
492 0x28a4c, 0x07ffffff, 0x4e000000,
493 0x28350, 0x3f3f3fff, 0x00000082,
494 0x30, 0x000000ff, 0x0040,
495 0x34, 0x00000040, 0x00004040,
496 0x9100, 0x07ffffff, 0x03000000,
497 0x9060, 0x0000007f, 0x00000020,
498 0x9508, 0x00010000, 0x00010000,
499 0xac14, 0x000003ff, 0x000000f3,
500 0xac10, 0xffffffff, 0x00000000,
501 0xac0c, 0xffffffff, 0x00003210,
502 0x88d4, 0x0000001f, 0x00000010,
503 0x15c0, 0x000c0fc0, 0x000c0400
504};
505
fffbdda4
AD		 506static const u32 hainan_golden_registers[] =
507{
508 0x9a10, 0x00010000, 0x00018208,
509 0x9830, 0xffffffff, 0x00000000,
510 0x9834, 0xf00fffff, 0x00000400,
511 0x9838, 0x0002021c, 0x00020200,
512 0xd0c0, 0xff000fff, 0x00000100,
513 0xd030, 0x000300c0, 0x00800040,
514 0xd8c0, 0xff000fff, 0x00000100,
515 0xd830, 0x000300c0, 0x00800040,
516 0x2ae4, 0x00073ffe, 0x000022a2,
517 0x240c, 0x000007ff, 0x00000000,
518 0x8a14, 0xf000001f, 0x00000007,
519 0x8b24, 0xffffffff, 0x00ffffff,
520 0x8b10, 0x0000ff0f, 0x00000000,
521 0x28a4c, 0x07ffffff, 0x4e000000,
522 0x28350, 0x3f3f3fff, 0x00000000,
523 0x30, 0x000000ff, 0x0040,
524 0x34, 0x00000040, 0x00004040,
525 0x9100, 0x03e00000, 0x03600000,
526 0x9060, 0x0000007f, 0x00000020,
527 0x9508, 0x00010000, 0x00010000,
528 0xac14, 0x000003ff, 0x000000f1,
529 0xac10, 0xffffffff, 0x00000000,
530 0xac0c, 0xffffffff, 0x00003210,
531 0x88d4, 0x0000001f, 0x00000010,
532 0x15c0, 0x000c0fc0, 0x000c0400
533};
534
535static const u32 hainan_golden_registers2[] =
536{
537 0x98f8, 0xffffffff, 0x02010001
538};
539
205996c0
AD		 540static const u32 tahiti_mgcg_cgcg_init[] =
541{
542 0xc400, 0xffffffff, 0xfffffffc,
543 0x802c, 0xffffffff, 0xe0000000,
544 0x9a60, 0xffffffff, 0x00000100,
545 0x92a4, 0xffffffff, 0x00000100,
546 0xc164, 0xffffffff, 0x00000100,
547 0x9774, 0xffffffff, 0x00000100,
548 0x8984, 0xffffffff, 0x06000100,
549 0x8a18, 0xffffffff, 0x00000100,
550 0x92a0, 0xffffffff, 0x00000100,
551 0xc380, 0xffffffff, 0x00000100,
552 0x8b28, 0xffffffff, 0x00000100,
553 0x9144, 0xffffffff, 0x00000100,
554 0x8d88, 0xffffffff, 0x00000100,
555 0x8d8c, 0xffffffff, 0x00000100,
556 0x9030, 0xffffffff, 0x00000100,
557 0x9034, 0xffffffff, 0x00000100,
558 0x9038, 0xffffffff, 0x00000100,
559 0x903c, 0xffffffff, 0x00000100,
560 0xad80, 0xffffffff, 0x00000100,
561 0xac54, 0xffffffff, 0x00000100,
562 0x897c, 0xffffffff, 0x06000100,
563 0x9868, 0xffffffff, 0x00000100,
564 0x9510, 0xffffffff, 0x00000100,
565 0xaf04, 0xffffffff, 0x00000100,
566 0xae04, 0xffffffff, 0x00000100,
567 0x949c, 0xffffffff, 0x00000100,
568 0x802c, 0xffffffff, 0xe0000000,
569 0x9160, 0xffffffff, 0x00010000,
570 0x9164, 0xffffffff, 0x00030002,
571 0x9168, 0xffffffff, 0x00040007,
572 0x916c, 0xffffffff, 0x00060005,
573 0x9170, 0xffffffff, 0x00090008,
574 0x9174, 0xffffffff, 0x00020001,
575 0x9178, 0xffffffff, 0x00040003,
576 0x917c, 0xffffffff, 0x00000007,
577 0x9180, 0xffffffff, 0x00060005,
578 0x9184, 0xffffffff, 0x00090008,
579 0x9188, 0xffffffff, 0x00030002,
580 0x918c, 0xffffffff, 0x00050004,
581 0x9190, 0xffffffff, 0x00000008,
582 0x9194, 0xffffffff, 0x00070006,
583 0x9198, 0xffffffff, 0x000a0009,
584 0x919c, 0xffffffff, 0x00040003,
585 0x91a0, 0xffffffff, 0x00060005,
586 0x91a4, 0xffffffff, 0x00000009,
587 0x91a8, 0xffffffff, 0x00080007,
588 0x91ac, 0xffffffff, 0x000b000a,
589 0x91b0, 0xffffffff, 0x00050004,
590 0x91b4, 0xffffffff, 0x00070006,
591 0x91b8, 0xffffffff, 0x0008000b,
592 0x91bc, 0xffffffff, 0x000a0009,
593 0x91c0, 0xffffffff, 0x000d000c,
594 0x91c4, 0xffffffff, 0x00060005,
595 0x91c8, 0xffffffff, 0x00080007,
596 0x91cc, 0xffffffff, 0x0000000b,
597 0x91d0, 0xffffffff, 0x000a0009,
598 0x91d4, 0xffffffff, 0x000d000c,
599 0x91d8, 0xffffffff, 0x00070006,
600 0x91dc, 0xffffffff, 0x00090008,
601 0x91e0, 0xffffffff, 0x0000000c,
602 0x91e4, 0xffffffff, 0x000b000a,
603 0x91e8, 0xffffffff, 0x000e000d,
604 0x91ec, 0xffffffff, 0x00080007,
605 0x91f0, 0xffffffff, 0x000a0009,
606 0x91f4, 0xffffffff, 0x0000000d,
607 0x91f8, 0xffffffff, 0x000c000b,
608 0x91fc, 0xffffffff, 0x000f000e,
609 0x9200, 0xffffffff, 0x00090008,
610 0x9204, 0xffffffff, 0x000b000a,
611 0x9208, 0xffffffff, 0x000c000f,
612 0x920c, 0xffffffff, 0x000e000d,
613 0x9210, 0xffffffff, 0x00110010,
614 0x9214, 0xffffffff, 0x000a0009,
615 0x9218, 0xffffffff, 0x000c000b,
616 0x921c, 0xffffffff, 0x0000000f,
617 0x9220, 0xffffffff, 0x000e000d,
618 0x9224, 0xffffffff, 0x00110010,
619 0x9228, 0xffffffff, 0x000b000a,
620 0x922c, 0xffffffff, 0x000d000c,
621 0x9230, 0xffffffff, 0x00000010,
622 0x9234, 0xffffffff, 0x000f000e,
623 0x9238, 0xffffffff, 0x00120011,
624 0x923c, 0xffffffff, 0x000c000b,
625 0x9240, 0xffffffff, 0x000e000d,
626 0x9244, 0xffffffff, 0x00000011,
627 0x9248, 0xffffffff, 0x0010000f,
628 0x924c, 0xffffffff, 0x00130012,
629 0x9250, 0xffffffff, 0x000d000c,
630 0x9254, 0xffffffff, 0x000f000e,
631 0x9258, 0xffffffff, 0x00100013,
632 0x925c, 0xffffffff, 0x00120011,
633 0x9260, 0xffffffff, 0x00150014,
634 0x9264, 0xffffffff, 0x000e000d,
635 0x9268, 0xffffffff, 0x0010000f,
636 0x926c, 0xffffffff, 0x00000013,
637 0x9270, 0xffffffff, 0x00120011,
638 0x9274, 0xffffffff, 0x00150014,
639 0x9278, 0xffffffff, 0x000f000e,
640 0x927c, 0xffffffff, 0x00110010,
641 0x9280, 0xffffffff, 0x00000014,
642 0x9284, 0xffffffff, 0x00130012,
643 0x9288, 0xffffffff, 0x00160015,
644 0x928c, 0xffffffff, 0x0010000f,
645 0x9290, 0xffffffff, 0x00120011,
646 0x9294, 0xffffffff, 0x00000015,
647 0x9298, 0xffffffff, 0x00140013,
648 0x929c, 0xffffffff, 0x00170016,
649 0x9150, 0xffffffff, 0x96940200,
650 0x8708, 0xffffffff, 0x00900100,
651 0xc478, 0xffffffff, 0x00000080,
652 0xc404, 0xffffffff, 0x0020003f,
653 0x30, 0xffffffff, 0x0000001c,
654 0x34, 0x000f0000, 0x000f0000,
655 0x160c, 0xffffffff, 0x00000100,
656 0x1024, 0xffffffff, 0x00000100,
657 0x102c, 0x00000101, 0x00000000,
658 0x20a8, 0xffffffff, 0x00000104,
659 0x264c, 0x000c0000, 0x000c0000,
660 0x2648, 0x000c0000, 0x000c0000,
661 0x55e4, 0xff000fff, 0x00000100,
662 0x55e8, 0x00000001, 0x00000001,
663 0x2f50, 0x00000001, 0x00000001,
664 0x30cc, 0xc0000fff, 0x00000104,
665 0xc1e4, 0x00000001, 0x00000001,
666 0xd0c0, 0xfffffff0, 0x00000100,
667 0xd8c0, 0xfffffff0, 0x00000100
668};
669
670static const u32 pitcairn_mgcg_cgcg_init[] =
671{
672 0xc400, 0xffffffff, 0xfffffffc,
673 0x802c, 0xffffffff, 0xe0000000,
674 0x9a60, 0xffffffff, 0x00000100,
675 0x92a4, 0xffffffff, 0x00000100,
676 0xc164, 0xffffffff, 0x00000100,
677 0x9774, 0xffffffff, 0x00000100,
678 0x8984, 0xffffffff, 0x06000100,
679 0x8a18, 0xffffffff, 0x00000100,
680 0x92a0, 0xffffffff, 0x00000100,
681 0xc380, 0xffffffff, 0x00000100,
682 0x8b28, 0xffffffff, 0x00000100,
683 0x9144, 0xffffffff, 0x00000100,
684 0x8d88, 0xffffffff, 0x00000100,
685 0x8d8c, 0xffffffff, 0x00000100,
686 0x9030, 0xffffffff, 0x00000100,
687 0x9034, 0xffffffff, 0x00000100,
688 0x9038, 0xffffffff, 0x00000100,
689 0x903c, 0xffffffff, 0x00000100,
690 0xad80, 0xffffffff, 0x00000100,
691 0xac54, 0xffffffff, 0x00000100,
692 0x897c, 0xffffffff, 0x06000100,
693 0x9868, 0xffffffff, 0x00000100,
694 0x9510, 0xffffffff, 0x00000100,
695 0xaf04, 0xffffffff, 0x00000100,
696 0xae04, 0xffffffff, 0x00000100,
697 0x949c, 0xffffffff, 0x00000100,
698 0x802c, 0xffffffff, 0xe0000000,
699 0x9160, 0xffffffff, 0x00010000,
700 0x9164, 0xffffffff, 0x00030002,
701 0x9168, 0xffffffff, 0x00040007,
702 0x916c, 0xffffffff, 0x00060005,
703 0x9170, 0xffffffff, 0x00090008,
704 0x9174, 0xffffffff, 0x00020001,
705 0x9178, 0xffffffff, 0x00040003,
706 0x917c, 0xffffffff, 0x00000007,
707 0x9180, 0xffffffff, 0x00060005,
708 0x9184, 0xffffffff, 0x00090008,
709 0x9188, 0xffffffff, 0x00030002,
710 0x918c, 0xffffffff, 0x00050004,
711 0x9190, 0xffffffff, 0x00000008,
712 0x9194, 0xffffffff, 0x00070006,
713 0x9198, 0xffffffff, 0x000a0009,
714 0x919c, 0xffffffff, 0x00040003,
715 0x91a0, 0xffffffff, 0x00060005,
716 0x91a4, 0xffffffff, 0x00000009,
717 0x91a8, 0xffffffff, 0x00080007,
718 0x91ac, 0xffffffff, 0x000b000a,
719 0x91b0, 0xffffffff, 0x00050004,
720 0x91b4, 0xffffffff, 0x00070006,
721 0x91b8, 0xffffffff, 0x0008000b,
722 0x91bc, 0xffffffff, 0x000a0009,
723 0x91c0, 0xffffffff, 0x000d000c,
724 0x9200, 0xffffffff, 0x00090008,
725 0x9204, 0xffffffff, 0x000b000a,
726 0x9208, 0xffffffff, 0x000c000f,
727 0x920c, 0xffffffff, 0x000e000d,
728 0x9210, 0xffffffff, 0x00110010,
729 0x9214, 0xffffffff, 0x000a0009,
730 0x9218, 0xffffffff, 0x000c000b,
731 0x921c, 0xffffffff, 0x0000000f,
732 0x9220, 0xffffffff, 0x000e000d,
733 0x9224, 0xffffffff, 0x00110010,
734 0x9228, 0xffffffff, 0x000b000a,
735 0x922c, 0xffffffff, 0x000d000c,
736 0x9230, 0xffffffff, 0x00000010,
737 0x9234, 0xffffffff, 0x000f000e,
738 0x9238, 0xffffffff, 0x00120011,
739 0x923c, 0xffffffff, 0x000c000b,
740 0x9240, 0xffffffff, 0x000e000d,
741 0x9244, 0xffffffff, 0x00000011,
742 0x9248, 0xffffffff, 0x0010000f,
743 0x924c, 0xffffffff, 0x00130012,
744 0x9250, 0xffffffff, 0x000d000c,
745 0x9254, 0xffffffff, 0x000f000e,
746 0x9258, 0xffffffff, 0x00100013,
747 0x925c, 0xffffffff, 0x00120011,
748 0x9260, 0xffffffff, 0x00150014,
749 0x9150, 0xffffffff, 0x96940200,
750 0x8708, 0xffffffff, 0x00900100,
751 0xc478, 0xffffffff, 0x00000080,
752 0xc404, 0xffffffff, 0x0020003f,
753 0x30, 0xffffffff, 0x0000001c,
754 0x34, 0x000f0000, 0x000f0000,
755 0x160c, 0xffffffff, 0x00000100,
756 0x1024, 0xffffffff, 0x00000100,
757 0x102c, 0x00000101, 0x00000000,
758 0x20a8, 0xffffffff, 0x00000104,
759 0x55e4, 0xff000fff, 0x00000100,
760 0x55e8, 0x00000001, 0x00000001,
761 0x2f50, 0x00000001, 0x00000001,
762 0x30cc, 0xc0000fff, 0x00000104,
763 0xc1e4, 0x00000001, 0x00000001,
764 0xd0c0, 0xfffffff0, 0x00000100,
765 0xd8c0, 0xfffffff0, 0x00000100
766};
767
768static const u32 verde_mgcg_cgcg_init[] =
769{
770 0xc400, 0xffffffff, 0xfffffffc,
771 0x802c, 0xffffffff, 0xe0000000,
772 0x9a60, 0xffffffff, 0x00000100,
773 0x92a4, 0xffffffff, 0x00000100,
774 0xc164, 0xffffffff, 0x00000100,
775 0x9774, 0xffffffff, 0x00000100,
776 0x8984, 0xffffffff, 0x06000100,
777 0x8a18, 0xffffffff, 0x00000100,
778 0x92a0, 0xffffffff, 0x00000100,
779 0xc380, 0xffffffff, 0x00000100,
780 0x8b28, 0xffffffff, 0x00000100,
781 0x9144, 0xffffffff, 0x00000100,
782 0x8d88, 0xffffffff, 0x00000100,
783 0x8d8c, 0xffffffff, 0x00000100,
784 0x9030, 0xffffffff, 0x00000100,
785 0x9034, 0xffffffff, 0x00000100,
786 0x9038, 0xffffffff, 0x00000100,
787 0x903c, 0xffffffff, 0x00000100,
788 0xad80, 0xffffffff, 0x00000100,
789 0xac54, 0xffffffff, 0x00000100,
790 0x897c, 0xffffffff, 0x06000100,
791 0x9868, 0xffffffff, 0x00000100,
792 0x9510, 0xffffffff, 0x00000100,
793 0xaf04, 0xffffffff, 0x00000100,
794 0xae04, 0xffffffff, 0x00000100,
795 0x949c, 0xffffffff, 0x00000100,
796 0x802c, 0xffffffff, 0xe0000000,
797 0x9160, 0xffffffff, 0x00010000,
798 0x9164, 0xffffffff, 0x00030002,
799 0x9168, 0xffffffff, 0x00040007,
800 0x916c, 0xffffffff, 0x00060005,
801 0x9170, 0xffffffff, 0x00090008,
802 0x9174, 0xffffffff, 0x00020001,
803 0x9178, 0xffffffff, 0x00040003,
804 0x917c, 0xffffffff, 0x00000007,
805 0x9180, 0xffffffff, 0x00060005,
806 0x9184, 0xffffffff, 0x00090008,
807 0x9188, 0xffffffff, 0x00030002,
808 0x918c, 0xffffffff, 0x00050004,
809 0x9190, 0xffffffff, 0x00000008,
810 0x9194, 0xffffffff, 0x00070006,
811 0x9198, 0xffffffff, 0x000a0009,
812 0x919c, 0xffffffff, 0x00040003,
813 0x91a0, 0xffffffff, 0x00060005,
814 0x91a4, 0xffffffff, 0x00000009,
815 0x91a8, 0xffffffff, 0x00080007,
816 0x91ac, 0xffffffff, 0x000b000a,
817 0x91b0, 0xffffffff, 0x00050004,
818 0x91b4, 0xffffffff, 0x00070006,
819 0x91b8, 0xffffffff, 0x0008000b,
820 0x91bc, 0xffffffff, 0x000a0009,
821 0x91c0, 0xffffffff, 0x000d000c,
822 0x9200, 0xffffffff, 0x00090008,
823 0x9204, 0xffffffff, 0x000b000a,
824 0x9208, 0xffffffff, 0x000c000f,
825 0x920c, 0xffffffff, 0x000e000d,
826 0x9210, 0xffffffff, 0x00110010,
827 0x9214, 0xffffffff, 0x000a0009,
828 0x9218, 0xffffffff, 0x000c000b,
829 0x921c, 0xffffffff, 0x0000000f,
830 0x9220, 0xffffffff, 0x000e000d,
831 0x9224, 0xffffffff, 0x00110010,
832 0x9228, 0xffffffff, 0x000b000a,
833 0x922c, 0xffffffff, 0x000d000c,
834 0x9230, 0xffffffff, 0x00000010,
835 0x9234, 0xffffffff, 0x000f000e,
836 0x9238, 0xffffffff, 0x00120011,
837 0x923c, 0xffffffff, 0x000c000b,
838 0x9240, 0xffffffff, 0x000e000d,
839 0x9244, 0xffffffff, 0x00000011,
840 0x9248, 0xffffffff, 0x0010000f,
841 0x924c, 0xffffffff, 0x00130012,
842 0x9250, 0xffffffff, 0x000d000c,
843 0x9254, 0xffffffff, 0x000f000e,
844 0x9258, 0xffffffff, 0x00100013,
845 0x925c, 0xffffffff, 0x00120011,
846 0x9260, 0xffffffff, 0x00150014,
847 0x9150, 0xffffffff, 0x96940200,
848 0x8708, 0xffffffff, 0x00900100,
849 0xc478, 0xffffffff, 0x00000080,
850 0xc404, 0xffffffff, 0x0020003f,
851 0x30, 0xffffffff, 0x0000001c,
852 0x34, 0x000f0000, 0x000f0000,
853 0x160c, 0xffffffff, 0x00000100,
854 0x1024, 0xffffffff, 0x00000100,
855 0x102c, 0x00000101, 0x00000000,
856 0x20a8, 0xffffffff, 0x00000104,
857 0x264c, 0x000c0000, 0x000c0000,
858 0x2648, 0x000c0000, 0x000c0000,
859 0x55e4, 0xff000fff, 0x00000100,
860 0x55e8, 0x00000001, 0x00000001,
861 0x2f50, 0x00000001, 0x00000001,
862 0x30cc, 0xc0000fff, 0x00000104,
863 0xc1e4, 0x00000001, 0x00000001,
864 0xd0c0, 0xfffffff0, 0x00000100,
865 0xd8c0, 0xfffffff0, 0x00000100
866};
867
868static const u32 oland_mgcg_cgcg_init[] =
869{
870 0xc400, 0xffffffff, 0xfffffffc,
871 0x802c, 0xffffffff, 0xe0000000,
872 0x9a60, 0xffffffff, 0x00000100,
873 0x92a4, 0xffffffff, 0x00000100,
874 0xc164, 0xffffffff, 0x00000100,
875 0x9774, 0xffffffff, 0x00000100,
876 0x8984, 0xffffffff, 0x06000100,
877 0x8a18, 0xffffffff, 0x00000100,
878 0x92a0, 0xffffffff, 0x00000100,
879 0xc380, 0xffffffff, 0x00000100,
880 0x8b28, 0xffffffff, 0x00000100,
881 0x9144, 0xffffffff, 0x00000100,
882 0x8d88, 0xffffffff, 0x00000100,
883 0x8d8c, 0xffffffff, 0x00000100,
884 0x9030, 0xffffffff, 0x00000100,
885 0x9034, 0xffffffff, 0x00000100,
886 0x9038, 0xffffffff, 0x00000100,
887 0x903c, 0xffffffff, 0x00000100,
888 0xad80, 0xffffffff, 0x00000100,
889 0xac54, 0xffffffff, 0x00000100,
890 0x897c, 0xffffffff, 0x06000100,
891 0x9868, 0xffffffff, 0x00000100,
892 0x9510, 0xffffffff, 0x00000100,
893 0xaf04, 0xffffffff, 0x00000100,
894 0xae04, 0xffffffff, 0x00000100,
895 0x949c, 0xffffffff, 0x00000100,
896 0x802c, 0xffffffff, 0xe0000000,
897 0x9160, 0xffffffff, 0x00010000,
898 0x9164, 0xffffffff, 0x00030002,
899 0x9168, 0xffffffff, 0x00040007,
900 0x916c, 0xffffffff, 0x00060005,
901 0x9170, 0xffffffff, 0x00090008,
902 0x9174, 0xffffffff, 0x00020001,
903 0x9178, 0xffffffff, 0x00040003,
904 0x917c, 0xffffffff, 0x00000007,
905 0x9180, 0xffffffff, 0x00060005,
906 0x9184, 0xffffffff, 0x00090008,
907 0x9188, 0xffffffff, 0x00030002,
908 0x918c, 0xffffffff, 0x00050004,
909 0x9190, 0xffffffff, 0x00000008,
910 0x9194, 0xffffffff, 0x00070006,
911 0x9198, 0xffffffff, 0x000a0009,
912 0x919c, 0xffffffff, 0x00040003,
913 0x91a0, 0xffffffff, 0x00060005,
914 0x91a4, 0xffffffff, 0x00000009,
915 0x91a8, 0xffffffff, 0x00080007,
916 0x91ac, 0xffffffff, 0x000b000a,
917 0x91b0, 0xffffffff, 0x00050004,
918 0x91b4, 0xffffffff, 0x00070006,
919 0x91b8, 0xffffffff, 0x0008000b,
920 0x91bc, 0xffffffff, 0x000a0009,
921 0x91c0, 0xffffffff, 0x000d000c,
922 0x91c4, 0xffffffff, 0x00060005,
923 0x91c8, 0xffffffff, 0x00080007,
924 0x91cc, 0xffffffff, 0x0000000b,
925 0x91d0, 0xffffffff, 0x000a0009,
926 0x91d4, 0xffffffff, 0x000d000c,
927 0x9150, 0xffffffff, 0x96940200,
928 0x8708, 0xffffffff, 0x00900100,
929 0xc478, 0xffffffff, 0x00000080,
930 0xc404, 0xffffffff, 0x0020003f,
931 0x30, 0xffffffff, 0x0000001c,
932 0x34, 0x000f0000, 0x000f0000,
933 0x160c, 0xffffffff, 0x00000100,
934 0x1024, 0xffffffff, 0x00000100,
935 0x102c, 0x00000101, 0x00000000,
936 0x20a8, 0xffffffff, 0x00000104,
937 0x264c, 0x000c0000, 0x000c0000,
938 0x2648, 0x000c0000, 0x000c0000,
939 0x55e4, 0xff000fff, 0x00000100,
940 0x55e8, 0x00000001, 0x00000001,
941 0x2f50, 0x00000001, 0x00000001,
942 0x30cc, 0xc0000fff, 0x00000104,
943 0xc1e4, 0x00000001, 0x00000001,
944 0xd0c0, 0xfffffff0, 0x00000100,
945 0xd8c0, 0xfffffff0, 0x00000100
946};
947
fffbdda4
AD		 948static const u32 hainan_mgcg_cgcg_init[] =
949{
950 0xc400, 0xffffffff, 0xfffffffc,
951 0x802c, 0xffffffff, 0xe0000000,
952 0x9a60, 0xffffffff, 0x00000100,
953 0x92a4, 0xffffffff, 0x00000100,
954 0xc164, 0xffffffff, 0x00000100,
955 0x9774, 0xffffffff, 0x00000100,
956 0x8984, 0xffffffff, 0x06000100,
957 0x8a18, 0xffffffff, 0x00000100,
958 0x92a0, 0xffffffff, 0x00000100,
959 0xc380, 0xffffffff, 0x00000100,
960 0x8b28, 0xffffffff, 0x00000100,
961 0x9144, 0xffffffff, 0x00000100,
962 0x8d88, 0xffffffff, 0x00000100,
963 0x8d8c, 0xffffffff, 0x00000100,
964 0x9030, 0xffffffff, 0x00000100,
965 0x9034, 0xffffffff, 0x00000100,
966 0x9038, 0xffffffff, 0x00000100,
967 0x903c, 0xffffffff, 0x00000100,
968 0xad80, 0xffffffff, 0x00000100,
969 0xac54, 0xffffffff, 0x00000100,
970 0x897c, 0xffffffff, 0x06000100,
971 0x9868, 0xffffffff, 0x00000100,
972 0x9510, 0xffffffff, 0x00000100,
973 0xaf04, 0xffffffff, 0x00000100,
974 0xae04, 0xffffffff, 0x00000100,
975 0x949c, 0xffffffff, 0x00000100,
976 0x802c, 0xffffffff, 0xe0000000,
977 0x9160, 0xffffffff, 0x00010000,
978 0x9164, 0xffffffff, 0x00030002,
979 0x9168, 0xffffffff, 0x00040007,
980 0x916c, 0xffffffff, 0x00060005,
981 0x9170, 0xffffffff, 0x00090008,
982 0x9174, 0xffffffff, 0x00020001,
983 0x9178, 0xffffffff, 0x00040003,
984 0x917c, 0xffffffff, 0x00000007,
985 0x9180, 0xffffffff, 0x00060005,
986 0x9184, 0xffffffff, 0x00090008,
987 0x9188, 0xffffffff, 0x00030002,
988 0x918c, 0xffffffff, 0x00050004,
989 0x9190, 0xffffffff, 0x00000008,
990 0x9194, 0xffffffff, 0x00070006,
991 0x9198, 0xffffffff, 0x000a0009,
992 0x919c, 0xffffffff, 0x00040003,
993 0x91a0, 0xffffffff, 0x00060005,
994 0x91a4, 0xffffffff, 0x00000009,
995 0x91a8, 0xffffffff, 0x00080007,
996 0x91ac, 0xffffffff, 0x000b000a,
997 0x91b0, 0xffffffff, 0x00050004,
998 0x91b4, 0xffffffff, 0x00070006,
999 0x91b8, 0xffffffff, 0x0008000b,
1000 0x91bc, 0xffffffff, 0x000a0009,
1001 0x91c0, 0xffffffff, 0x000d000c,
1002 0x91c4, 0xffffffff, 0x00060005,
1003 0x91c8, 0xffffffff, 0x00080007,
1004 0x91cc, 0xffffffff, 0x0000000b,
1005 0x91d0, 0xffffffff, 0x000a0009,
1006 0x91d4, 0xffffffff, 0x000d000c,
1007 0x9150, 0xffffffff, 0x96940200,
1008 0x8708, 0xffffffff, 0x00900100,
1009 0xc478, 0xffffffff, 0x00000080,
1010 0xc404, 0xffffffff, 0x0020003f,
1011 0x30, 0xffffffff, 0x0000001c,
1012 0x34, 0x000f0000, 0x000f0000,
1013 0x160c, 0xffffffff, 0x00000100,
1014 0x1024, 0xffffffff, 0x00000100,
1015 0x20a8, 0xffffffff, 0x00000104,
1016 0x264c, 0x000c0000, 0x000c0000,
1017 0x2648, 0x000c0000, 0x000c0000,
1018 0x2f50, 0x00000001, 0x00000001,
1019 0x30cc, 0xc0000fff, 0x00000104,
1020 0xc1e4, 0x00000001, 0x00000001,
1021 0xd0c0, 0xfffffff0, 0x00000100,
1022 0xd8c0, 0xfffffff0, 0x00000100
1023};
1024
205996c0
AD		 1025static u32 verde_pg_init[] =
1026{
1027 0x353c, 0xffffffff, 0x40000,
1028 0x3538, 0xffffffff, 0x200010ff,
1029 0x353c, 0xffffffff, 0x0,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x0,
1034 0x353c, 0xffffffff, 0x7007,
1035 0x3538, 0xffffffff, 0x300010ff,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x0,
1041 0x353c, 0xffffffff, 0x400000,
1042 0x3538, 0xffffffff, 0x100010ff,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x0,
1048 0x353c, 0xffffffff, 0x120200,
1049 0x3538, 0xffffffff, 0x500010ff,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x0,
1055 0x353c, 0xffffffff, 0x1e1e16,
1056 0x3538, 0xffffffff, 0x600010ff,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x0,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x0,
1061 0x353c, 0xffffffff, 0x0,
1062 0x353c, 0xffffffff, 0x171f1e,
1063 0x3538, 0xffffffff, 0x700010ff,
1064 0x353c, 0xffffffff, 0x0,
1065 0x353c, 0xffffffff, 0x0,
1066 0x353c, 0xffffffff, 0x0,
1067 0x353c, 0xffffffff, 0x0,
1068 0x353c, 0xffffffff, 0x0,
1069 0x353c, 0xffffffff, 0x0,
1070 0x3538, 0xffffffff, 0x9ff,
1071 0x3500, 0xffffffff, 0x0,
1072 0x3504, 0xffffffff, 0x10000800,
1073 0x3504, 0xffffffff, 0xf,
1074 0x3504, 0xffffffff, 0xf,
1075 0x3500, 0xffffffff, 0x4,
1076 0x3504, 0xffffffff, 0x1000051e,
1077 0x3504, 0xffffffff, 0xffff,
1078 0x3504, 0xffffffff, 0xffff,
1079 0x3500, 0xffffffff, 0x8,
1080 0x3504, 0xffffffff, 0x80500,
1081 0x3500, 0xffffffff, 0x12,
1082 0x3504, 0xffffffff, 0x9050c,
1083 0x3500, 0xffffffff, 0x1d,
1084 0x3504, 0xffffffff, 0xb052c,
1085 0x3500, 0xffffffff, 0x2a,
1086 0x3504, 0xffffffff, 0x1053e,
1087 0x3500, 0xffffffff, 0x2d,
1088 0x3504, 0xffffffff, 0x10546,
1089 0x3500, 0xffffffff, 0x30,
1090 0x3504, 0xffffffff, 0xa054e,
1091 0x3500, 0xffffffff, 0x3c,
1092 0x3504, 0xffffffff, 0x1055f,
1093 0x3500, 0xffffffff, 0x3f,
1094 0x3504, 0xffffffff, 0x10567,
1095 0x3500, 0xffffffff, 0x42,
1096 0x3504, 0xffffffff, 0x1056f,
1097 0x3500, 0xffffffff, 0x45,
1098 0x3504, 0xffffffff, 0x10572,
1099 0x3500, 0xffffffff, 0x48,
1100 0x3504, 0xffffffff, 0x20575,
1101 0x3500, 0xffffffff, 0x4c,
1102 0x3504, 0xffffffff, 0x190801,
1103 0x3500, 0xffffffff, 0x67,
1104 0x3504, 0xffffffff, 0x1082a,
1105 0x3500, 0xffffffff, 0x6a,
1106 0x3504, 0xffffffff, 0x1b082d,
1107 0x3500, 0xffffffff, 0x87,
1108 0x3504, 0xffffffff, 0x310851,
1109 0x3500, 0xffffffff, 0xba,
1110 0x3504, 0xffffffff, 0x891,
1111 0x3500, 0xffffffff, 0xbc,
1112 0x3504, 0xffffffff, 0x893,
1113 0x3500, 0xffffffff, 0xbe,
1114 0x3504, 0xffffffff, 0x20895,
1115 0x3500, 0xffffffff, 0xc2,
1116 0x3504, 0xffffffff, 0x20899,
1117 0x3500, 0xffffffff, 0xc6,
1118 0x3504, 0xffffffff, 0x2089d,
1119 0x3500, 0xffffffff, 0xca,
1120 0x3504, 0xffffffff, 0x8a1,
1121 0x3500, 0xffffffff, 0xcc,
1122 0x3504, 0xffffffff, 0x8a3,
1123 0x3500, 0xffffffff, 0xce,
1124 0x3504, 0xffffffff, 0x308a5,
1125 0x3500, 0xffffffff, 0xd3,
1126 0x3504, 0xffffffff, 0x6d08cd,
1127 0x3500, 0xffffffff, 0x142,
1128 0x3504, 0xffffffff, 0x2000095a,
1129 0x3504, 0xffffffff, 0x1,
1130 0x3500, 0xffffffff, 0x144,
1131 0x3504, 0xffffffff, 0x301f095b,
1132 0x3500, 0xffffffff, 0x165,
1133 0x3504, 0xffffffff, 0xc094d,
1134 0x3500, 0xffffffff, 0x173,
1135 0x3504, 0xffffffff, 0xf096d,
1136 0x3500, 0xffffffff, 0x184,
1137 0x3504, 0xffffffff, 0x15097f,
1138 0x3500, 0xffffffff, 0x19b,
1139 0x3504, 0xffffffff, 0xc0998,
1140 0x3500, 0xffffffff, 0x1a9,
1141 0x3504, 0xffffffff, 0x409a7,
1142 0x3500, 0xffffffff, 0x1af,
1143 0x3504, 0xffffffff, 0xcdc,
1144 0x3500, 0xffffffff, 0x1b1,
1145 0x3504, 0xffffffff, 0x800,
1146 0x3508, 0xffffffff, 0x6c9b2000,
1147 0x3510, 0xfc00, 0x2000,
1148 0x3544, 0xffffffff, 0xfc0,
1149 0x28d4, 0x00000100, 0x100
1150};
1151
1152static void si_init_golden_registers(struct radeon_device *rdev)
1153{
1154 switch (rdev->family) {
1155 case CHIP_TAHITI:
1156 radeon_program_register_sequence(rdev,
1157 tahiti_golden_registers,
1158 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1159 radeon_program_register_sequence(rdev,
1160 tahiti_golden_rlc_registers,
1161 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1162 radeon_program_register_sequence(rdev,
1163 tahiti_mgcg_cgcg_init,
1164 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1165 radeon_program_register_sequence(rdev,
1166 tahiti_golden_registers2,
1167 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1168 break;
1169 case CHIP_PITCAIRN:
1170 radeon_program_register_sequence(rdev,
1171 pitcairn_golden_registers,
1172 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1173 radeon_program_register_sequence(rdev,
1174 pitcairn_golden_rlc_registers,
1175 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1176 radeon_program_register_sequence(rdev,
1177 pitcairn_mgcg_cgcg_init,
1178 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1179 break;
1180 case CHIP_VERDE:
1181 radeon_program_register_sequence(rdev,
1182 verde_golden_registers,
1183 (const u32)ARRAY_SIZE(verde_golden_registers));
1184 radeon_program_register_sequence(rdev,
1185 verde_golden_rlc_registers,
1186 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1187 radeon_program_register_sequence(rdev,
1188 verde_mgcg_cgcg_init,
1189 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1190 radeon_program_register_sequence(rdev,
1191 verde_pg_init,
1192 (const u32)ARRAY_SIZE(verde_pg_init));
1193 break;
1194 case CHIP_OLAND:
1195 radeon_program_register_sequence(rdev,
1196 oland_golden_registers,
1197 (const u32)ARRAY_SIZE(oland_golden_registers));
1198 radeon_program_register_sequence(rdev,
1199 oland_golden_rlc_registers,
1200 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1201 radeon_program_register_sequence(rdev,
1202 oland_mgcg_cgcg_init,
1203 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1204 break;
fffbdda4
AD		 1205 case CHIP_HAINAN:
1206 radeon_program_register_sequence(rdev,
1207 hainan_golden_registers,
1208 (const u32)ARRAY_SIZE(hainan_golden_registers));
1209 radeon_program_register_sequence(rdev,
1210 hainan_golden_registers2,
1211 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1212 radeon_program_register_sequence(rdev,
1213 hainan_mgcg_cgcg_init,
1214 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1215 break;
205996c0
AD		 1216 default:
1217 break;
1218 }
1219}
1220
454d2e2a
AD		 1221#define PCIE_BUS_CLK 10000
1222#define TCLK (PCIE_BUS_CLK / 10)
1223
1224/**
1225 * si_get_xclk - get the xclk
1226 *
1227 * @rdev: radeon_device pointer
1228 *
1229 * Returns the reference clock used by the gfx engine
1230 * (SI).
1231 */
1232u32 si_get_xclk(struct radeon_device *rdev)
1233{
1234 u32 reference_clock = rdev->clock.spll.reference_freq;
1235 u32 tmp;
1236
1237 tmp = RREG32(CG_CLKPIN_CNTL_2);
1238 if (tmp & MUX_TCLK_TO_XCLK)
1239 return TCLK;
1240
1241 tmp = RREG32(CG_CLKPIN_CNTL);
1242 if (tmp & XTALIN_DIVIDE)
1243 return reference_clock / 4;
1244
1245 return reference_clock;
1246}
1247
1bd47d2e
AD		 1248/* get temperature in millidegrees */
1249int si_get_temp(struct radeon_device *rdev)
1250{
1251 u32 temp;
1252 int actual_temp = 0;
1253
1254 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1255 CTF_TEMP_SHIFT;
1256
1257 if (temp & 0x200)
1258 actual_temp = 255;
1259 else
1260 actual_temp = temp & 0x1ff;
1261
1262 actual_temp = (actual_temp * 1000);
1263
1264 return actual_temp;
1265}
1266
8b074dd6
AD		 1267#define TAHITI_IO_MC_REGS_SIZE 36
1268
1269static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1270 {0x0000006f, 0x03044000},
1271 {0x00000070, 0x0480c018},
1272 {0x00000071, 0x00000040},
1273 {0x00000072, 0x01000000},
1274 {0x00000074, 0x000000ff},
1275 {0x00000075, 0x00143400},
1276 {0x00000076, 0x08ec0800},
1277 {0x00000077, 0x040000cc},
1278 {0x00000079, 0x00000000},
1279 {0x0000007a, 0x21000409},
1280 {0x0000007c, 0x00000000},
1281 {0x0000007d, 0xe8000000},
1282 {0x0000007e, 0x044408a8},
1283 {0x0000007f, 0x00000003},
1284 {0x00000080, 0x00000000},
1285 {0x00000081, 0x01000000},
1286 {0x00000082, 0x02000000},
1287 {0x00000083, 0x00000000},
1288 {0x00000084, 0xe3f3e4f4},
1289 {0x00000085, 0x00052024},
1290 {0x00000087, 0x00000000},
1291 {0x00000088, 0x66036603},
1292 {0x00000089, 0x01000000},
1293 {0x0000008b, 0x1c0a0000},
1294 {0x0000008c, 0xff010000},
1295 {0x0000008e, 0xffffefff},
1296 {0x0000008f, 0xfff3efff},
1297 {0x00000090, 0xfff3efbf},
1298 {0x00000094, 0x00101101},
1299 {0x00000095, 0x00000fff},
1300 {0x00000096, 0x00116fff},
1301 {0x00000097, 0x60010000},
1302 {0x00000098, 0x10010000},
1303 {0x00000099, 0x00006000},
1304 {0x0000009a, 0x00001000},
1305 {0x0000009f, 0x00a77400}
1306};
1307
1308static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1309 {0x0000006f, 0x03044000},
1310 {0x00000070, 0x0480c018},
1311 {0x00000071, 0x00000040},
1312 {0x00000072, 0x01000000},
1313 {0x00000074, 0x000000ff},
1314 {0x00000075, 0x00143400},
1315 {0x00000076, 0x08ec0800},
1316 {0x00000077, 0x040000cc},
1317 {0x00000079, 0x00000000},
1318 {0x0000007a, 0x21000409},
1319 {0x0000007c, 0x00000000},
1320 {0x0000007d, 0xe8000000},
1321 {0x0000007e, 0x044408a8},
1322 {0x0000007f, 0x00000003},
1323 {0x00000080, 0x00000000},
1324 {0x00000081, 0x01000000},
1325 {0x00000082, 0x02000000},
1326 {0x00000083, 0x00000000},
1327 {0x00000084, 0xe3f3e4f4},
1328 {0x00000085, 0x00052024},
1329 {0x00000087, 0x00000000},
1330 {0x00000088, 0x66036603},
1331 {0x00000089, 0x01000000},
1332 {0x0000008b, 0x1c0a0000},
1333 {0x0000008c, 0xff010000},
1334 {0x0000008e, 0xffffefff},
1335 {0x0000008f, 0xfff3efff},
1336 {0x00000090, 0xfff3efbf},
1337 {0x00000094, 0x00101101},
1338 {0x00000095, 0x00000fff},
1339 {0x00000096, 0x00116fff},
1340 {0x00000097, 0x60010000},
1341 {0x00000098, 0x10010000},
1342 {0x00000099, 0x00006000},
1343 {0x0000009a, 0x00001000},
1344 {0x0000009f, 0x00a47400}
1345};
1346
1347static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1348 {0x0000006f, 0x03044000},
1349 {0x00000070, 0x0480c018},
1350 {0x00000071, 0x00000040},
1351 {0x00000072, 0x01000000},
1352 {0x00000074, 0x000000ff},
1353 {0x00000075, 0x00143400},
1354 {0x00000076, 0x08ec0800},
1355 {0x00000077, 0x040000cc},
1356 {0x00000079, 0x00000000},
1357 {0x0000007a, 0x21000409},
1358 {0x0000007c, 0x00000000},
1359 {0x0000007d, 0xe8000000},
1360 {0x0000007e, 0x044408a8},
1361 {0x0000007f, 0x00000003},
1362 {0x00000080, 0x00000000},
1363 {0x00000081, 0x01000000},
1364 {0x00000082, 0x02000000},
1365 {0x00000083, 0x00000000},
1366 {0x00000084, 0xe3f3e4f4},
1367 {0x00000085, 0x00052024},
1368 {0x00000087, 0x00000000},
1369 {0x00000088, 0x66036603},
1370 {0x00000089, 0x01000000},
1371 {0x0000008b, 0x1c0a0000},
1372 {0x0000008c, 0xff010000},
1373 {0x0000008e, 0xffffefff},
1374 {0x0000008f, 0xfff3efff},
1375 {0x00000090, 0xfff3efbf},
1376 {0x00000094, 0x00101101},
1377 {0x00000095, 0x00000fff},
1378 {0x00000096, 0x00116fff},
1379 {0x00000097, 0x60010000},
1380 {0x00000098, 0x10010000},
1381 {0x00000099, 0x00006000},
1382 {0x0000009a, 0x00001000},
1383 {0x0000009f, 0x00a37400}
1384};
1385
bcc7f5d2
AD		 1386static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1387 {0x0000006f, 0x03044000},
1388 {0x00000070, 0x0480c018},
1389 {0x00000071, 0x00000040},
1390 {0x00000072, 0x01000000},
1391 {0x00000074, 0x000000ff},
1392 {0x00000075, 0x00143400},
1393 {0x00000076, 0x08ec0800},
1394 {0x00000077, 0x040000cc},
1395 {0x00000079, 0x00000000},
1396 {0x0000007a, 0x21000409},
1397 {0x0000007c, 0x00000000},
1398 {0x0000007d, 0xe8000000},
1399 {0x0000007e, 0x044408a8},
1400 {0x0000007f, 0x00000003},
1401 {0x00000080, 0x00000000},
1402 {0x00000081, 0x01000000},
1403 {0x00000082, 0x02000000},
1404 {0x00000083, 0x00000000},
1405 {0x00000084, 0xe3f3e4f4},
1406 {0x00000085, 0x00052024},
1407 {0x00000087, 0x00000000},
1408 {0x00000088, 0x66036603},
1409 {0x00000089, 0x01000000},
1410 {0x0000008b, 0x1c0a0000},
1411 {0x0000008c, 0xff010000},
1412 {0x0000008e, 0xffffefff},
1413 {0x0000008f, 0xfff3efff},
1414 {0x00000090, 0xfff3efbf},
1415 {0x00000094, 0x00101101},
1416 {0x00000095, 0x00000fff},
1417 {0x00000096, 0x00116fff},
1418 {0x00000097, 0x60010000},
1419 {0x00000098, 0x10010000},
1420 {0x00000099, 0x00006000},
1421 {0x0000009a, 0x00001000},
1422 {0x0000009f, 0x00a17730}
1423};
1424
c04c00b4
AD		 1425static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1426 {0x0000006f, 0x03044000},
1427 {0x00000070, 0x0480c018},
1428 {0x00000071, 0x00000040},
1429 {0x00000072, 0x01000000},
1430 {0x00000074, 0x000000ff},
1431 {0x00000075, 0x00143400},
1432 {0x00000076, 0x08ec0800},
1433 {0x00000077, 0x040000cc},
1434 {0x00000079, 0x00000000},
1435 {0x0000007a, 0x21000409},
1436 {0x0000007c, 0x00000000},
1437 {0x0000007d, 0xe8000000},
1438 {0x0000007e, 0x044408a8},
1439 {0x0000007f, 0x00000003},
1440 {0x00000080, 0x00000000},
1441 {0x00000081, 0x01000000},
1442 {0x00000082, 0x02000000},
1443 {0x00000083, 0x00000000},
1444 {0x00000084, 0xe3f3e4f4},
1445 {0x00000085, 0x00052024},
1446 {0x00000087, 0x00000000},
1447 {0x00000088, 0x66036603},
1448 {0x00000089, 0x01000000},
1449 {0x0000008b, 0x1c0a0000},
1450 {0x0000008c, 0xff010000},
1451 {0x0000008e, 0xffffefff},
1452 {0x0000008f, 0xfff3efff},
1453 {0x00000090, 0xfff3efbf},
1454 {0x00000094, 0x00101101},
1455 {0x00000095, 0x00000fff},
1456 {0x00000096, 0x00116fff},
1457 {0x00000097, 0x60010000},
1458 {0x00000098, 0x10010000},
1459 {0x00000099, 0x00006000},
1460 {0x0000009a, 0x00001000},
1461 {0x0000009f, 0x00a07730}
1462};
1463
8b074dd6
AD		 1464/* ucode loading */
1465static int si_mc_load_microcode(struct radeon_device *rdev)
1466{
1467 const __be32 *fw_data;
1468 u32 running, blackout = 0;
1469 u32 *io_mc_regs;
1470 int i, ucode_size, regs_size;
1471
1472 if (!rdev->mc_fw)
1473 return -EINVAL;
1474
1475 switch (rdev->family) {
1476 case CHIP_TAHITI:
1477 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1478 ucode_size = SI_MC_UCODE_SIZE;
1479 regs_size = TAHITI_IO_MC_REGS_SIZE;
1480 break;
1481 case CHIP_PITCAIRN:
1482 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1483 ucode_size = SI_MC_UCODE_SIZE;
1484 regs_size = TAHITI_IO_MC_REGS_SIZE;
1485 break;
1486 case CHIP_VERDE:
1487 default:
1488 io_mc_regs = (u32 *)&verde_io_mc_regs;
1489 ucode_size = SI_MC_UCODE_SIZE;
1490 regs_size = TAHITI_IO_MC_REGS_SIZE;
1491 break;
bcc7f5d2
AD		 1492 case CHIP_OLAND:
1493 io_mc_regs = (u32 *)&oland_io_mc_regs;
1494 ucode_size = OLAND_MC_UCODE_SIZE;
1495 regs_size = TAHITI_IO_MC_REGS_SIZE;
1496 break;
c04c00b4
AD		 1497 case CHIP_HAINAN:
1498 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1499 ucode_size = OLAND_MC_UCODE_SIZE;
1500 regs_size = TAHITI_IO_MC_REGS_SIZE;
1501 break;
8b074dd6
AD		 1502 }
1503
1504 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1505
1506 if (running == 0) {
1507 if (running) {
1508 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1509 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1510 }
1511
1512 /* reset the engine and set to writable */
1513 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1514 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1515
1516 /* load mc io regs */
1517 for (i = 0; i < regs_size; i++) {
1518 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1519 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1520 }
1521 /* load the MC ucode */
1522 fw_data = (const __be32 *)rdev->mc_fw->data;
1523 for (i = 0; i < ucode_size; i++)
1524 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1525
1526 /* put the engine back into the active state */
1527 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1528 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1529 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1530
1531 /* wait for training to complete */
1532 for (i = 0; i < rdev->usec_timeout; i++) {
1533 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1534 break;
1535 udelay(1);
1536 }
1537 for (i = 0; i < rdev->usec_timeout; i++) {
1538 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1539 break;
1540 udelay(1);
1541 }
1542
1543 if (running)
1544 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1545 }
1546
1547 return 0;
1548}
1549
0f0de06c
AD		 1550static int si_init_microcode(struct radeon_device *rdev)
1551{
0f0de06c
AD		 1552 const char *chip_name;
1553 const char *rlc_chip_name;
1554 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
a9e61410 1555 size_t smc_req_size;
0f0de06c
AD		 1556 char fw_name[30];
1557 int err;
1558
1559 DRM_DEBUG("\n");
1560
0f0de06c
AD		 1561 switch (rdev->family) {
1562 case CHIP_TAHITI:
1563 chip_name = "TAHITI";
1564 rlc_chip_name = "TAHITI";
1565 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1566 me_req_size = SI_PM4_UCODE_SIZE * 4;
1567 ce_req_size = SI_CE_UCODE_SIZE * 4;
1568 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1569 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1570 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1571 break;
1572 case CHIP_PITCAIRN:
1573 chip_name = "PITCAIRN";
1574 rlc_chip_name = "PITCAIRN";
1575 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1576 me_req_size = SI_PM4_UCODE_SIZE * 4;
1577 ce_req_size = SI_CE_UCODE_SIZE * 4;
1578 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1579 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1580 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1581 break;
1582 case CHIP_VERDE:
1583 chip_name = "VERDE";
1584 rlc_chip_name = "VERDE";
1585 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1586 me_req_size = SI_PM4_UCODE_SIZE * 4;
1587 ce_req_size = SI_CE_UCODE_SIZE * 4;
1588 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1589 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1590 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
0f0de06c 1591 break;
bcc7f5d2
AD		 1592 case CHIP_OLAND:
1593 chip_name = "OLAND";
1594 rlc_chip_name = "OLAND";
1595 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1596 me_req_size = SI_PM4_UCODE_SIZE * 4;
1597 ce_req_size = SI_CE_UCODE_SIZE * 4;
1598 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1599 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1600 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
bcc7f5d2 1601 break;
c04c00b4
AD		 1602 case CHIP_HAINAN:
1603 chip_name = "HAINAN";
1604 rlc_chip_name = "HAINAN";
1605 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1606 me_req_size = SI_PM4_UCODE_SIZE * 4;
1607 ce_req_size = SI_CE_UCODE_SIZE * 4;
1608 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1609 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1610 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
c04c00b4 1611 break;
0f0de06c
AD		 1612 default: BUG();
1613 }
1614
1615 DRM_INFO("Loading %s Microcode\n", chip_name);
1616
1617 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
0a168933 1618 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
0f0de06c
AD		 1619 if (err)
1620 goto out;
1621 if (rdev->pfp_fw->size != pfp_req_size) {
1622 printk(KERN_ERR
1623 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1624 rdev->pfp_fw->size, fw_name);
1625 err = -EINVAL;
1626 goto out;
1627 }
1628
1629 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
0a168933 1630 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
0f0de06c
AD		 1631 if (err)
1632 goto out;
1633 if (rdev->me_fw->size != me_req_size) {
1634 printk(KERN_ERR
1635 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1636 rdev->me_fw->size, fw_name);
1637 err = -EINVAL;
1638 }
1639
1640 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
0a168933 1641 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
0f0de06c
AD		 1642 if (err)
1643 goto out;
1644 if (rdev->ce_fw->size != ce_req_size) {
1645 printk(KERN_ERR
1646 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1647 rdev->ce_fw->size, fw_name);
1648 err = -EINVAL;
1649 }
1650
1651 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
0a168933 1652 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
0f0de06c
AD		 1653 if (err)
1654 goto out;
1655 if (rdev->rlc_fw->size != rlc_req_size) {
1656 printk(KERN_ERR
1657 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1658 rdev->rlc_fw->size, fw_name);
1659 err = -EINVAL;
1660 }
1661
1662 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
0a168933 1663 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
0f0de06c
AD		 1664 if (err)
1665 goto out;
1666 if (rdev->mc_fw->size != mc_req_size) {
1667 printk(KERN_ERR
1668 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1669 rdev->mc_fw->size, fw_name);
1670 err = -EINVAL;
1671 }
1672
a9e61410 1673 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
0a168933 1674 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
8a53fa23
AD		 1675 if (err) {
1676 printk(KERN_ERR
1677 "smc: error loading firmware \"%s\"\n",
1678 fw_name);
1679 release_firmware(rdev->smc_fw);
1680 rdev->smc_fw = NULL;
1681 } else if (rdev->smc_fw->size != smc_req_size) {
a9e61410
AD		 1682 printk(KERN_ERR
1683 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1684 rdev->smc_fw->size, fw_name);
1685 err = -EINVAL;
1686 }
1687
0f0de06c 1688out:
0f0de06c
AD		 1689 if (err) {
1690 if (err != -EINVAL)
1691 printk(KERN_ERR
1692 "si_cp: Failed to load firmware \"%s\"\n",
1693 fw_name);
1694 release_firmware(rdev->pfp_fw);
1695 rdev->pfp_fw = NULL;
1696 release_firmware(rdev->me_fw);
1697 rdev->me_fw = NULL;
1698 release_firmware(rdev->ce_fw);
1699 rdev->ce_fw = NULL;
1700 release_firmware(rdev->rlc_fw);
1701 rdev->rlc_fw = NULL;
1702 release_firmware(rdev->mc_fw);
1703 rdev->mc_fw = NULL;
a9e61410
AD		 1704 release_firmware(rdev->smc_fw);
1705 rdev->smc_fw = NULL;
0f0de06c
AD		 1706 }
1707 return err;
1708}
1709
43b3cd99
AD		 1710/* watermark setup */
1711static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1712 struct radeon_crtc *radeon_crtc,
1713 struct drm_display_mode *mode,
1714 struct drm_display_mode *other_mode)
1715{
290d2457
AD		 1716 u32 tmp, buffer_alloc, i;
1717 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
43b3cd99
AD		 1718 /*
1719 * Line Buffer Setup
1720 * There are 3 line buffers, each one shared by 2 display controllers.
1721 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1722 * the display controllers. The paritioning is done via one of four
1723 * preset allocations specified in bits 21:20:
1724 * 0 - half lb
1725 * 2 - whole lb, other crtc must be disabled
1726 */
1727 /* this can get tricky if we have two large displays on a paired group
1728 * of crtcs. Ideally for multiple large displays we'd assign them to
1729 * non-linked crtcs for maximum line buffer allocation.
1730 */
1731 if (radeon_crtc->base.enabled && mode) {
290d2457 1732 if (other_mode) {
43b3cd99 1733 tmp = 0; /* 1/2 */
290d2457
AD		 1734 buffer_alloc = 1;
1735 } else {
43b3cd99 1736 tmp = 2; /* whole */
290d2457
AD		 1737 buffer_alloc = 2;
1738 }
1739 } else {
43b3cd99 1740 tmp = 0;
290d2457
AD		 1741 buffer_alloc = 0;
1742 }
43b3cd99
AD		 1743
1744 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1745 DC_LB_MEMORY_CONFIG(tmp));
1746
290d2457
AD		 1747 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1748 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1749 for (i = 0; i < rdev->usec_timeout; i++) {
1750 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1751 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1752 break;
1753 udelay(1);
1754 }
1755
43b3cd99
AD		 1756 if (radeon_crtc->base.enabled && mode) {
1757 switch (tmp) {
1758 case 0:
1759 default:
1760 return 4096 * 2;
1761 case 2:
1762 return 8192 * 2;
1763 }
1764 }
1765
1766 /* controller not enabled, so no lb used */
1767 return 0;
1768}
1769
ca7db22b 1770static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 1771{
1772 u32 tmp = RREG32(MC_SHARED_CHMAP);
1773
1774 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1775 case 0:
1776 default:
1777 return 1;
1778 case 1:
1779 return 2;
1780 case 2:
1781 return 4;
1782 case 3:
1783 return 8;
1784 case 4:
1785 return 3;
1786 case 5:
1787 return 6;
1788 case 6:
1789 return 10;
1790 case 7:
1791 return 12;
1792 case 8:
1793 return 16;
1794 }
1795}
1796
1797struct dce6_wm_params {
1798 u32 dram_channels; /* number of dram channels */
1799 u32 yclk; /* bandwidth per dram data pin in kHz */
1800 u32 sclk; /* engine clock in kHz */
1801 u32 disp_clk; /* display clock in kHz */
1802 u32 src_width; /* viewport width */
1803 u32 active_time; /* active display time in ns */
1804 u32 blank_time; /* blank time in ns */
1805 bool interlaced; /* mode is interlaced */
1806 fixed20_12 vsc; /* vertical scale ratio */
1807 u32 num_heads; /* number of active crtcs */
1808 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1809 u32 lb_size; /* line buffer allocated to pipe */
1810 u32 vtaps; /* vertical scaler taps */
1811};
1812
1813static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1814{
1815 /* Calculate raw DRAM Bandwidth */
1816 fixed20_12 dram_efficiency; /* 0.7 */
1817 fixed20_12 yclk, dram_channels, bandwidth;
1818 fixed20_12 a;
1819
1820 a.full = dfixed_const(1000);
1821 yclk.full = dfixed_const(wm->yclk);
1822 yclk.full = dfixed_div(yclk, a);
1823 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1824 a.full = dfixed_const(10);
1825 dram_efficiency.full = dfixed_const(7);
1826 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1827 bandwidth.full = dfixed_mul(dram_channels, yclk);
1828 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1829
1830 return dfixed_trunc(bandwidth);
1831}
1832
1833static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1834{
1835 /* Calculate DRAM Bandwidth and the part allocated to display. */
1836 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1837 fixed20_12 yclk, dram_channels, bandwidth;
1838 fixed20_12 a;
1839
1840 a.full = dfixed_const(1000);
1841 yclk.full = dfixed_const(wm->yclk);
1842 yclk.full = dfixed_div(yclk, a);
1843 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1844 a.full = dfixed_const(10);
1845 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1846 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1847 bandwidth.full = dfixed_mul(dram_channels, yclk);
1848 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1849
1850 return dfixed_trunc(bandwidth);
1851}
1852
1853static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1854{
1855 /* Calculate the display Data return Bandwidth */
1856 fixed20_12 return_efficiency; /* 0.8 */
1857 fixed20_12 sclk, bandwidth;
1858 fixed20_12 a;
1859
1860 a.full = dfixed_const(1000);
1861 sclk.full = dfixed_const(wm->sclk);
1862 sclk.full = dfixed_div(sclk, a);
1863 a.full = dfixed_const(10);
1864 return_efficiency.full = dfixed_const(8);
1865 return_efficiency.full = dfixed_div(return_efficiency, a);
1866 a.full = dfixed_const(32);
1867 bandwidth.full = dfixed_mul(a, sclk);
1868 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1869
1870 return dfixed_trunc(bandwidth);
1871}
1872
1873static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1874{
1875 return 32;
1876}
1877
1878static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1879{
1880 /* Calculate the DMIF Request Bandwidth */
1881 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1882 fixed20_12 disp_clk, sclk, bandwidth;
1883 fixed20_12 a, b1, b2;
1884 u32 min_bandwidth;
1885
1886 a.full = dfixed_const(1000);
1887 disp_clk.full = dfixed_const(wm->disp_clk);
1888 disp_clk.full = dfixed_div(disp_clk, a);
1889 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1890 b1.full = dfixed_mul(a, disp_clk);
1891
1892 a.full = dfixed_const(1000);
1893 sclk.full = dfixed_const(wm->sclk);
1894 sclk.full = dfixed_div(sclk, a);
1895 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1896 b2.full = dfixed_mul(a, sclk);
1897
1898 a.full = dfixed_const(10);
1899 disp_clk_request_efficiency.full = dfixed_const(8);
1900 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1901
1902 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1903
1904 a.full = dfixed_const(min_bandwidth);
1905 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1906
1907 return dfixed_trunc(bandwidth);
1908}
1909
1910static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1911{
1912 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1913 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1914 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1915 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1916
1917 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1918}
1919
1920static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1921{
1922 /* Calculate the display mode Average Bandwidth
1923 * DisplayMode should contain the source and destination dimensions,
1924 * timing, etc.
1925 */
1926 fixed20_12 bpp;
1927 fixed20_12 line_time;
1928 fixed20_12 src_width;
1929 fixed20_12 bandwidth;
1930 fixed20_12 a;
1931
1932 a.full = dfixed_const(1000);
1933 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1934 line_time.full = dfixed_div(line_time, a);
1935 bpp.full = dfixed_const(wm->bytes_per_pixel);
1936 src_width.full = dfixed_const(wm->src_width);
1937 bandwidth.full = dfixed_mul(src_width, bpp);
1938 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1939 bandwidth.full = dfixed_div(bandwidth, line_time);
1940
1941 return dfixed_trunc(bandwidth);
1942}
1943
1944static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1945{
1946 /* First calcualte the latency in ns */
1947 u32 mc_latency = 2000; /* 2000 ns. */
1948 u32 available_bandwidth = dce6_available_bandwidth(wm);
1949 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1950 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1951 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1952 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1953 (wm->num_heads * cursor_line_pair_return_time);
1954 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1955 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1956 u32 tmp, dmif_size = 12288;
1957 fixed20_12 a, b, c;
1958
1959 if (wm->num_heads == 0)
1960 return 0;
1961
1962 a.full = dfixed_const(2);
1963 b.full = dfixed_const(1);
1964 if ((wm->vsc.full > a.full) ||
1965 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1966 (wm->vtaps >= 5) ||
1967 ((wm->vsc.full >= a.full) && wm->interlaced))
1968 max_src_lines_per_dst_line = 4;
1969 else
1970 max_src_lines_per_dst_line = 2;
1971
1972 a.full = dfixed_const(available_bandwidth);
1973 b.full = dfixed_const(wm->num_heads);
1974 a.full = dfixed_div(a, b);
1975
1976 b.full = dfixed_const(mc_latency + 512);
1977 c.full = dfixed_const(wm->disp_clk);
1978 b.full = dfixed_div(b, c);
1979
1980 c.full = dfixed_const(dmif_size);
1981 b.full = dfixed_div(c, b);
1982
1983 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1984
1985 b.full = dfixed_const(1000);
1986 c.full = dfixed_const(wm->disp_clk);
1987 b.full = dfixed_div(c, b);
1988 c.full = dfixed_const(wm->bytes_per_pixel);
1989 b.full = dfixed_mul(b, c);
1990
1991 lb_fill_bw = min(tmp, dfixed_trunc(b));
1992
1993 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1994 b.full = dfixed_const(1000);
1995 c.full = dfixed_const(lb_fill_bw);
1996 b.full = dfixed_div(c, b);
1997 a.full = dfixed_div(a, b);
1998 line_fill_time = dfixed_trunc(a);
1999
2000 if (line_fill_time < wm->active_time)
2001 return latency;
2002 else
2003 return latency + (line_fill_time - wm->active_time);
2004
2005}
2006
2007static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2008{
2009 if (dce6_average_bandwidth(wm) <=
2010 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2011 return true;
2012 else
2013 return false;
2014};
2015
2016static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2017{
2018 if (dce6_average_bandwidth(wm) <=
2019 (dce6_available_bandwidth(wm) / wm->num_heads))
2020 return true;
2021 else
2022 return false;
2023};
2024
2025static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2026{
2027 u32 lb_partitions = wm->lb_size / wm->src_width;
2028 u32 line_time = wm->active_time + wm->blank_time;
2029 u32 latency_tolerant_lines;
2030 u32 latency_hiding;
2031 fixed20_12 a;
2032
2033 a.full = dfixed_const(1);
2034 if (wm->vsc.full > a.full)
2035 latency_tolerant_lines = 1;
2036 else {
2037 if (lb_partitions <= (wm->vtaps + 1))
2038 latency_tolerant_lines = 1;
2039 else
2040 latency_tolerant_lines = 2;
2041 }
2042
2043 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2044
2045 if (dce6_latency_watermark(wm) <= latency_hiding)
2046 return true;
2047 else
2048 return false;
2049}
2050
2051static void dce6_program_watermarks(struct radeon_device *rdev,
2052 struct radeon_crtc *radeon_crtc,
2053 u32 lb_size, u32 num_heads)
2054{
2055 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 2056 struct dce6_wm_params wm_low, wm_high;
2057 u32 dram_channels;
43b3cd99
AD		 2058 u32 pixel_period;
2059 u32 line_time = 0;
2060 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2061 u32 priority_a_mark = 0, priority_b_mark = 0;
2062 u32 priority_a_cnt = PRIORITY_OFF;
2063 u32 priority_b_cnt = PRIORITY_OFF;
2064 u32 tmp, arb_control3;
2065 fixed20_12 a, b, c;
2066
2067 if (radeon_crtc->base.enabled && num_heads && mode) {
2068 pixel_period = 1000000 / (u32)mode->clock;
2069 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2070 priority_a_cnt = 0;
2071 priority_b_cnt = 0;
2072
ca7db22b 2073 if (rdev->family == CHIP_ARUBA)
c696e53f 2074 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 2075 else
c696e53f
AD		 2076 dram_channels = si_get_number_of_dram_channels(rdev);
2077
2078 /* watermark for high clocks */
2079 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2080 wm_high.yclk =
2081 radeon_dpm_get_mclk(rdev, false) * 10;
2082 wm_high.sclk =
2083 radeon_dpm_get_sclk(rdev, false) * 10;
2084 } else {
2085 wm_high.yclk = rdev->pm.current_mclk * 10;
2086 wm_high.sclk = rdev->pm.current_sclk * 10;
2087 }
2088
2089 wm_high.disp_clk = mode->clock;
2090 wm_high.src_width = mode->crtc_hdisplay;
2091 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2092 wm_high.blank_time = line_time - wm_high.active_time;
2093 wm_high.interlaced = false;
2094 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2095 wm_high.interlaced = true;
2096 wm_high.vsc = radeon_crtc->vsc;
2097 wm_high.vtaps = 1;
2098 if (radeon_crtc->rmx_type != RMX_OFF)
2099 wm_high.vtaps = 2;
2100 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2101 wm_high.lb_size = lb_size;
2102 wm_high.dram_channels = dram_channels;
2103 wm_high.num_heads = num_heads;
2104
2105 /* watermark for low clocks */
2106 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2107 wm_low.yclk =
2108 radeon_dpm_get_mclk(rdev, true) * 10;
2109 wm_low.sclk =
2110 radeon_dpm_get_sclk(rdev, true) * 10;
2111 } else {
2112 wm_low.yclk = rdev->pm.current_mclk * 10;
2113 wm_low.sclk = rdev->pm.current_sclk * 10;
2114 }
2115
2116 wm_low.disp_clk = mode->clock;
2117 wm_low.src_width = mode->crtc_hdisplay;
2118 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2119 wm_low.blank_time = line_time - wm_low.active_time;
2120 wm_low.interlaced = false;
2121 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2122 wm_low.interlaced = true;
2123 wm_low.vsc = radeon_crtc->vsc;
2124 wm_low.vtaps = 1;
2125 if (radeon_crtc->rmx_type != RMX_OFF)
2126 wm_low.vtaps = 2;
2127 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2128 wm_low.lb_size = lb_size;
2129 wm_low.dram_channels = dram_channels;
2130 wm_low.num_heads = num_heads;
43b3cd99
AD		 2131
2132 /* set for high clocks */
c696e53f 2133 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 2134 /* set for low clocks */
c696e53f 2135 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 2136
2137 /* possibly force display priority to high */
2138 /* should really do this at mode validation time... */
c696e53f
AD		 2139 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2140 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2141 !dce6_check_latency_hiding(&wm_high) ||
2142 (rdev->disp_priority == 2)) {
2143 DRM_DEBUG_KMS("force priority to high\n");
2144 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2145 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2146 }
2147 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2148 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2149 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 2150 (rdev->disp_priority == 2)) {
2151 DRM_DEBUG_KMS("force priority to high\n");
2152 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2153 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2154 }
2155
2156 a.full = dfixed_const(1000);
2157 b.full = dfixed_const(mode->clock);
2158 b.full = dfixed_div(b, a);
2159 c.full = dfixed_const(latency_watermark_a);
2160 c.full = dfixed_mul(c, b);
2161 c.full = dfixed_mul(c, radeon_crtc->hsc);
2162 c.full = dfixed_div(c, a);
2163 a.full = dfixed_const(16);
2164 c.full = dfixed_div(c, a);
2165 priority_a_mark = dfixed_trunc(c);
2166 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2167
2168 a.full = dfixed_const(1000);
2169 b.full = dfixed_const(mode->clock);
2170 b.full = dfixed_div(b, a);
2171 c.full = dfixed_const(latency_watermark_b);
2172 c.full = dfixed_mul(c, b);
2173 c.full = dfixed_mul(c, radeon_crtc->hsc);
2174 c.full = dfixed_div(c, a);
2175 a.full = dfixed_const(16);
2176 c.full = dfixed_div(c, a);
2177 priority_b_mark = dfixed_trunc(c);
2178 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2179 }
2180
2181 /* select wm A */
2182 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2183 tmp = arb_control3;
2184 tmp &= ~LATENCY_WATERMARK_MASK(3);
2185 tmp |= LATENCY_WATERMARK_MASK(1);
2186 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2187 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2188 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2189 LATENCY_HIGH_WATERMARK(line_time)));
2190 /* select wm B */
2191 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2192 tmp &= ~LATENCY_WATERMARK_MASK(3);
2193 tmp |= LATENCY_WATERMARK_MASK(2);
2194 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2195 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2196 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2197 LATENCY_HIGH_WATERMARK(line_time)));
2198 /* restore original selection */
2199 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2200
2201 /* write the priority marks */
2202 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2203 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2204
7178d2a6
AD		 2205 /* save values for DPM */
2206 radeon_crtc->line_time = line_time;
2207 radeon_crtc->wm_high = latency_watermark_a;
2208 radeon_crtc->wm_low = latency_watermark_b;
43b3cd99
AD		 2209}
2210
2211void dce6_bandwidth_update(struct radeon_device *rdev)
2212{
2213 struct drm_display_mode *mode0 = NULL;
2214 struct drm_display_mode *mode1 = NULL;
2215 u32 num_heads = 0, lb_size;
2216 int i;
2217
2218 radeon_update_display_priority(rdev);
2219
2220 for (i = 0; i < rdev->num_crtc; i++) {
2221 if (rdev->mode_info.crtcs[i]->base.enabled)
2222 num_heads++;
2223 }
2224 for (i = 0; i < rdev->num_crtc; i += 2) {
2225 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2226 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2227 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2228 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2229 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2230 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2231 }
2232}
2233
0a96d72b
AD		 2234/*
2235 * Core functions
2236 */
0a96d72b
AD		 2237static void si_tiling_mode_table_init(struct radeon_device *rdev)
2238{
2239 const u32 num_tile_mode_states = 32;
2240 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2241
2242 switch (rdev->config.si.mem_row_size_in_kb) {
2243 case 1:
2244 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2245 break;
2246 case 2:
2247 default:
2248 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2249 break;
2250 case 4:
2251 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2252 break;
2253 }
2254
2255 if ((rdev->family == CHIP_TAHITI) ||
2256 (rdev->family == CHIP_PITCAIRN)) {
2257 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2258 switch (reg_offset) {
2259 case 0: /* non-AA compressed depth or any compressed stencil */
2260 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2261 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2262 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2263 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2264 NUM_BANKS(ADDR_SURF_16_BANK) |
2265 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2266 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2267 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2268 break;
2269 case 1: /* 2xAA/4xAA compressed depth only */
2270 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2271 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2272 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2273 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2274 NUM_BANKS(ADDR_SURF_16_BANK) |
2275 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2276 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2277 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2278 break;
2279 case 2: /* 8xAA compressed depth only */
2280 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2281 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2282 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2283 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2284 NUM_BANKS(ADDR_SURF_16_BANK) |
2285 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2286 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2287 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2288 break;
2289 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2290 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2291 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2292 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2293 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2294 NUM_BANKS(ADDR_SURF_16_BANK) |
2295 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2296 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2297 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2298 break;
2299 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2300 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2301 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2302 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2303 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2304 NUM_BANKS(ADDR_SURF_16_BANK) |
2305 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2306 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2307 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2308 break;
2309 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2310 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2311 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2312 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2313 TILE_SPLIT(split_equal_to_row_size) |
2314 NUM_BANKS(ADDR_SURF_16_BANK) |
2315 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2316 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2317 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2318 break;
2319 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2320 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2321 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2322 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2323 TILE_SPLIT(split_equal_to_row_size) |
2324 NUM_BANKS(ADDR_SURF_16_BANK) |
2325 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2326 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2327 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2328 break;
2329 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2330 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2331 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2332 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2333 TILE_SPLIT(split_equal_to_row_size) |
2334 NUM_BANKS(ADDR_SURF_16_BANK) |
2335 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2336 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2337 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2338 break;
2339 case 8: /* 1D and 1D Array Surfaces */
2340 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2341 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2342 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2343 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2344 NUM_BANKS(ADDR_SURF_16_BANK) |
2345 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2346 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2347 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2348 break;
2349 case 9: /* Displayable maps. */
2350 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2351 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2352 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2353 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2354 NUM_BANKS(ADDR_SURF_16_BANK) |
2355 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2358 break;
2359 case 10: /* Display 8bpp. */
2360 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2361 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2362 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2363 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2364 NUM_BANKS(ADDR_SURF_16_BANK) |
2365 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2366 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2367 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2368 break;
2369 case 11: /* Display 16bpp. */
2370 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2371 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2372 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2373 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2374 NUM_BANKS(ADDR_SURF_16_BANK) |
2375 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2376 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2377 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2378 break;
2379 case 12: /* Display 32bpp. */
2380 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2381 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2382 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2383 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2384 NUM_BANKS(ADDR_SURF_16_BANK) |
2385 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2387 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2388 break;
2389 case 13: /* Thin. */
2390 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2391 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2392 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2393 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2394 NUM_BANKS(ADDR_SURF_16_BANK) |
2395 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2396 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2397 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2398 break;
2399 case 14: /* Thin 8 bpp. */
2400 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2401 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2402 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2403 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2404 NUM_BANKS(ADDR_SURF_16_BANK) |
2405 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2406 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2407 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2408 break;
2409 case 15: /* Thin 16 bpp. */
2410 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2411 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2412 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2413 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2414 NUM_BANKS(ADDR_SURF_16_BANK) |
2415 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2416 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2417 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2418 break;
2419 case 16: /* Thin 32 bpp. */
2420 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2421 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2422 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2423 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2424 NUM_BANKS(ADDR_SURF_16_BANK) |
2425 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2426 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2427 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2428 break;
2429 case 17: /* Thin 64 bpp. */
2430 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2431 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2432 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2433 TILE_SPLIT(split_equal_to_row_size) |
2434 NUM_BANKS(ADDR_SURF_16_BANK) |
2435 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2436 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2437 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2438 break;
2439 case 21: /* 8 bpp PRT. */
2440 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2441 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2442 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2443 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2444 NUM_BANKS(ADDR_SURF_16_BANK) |
2445 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2446 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2447 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2448 break;
2449 case 22: /* 16 bpp PRT */
2450 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2451 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2452 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2453 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2454 NUM_BANKS(ADDR_SURF_16_BANK) |
2455 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2456 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2457 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2458 break;
2459 case 23: /* 32 bpp PRT */
2460 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2461 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2462 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2463 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2464 NUM_BANKS(ADDR_SURF_16_BANK) |
2465 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2466 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2467 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2468 break;
2469 case 24: /* 64 bpp PRT */
2470 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2471 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2472 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2473 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2474 NUM_BANKS(ADDR_SURF_16_BANK) |
2475 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2476 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2477 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2478 break;
2479 case 25: /* 128 bpp PRT */
2480 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2481 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2482 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2483 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2484 NUM_BANKS(ADDR_SURF_8_BANK) |
2485 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2488 break;
2489 default:
2490 gb_tile_moden = 0;
2491 break;
2492 }
64d7b8be 2493 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2494 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2495 }
d0ae7fcc 2496 } else if ((rdev->family == CHIP_VERDE) ||
8b02859d
AD		 2497 (rdev->family == CHIP_OLAND) ||
2498 (rdev->family == CHIP_HAINAN)) {
0a96d72b
AD		 2499 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2500 switch (reg_offset) {
2501 case 0: /* non-AA compressed depth or any compressed stencil */
2502 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2503 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2504 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2505 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2506 NUM_BANKS(ADDR_SURF_16_BANK) |
2507 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2508 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2509 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2510 break;
2511 case 1: /* 2xAA/4xAA compressed depth only */
2512 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2513 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2514 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2515 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2516 NUM_BANKS(ADDR_SURF_16_BANK) |
2517 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2520 break;
2521 case 2: /* 8xAA compressed depth only */
2522 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2523 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2524 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2525 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2526 NUM_BANKS(ADDR_SURF_16_BANK) |
2527 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2528 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2529 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2530 break;
2531 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2532 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2533 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2534 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2535 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2536 NUM_BANKS(ADDR_SURF_16_BANK) |
2537 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2538 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2539 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2540 break;
2541 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2542 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2543 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2544 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2545 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2546 NUM_BANKS(ADDR_SURF_16_BANK) |
2547 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2548 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2549 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2550 break;
2551 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2552 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2553 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2554 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2555 TILE_SPLIT(split_equal_to_row_size) |
2556 NUM_BANKS(ADDR_SURF_16_BANK) |
2557 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2558 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2559 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2560 break;
2561 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2562 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2563 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2564 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2565 TILE_SPLIT(split_equal_to_row_size) |
2566 NUM_BANKS(ADDR_SURF_16_BANK) |
2567 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2568 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2569 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2570 break;
2571 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2572 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2573 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2574 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2575 TILE_SPLIT(split_equal_to_row_size) |
2576 NUM_BANKS(ADDR_SURF_16_BANK) |
2577 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2578 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2579 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2580 break;
2581 case 8: /* 1D and 1D Array Surfaces */
2582 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2583 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2584 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2585 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2586 NUM_BANKS(ADDR_SURF_16_BANK) |
2587 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2588 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2589 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2590 break;
2591 case 9: /* Displayable maps. */
2592 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2593 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2594 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2595 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2596 NUM_BANKS(ADDR_SURF_16_BANK) |
2597 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2598 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2599 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2600 break;
2601 case 10: /* Display 8bpp. */
2602 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2603 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2604 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2605 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2606 NUM_BANKS(ADDR_SURF_16_BANK) |
2607 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2608 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2609 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2610 break;
2611 case 11: /* Display 16bpp. */
2612 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2613 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2614 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2615 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2616 NUM_BANKS(ADDR_SURF_16_BANK) |
2617 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2618 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2619 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2620 break;
2621 case 12: /* Display 32bpp. */
2622 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2623 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2624 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2625 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2626 NUM_BANKS(ADDR_SURF_16_BANK) |
2627 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2629 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2630 break;
2631 case 13: /* Thin. */
2632 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2633 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2634 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2635 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2636 NUM_BANKS(ADDR_SURF_16_BANK) |
2637 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2638 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2639 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2640 break;
2641 case 14: /* Thin 8 bpp. */
2642 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2643 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2644 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2645 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2646 NUM_BANKS(ADDR_SURF_16_BANK) |
2647 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2648 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2649 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2650 break;
2651 case 15: /* Thin 16 bpp. */
2652 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2653 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2654 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2655 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2656 NUM_BANKS(ADDR_SURF_16_BANK) |
2657 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2658 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2659 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2660 break;
2661 case 16: /* Thin 32 bpp. */
2662 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2663 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2664 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2665 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2666 NUM_BANKS(ADDR_SURF_16_BANK) |
2667 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2668 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2669 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2670 break;
2671 case 17: /* Thin 64 bpp. */
2672 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2673 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2674 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2675 TILE_SPLIT(split_equal_to_row_size) |
2676 NUM_BANKS(ADDR_SURF_16_BANK) |
2677 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2678 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2679 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2680 break;
2681 case 21: /* 8 bpp PRT. */
2682 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2683 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2684 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2685 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2686 NUM_BANKS(ADDR_SURF_16_BANK) |
2687 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2688 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2689 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2690 break;
2691 case 22: /* 16 bpp PRT */
2692 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2693 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2694 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2695 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2696 NUM_BANKS(ADDR_SURF_16_BANK) |
2697 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2698 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2699 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2700 break;
2701 case 23: /* 32 bpp PRT */
2702 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2703 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2704 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2705 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2706 NUM_BANKS(ADDR_SURF_16_BANK) |
2707 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2708 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2709 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2710 break;
2711 case 24: /* 64 bpp PRT */
2712 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2713 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2714 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2715 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2716 NUM_BANKS(ADDR_SURF_16_BANK) |
2717 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2718 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2719 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2720 break;
2721 case 25: /* 128 bpp PRT */
2722 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2723 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2724 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2725 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2726 NUM_BANKS(ADDR_SURF_8_BANK) |
2727 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2728 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2729 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2730 break;
2731 default:
2732 gb_tile_moden = 0;
2733 break;
2734 }
64d7b8be 2735 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2736 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2737 }
2738 } else
2739 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2740}
2741
1a8ca750
AD		 2742static void si_select_se_sh(struct radeon_device *rdev,
2743 u32 se_num, u32 sh_num)
2744{
2745 u32 data = INSTANCE_BROADCAST_WRITES;
2746
2747 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2748 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2749 else if (se_num == 0xffffffff)
2750 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2751 else if (sh_num == 0xffffffff)
2752 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2753 else
2754 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2755 WREG32(GRBM_GFX_INDEX, data);
2756}
2757
2758static u32 si_create_bitmask(u32 bit_width)
2759{
2760 u32 i, mask = 0;
2761
2762 for (i = 0; i < bit_width; i++) {
2763 mask <<= 1;
2764 mask |= 1;
2765 }
2766 return mask;
2767}
2768
2769static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2770{
2771 u32 data, mask;
2772
2773 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2774 if (data & 1)
2775 data &= INACTIVE_CUS_MASK;
2776 else
2777 data = 0;
2778 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2779
2780 data >>= INACTIVE_CUS_SHIFT;
2781
2782 mask = si_create_bitmask(cu_per_sh);
2783
2784 return ~data & mask;
2785}
2786
2787static void si_setup_spi(struct radeon_device *rdev,
2788 u32 se_num, u32 sh_per_se,
2789 u32 cu_per_sh)
2790{
2791 int i, j, k;
2792 u32 data, mask, active_cu;
2793
2794 for (i = 0; i < se_num; i++) {
2795 for (j = 0; j < sh_per_se; j++) {
2796 si_select_se_sh(rdev, i, j);
2797 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2798 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2799
2800 mask = 1;
2801 for (k = 0; k < 16; k++) {
2802 mask <<= k;
2803 if (active_cu & mask) {
2804 data &= ~mask;
2805 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2806 break;
2807 }
2808 }
2809 }
2810 }
2811 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2812}
2813
2814static u32 si_get_rb_disabled(struct radeon_device *rdev,
2815 u32 max_rb_num, u32 se_num,
2816 u32 sh_per_se)
2817{
2818 u32 data, mask;
2819
2820 data = RREG32(CC_RB_BACKEND_DISABLE);
2821 if (data & 1)
2822 data &= BACKEND_DISABLE_MASK;
2823 else
2824 data = 0;
2825 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2826
2827 data >>= BACKEND_DISABLE_SHIFT;
2828
2829 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2830
2831 return data & mask;
2832}
2833
2834static void si_setup_rb(struct radeon_device *rdev,
2835 u32 se_num, u32 sh_per_se,
2836 u32 max_rb_num)
2837{
2838 int i, j;
2839 u32 data, mask;
2840 u32 disabled_rbs = 0;
2841 u32 enabled_rbs = 0;
2842
2843 for (i = 0; i < se_num; i++) {
2844 for (j = 0; j < sh_per_se; j++) {
2845 si_select_se_sh(rdev, i, j);
2846 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2847 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2848 }
2849 }
2850 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2851
2852 mask = 1;
2853 for (i = 0; i < max_rb_num; i++) {
2854 if (!(disabled_rbs & mask))
2855 enabled_rbs |= mask;
2856 mask <<= 1;
2857 }
2858
2859 for (i = 0; i < se_num; i++) {
2860 si_select_se_sh(rdev, i, 0xffffffff);
2861 data = 0;
2862 for (j = 0; j < sh_per_se; j++) {
2863 switch (enabled_rbs & 3) {
2864 case 1:
2865 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2866 break;
2867 case 2:
2868 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2869 break;
2870 case 3:
2871 default:
2872 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2873 break;
2874 }
2875 enabled_rbs >>= 2;
2876 }
2877 WREG32(PA_SC_RASTER_CONFIG, data);
2878 }
2879 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2880}
2881
0a96d72b
AD		 2882static void si_gpu_init(struct radeon_device *rdev)
2883{
0a96d72b
AD		 2884 u32 gb_addr_config = 0;
2885 u32 mc_shared_chmap, mc_arb_ramcfg;
0a96d72b 2886 u32 sx_debug_1;
0a96d72b
AD		 2887 u32 hdp_host_path_cntl;
2888 u32 tmp;
2889 int i, j;
2890
2891 switch (rdev->family) {
2892 case CHIP_TAHITI:
2893 rdev->config.si.max_shader_engines = 2;
0a96d72b 2894 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 2895 rdev->config.si.max_cu_per_sh = 8;
2896 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2897 rdev->config.si.max_backends_per_se = 4;
2898 rdev->config.si.max_texture_channel_caches = 12;
2899 rdev->config.si.max_gprs = 256;
2900 rdev->config.si.max_gs_threads = 32;
2901 rdev->config.si.max_hw_contexts = 8;
2902
2903 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2904 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2905 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2906 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2907 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2908 break;
2909 case CHIP_PITCAIRN:
2910 rdev->config.si.max_shader_engines = 2;
0a96d72b 2911 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 2912 rdev->config.si.max_cu_per_sh = 5;
2913 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2914 rdev->config.si.max_backends_per_se = 4;
2915 rdev->config.si.max_texture_channel_caches = 8;
2916 rdev->config.si.max_gprs = 256;
2917 rdev->config.si.max_gs_threads = 32;
2918 rdev->config.si.max_hw_contexts = 8;
2919
2920 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2921 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2922 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2923 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2924 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2925 break;
2926 case CHIP_VERDE:
2927 default:
2928 rdev->config.si.max_shader_engines = 1;
0a96d72b 2929 rdev->config.si.max_tile_pipes = 4;
468ef1a5 2930 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 2931 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2932 rdev->config.si.max_backends_per_se = 4;
2933 rdev->config.si.max_texture_channel_caches = 4;
2934 rdev->config.si.max_gprs = 256;
2935 rdev->config.si.max_gs_threads = 32;
2936 rdev->config.si.max_hw_contexts = 8;
2937
d0ae7fcc
AD		 2938 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2939 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2940 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2941 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2942 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2943 break;
2944 case CHIP_OLAND:
2945 rdev->config.si.max_shader_engines = 1;
2946 rdev->config.si.max_tile_pipes = 4;
2947 rdev->config.si.max_cu_per_sh = 6;
2948 rdev->config.si.max_sh_per_se = 1;
2949 rdev->config.si.max_backends_per_se = 2;
2950 rdev->config.si.max_texture_channel_caches = 4;
2951 rdev->config.si.max_gprs = 256;
2952 rdev->config.si.max_gs_threads = 16;
2953 rdev->config.si.max_hw_contexts = 8;
2954
0a96d72b
AD		 2955 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2956 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2957 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2958 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2959 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 2960 break;
8b02859d
AD		 2961 case CHIP_HAINAN:
2962 rdev->config.si.max_shader_engines = 1;
2963 rdev->config.si.max_tile_pipes = 4;
2964 rdev->config.si.max_cu_per_sh = 5;
2965 rdev->config.si.max_sh_per_se = 1;
2966 rdev->config.si.max_backends_per_se = 1;
2967 rdev->config.si.max_texture_channel_caches = 2;
2968 rdev->config.si.max_gprs = 256;
2969 rdev->config.si.max_gs_threads = 16;
2970 rdev->config.si.max_hw_contexts = 8;
2971
2972 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2973 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2974 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2975 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2976 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2977 break;
0a96d72b
AD		 2978 }
2979
2980 /* Initialize HDP */
2981 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2982 WREG32((0x2c14 + j), 0x00000000);
2983 WREG32((0x2c18 + j), 0x00000000);
2984 WREG32((0x2c1c + j), 0x00000000);
2985 WREG32((0x2c20 + j), 0x00000000);
2986 WREG32((0x2c24 + j), 0x00000000);
2987 }
2988
2989 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2990
2991 evergreen_fix_pci_max_read_req_size(rdev);
2992
2993 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2994
2995 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2996 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2997
0a96d72b 2998 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 2999 rdev->config.si.mem_max_burst_length_bytes = 256;
3000 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3001 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3002 if (rdev->config.si.mem_row_size_in_kb > 4)
3003 rdev->config.si.mem_row_size_in_kb = 4;
3004 /* XXX use MC settings? */
3005 rdev->config.si.shader_engine_tile_size = 32;
3006 rdev->config.si.num_gpus = 1;
3007 rdev->config.si.multi_gpu_tile_size = 64;
3008
1a8ca750
AD		 3009 /* fix up row size */
3010 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 3011 switch (rdev->config.si.mem_row_size_in_kb) {
3012 case 1:
3013 default:
3014 gb_addr_config |= ROW_SIZE(0);
3015 break;
3016 case 2:
3017 gb_addr_config |= ROW_SIZE(1);
3018 break;
3019 case 4:
3020 gb_addr_config |= ROW_SIZE(2);
3021 break;
3022 }
3023
0a96d72b
AD		 3024 /* setup tiling info dword. gb_addr_config is not adequate since it does
3025 * not have bank info, so create a custom tiling dword.
3026 * bits 3:0 num_pipes
3027 * bits 7:4 num_banks
3028 * bits 11:8 group_size
3029 * bits 15:12 row_size
3030 */
3031 rdev->config.si.tile_config = 0;
3032 switch (rdev->config.si.num_tile_pipes) {
3033 case 1:
3034 rdev->config.si.tile_config |= (0 << 0);
3035 break;
3036 case 2:
3037 rdev->config.si.tile_config |= (1 << 0);
3038 break;
3039 case 4:
3040 rdev->config.si.tile_config |= (2 << 0);
3041 break;
3042 case 8:
3043 default:
3044 /* XXX what about 12? */
3045 rdev->config.si.tile_config |= (3 << 0);
3046 break;
dca571a6
CK		 3047 }
3048 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3049 case 0: /* four banks */
1a8ca750 3050 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 3051 break;
3052 case 1: /* eight banks */
3053 rdev->config.si.tile_config |= 1 << 4;
3054 break;
3055 case 2: /* sixteen banks */
3056 default:
3057 rdev->config.si.tile_config |= 2 << 4;
3058 break;
3059 }
0a96d72b
AD		 3060 rdev->config.si.tile_config |=
3061 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3062 rdev->config.si.tile_config |=
3063 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3064
0a96d72b
AD		 3065 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3066 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 3067 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 3068 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 3069 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3070 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 3071 if (rdev->has_uvd) {
3072 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3073 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3074 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3075 }
0a96d72b 3076
1a8ca750 3077 si_tiling_mode_table_init(rdev);
0a96d72b 3078
1a8ca750
AD		 3079 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3080 rdev->config.si.max_sh_per_se,
3081 rdev->config.si.max_backends_per_se);
0a96d72b 3082
1a8ca750
AD		 3083 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3084 rdev->config.si.max_sh_per_se,
3085 rdev->config.si.max_cu_per_sh);
0a96d72b 3086
0a96d72b
AD		 3087
3088 /* set HW defaults for 3D engine */
3089 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3090 ROQ_IB2_START(0x2b)));
3091 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3092
3093 sx_debug_1 = RREG32(SX_DEBUG_1);
3094 WREG32(SX_DEBUG_1, sx_debug_1);
3095
3096 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3097
3098 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3099 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3100 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3101 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3102
3103 WREG32(VGT_NUM_INSTANCES, 1);
3104
3105 WREG32(CP_PERFMON_CNTL, 0);
3106
3107 WREG32(SQ_CONFIG, 0);
3108
3109 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3110 FORCE_EOV_MAX_REZ_CNT(255)));
3111
3112 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3113 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3114
3115 WREG32(VGT_GS_VERTEX_REUSE, 16);
3116 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3117
3118 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3119 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3120 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3121 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3122 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3123 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3124 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3125 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3126
3127 tmp = RREG32(HDP_MISC_CNTL);
3128 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3129 WREG32(HDP_MISC_CNTL, tmp);
3130
3131 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3132 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3133
3134 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3135
3136 udelay(50);
3137}
c476dde2 3138
2ece2e8b
AD		 3139/*
3140 * GPU scratch registers helpers function.
3141 */
3142static void si_scratch_init(struct radeon_device *rdev)
3143{
3144 int i;
3145
3146 rdev->scratch.num_reg = 7;
3147 rdev->scratch.reg_base = SCRATCH_REG0;
3148 for (i = 0; i < rdev->scratch.num_reg; i++) {
3149 rdev->scratch.free[i] = true;
3150 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3151 }
3152}
3153
3154void si_fence_ring_emit(struct radeon_device *rdev,
3155 struct radeon_fence *fence)
3156{
3157 struct radeon_ring *ring = &rdev->ring[fence->ring];
3158 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3159
3160 /* flush read cache over gart */
3161 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3162 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3163 radeon_ring_write(ring, 0);
3164 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3165 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3166 PACKET3_TC_ACTION_ENA |
3167 PACKET3_SH_KCACHE_ACTION_ENA |
3168 PACKET3_SH_ICACHE_ACTION_ENA);
3169 radeon_ring_write(ring, 0xFFFFFFFF);
3170 radeon_ring_write(ring, 0);
3171 radeon_ring_write(ring, 10); /* poll interval */
3172 /* EVENT_WRITE_EOP - flush caches, send int */
3173 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3174 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3175 radeon_ring_write(ring, addr & 0xffffffff);
3176 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3177 radeon_ring_write(ring, fence->seq);
3178 radeon_ring_write(ring, 0);
3179}
3180
3181/*
3182 * IB stuff
3183 */
3184void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3185{
876dc9f3 3186 struct radeon_ring *ring = &rdev->ring[ib->ring];
2ece2e8b
AD		 3187 u32 header;
3188
a85a7da4
AD		 3189 if (ib->is_const_ib) {
3190 /* set switch buffer packet before const IB */
3191 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3192 radeon_ring_write(ring, 0);
45df6803 3193
2ece2e8b 3194 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 3195 } else {
89d35807 3196 u32 next_rptr;
a85a7da4 3197 if (ring->rptr_save_reg) {
89d35807 3198 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 3199 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3200 radeon_ring_write(ring, ((ring->rptr_save_reg -
3201 PACKET3_SET_CONFIG_REG_START) >> 2));
3202 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3203 } else if (rdev->wb.enabled) {
3204 next_rptr = ring->wptr + 5 + 4 + 8;
3205 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3206 radeon_ring_write(ring, (1 << 8));
3207 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3208 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3209 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 3210 }
3211
2ece2e8b 3212 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 3213 }
2ece2e8b
AD		 3214
3215 radeon_ring_write(ring, header);
3216 radeon_ring_write(ring,
3217#ifdef __BIG_ENDIAN
3218 (2 << 0) |
3219#endif
3220 (ib->gpu_addr & 0xFFFFFFFC));
3221 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4bf3dd92
CK		 3222 radeon_ring_write(ring, ib->length_dw |
3223 (ib->vm ? (ib->vm->id << 24) : 0));
2ece2e8b 3224
a85a7da4
AD		 3225 if (!ib->is_const_ib) {
3226 /* flush read cache over gart for this vmid */
3227 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3228 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 3229 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
a85a7da4
AD		 3230 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3231 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3232 PACKET3_TC_ACTION_ENA |
3233 PACKET3_SH_KCACHE_ACTION_ENA |
3234 PACKET3_SH_ICACHE_ACTION_ENA);
3235 radeon_ring_write(ring, 0xFFFFFFFF);
3236 radeon_ring_write(ring, 0);
3237 radeon_ring_write(ring, 10); /* poll interval */
3238 }
2ece2e8b
AD		 3239}
3240
48c0c902
AD		 3241/*
3242 * CP.
3243 */
3244static void si_cp_enable(struct radeon_device *rdev, bool enable)
3245{
3246 if (enable)
3247 WREG32(CP_ME_CNTL, 0);
3248 else {
3249 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3250 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3251 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 3252 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3253 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3254 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 3255 }
3256 udelay(50);
3257}
3258
3259static int si_cp_load_microcode(struct radeon_device *rdev)
3260{
3261 const __be32 *fw_data;
3262 int i;
3263
3264 if (!rdev->me_fw || !rdev->pfp_fw)
3265 return -EINVAL;
3266
3267 si_cp_enable(rdev, false);
3268
3269 /* PFP */
3270 fw_data = (const __be32 *)rdev->pfp_fw->data;
3271 WREG32(CP_PFP_UCODE_ADDR, 0);
3272 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3273 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3274 WREG32(CP_PFP_UCODE_ADDR, 0);
3275
3276 /* CE */
3277 fw_data = (const __be32 *)rdev->ce_fw->data;
3278 WREG32(CP_CE_UCODE_ADDR, 0);
3279 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3280 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3281 WREG32(CP_CE_UCODE_ADDR, 0);
3282
3283 /* ME */
3284 fw_data = (const __be32 *)rdev->me_fw->data;
3285 WREG32(CP_ME_RAM_WADDR, 0);
3286 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3287 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3288 WREG32(CP_ME_RAM_WADDR, 0);
3289
3290 WREG32(CP_PFP_UCODE_ADDR, 0);
3291 WREG32(CP_CE_UCODE_ADDR, 0);
3292 WREG32(CP_ME_RAM_WADDR, 0);
3293 WREG32(CP_ME_RAM_RADDR, 0);
3294 return 0;
3295}
3296
3297static int si_cp_start(struct radeon_device *rdev)
3298{
3299 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3300 int r, i;
3301
3302 r = radeon_ring_lock(rdev, ring, 7 + 4);
3303 if (r) {
3304 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3305 return r;
3306 }
3307 /* init the CP */
3308 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3309 radeon_ring_write(ring, 0x1);
3310 radeon_ring_write(ring, 0x0);
3311 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3312 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3313 radeon_ring_write(ring, 0);
3314 radeon_ring_write(ring, 0);
3315
3316 /* init the CE partitions */
3317 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3318 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3319 radeon_ring_write(ring, 0xc000);
3320 radeon_ring_write(ring, 0xe000);
3321 radeon_ring_unlock_commit(rdev, ring);
3322
3323 si_cp_enable(rdev, true);
3324
3325 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3326 if (r) {
3327 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3328 return r;
3329 }
3330
3331 /* setup clear context state */
3332 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3333 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3334
3335 for (i = 0; i < si_default_size; i++)
3336 radeon_ring_write(ring, si_default_state[i]);
3337
3338 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3339 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3340
3341 /* set clear context state */
3342 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3343 radeon_ring_write(ring, 0);
3344
3345 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3346 radeon_ring_write(ring, 0x00000316);
3347 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3348 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3349
3350 radeon_ring_unlock_commit(rdev, ring);
3351
3352 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3353 ring = &rdev->ring[i];
3354 r = radeon_ring_lock(rdev, ring, 2);
3355
3356 /* clear the compute context state */
3357 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3358 radeon_ring_write(ring, 0);
3359
3360 radeon_ring_unlock_commit(rdev, ring);
3361 }
3362
3363 return 0;
3364}
3365
3366static void si_cp_fini(struct radeon_device *rdev)
3367{
45df6803 3368 struct radeon_ring *ring;
48c0c902 3369 si_cp_enable(rdev, false);
45df6803
CK		 3370
3371 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3372 radeon_ring_fini(rdev, ring);
3373 radeon_scratch_free(rdev, ring->rptr_save_reg);
3374
3375 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3376 radeon_ring_fini(rdev, ring);
3377 radeon_scratch_free(rdev, ring->rptr_save_reg);
3378
3379 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3380 radeon_ring_fini(rdev, ring);
3381 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3382}
3383
3384static int si_cp_resume(struct radeon_device *rdev)
3385{
3386 struct radeon_ring *ring;
3387 u32 tmp;
3388 u32 rb_bufsz;
3389 int r;
3390
811e4d58
AD		 3391 si_enable_gui_idle_interrupt(rdev, false);
3392
48c0c902
AD		 3393 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3394 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3395
3396 /* Set the write pointer delay */
3397 WREG32(CP_RB_WPTR_DELAY, 0);
3398
3399 WREG32(CP_DEBUG, 0);
3400 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3401
3402 /* ring 0 - compute and gfx */
3403 /* Set ring buffer size */
3404 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 3405 rb_bufsz = order_base_2(ring->ring_size / 8);
3406 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3407#ifdef __BIG_ENDIAN
3408 tmp |= BUF_SWAP_32BIT;
3409#endif
3410 WREG32(CP_RB0_CNTL, tmp);
3411
3412 /* Initialize the ring buffer's read and write pointers */
3413 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3414 ring->wptr = 0;
3415 WREG32(CP_RB0_WPTR, ring->wptr);
3416
48fc7f7e 3417 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3418 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3419 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3420
3421 if (rdev->wb.enabled)
3422 WREG32(SCRATCH_UMSK, 0xff);
3423 else {
3424 tmp |= RB_NO_UPDATE;
3425 WREG32(SCRATCH_UMSK, 0);
3426 }
3427
3428 mdelay(1);
3429 WREG32(CP_RB0_CNTL, tmp);
3430
3431 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3432
3433 ring->rptr = RREG32(CP_RB0_RPTR);
3434
3435 /* ring1 - compute only */
3436 /* Set ring buffer size */
3437 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
b72a8925
DV		 3438 rb_bufsz = order_base_2(ring->ring_size / 8);
3439 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3440#ifdef __BIG_ENDIAN
3441 tmp |= BUF_SWAP_32BIT;
3442#endif
3443 WREG32(CP_RB1_CNTL, tmp);
3444
3445 /* Initialize the ring buffer's read and write pointers */
3446 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3447 ring->wptr = 0;
3448 WREG32(CP_RB1_WPTR, ring->wptr);
3449
48fc7f7e 3450 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3451 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3452 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3453
3454 mdelay(1);
3455 WREG32(CP_RB1_CNTL, tmp);
3456
3457 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3458
3459 ring->rptr = RREG32(CP_RB1_RPTR);
3460
3461 /* ring2 - compute only */
3462 /* Set ring buffer size */
3463 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
b72a8925
DV		 3464 rb_bufsz = order_base_2(ring->ring_size / 8);
3465 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3466#ifdef __BIG_ENDIAN
3467 tmp |= BUF_SWAP_32BIT;
3468#endif
3469 WREG32(CP_RB2_CNTL, tmp);
3470
3471 /* Initialize the ring buffer's read and write pointers */
3472 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3473 ring->wptr = 0;
3474 WREG32(CP_RB2_WPTR, ring->wptr);
3475
48fc7f7e 3476 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3477 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3478 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3479
3480 mdelay(1);
3481 WREG32(CP_RB2_CNTL, tmp);
3482
3483 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3484
3485 ring->rptr = RREG32(CP_RB2_RPTR);
3486
3487 /* start the rings */
3488 si_cp_start(rdev);
3489 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3490 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3491 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3492 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3493 if (r) {
3494 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3495 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3496 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3497 return r;
3498 }
3499 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3500 if (r) {
3501 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3502 }
3503 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3504 if (r) {
3505 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3506 }
3507
811e4d58
AD		 3508 si_enable_gui_idle_interrupt(rdev, true);
3509
48c0c902
AD		 3510 return 0;
3511}
3512
2483b4ea 3513u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3514{
014bb209 3515 u32 reset_mask = 0;
1c534671 3516 u32 tmp;
06bc6df0 3517
014bb209
AD		 3518 /* GRBM_STATUS */
3519 tmp = RREG32(GRBM_STATUS);
3520 if (tmp & (PA_BUSY | SC_BUSY |
3521 BCI_BUSY | SX_BUSY |
3522 TA_BUSY | VGT_BUSY |
3523 DB_BUSY | CB_BUSY |
3524 GDS_BUSY | SPI_BUSY |
3525 IA_BUSY | IA_BUSY_NO_DMA))
3526 reset_mask |= RADEON_RESET_GFX;
3527
3528 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3529 CP_BUSY | CP_COHERENCY_BUSY))
3530 reset_mask |= RADEON_RESET_CP;
3531
3532 if (tmp & GRBM_EE_BUSY)
3533 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3534
3535 /* GRBM_STATUS2 */
3536 tmp = RREG32(GRBM_STATUS2);
3537 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3538 reset_mask |= RADEON_RESET_RLC;
3539
3540 /* DMA_STATUS_REG 0 */
3541 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3542 if (!(tmp & DMA_IDLE))
3543 reset_mask |= RADEON_RESET_DMA;
3544
3545 /* DMA_STATUS_REG 1 */
3546 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3547 if (!(tmp & DMA_IDLE))
3548 reset_mask |= RADEON_RESET_DMA1;
3549
3550 /* SRBM_STATUS2 */
3551 tmp = RREG32(SRBM_STATUS2);
3552 if (tmp & DMA_BUSY)
3553 reset_mask |= RADEON_RESET_DMA;
3554
3555 if (tmp & DMA1_BUSY)
3556 reset_mask |= RADEON_RESET_DMA1;
3557
3558 /* SRBM_STATUS */
3559 tmp = RREG32(SRBM_STATUS);
3560
3561 if (tmp & IH_BUSY)
3562 reset_mask |= RADEON_RESET_IH;
3563
3564 if (tmp & SEM_BUSY)
3565 reset_mask |= RADEON_RESET_SEM;
3566
3567 if (tmp & GRBM_RQ_PENDING)
3568 reset_mask |= RADEON_RESET_GRBM;
3569
3570 if (tmp & VMC_BUSY)
3571 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3572
014bb209
AD		 3573 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3574 MCC_BUSY | MCD_BUSY))
3575 reset_mask |= RADEON_RESET_MC;
3576
3577 if (evergreen_is_display_hung(rdev))
3578 reset_mask |= RADEON_RESET_DISPLAY;
3579
3580 /* VM_L2_STATUS */
3581 tmp = RREG32(VM_L2_STATUS);
3582 if (tmp & L2_BUSY)
3583 reset_mask |= RADEON_RESET_VMC;
3584
d808fc88
AD		 3585 /* Skip MC reset as it's mostly likely not hung, just busy */
3586 if (reset_mask & RADEON_RESET_MC) {
3587 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3588 reset_mask &= ~RADEON_RESET_MC;
3589 }
3590
014bb209
AD		 3591 return reset_mask;
3592}
3593
3594static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3595{
3596 struct evergreen_mc_save save;
3597 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3598 u32 tmp;
19fc42ed 3599
06bc6df0 3600 if (reset_mask == 0)
014bb209 3601 return;
06bc6df0
AD		 3602
3603 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3604
1c534671 3605 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3606 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3607 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3608 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3609 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3610
1c534671
AD		 3611 /* Disable CP parsing/prefetching */
3612 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3613
3614 if (reset_mask & RADEON_RESET_DMA) {
3615 /* dma0 */
3616 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3617 tmp &= ~DMA_RB_ENABLE;
3618 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3619 }
3620 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3621 /* dma1 */
3622 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3623 tmp &= ~DMA_RB_ENABLE;
3624 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3625 }
3626
f770d78a
AD		 3627 udelay(50);
3628
3629 evergreen_mc_stop(rdev, &save);
3630 if (evergreen_mc_wait_for_idle(rdev)) {
3631 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3632 }
3633
1c534671
AD		 3634 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3635 grbm_soft_reset = SOFT_RESET_CB |
3636 SOFT_RESET_DB |
3637 SOFT_RESET_GDS |
3638 SOFT_RESET_PA |
3639 SOFT_RESET_SC |
3640 SOFT_RESET_BCI |
3641 SOFT_RESET_SPI |
3642 SOFT_RESET_SX |
3643 SOFT_RESET_TC |
3644 SOFT_RESET_TA |
3645 SOFT_RESET_VGT |
3646 SOFT_RESET_IA;
3647 }
3648
3649 if (reset_mask & RADEON_RESET_CP) {
3650 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3651
3652 srbm_soft_reset |= SOFT_RESET_GRBM;
3653 }
06bc6df0
AD		 3654
3655 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3656 srbm_soft_reset |= SOFT_RESET_DMA;
3657
3658 if (reset_mask & RADEON_RESET_DMA1)
3659 srbm_soft_reset |= SOFT_RESET_DMA1;
3660
3661 if (reset_mask & RADEON_RESET_DISPLAY)
3662 srbm_soft_reset |= SOFT_RESET_DC;
3663
3664 if (reset_mask & RADEON_RESET_RLC)
3665 grbm_soft_reset |= SOFT_RESET_RLC;
3666
3667 if (reset_mask & RADEON_RESET_SEM)
3668 srbm_soft_reset |= SOFT_RESET_SEM;
3669
3670 if (reset_mask & RADEON_RESET_IH)
3671 srbm_soft_reset |= SOFT_RESET_IH;
3672
3673 if (reset_mask & RADEON_RESET_GRBM)
3674 srbm_soft_reset |= SOFT_RESET_GRBM;
3675
3676 if (reset_mask & RADEON_RESET_VMC)
3677 srbm_soft_reset |= SOFT_RESET_VMC;
3678
3679 if (reset_mask & RADEON_RESET_MC)
3680 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3681
3682 if (grbm_soft_reset) {
3683 tmp = RREG32(GRBM_SOFT_RESET);
3684 tmp |= grbm_soft_reset;
3685 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3686 WREG32(GRBM_SOFT_RESET, tmp);
3687 tmp = RREG32(GRBM_SOFT_RESET);
3688
3689 udelay(50);
3690
3691 tmp &= ~grbm_soft_reset;
3692 WREG32(GRBM_SOFT_RESET, tmp);
3693 tmp = RREG32(GRBM_SOFT_RESET);
3694 }
3695
3696 if (srbm_soft_reset) {
3697 tmp = RREG32(SRBM_SOFT_RESET);
3698 tmp |= srbm_soft_reset;
3699 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3700 WREG32(SRBM_SOFT_RESET, tmp);
3701 tmp = RREG32(SRBM_SOFT_RESET);
3702
3703 udelay(50);
3704
3705 tmp &= ~srbm_soft_reset;
3706 WREG32(SRBM_SOFT_RESET, tmp);
3707 tmp = RREG32(SRBM_SOFT_RESET);
3708 }
06bc6df0
AD		 3709
3710 /* Wait a little for things to settle down */
3711 udelay(50);
3712
c476dde2 3713 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3714 udelay(50);
3715
1c534671 3716 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3717}
3718
3719int si_asic_reset(struct radeon_device *rdev)
3720{
014bb209
AD		 3721 u32 reset_mask;
3722
3723 reset_mask = si_gpu_check_soft_reset(rdev);
3724
3725 if (reset_mask)
3726 r600_set_bios_scratch_engine_hung(rdev, true);
3727
3728 si_gpu_soft_reset(rdev, reset_mask);
3729
3730 reset_mask = si_gpu_check_soft_reset(rdev);
3731
3732 if (!reset_mask)
3733 r600_set_bios_scratch_engine_hung(rdev, false);
3734
3735 return 0;
c476dde2
AD		 3736}
3737
123bc183
AD		 3738/**
3739 * si_gfx_is_lockup - Check if the GFX engine is locked up
3740 *
3741 * @rdev: radeon_device pointer
3742 * @ring: radeon_ring structure holding ring information
3743 *
3744 * Check if the GFX engine is locked up.
3745 * Returns true if the engine appears to be locked up, false if not.
3746 */
3747bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3748{
3749 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3750
3751 if (!(reset_mask & (RADEON_RESET_GFX |
3752 RADEON_RESET_COMPUTE |
3753 RADEON_RESET_CP))) {
3754 radeon_ring_lockup_update(ring);
3755 return false;
3756 }
3757 /* force CP activities */
3758 radeon_ring_force_activity(rdev, ring);
3759 return radeon_ring_test_lockup(rdev, ring);
3760}
3761
d2800ee5
AD		 3762/* MC */
3763static void si_mc_program(struct radeon_device *rdev)
3764{
3765 struct evergreen_mc_save save;
3766 u32 tmp;
3767 int i, j;
3768
3769 /* Initialize HDP */
3770 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3771 WREG32((0x2c14 + j), 0x00000000);
3772 WREG32((0x2c18 + j), 0x00000000);
3773 WREG32((0x2c1c + j), 0x00000000);
3774 WREG32((0x2c20 + j), 0x00000000);
3775 WREG32((0x2c24 + j), 0x00000000);
3776 }
3777 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3778
3779 evergreen_mc_stop(rdev, &save);
3780 if (radeon_mc_wait_for_idle(rdev)) {
3781 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3782 }
5153550a
AD		 3783 if (!ASIC_IS_NODCE(rdev))
3784 /* Lockout access through VGA aperture*/
3785 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 3786 /* Update configuration */
3787 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3788 rdev->mc.vram_start >> 12);
3789 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3790 rdev->mc.vram_end >> 12);
3791 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3792 rdev->vram_scratch.gpu_addr >> 12);
3793 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3794 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3795 WREG32(MC_VM_FB_LOCATION, tmp);
3796 /* XXX double check these! */
3797 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3798 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3799 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3800 WREG32(MC_VM_AGP_BASE, 0);
3801 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3802 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3803 if (radeon_mc_wait_for_idle(rdev)) {
3804 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3805 }
3806 evergreen_mc_resume(rdev, &save);
5153550a
AD		 3807 if (!ASIC_IS_NODCE(rdev)) {
3808 /* we need to own VRAM, so turn off the VGA renderer here
3809 * to stop it overwriting our objects */
3810 rv515_vga_render_disable(rdev);
3811 }
d2800ee5
AD		 3812}
3813
1c49165d
AD		 3814void si_vram_gtt_location(struct radeon_device *rdev,
3815 struct radeon_mc *mc)
d2800ee5
AD		 3816{
3817 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3818 /* leave room for at least 1024M GTT */
3819 dev_warn(rdev->dev, "limiting VRAM\n");
3820 mc->real_vram_size = 0xFFC0000000ULL;
3821 mc->mc_vram_size = 0xFFC0000000ULL;
3822 }
9ed8b1f9 3823 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 3824 rdev->mc.gtt_base_align = 0;
9ed8b1f9 3825 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 3826}
3827
3828static int si_mc_init(struct radeon_device *rdev)
3829{
3830 u32 tmp;
3831 int chansize, numchan;
3832
3833 /* Get VRAM informations */
3834 rdev->mc.vram_is_ddr = true;
3835 tmp = RREG32(MC_ARB_RAMCFG);
3836 if (tmp & CHANSIZE_OVERRIDE) {
3837 chansize = 16;
3838 } else if (tmp & CHANSIZE_MASK) {
3839 chansize = 64;
3840 } else {
3841 chansize = 32;
3842 }
3843 tmp = RREG32(MC_SHARED_CHMAP);
3844 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3845 case 0:
3846 default:
3847 numchan = 1;
3848 break;
3849 case 1:
3850 numchan = 2;
3851 break;
3852 case 2:
3853 numchan = 4;
3854 break;
3855 case 3:
3856 numchan = 8;
3857 break;
3858 case 4:
3859 numchan = 3;
3860 break;
3861 case 5:
3862 numchan = 6;
3863 break;
3864 case 6:
3865 numchan = 10;
3866 break;
3867 case 7:
3868 numchan = 12;
3869 break;
3870 case 8:
3871 numchan = 16;
3872 break;
3873 }
3874 rdev->mc.vram_width = numchan * chansize;
3875 /* Could aper size report 0 ? */
3876 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3877 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3878 /* size in MB on si */
fc986034
NOS		 3879 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3880 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
d2800ee5
AD		 3881 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3882 si_vram_gtt_location(rdev, &rdev->mc);
3883 radeon_update_bandwidth_info(rdev);
3884
3885 return 0;
3886}
3887
3888/*
3889 * GART
3890 */
3891void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3892{
3893 /* flush hdp cache */
3894 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3895
3896 /* bits 0-15 are the VM contexts0-15 */
3897 WREG32(VM_INVALIDATE_REQUEST, 1);
3898}
3899
1109ca09 3900static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 3901{
3902 int r, i;
3903
3904 if (rdev->gart.robj == NULL) {
3905 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3906 return -EINVAL;
3907 }
3908 r = radeon_gart_table_vram_pin(rdev);
3909 if (r)
3910 return r;
3911 radeon_gart_restore(rdev);
3912 /* Setup TLB control */
3913 WREG32(MC_VM_MX_L1_TLB_CNTL,
3914 (0xA << 7) |
3915 ENABLE_L1_TLB |
3916 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3917 ENABLE_ADVANCED_DRIVER_MODEL |
3918 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3919 /* Setup L2 cache */
3920 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3921 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3922 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3923 EFFECTIVE_L2_QUEUE_SIZE(7) |
3924 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3925 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3926 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3927 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3928 /* setup context0 */
3929 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3930 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3931 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3932 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3933 (u32)(rdev->dummy_page.addr >> 12));
3934 WREG32(VM_CONTEXT0_CNTL2, 0);
3935 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3936 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3937
3938 WREG32(0x15D4, 0);
3939 WREG32(0x15D8, 0);
3940 WREG32(0x15DC, 0);
3941
3942 /* empty context1-15 */
d2800ee5
AD		 3943 /* set vm size, must be a multiple of 4 */
3944 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
c21b328e 3945 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
23d4f1f2
AD		 3946 /* Assign the pt base to something valid for now; the pts used for
3947 * the VMs are determined by the application and setup and assigned
3948 * on the fly in the vm part of radeon_gart.c
3949 */
d2800ee5
AD		 3950 for (i = 1; i < 16; i++) {
3951 if (i < 8)
3952 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3953 rdev->gart.table_addr >> 12);
3954 else
3955 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3956 rdev->gart.table_addr >> 12);
3957 }
3958
3959 /* enable context1-15 */
3960 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3961 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 3962 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 3963 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
ae133a11
CK		 3964 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3965 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3966 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3967 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3968 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3969 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3970 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3971 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3972 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3973 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3974 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3975 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 3976
3977 si_pcie_gart_tlb_flush(rdev);
3978 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3979 (unsigned)(rdev->mc.gtt_size >> 20),
3980 (unsigned long long)rdev->gart.table_addr);
3981 rdev->gart.ready = true;
3982 return 0;
3983}
3984
1109ca09 3985static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5
AD		 3986{
3987 /* Disable all tables */
3988 WREG32(VM_CONTEXT0_CNTL, 0);
3989 WREG32(VM_CONTEXT1_CNTL, 0);
3990 /* Setup TLB control */
3991 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3992 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3993 /* Setup L2 cache */
3994 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3995 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3996 EFFECTIVE_L2_QUEUE_SIZE(7) |
3997 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3998 WREG32(VM_L2_CNTL2, 0);
3999 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4000 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4001 radeon_gart_table_vram_unpin(rdev);
4002}
4003
1109ca09 4004static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 4005{
4006 si_pcie_gart_disable(rdev);
4007 radeon_gart_table_vram_free(rdev);
4008 radeon_gart_fini(rdev);
4009}
4010
498dd8b3
AD		 4011/* vm parser */
4012static bool si_vm_reg_valid(u32 reg)
4013{
4014 /* context regs are fine */
4015 if (reg >= 0x28000)
4016 return true;
4017
4018 /* check config regs */
4019 switch (reg) {
4020 case GRBM_GFX_INDEX:
f418b88a 4021 case CP_STRMOUT_CNTL:
498dd8b3
AD		 4022 case VGT_VTX_VECT_EJECT_REG:
4023 case VGT_CACHE_INVALIDATION:
4024 case VGT_ESGS_RING_SIZE:
4025 case VGT_GSVS_RING_SIZE:
4026 case VGT_GS_VERTEX_REUSE:
4027 case VGT_PRIMITIVE_TYPE:
4028 case VGT_INDEX_TYPE:
4029 case VGT_NUM_INDICES:
4030 case VGT_NUM_INSTANCES:
4031 case VGT_TF_RING_SIZE:
4032 case VGT_HS_OFFCHIP_PARAM:
4033 case VGT_TF_MEMORY_BASE:
4034 case PA_CL_ENHANCE:
4035 case PA_SU_LINE_STIPPLE_VALUE:
4036 case PA_SC_LINE_STIPPLE_STATE:
4037 case PA_SC_ENHANCE:
4038 case SQC_CACHES:
4039 case SPI_STATIC_THREAD_MGMT_1:
4040 case SPI_STATIC_THREAD_MGMT_2:
4041 case SPI_STATIC_THREAD_MGMT_3:
4042 case SPI_PS_MAX_WAVE_ID:
4043 case SPI_CONFIG_CNTL:
4044 case SPI_CONFIG_CNTL_1:
4045 case TA_CNTL_AUX:
4046 return true;
4047 default:
4048 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4049 return false;
4050 }
4051}
4052
4053static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4054 u32 *ib, struct radeon_cs_packet *pkt)
4055{
4056 switch (pkt->opcode) {
4057 case PACKET3_NOP:
4058 case PACKET3_SET_BASE:
4059 case PACKET3_SET_CE_DE_COUNTERS:
4060 case PACKET3_LOAD_CONST_RAM:
4061 case PACKET3_WRITE_CONST_RAM:
4062 case PACKET3_WRITE_CONST_RAM_OFFSET:
4063 case PACKET3_DUMP_CONST_RAM:
4064 case PACKET3_INCREMENT_CE_COUNTER:
4065 case PACKET3_WAIT_ON_DE_COUNTER:
4066 case PACKET3_CE_WRITE:
4067 break;
4068 default:
4069 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4070 return -EINVAL;
4071 }
4072 return 0;
4073}
4074
e5b9e750
TS		 4075static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4076{
4077 u32 start_reg, reg, i;
4078 u32 command = ib[idx + 4];
4079 u32 info = ib[idx + 1];
4080 u32 idx_value = ib[idx];
4081 if (command & PACKET3_CP_DMA_CMD_SAS) {
4082 /* src address space is register */
4083 if (((info & 0x60000000) >> 29) == 0) {
4084 start_reg = idx_value << 2;
4085 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4086 reg = start_reg;
4087 if (!si_vm_reg_valid(reg)) {
4088 DRM_ERROR("CP DMA Bad SRC register\n");
4089 return -EINVAL;
4090 }
4091 } else {
4092 for (i = 0; i < (command & 0x1fffff); i++) {
4093 reg = start_reg + (4 * i);
4094 if (!si_vm_reg_valid(reg)) {
4095 DRM_ERROR("CP DMA Bad SRC register\n");
4096 return -EINVAL;
4097 }
4098 }
4099 }
4100 }
4101 }
4102 if (command & PACKET3_CP_DMA_CMD_DAS) {
4103 /* dst address space is register */
4104 if (((info & 0x00300000) >> 20) == 0) {
4105 start_reg = ib[idx + 2];
4106 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4107 reg = start_reg;
4108 if (!si_vm_reg_valid(reg)) {
4109 DRM_ERROR("CP DMA Bad DST register\n");
4110 return -EINVAL;
4111 }
4112 } else {
4113 for (i = 0; i < (command & 0x1fffff); i++) {
4114 reg = start_reg + (4 * i);
4115 if (!si_vm_reg_valid(reg)) {
4116 DRM_ERROR("CP DMA Bad DST register\n");
4117 return -EINVAL;
4118 }
4119 }
4120 }
4121 }
4122 }
4123 return 0;
4124}
4125
498dd8b3
AD		 4126static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4127 u32 *ib, struct radeon_cs_packet *pkt)
4128{
e5b9e750 4129 int r;
498dd8b3
AD		 4130 u32 idx = pkt->idx + 1;
4131 u32 idx_value = ib[idx];
4132 u32 start_reg, end_reg, reg, i;
4133
4134 switch (pkt->opcode) {
4135 case PACKET3_NOP:
4136 case PACKET3_SET_BASE:
4137 case PACKET3_CLEAR_STATE:
4138 case PACKET3_INDEX_BUFFER_SIZE:
4139 case PACKET3_DISPATCH_DIRECT:
4140 case PACKET3_DISPATCH_INDIRECT:
4141 case PACKET3_ALLOC_GDS:
4142 case PACKET3_WRITE_GDS_RAM:
4143 case PACKET3_ATOMIC_GDS:
4144 case PACKET3_ATOMIC:
4145 case PACKET3_OCCLUSION_QUERY:
4146 case PACKET3_SET_PREDICATION:
4147 case PACKET3_COND_EXEC:
4148 case PACKET3_PRED_EXEC:
4149 case PACKET3_DRAW_INDIRECT:
4150 case PACKET3_DRAW_INDEX_INDIRECT:
4151 case PACKET3_INDEX_BASE:
4152 case PACKET3_DRAW_INDEX_2:
4153 case PACKET3_CONTEXT_CONTROL:
4154 case PACKET3_INDEX_TYPE:
4155 case PACKET3_DRAW_INDIRECT_MULTI:
4156 case PACKET3_DRAW_INDEX_AUTO:
4157 case PACKET3_DRAW_INDEX_IMMD:
4158 case PACKET3_NUM_INSTANCES:
4159 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4160 case PACKET3_STRMOUT_BUFFER_UPDATE:
4161 case PACKET3_DRAW_INDEX_OFFSET_2:
4162 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4163 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4164 case PACKET3_MPEG_INDEX:
4165 case PACKET3_WAIT_REG_MEM:
4166 case PACKET3_MEM_WRITE:
4167 case PACKET3_PFP_SYNC_ME:
4168 case PACKET3_SURFACE_SYNC:
4169 case PACKET3_EVENT_WRITE:
4170 case PACKET3_EVENT_WRITE_EOP:
4171 case PACKET3_EVENT_WRITE_EOS:
4172 case PACKET3_SET_CONTEXT_REG:
4173 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4174 case PACKET3_SET_SH_REG:
4175 case PACKET3_SET_SH_REG_OFFSET:
4176 case PACKET3_INCREMENT_DE_COUNTER:
4177 case PACKET3_WAIT_ON_CE_COUNTER:
4178 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4179 case PACKET3_ME_WRITE:
4180 break;
4181 case PACKET3_COPY_DATA:
4182 if ((idx_value & 0xf00) == 0) {
4183 reg = ib[idx + 3] * 4;
4184 if (!si_vm_reg_valid(reg))
4185 return -EINVAL;
4186 }
4187 break;
4188 case PACKET3_WRITE_DATA:
4189 if ((idx_value & 0xf00) == 0) {
4190 start_reg = ib[idx + 1] * 4;
4191 if (idx_value & 0x10000) {
4192 if (!si_vm_reg_valid(start_reg))
4193 return -EINVAL;
4194 } else {
4195 for (i = 0; i < (pkt->count - 2); i++) {
4196 reg = start_reg + (4 * i);
4197 if (!si_vm_reg_valid(reg))
4198 return -EINVAL;
4199 }
4200 }
4201 }
4202 break;
4203 case PACKET3_COND_WRITE:
4204 if (idx_value & 0x100) {
4205 reg = ib[idx + 5] * 4;
4206 if (!si_vm_reg_valid(reg))
4207 return -EINVAL;
4208 }
4209 break;
4210 case PACKET3_COPY_DW:
4211 if (idx_value & 0x2) {
4212 reg = ib[idx + 3] * 4;
4213 if (!si_vm_reg_valid(reg))
4214 return -EINVAL;
4215 }
4216 break;
4217 case PACKET3_SET_CONFIG_REG:
4218 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4219 end_reg = 4 * pkt->count + start_reg - 4;
4220 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4221 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4222 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4223 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4224 return -EINVAL;
4225 }
4226 for (i = 0; i < pkt->count; i++) {
4227 reg = start_reg + (4 * i);
4228 if (!si_vm_reg_valid(reg))
4229 return -EINVAL;
4230 }
4231 break;
5aa709be 4232 case PACKET3_CP_DMA:
e5b9e750
TS		 4233 r = si_vm_packet3_cp_dma_check(ib, idx);
4234 if (r)
4235 return r;
5aa709be 4236 break;
498dd8b3
AD		 4237 default:
4238 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4239 return -EINVAL;
4240 }
4241 return 0;
4242}
4243
4244static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4245 u32 *ib, struct radeon_cs_packet *pkt)
4246{
e5b9e750 4247 int r;
498dd8b3
AD		 4248 u32 idx = pkt->idx + 1;
4249 u32 idx_value = ib[idx];
4250 u32 start_reg, reg, i;
4251
4252 switch (pkt->opcode) {
4253 case PACKET3_NOP:
4254 case PACKET3_SET_BASE:
4255 case PACKET3_CLEAR_STATE:
4256 case PACKET3_DISPATCH_DIRECT:
4257 case PACKET3_DISPATCH_INDIRECT:
4258 case PACKET3_ALLOC_GDS:
4259 case PACKET3_WRITE_GDS_RAM:
4260 case PACKET3_ATOMIC_GDS:
4261 case PACKET3_ATOMIC:
4262 case PACKET3_OCCLUSION_QUERY:
4263 case PACKET3_SET_PREDICATION:
4264 case PACKET3_COND_EXEC:
4265 case PACKET3_PRED_EXEC:
4266 case PACKET3_CONTEXT_CONTROL:
4267 case PACKET3_STRMOUT_BUFFER_UPDATE:
4268 case PACKET3_WAIT_REG_MEM:
4269 case PACKET3_MEM_WRITE:
4270 case PACKET3_PFP_SYNC_ME:
4271 case PACKET3_SURFACE_SYNC:
4272 case PACKET3_EVENT_WRITE:
4273 case PACKET3_EVENT_WRITE_EOP:
4274 case PACKET3_EVENT_WRITE_EOS:
4275 case PACKET3_SET_CONTEXT_REG:
4276 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4277 case PACKET3_SET_SH_REG:
4278 case PACKET3_SET_SH_REG_OFFSET:
4279 case PACKET3_INCREMENT_DE_COUNTER:
4280 case PACKET3_WAIT_ON_CE_COUNTER:
4281 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4282 case PACKET3_ME_WRITE:
4283 break;
4284 case PACKET3_COPY_DATA:
4285 if ((idx_value & 0xf00) == 0) {
4286 reg = ib[idx + 3] * 4;
4287 if (!si_vm_reg_valid(reg))
4288 return -EINVAL;
4289 }
4290 break;
4291 case PACKET3_WRITE_DATA:
4292 if ((idx_value & 0xf00) == 0) {
4293 start_reg = ib[idx + 1] * 4;
4294 if (idx_value & 0x10000) {
4295 if (!si_vm_reg_valid(start_reg))
4296 return -EINVAL;
4297 } else {
4298 for (i = 0; i < (pkt->count - 2); i++) {
4299 reg = start_reg + (4 * i);
4300 if (!si_vm_reg_valid(reg))
4301 return -EINVAL;
4302 }
4303 }
4304 }
4305 break;
4306 case PACKET3_COND_WRITE:
4307 if (idx_value & 0x100) {
4308 reg = ib[idx + 5] * 4;
4309 if (!si_vm_reg_valid(reg))
4310 return -EINVAL;
4311 }
4312 break;
4313 case PACKET3_COPY_DW:
4314 if (idx_value & 0x2) {
4315 reg = ib[idx + 3] * 4;
4316 if (!si_vm_reg_valid(reg))
4317 return -EINVAL;
4318 }
4319 break;
e5b9e750
TS		 4320 case PACKET3_CP_DMA:
4321 r = si_vm_packet3_cp_dma_check(ib, idx);
4322 if (r)
4323 return r;
4324 break;
498dd8b3
AD		 4325 default:
4326 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4327 return -EINVAL;
4328 }
4329 return 0;
4330}
4331
4332int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4333{
4334 int ret = 0;
4335 u32 idx = 0;
4336 struct radeon_cs_packet pkt;
4337
4338 do {
4339 pkt.idx = idx;
4e872ae2
IH		 4340 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4341 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4342 pkt.one_reg_wr = 0;
4343 switch (pkt.type) {
4e872ae2 4344 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 4345 dev_err(rdev->dev, "Packet0 not allowed!\n");
4346 ret = -EINVAL;
4347 break;
4e872ae2 4348 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4349 idx += 1;
4350 break;
4e872ae2
IH		 4351 case RADEON_PACKET_TYPE3:
4352 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4353 if (ib->is_const_ib)
4354 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4355 else {
876dc9f3 4356 switch (ib->ring) {
498dd8b3
AD		 4357 case RADEON_RING_TYPE_GFX_INDEX:
4358 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4359 break;
4360 case CAYMAN_RING_TYPE_CP1_INDEX:
4361 case CAYMAN_RING_TYPE_CP2_INDEX:
4362 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4363 break;
4364 default:
876dc9f3 4365 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4366 ret = -EINVAL;
4367 break;
4368 }
4369 }
4370 idx += pkt.count + 2;
4371 break;
4372 default:
4373 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4374 ret = -EINVAL;
4375 break;
4376 }
4377 if (ret)
4378 break;
4379 } while (idx < ib->length_dw);
4380
4381 return ret;
4382}
4383
d2800ee5
AD		 4384/*
4385 * vm
4386 */
4387int si_vm_init(struct radeon_device *rdev)
4388{
4389 /* number of VMs */
4390 rdev->vm_manager.nvm = 16;
4391 /* base offset of vram pages */
4392 rdev->vm_manager.vram_base_offset = 0;
4393
4394 return 0;
4395}
4396
4397void si_vm_fini(struct radeon_device *rdev)
4398{
4399}
4400
fbf6dc7a
AD		 4401/**
4402 * si_vm_decode_fault - print human readable fault info
4403 *
4404 * @rdev: radeon_device pointer
4405 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4406 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4407 *
4408 * Print human readable fault information (SI).
4409 */
4410static void si_vm_decode_fault(struct radeon_device *rdev,
4411 u32 status, u32 addr)
4412{
4413 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4414 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4415 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4416 char *block;
4417
4418 if (rdev->family == CHIP_TAHITI) {
4419 switch (mc_id) {
4420 case 160:
4421 case 144:
4422 case 96:
4423 case 80:
4424 case 224:
4425 case 208:
4426 case 32:
4427 case 16:
4428 block = "CB";
4429 break;
4430 case 161:
4431 case 145:
4432 case 97:
4433 case 81:
4434 case 225:
4435 case 209:
4436 case 33:
4437 case 17:
4438 block = "CB_FMASK";
4439 break;
4440 case 162:
4441 case 146:
4442 case 98:
4443 case 82:
4444 case 226:
4445 case 210:
4446 case 34:
4447 case 18:
4448 block = "CB_CMASK";
4449 break;
4450 case 163:
4451 case 147:
4452 case 99:
4453 case 83:
4454 case 227:
4455 case 211:
4456 case 35:
4457 case 19:
4458 block = "CB_IMMED";
4459 break;
4460 case 164:
4461 case 148:
4462 case 100:
4463 case 84:
4464 case 228:
4465 case 212:
4466 case 36:
4467 case 20:
4468 block = "DB";
4469 break;
4470 case 165:
4471 case 149:
4472 case 101:
4473 case 85:
4474 case 229:
4475 case 213:
4476 case 37:
4477 case 21:
4478 block = "DB_HTILE";
4479 break;
4480 case 167:
4481 case 151:
4482 case 103:
4483 case 87:
4484 case 231:
4485 case 215:
4486 case 39:
4487 case 23:
4488 block = "DB_STEN";
4489 break;
4490 case 72:
4491 case 68:
4492 case 64:
4493 case 8:
4494 case 4:
4495 case 0:
4496 case 136:
4497 case 132:
4498 case 128:
4499 case 200:
4500 case 196:
4501 case 192:
4502 block = "TC";
4503 break;
4504 case 112:
4505 case 48:
4506 block = "CP";
4507 break;
4508 case 49:
4509 case 177:
4510 case 50:
4511 case 178:
4512 block = "SH";
4513 break;
4514 case 53:
4515 case 190:
4516 block = "VGT";
4517 break;
4518 case 117:
4519 block = "IH";
4520 break;
4521 case 51:
4522 case 115:
4523 block = "RLC";
4524 break;
4525 case 119:
4526 case 183:
4527 block = "DMA0";
4528 break;
4529 case 61:
4530 block = "DMA1";
4531 break;
4532 case 248:
4533 case 120:
4534 block = "HDP";
4535 break;
4536 default:
4537 block = "unknown";
4538 break;
4539 }
4540 } else {
4541 switch (mc_id) {
4542 case 32:
4543 case 16:
4544 case 96:
4545 case 80:
4546 case 160:
4547 case 144:
4548 case 224:
4549 case 208:
4550 block = "CB";
4551 break;
4552 case 33:
4553 case 17:
4554 case 97:
4555 case 81:
4556 case 161:
4557 case 145:
4558 case 225:
4559 case 209:
4560 block = "CB_FMASK";
4561 break;
4562 case 34:
4563 case 18:
4564 case 98:
4565 case 82:
4566 case 162:
4567 case 146:
4568 case 226:
4569 case 210:
4570 block = "CB_CMASK";
4571 break;
4572 case 35:
4573 case 19:
4574 case 99:
4575 case 83:
4576 case 163:
4577 case 147:
4578 case 227:
4579 case 211:
4580 block = "CB_IMMED";
4581 break;
4582 case 36:
4583 case 20:
4584 case 100:
4585 case 84:
4586 case 164:
4587 case 148:
4588 case 228:
4589 case 212:
4590 block = "DB";
4591 break;
4592 case 37:
4593 case 21:
4594 case 101:
4595 case 85:
4596 case 165:
4597 case 149:
4598 case 229:
4599 case 213:
4600 block = "DB_HTILE";
4601 break;
4602 case 39:
4603 case 23:
4604 case 103:
4605 case 87:
4606 case 167:
4607 case 151:
4608 case 231:
4609 case 215:
4610 block = "DB_STEN";
4611 break;
4612 case 72:
4613 case 68:
4614 case 8:
4615 case 4:
4616 case 136:
4617 case 132:
4618 case 200:
4619 case 196:
4620 block = "TC";
4621 break;
4622 case 112:
4623 case 48:
4624 block = "CP";
4625 break;
4626 case 49:
4627 case 177:
4628 case 50:
4629 case 178:
4630 block = "SH";
4631 break;
4632 case 53:
4633 block = "VGT";
4634 break;
4635 case 117:
4636 block = "IH";
4637 break;
4638 case 51:
4639 case 115:
4640 block = "RLC";
4641 break;
4642 case 119:
4643 case 183:
4644 block = "DMA0";
4645 break;
4646 case 61:
4647 block = "DMA1";
4648 break;
4649 case 248:
4650 case 120:
4651 block = "HDP";
4652 break;
4653 default:
4654 block = "unknown";
4655 break;
4656 }
4657 }
4658
4659 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4660 protections, vmid, addr,
4661 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4662 block, mc_id);
4663}
4664
82ffd92b
AD		 4665/**
4666 * si_vm_set_page - update the page tables using the CP
4667 *
4668 * @rdev: radeon_device pointer
43f1214a 4669 * @ib: indirect buffer to fill with commands
82ffd92b
AD		 4670 * @pe: addr of the page entry
4671 * @addr: dst addr to write into pe
4672 * @count: number of page entries to update
4673 * @incr: increase next addr by incr bytes
4674 * @flags: access flags
4675 *
43f1214a 4676 * Update the page tables using the CP (SI).
82ffd92b 4677 */
43f1214a
AD		 4678void si_vm_set_page(struct radeon_device *rdev,
4679 struct radeon_ib *ib,
4680 uint64_t pe,
82ffd92b
AD		 4681 uint64_t addr, unsigned count,
4682 uint32_t incr, uint32_t flags)
d2800ee5 4683{
82ffd92b 4684 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
deab48f1
AD		 4685 uint64_t value;
4686 unsigned ndw;
4687
4688 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4689 while (count) {
4690 ndw = 2 + count * 2;
4691 if (ndw > 0x3FFE)
4692 ndw = 0x3FFE;
4693
43f1214a
AD		 4694 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4695 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4696 WRITE_DATA_DST_SEL(1));
4697 ib->ptr[ib->length_dw++] = pe;
4698 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
deab48f1
AD		 4699 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4700 if (flags & RADEON_VM_PAGE_SYSTEM) {
4701 value = radeon_vm_map_gart(rdev, addr);
4702 value &= 0xFFFFFFFFFFFFF000ULL;
4703 } else if (flags & RADEON_VM_PAGE_VALID) {
4704 value = addr;
4705 } else {
4706 value = 0;
4707 }
4708 addr += incr;
4709 value |= r600_flags;
43f1214a
AD		 4710 ib->ptr[ib->length_dw++] = value;
4711 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4712 }
4713 }
4714 } else {
4715 /* DMA */
2483b4ea 4716 si_dma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
82ffd92b 4717 }
d2800ee5
AD		 4718}
4719
498522b4 4720void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
d2800ee5 4721{
498522b4 4722 struct radeon_ring *ring = &rdev->ring[ridx];
d2800ee5 4723
ee60e29f 4724 if (vm == NULL)
d2800ee5
AD		 4725 return;
4726
76c44f2c
AD		 4727 /* write new base address */
4728 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4729 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4730 WRITE_DATA_DST_SEL(0)));
4731
ee60e29f 4732 if (vm->id < 8) {
76c44f2c
AD		 4733 radeon_ring_write(ring,
4734 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
ee60e29f 4735 } else {
76c44f2c
AD		 4736 radeon_ring_write(ring,
4737 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
ee60e29f 4738 }
76c44f2c 4739 radeon_ring_write(ring, 0);
fa87e62d 4740 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 4741
d2800ee5 4742 /* flush hdp cache */
76c44f2c
AD		 4743 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4744 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4745 WRITE_DATA_DST_SEL(0)));
4746 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4747 radeon_ring_write(ring, 0);
ee60e29f
CK		 4748 radeon_ring_write(ring, 0x1);
4749
d2800ee5 4750 /* bits 0-15 are the VM contexts0-15 */
76c44f2c
AD		 4751 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4752 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4753 WRITE_DATA_DST_SEL(0)));
4754 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4755 radeon_ring_write(ring, 0);
498522b4 4756 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 4757
4758 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4759 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4760 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 4761}
4762
f8f84ac5
AD		 4763/*
4764 * Power and clock gating
4765 */
4766static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4767{
4768 int i;
4769
4770 for (i = 0; i < rdev->usec_timeout; i++) {
4771 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4772 break;
4773 udelay(1);
4774 }
4775
4776 for (i = 0; i < rdev->usec_timeout; i++) {
4777 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4778 break;
4779 udelay(1);
4780 }
4781}
4782
4783static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4784 bool enable)
4785{
4786 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4787 u32 mask;
4788 int i;
4789
4790 if (enable)
4791 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4792 else
4793 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4794 WREG32(CP_INT_CNTL_RING0, tmp);
4795
4796 if (!enable) {
4797 /* read a gfx register */
4798 tmp = RREG32(DB_DEPTH_INFO);
4799
4800 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4801 for (i = 0; i < rdev->usec_timeout; i++) {
4802 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4803 break;
4804 udelay(1);
4805 }
4806 }
4807}
4808
4809static void si_set_uvd_dcm(struct radeon_device *rdev,
4810 bool sw_mode)
4811{
4812 u32 tmp, tmp2;
4813
4814 tmp = RREG32(UVD_CGC_CTRL);
4815 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4816 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4817
4818 if (sw_mode) {
4819 tmp &= ~0x7ffff800;
4820 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4821 } else {
4822 tmp |= 0x7ffff800;
4823 tmp2 = 0;
4824 }
4825
4826 WREG32(UVD_CGC_CTRL, tmp);
4827 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4828}
4829
22c775ce 4830void si_init_uvd_internal_cg(struct radeon_device *rdev)
f8f84ac5
AD		 4831{
4832 bool hw_mode = true;
4833
4834 if (hw_mode) {
4835 si_set_uvd_dcm(rdev, false);
4836 } else {
4837 u32 tmp = RREG32(UVD_CGC_CTRL);
4838 tmp &= ~DCM;
4839 WREG32(UVD_CGC_CTRL, tmp);
4840 }
4841}
4842
4843static u32 si_halt_rlc(struct radeon_device *rdev)
4844{
4845 u32 data, orig;
4846
4847 orig = data = RREG32(RLC_CNTL);
4848
4849 if (data & RLC_ENABLE) {
4850 data &= ~RLC_ENABLE;
4851 WREG32(RLC_CNTL, data);
4852
4853 si_wait_for_rlc_serdes(rdev);
4854 }
4855
4856 return orig;
4857}
4858
4859static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4860{
4861 u32 tmp;
4862
4863 tmp = RREG32(RLC_CNTL);
4864 if (tmp != rlc)
4865 WREG32(RLC_CNTL, rlc);
4866}
4867
4868static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4869{
4870 u32 data, orig;
4871
4872 orig = data = RREG32(DMA_PG);
e16866ec 4873 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
f8f84ac5
AD		 4874 data |= PG_CNTL_ENABLE;
4875 else
4876 data &= ~PG_CNTL_ENABLE;
4877 if (orig != data)
4878 WREG32(DMA_PG, data);
4879}
4880
4881static void si_init_dma_pg(struct radeon_device *rdev)
4882{
4883 u32 tmp;
4884
4885 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4886 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4887
4888 for (tmp = 0; tmp < 5; tmp++)
4889 WREG32(DMA_PGFSM_WRITE, 0);
4890}
4891
4892static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4893 bool enable)
4894{
4895 u32 tmp;
4896
e16866ec 4897 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_CG)) {
f8f84ac5
AD		 4898 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4899 WREG32(RLC_TTOP_D, tmp);
4900
4901 tmp = RREG32(RLC_PG_CNTL);
4902 tmp |= GFX_PG_ENABLE;
4903 WREG32(RLC_PG_CNTL, tmp);
4904
4905 tmp = RREG32(RLC_AUTO_PG_CTRL);
4906 tmp |= AUTO_PG_EN;
4907 WREG32(RLC_AUTO_PG_CTRL, tmp);
4908 } else {
4909 tmp = RREG32(RLC_AUTO_PG_CTRL);
4910 tmp &= ~AUTO_PG_EN;
4911 WREG32(RLC_AUTO_PG_CTRL, tmp);
4912
4913 tmp = RREG32(DB_RENDER_CONTROL);
4914 }
4915}
4916
4917static void si_init_gfx_cgpg(struct radeon_device *rdev)
4918{
4919 u32 tmp;
4920
4921 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4922
4923 tmp = RREG32(RLC_PG_CNTL);
4924 tmp |= GFX_PG_SRC;
4925 WREG32(RLC_PG_CNTL, tmp);
4926
4927 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4928
4929 tmp = RREG32(RLC_AUTO_PG_CTRL);
4930
4931 tmp &= ~GRBM_REG_SGIT_MASK;
4932 tmp |= GRBM_REG_SGIT(0x700);
4933 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
4934 WREG32(RLC_AUTO_PG_CTRL, tmp);
4935}
4936
ba19031a 4937static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
f8f84ac5
AD		 4938{
4939 u32 mask = 0, tmp, tmp1;
4940 int i;
4941
4942 si_select_se_sh(rdev, se, sh);
4943 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
4944 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
4945 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4946
4947 tmp &= 0xffff0000;
4948
4949 tmp |= tmp1;
4950 tmp >>= 16;
4951
4952 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
4953 mask <<= 1;
4954 mask |= 1;
4955 }
4956
4957 return (~tmp) & mask;
4958}
4959
4960static void si_init_ao_cu_mask(struct radeon_device *rdev)
4961{
4962 u32 i, j, k, active_cu_number = 0;
4963 u32 mask, counter, cu_bitmap;
4964 u32 tmp = 0;
4965
4966 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
4967 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
4968 mask = 1;
4969 cu_bitmap = 0;
4970 counter = 0;
4971 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
ba19031a 4972 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
f8f84ac5
AD		 4973 if (counter < 2)
4974 cu_bitmap |= mask;
4975 counter++;
4976 }
4977 mask <<= 1;
4978 }
4979
4980 active_cu_number += counter;
4981 tmp |= (cu_bitmap << (i * 16 + j * 8));
4982 }
4983 }
4984
4985 WREG32(RLC_PG_AO_CU_MASK, tmp);
4986
4987 tmp = RREG32(RLC_MAX_PG_CU);
4988 tmp &= ~MAX_PU_CU_MASK;
4989 tmp |= MAX_PU_CU(active_cu_number);
4990 WREG32(RLC_MAX_PG_CU, tmp);
4991}
4992
4993static void si_enable_cgcg(struct radeon_device *rdev,
4994 bool enable)
4995{
4996 u32 data, orig, tmp;
4997
4998 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
4999
e16866ec 5000 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5594a558 5001 si_enable_gui_idle_interrupt(rdev, true);
f8f84ac5 5002
f8f84ac5
AD		 5003 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5004
5005 tmp = si_halt_rlc(rdev);
5006
5007 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5008 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5009 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5010
5011 si_wait_for_rlc_serdes(rdev);
5012
5013 si_update_rlc(rdev, tmp);
5014
5015 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5016
5017 data |= CGCG_EN | CGLS_EN;
5018 } else {
5594a558
AD		 5019 si_enable_gui_idle_interrupt(rdev, false);
5020
f8f84ac5
AD		 5021 RREG32(CB_CGTT_SCLK_CTRL);
5022 RREG32(CB_CGTT_SCLK_CTRL);
5023 RREG32(CB_CGTT_SCLK_CTRL);
5024 RREG32(CB_CGTT_SCLK_CTRL);
5025
5026 data &= ~(CGCG_EN | CGLS_EN);
5027 }
5028
5029 if (orig != data)
5030 WREG32(RLC_CGCG_CGLS_CTRL, data);
5031}
5032
5033static void si_enable_mgcg(struct radeon_device *rdev,
5034 bool enable)
5035{
5036 u32 data, orig, tmp = 0;
5037
e16866ec 5038 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
f8f84ac5
AD		 5039 orig = data = RREG32(CGTS_SM_CTRL_REG);
5040 data = 0x96940200;
5041 if (orig != data)
5042 WREG32(CGTS_SM_CTRL_REG, data);
5043
e16866ec
AD		 5044 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5045 orig = data = RREG32(CP_MEM_SLP_CNTL);
5046 data |= CP_MEM_LS_EN;
5047 if (orig != data)
5048 WREG32(CP_MEM_SLP_CNTL, data);
5049 }
f8f84ac5
AD		 5050
5051 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5052 data &= 0xffffffc0;
5053 if (orig != data)
5054 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5055
5056 tmp = si_halt_rlc(rdev);
5057
5058 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5059 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5060 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5061
5062 si_update_rlc(rdev, tmp);
5063 } else {
5064 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5065 data |= 0x00000003;
5066 if (orig != data)
5067 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5068
5069 data = RREG32(CP_MEM_SLP_CNTL);
5070 if (data & CP_MEM_LS_EN) {
5071 data &= ~CP_MEM_LS_EN;
5072 WREG32(CP_MEM_SLP_CNTL, data);
5073 }
5074 orig = data = RREG32(CGTS_SM_CTRL_REG);
5075 data |= LS_OVERRIDE | OVERRIDE;
5076 if (orig != data)
5077 WREG32(CGTS_SM_CTRL_REG, data);
5078
5079 tmp = si_halt_rlc(rdev);
5080
5081 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5082 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5083 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5084
5085 si_update_rlc(rdev, tmp);
5086 }
5087}
5088
5089static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5090 bool enable)
5091{
5092 u32 orig, data, tmp;
5093
e16866ec 5094 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
f8f84ac5
AD		 5095 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5096 tmp |= 0x3fff;
5097 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5098
5099 orig = data = RREG32(UVD_CGC_CTRL);
5100 data |= DCM;
5101 if (orig != data)
5102 WREG32(UVD_CGC_CTRL, data);
5103
5104 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5105 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5106 } else {
5107 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5108 tmp &= ~0x3fff;
5109 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5110
5111 orig = data = RREG32(UVD_CGC_CTRL);
5112 data &= ~DCM;
5113 if (orig != data)
5114 WREG32(UVD_CGC_CTRL, data);
5115
5116 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5117 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5118 }
5119}
5120
5121static const u32 mc_cg_registers[] =
5122{
5123 MC_HUB_MISC_HUB_CG,
5124 MC_HUB_MISC_SIP_CG,
5125 MC_HUB_MISC_VM_CG,
5126 MC_XPB_CLK_GAT,
5127 ATC_MISC_CG,
5128 MC_CITF_MISC_WR_CG,
5129 MC_CITF_MISC_RD_CG,
5130 MC_CITF_MISC_VM_CG,
5131 VM_L2_CG,
5132};
5133
5134static void si_enable_mc_ls(struct radeon_device *rdev,
5135 bool enable)
5136{
5137 int i;
5138 u32 orig, data;
5139
5140 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5141 orig = data = RREG32(mc_cg_registers[i]);
e16866ec 5142 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
f8f84ac5
AD		 5143 data |= MC_LS_ENABLE;
5144 else
5145 data &= ~MC_LS_ENABLE;
5146 if (data != orig)
5147 WREG32(mc_cg_registers[i], data);
5148 }
5149}
5150
e16866ec
AD		 5151static void si_enable_mc_mgcg(struct radeon_device *rdev,
5152 bool enable)
f8f84ac5 5153{
e16866ec
AD		 5154 int i;
5155 u32 orig, data;
5156
5157 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5158 orig = data = RREG32(mc_cg_registers[i]);
5159 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5160 data |= MC_CG_ENABLE;
5161 else
5162 data &= ~MC_CG_ENABLE;
5163 if (data != orig)
5164 WREG32(mc_cg_registers[i], data);
f8f84ac5
AD		 5165 }
5166}
5167
e16866ec
AD		 5168static void si_enable_dma_mgcg(struct radeon_device *rdev,
5169 bool enable)
f8f84ac5 5170{
e16866ec
AD		 5171 u32 orig, data, offset;
5172 int i;
f8f84ac5 5173
e16866ec
AD		 5174 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5175 for (i = 0; i < 2; i++) {
5176 if (i == 0)
5177 offset = DMA0_REGISTER_OFFSET;
5178 else
5179 offset = DMA1_REGISTER_OFFSET;
5180 orig = data = RREG32(DMA_POWER_CNTL + offset);
5181 data &= ~MEM_POWER_OVERRIDE;
5182 if (data != orig)
5183 WREG32(DMA_POWER_CNTL + offset, data);
5184 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5185 }
5186 } else {
5187 for (i = 0; i < 2; i++) {
5188 if (i == 0)
5189 offset = DMA0_REGISTER_OFFSET;
5190 else
5191 offset = DMA1_REGISTER_OFFSET;
5192 orig = data = RREG32(DMA_POWER_CNTL + offset);
5193 data |= MEM_POWER_OVERRIDE;
5194 if (data != orig)
5195 WREG32(DMA_POWER_CNTL + offset, data);
5196
5197 orig = data = RREG32(DMA_CLK_CTRL + offset);
5198 data = 0xff000000;
5199 if (data != orig)
5200 WREG32(DMA_CLK_CTRL + offset, data);
5201 }
5202 }
f8f84ac5
AD		 5203}
5204
e16866ec
AD		 5205static void si_enable_bif_mgls(struct radeon_device *rdev,
5206 bool enable)
f8f84ac5 5207{
e16866ec 5208 u32 orig, data;
f8f84ac5 5209
e16866ec 5210 orig = data = RREG32_PCIE(PCIE_CNTL2);
f8f84ac5 5211
e16866ec
AD		 5212 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5213 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5214 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5215 else
5216 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5217 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5218
5219 if (orig != data)
5220 WREG32_PCIE(PCIE_CNTL2, data);
f8f84ac5
AD		 5221}
5222
e16866ec
AD		 5223static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5224 bool enable)
f8f84ac5 5225{
e16866ec 5226 u32 orig, data;
f8f84ac5 5227
e16866ec 5228 orig = data = RREG32(HDP_HOST_PATH_CNTL);
f8f84ac5 5229
e16866ec
AD		 5230 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5231 data &= ~CLOCK_GATING_DIS;
5232 else
5233 data |= CLOCK_GATING_DIS;
5234
5235 if (orig != data)
5236 WREG32(HDP_HOST_PATH_CNTL, data);
f8f84ac5
AD		 5237}
5238
e16866ec
AD		 5239static void si_enable_hdp_ls(struct radeon_device *rdev,
5240 bool enable)
347e7592 5241{
e16866ec 5242 u32 orig, data;
347e7592 5243
e16866ec 5244 orig = data = RREG32(HDP_MEM_POWER_LS);
347e7592 5245
e16866ec
AD		 5246 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5247 data |= HDP_LS_ENABLE;
5248 else
5249 data &= ~HDP_LS_ENABLE;
347e7592 5250
e16866ec
AD		 5251 if (orig != data)
5252 WREG32(HDP_MEM_POWER_LS, data);
347e7592
AD		 5253}
5254
e16866ec
AD		 5255void si_update_cg(struct radeon_device *rdev,
5256 u32 block, bool enable)
347e7592 5257{
e16866ec 5258 if (block & RADEON_CG_BLOCK_GFX) {
811e4d58 5259 si_enable_gui_idle_interrupt(rdev, false);
e16866ec
AD		 5260 /* order matters! */
5261 if (enable) {
5262 si_enable_mgcg(rdev, true);
5263 si_enable_cgcg(rdev, true);
5264 } else {
5265 si_enable_cgcg(rdev, false);
5266 si_enable_mgcg(rdev, false);
347e7592 5267 }
811e4d58 5268 si_enable_gui_idle_interrupt(rdev, true);
347e7592
AD		 5269 }
5270
e16866ec
AD		 5271 if (block & RADEON_CG_BLOCK_MC) {
5272 si_enable_mc_mgcg(rdev, enable);
5273 si_enable_mc_ls(rdev, enable);
347e7592 5274 }
e16866ec
AD		 5275
5276 if (block & RADEON_CG_BLOCK_SDMA) {
5277 si_enable_dma_mgcg(rdev, enable);
347e7592
AD		 5278 }
5279
e16866ec
AD		 5280 if (block & RADEON_CG_BLOCK_BIF) {
5281 si_enable_bif_mgls(rdev, enable);
6d8cf000 5282 }
6d8cf000 5283
e16866ec
AD		 5284 if (block & RADEON_CG_BLOCK_UVD) {
5285 if (rdev->has_uvd) {
5286 si_enable_uvd_mgcg(rdev, enable);
bd8cd539
AD		 5287 }
5288 }
bd8cd539 5289
e16866ec
AD		 5290 if (block & RADEON_CG_BLOCK_HDP) {
5291 si_enable_hdp_mgcg(rdev, enable);
5292 si_enable_hdp_ls(rdev, enable);
347e7592 5293 }
e16866ec 5294}
f8f84ac5
AD		 5295
5296static void si_init_cg(struct radeon_device *rdev)
5297{
e16866ec
AD		 5298 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5299 RADEON_CG_BLOCK_MC |
5300 RADEON_CG_BLOCK_SDMA |
5301 RADEON_CG_BLOCK_BIF |
5302 RADEON_CG_BLOCK_HDP), true);
b2d70917 5303 if (rdev->has_uvd) {
e16866ec 5304 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
f8f84ac5 5305 si_init_uvd_internal_cg(rdev);
347e7592 5306 }
f8f84ac5 5307}
bd8cd539 5308
f8f84ac5
AD		 5309static void si_fini_cg(struct radeon_device *rdev)
5310{
0116e1ef 5311 if (rdev->has_uvd) {
e16866ec 5312 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
347e7592 5313 }
e16866ec
AD		 5314 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5315 RADEON_CG_BLOCK_MC |
5316 RADEON_CG_BLOCK_SDMA |
5317 RADEON_CG_BLOCK_BIF |
5318 RADEON_CG_BLOCK_HDP), false);
5319}
5320
59a82d0e
AD		 5321u32 si_get_csb_size(struct radeon_device *rdev)
5322{
5323 u32 count = 0;
5324 const struct cs_section_def *sect = NULL;
5325 const struct cs_extent_def *ext = NULL;
5326
5327 if (rdev->rlc.cs_data == NULL)
5328 return 0;
5329
5330 /* begin clear state */
5331 count += 2;
5332 /* context control state */
5333 count += 3;
5334
5335 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5336 for (ext = sect->section; ext->extent != NULL; ++ext) {
5337 if (sect->id == SECT_CONTEXT)
5338 count += 2 + ext->reg_count;
5339 else
5340 return 0;
5341 }
bd8cd539 5342 }
59a82d0e
AD		 5343 /* pa_sc_raster_config */
5344 count += 3;
5345 /* end clear state */
5346 count += 2;
5347 /* clear state */
5348 count += 2;
5349
5350 return count;
5351}
5352
5353void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5354{
5355 u32 count = 0, i;
5356 const struct cs_section_def *sect = NULL;
5357 const struct cs_extent_def *ext = NULL;
5358
5359 if (rdev->rlc.cs_data == NULL)
5360 return;
5361 if (buffer == NULL)
5362 return;
5363
5364 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5365 buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE;
5366
5367 buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1);
5368 buffer[count++] = 0x80000000;
5369 buffer[count++] = 0x80000000;
5370
5371 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5372 for (ext = sect->section; ext->extent != NULL; ++ext) {
5373 if (sect->id == SECT_CONTEXT) {
5374 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count);
5375 buffer[count++] = ext->reg_index - 0xa000;
5376 for (i = 0; i < ext->reg_count; i++)
5377 buffer[count++] = ext->extent[i];
5378 } else {
5379 return;
bd8cd539 5380 }
bd8cd539
AD		 5381 }
5382 }
bd8cd539 5383
59a82d0e
AD		 5384 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 1);
5385 buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START;
5386 switch (rdev->family) {
5387 case CHIP_TAHITI:
5388 case CHIP_PITCAIRN:
5389 buffer[count++] = 0x2a00126a;
5390 break;
5391 case CHIP_VERDE:
5392 buffer[count++] = 0x0000124a;
5393 break;
5394 case CHIP_OLAND:
5395 buffer[count++] = 0x00000082;
5396 break;
5397 case CHIP_HAINAN:
5398 buffer[count++] = 0x00000000;
5399 break;
5400 default:
5401 buffer[count++] = 0x00000000;
5402 break;
5403 }
5404
5405 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5406 buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE;
347e7592 5407
59a82d0e
AD		 5408 buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0);
5409 buffer[count++] = 0;
5410}
5411
f8f84ac5
AD		 5412static void si_init_pg(struct radeon_device *rdev)
5413{
0116e1ef
AD		 5414 if (rdev->pg_flags) {
5415 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5416 si_init_dma_pg(rdev);
0116e1ef 5417 }
f8f84ac5 5418 si_init_ao_cu_mask(rdev);
0116e1ef
AD		 5419 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_CG) {
5420 si_init_gfx_cgpg(rdev);
0116e1ef 5421 }
ca6ebb39
AD		 5422 si_enable_dma_pg(rdev, true);
5423 si_enable_gfx_cgpg(rdev, true);
f8f84ac5
AD		 5424 } else {
5425 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5426 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5427 }
5428}
5429
5430static void si_fini_pg(struct radeon_device *rdev)
5431{
0116e1ef 5432 if (rdev->pg_flags) {
ca6ebb39
AD		 5433 si_enable_dma_pg(rdev, false);
5434 si_enable_gfx_cgpg(rdev, false);
f8f84ac5 5435 }
347e7592
AD		 5436}
5437
347e7592
AD		 5438/*
5439 * RLC
5440 */
866d83de 5441void si_rlc_reset(struct radeon_device *rdev)
d719cef3 5442{
f8f84ac5 5443 u32 tmp = RREG32(GRBM_SOFT_RESET);
d719cef3 5444
f8f84ac5
AD		 5445 tmp |= SOFT_RESET_RLC;
5446 WREG32(GRBM_SOFT_RESET, tmp);
5447 udelay(50);
5448 tmp &= ~SOFT_RESET_RLC;
5449 WREG32(GRBM_SOFT_RESET, tmp);
5450 udelay(50);
d719cef3
AD		 5451}
5452
347e7592
AD		 5453static void si_rlc_stop(struct radeon_device *rdev)
5454{
5455 WREG32(RLC_CNTL, 0);
d719cef3
AD		 5456
5457 si_enable_gui_idle_interrupt(rdev, false);
5458
5459 si_wait_for_rlc_serdes(rdev);
347e7592
AD		 5460}
5461
5462static void si_rlc_start(struct radeon_device *rdev)
5463{
5464 WREG32(RLC_CNTL, RLC_ENABLE);
d719cef3
AD		 5465
5466 si_enable_gui_idle_interrupt(rdev, true);
5467
5468 udelay(50);
5469}
5470
5471static bool si_lbpw_supported(struct radeon_device *rdev)
5472{
5473 u32 tmp;
5474
5475 /* Enable LBPW only for DDR3 */
5476 tmp = RREG32(MC_SEQ_MISC0);
5477 if ((tmp & 0xF0000000) == 0xB0000000)
5478 return true;
5479 return false;
5480}
5481
5482static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5483{
5484 u32 tmp;
5485
5486 tmp = RREG32(RLC_LB_CNTL);
5487 if (enable)
5488 tmp |= LOAD_BALANCE_ENABLE;
5489 else
5490 tmp &= ~LOAD_BALANCE_ENABLE;
5491 WREG32(RLC_LB_CNTL, tmp);
5492
5493 if (!enable) {
5494 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5495 WREG32(SPI_LB_CU_MASK, 0x00ff);
5496 }
347e7592
AD		 5497}
5498
5499static int si_rlc_resume(struct radeon_device *rdev)
5500{
5501 u32 i;
5502 const __be32 *fw_data;
5503
5504 if (!rdev->rlc_fw)
5505 return -EINVAL;
5506
5507 si_rlc_stop(rdev);
5508
f8f84ac5
AD		 5509 si_rlc_reset(rdev);
5510
5511 si_init_pg(rdev);
5512
5513 si_init_cg(rdev);
5514
347e7592
AD		 5515 WREG32(RLC_RL_BASE, 0);
5516 WREG32(RLC_RL_SIZE, 0);
5517 WREG32(RLC_LB_CNTL, 0);
5518 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5519 WREG32(RLC_LB_CNTR_INIT, 0);
d719cef3 5520 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
347e7592 5521
347e7592
AD		 5522 WREG32(RLC_MC_CNTL, 0);
5523 WREG32(RLC_UCODE_CNTL, 0);
5524
5525 fw_data = (const __be32 *)rdev->rlc_fw->data;
5526 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5527 WREG32(RLC_UCODE_ADDR, i);
5528 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5529 }
5530 WREG32(RLC_UCODE_ADDR, 0);
5531
d719cef3
AD		 5532 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5533
347e7592
AD		 5534 si_rlc_start(rdev);
5535
5536 return 0;
5537}
5538
25a857fb
AD		 5539static void si_enable_interrupts(struct radeon_device *rdev)
5540{
5541 u32 ih_cntl = RREG32(IH_CNTL);
5542 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5543
5544 ih_cntl |= ENABLE_INTR;
5545 ih_rb_cntl |= IH_RB_ENABLE;
5546 WREG32(IH_CNTL, ih_cntl);
5547 WREG32(IH_RB_CNTL, ih_rb_cntl);
5548 rdev->ih.enabled = true;
5549}
5550
5551static void si_disable_interrupts(struct radeon_device *rdev)
5552{
5553 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5554 u32 ih_cntl = RREG32(IH_CNTL);
5555
5556 ih_rb_cntl &= ~IH_RB_ENABLE;
5557 ih_cntl &= ~ENABLE_INTR;
5558 WREG32(IH_RB_CNTL, ih_rb_cntl);
5559 WREG32(IH_CNTL, ih_cntl);
5560 /* set rptr, wptr to 0 */
5561 WREG32(IH_RB_RPTR, 0);
5562 WREG32(IH_RB_WPTR, 0);
5563 rdev->ih.enabled = false;
25a857fb
AD		 5564 rdev->ih.rptr = 0;
5565}
5566
5567static void si_disable_interrupt_state(struct radeon_device *rdev)
5568{
5569 u32 tmp;
5570
811e4d58
AD		 5571 tmp = RREG32(CP_INT_CNTL_RING0) &
5572 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5573 WREG32(CP_INT_CNTL_RING0, tmp);
25a857fb
AD		 5574 WREG32(CP_INT_CNTL_RING1, 0);
5575 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 5576 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5577 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5578 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5579 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 5580 WREG32(GRBM_INT_CNTL, 0);
5153550a
AD		 5581 if (rdev->num_crtc >= 2) {
5582 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5583 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5584 }
25a857fb
AD		 5585 if (rdev->num_crtc >= 4) {
5586 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5587 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5588 }
5589 if (rdev->num_crtc >= 6) {
5590 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5591 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5592 }
5593
5153550a
AD		 5594 if (rdev->num_crtc >= 2) {
5595 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5596 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5597 }
25a857fb
AD		 5598 if (rdev->num_crtc >= 4) {
5599 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5600 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5601 }
5602 if (rdev->num_crtc >= 6) {
5603 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5604 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5605 }
5606
5153550a
AD		 5607 if (!ASIC_IS_NODCE(rdev)) {
5608 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
5609
5610 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5611 WREG32(DC_HPD1_INT_CONTROL, tmp);
5612 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5613 WREG32(DC_HPD2_INT_CONTROL, tmp);
5614 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5615 WREG32(DC_HPD3_INT_CONTROL, tmp);
5616 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5617 WREG32(DC_HPD4_INT_CONTROL, tmp);
5618 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5619 WREG32(DC_HPD5_INT_CONTROL, tmp);
5620 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5621 WREG32(DC_HPD6_INT_CONTROL, tmp);
5622 }
25a857fb
AD		 5623}
5624
5625static int si_irq_init(struct radeon_device *rdev)
5626{
5627 int ret = 0;
5628 int rb_bufsz;
5629 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5630
5631 /* allocate ring */
5632 ret = r600_ih_ring_alloc(rdev);
5633 if (ret)
5634 return ret;
5635
5636 /* disable irqs */
5637 si_disable_interrupts(rdev);
5638
5639 /* init rlc */
5640 ret = si_rlc_resume(rdev);
5641 if (ret) {
5642 r600_ih_ring_fini(rdev);
5643 return ret;
5644 }
5645
5646 /* setup interrupt control */
5647 /* set dummy read address to ring address */
5648 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5649 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5650 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5651 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5652 */
5653 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5654 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5655 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5656 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5657
5658 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 5659 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
25a857fb
AD		 5660
5661 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5662 IH_WPTR_OVERFLOW_CLEAR |
5663 (rb_bufsz << 1));
5664
5665 if (rdev->wb.enabled)
5666 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5667
5668 /* set the writeback address whether it's enabled or not */
5669 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5670 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5671
5672 WREG32(IH_RB_CNTL, ih_rb_cntl);
5673
5674 /* set rptr, wptr to 0 */
5675 WREG32(IH_RB_RPTR, 0);
5676 WREG32(IH_RB_WPTR, 0);
5677
5678 /* Default settings for IH_CNTL (disabled at first) */
5679 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5680 /* RPTR_REARM only works if msi's are enabled */
5681 if (rdev->msi_enabled)
5682 ih_cntl |= RPTR_REARM;
5683 WREG32(IH_CNTL, ih_cntl);
5684
5685 /* force the active interrupt state to all disabled */
5686 si_disable_interrupt_state(rdev);
5687
2099810f
DA		 5688 pci_set_master(rdev->pdev);
5689
25a857fb
AD		 5690 /* enable irqs */
5691 si_enable_interrupts(rdev);
5692
5693 return ret;
5694}
5695
5696int si_irq_set(struct radeon_device *rdev)
5697{
811e4d58 5698 u32 cp_int_cntl;
25a857fb
AD		 5699 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5700 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5153550a 5701 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
25a857fb
AD		 5702 u32 grbm_int_cntl = 0;
5703 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
8c5fd7ef 5704 u32 dma_cntl, dma_cntl1;
a9e61410 5705 u32 thermal_int = 0;
25a857fb
AD		 5706
5707 if (!rdev->irq.installed) {
5708 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5709 return -EINVAL;
5710 }
5711 /* don't enable anything if the ih is disabled */
5712 if (!rdev->ih.enabled) {
5713 si_disable_interrupts(rdev);
5714 /* force the active interrupt state to all disabled */
5715 si_disable_interrupt_state(rdev);
5716 return 0;
5717 }
5718
811e4d58
AD		 5719 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
5720 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5721
5153550a
AD		 5722 if (!ASIC_IS_NODCE(rdev)) {
5723 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5724 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5725 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5726 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5727 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5728 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5729 }
25a857fb 5730
8c5fd7ef
AD		 5731 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5732 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5733
a9e61410
AD		 5734 thermal_int = RREG32(CG_THERMAL_INT) &
5735 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5736
25a857fb 5737 /* enable CP interrupts on all rings */
736fc37f 5738 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 5739 DRM_DEBUG("si_irq_set: sw int gfx\n");
5740 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5741 }
736fc37f 5742 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 5743 DRM_DEBUG("si_irq_set: sw int cp1\n");
5744 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5745 }
736fc37f 5746 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 5747 DRM_DEBUG("si_irq_set: sw int cp2\n");
5748 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5749 }
8c5fd7ef
AD		 5750 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5751 DRM_DEBUG("si_irq_set: sw int dma\n");
5752 dma_cntl |= TRAP_ENABLE;
5753 }
5754
5755 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5756 DRM_DEBUG("si_irq_set: sw int dma1\n");
5757 dma_cntl1 |= TRAP_ENABLE;
5758 }
25a857fb 5759 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 5760 atomic_read(&rdev->irq.pflip[0])) {
25a857fb
AD		 5761 DRM_DEBUG("si_irq_set: vblank 0\n");
5762 crtc1 |= VBLANK_INT_MASK;
5763 }
5764 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 5765 atomic_read(&rdev->irq.pflip[1])) {
25a857fb
AD		 5766 DRM_DEBUG("si_irq_set: vblank 1\n");
5767 crtc2 |= VBLANK_INT_MASK;
5768 }
5769 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 5770 atomic_read(&rdev->irq.pflip[2])) {
25a857fb
AD		 5771 DRM_DEBUG("si_irq_set: vblank 2\n");
5772 crtc3 |= VBLANK_INT_MASK;
5773 }
5774 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 5775 atomic_read(&rdev->irq.pflip[3])) {
25a857fb
AD		 5776 DRM_DEBUG("si_irq_set: vblank 3\n");
5777 crtc4 |= VBLANK_INT_MASK;
5778 }
5779 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 5780 atomic_read(&rdev->irq.pflip[4])) {
25a857fb
AD		 5781 DRM_DEBUG("si_irq_set: vblank 4\n");
5782 crtc5 |= VBLANK_INT_MASK;
5783 }
5784 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 5785 atomic_read(&rdev->irq.pflip[5])) {
25a857fb
AD		 5786 DRM_DEBUG("si_irq_set: vblank 5\n");
5787 crtc6 |= VBLANK_INT_MASK;
5788 }
5789 if (rdev->irq.hpd[0]) {
5790 DRM_DEBUG("si_irq_set: hpd 1\n");
5791 hpd1 |= DC_HPDx_INT_EN;
5792 }
5793 if (rdev->irq.hpd[1]) {
5794 DRM_DEBUG("si_irq_set: hpd 2\n");
5795 hpd2 |= DC_HPDx_INT_EN;
5796 }
5797 if (rdev->irq.hpd[2]) {
5798 DRM_DEBUG("si_irq_set: hpd 3\n");
5799 hpd3 |= DC_HPDx_INT_EN;
5800 }
5801 if (rdev->irq.hpd[3]) {
5802 DRM_DEBUG("si_irq_set: hpd 4\n");
5803 hpd4 |= DC_HPDx_INT_EN;
5804 }
5805 if (rdev->irq.hpd[4]) {
5806 DRM_DEBUG("si_irq_set: hpd 5\n");
5807 hpd5 |= DC_HPDx_INT_EN;
5808 }
5809 if (rdev->irq.hpd[5]) {
5810 DRM_DEBUG("si_irq_set: hpd 6\n");
5811 hpd6 |= DC_HPDx_INT_EN;
5812 }
25a857fb
AD		 5813
5814 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5815 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5816 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5817
8c5fd7ef
AD		 5818 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5819 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5820
25a857fb
AD		 5821 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5822
a9e61410
AD		 5823 if (rdev->irq.dpm_thermal) {
5824 DRM_DEBUG("dpm thermal\n");
5825 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5826 }
5827
5153550a
AD		 5828 if (rdev->num_crtc >= 2) {
5829 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5830 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5831 }
25a857fb
AD		 5832 if (rdev->num_crtc >= 4) {
5833 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5834 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5835 }
5836 if (rdev->num_crtc >= 6) {
5837 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5838 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5839 }
5840
5153550a
AD		 5841 if (rdev->num_crtc >= 2) {
5842 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5843 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5844 }
25a857fb
AD		 5845 if (rdev->num_crtc >= 4) {
5846 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5847 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5848 }
5849 if (rdev->num_crtc >= 6) {
5850 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5851 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5852 }
5853
5153550a
AD		 5854 if (!ASIC_IS_NODCE(rdev)) {
5855 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5856 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5857 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5858 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5859 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5860 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5861 }
25a857fb 5862
a9e61410
AD		 5863 WREG32(CG_THERMAL_INT, thermal_int);
5864
25a857fb
AD		 5865 return 0;
5866}
5867
5868static inline void si_irq_ack(struct radeon_device *rdev)
5869{
5870 u32 tmp;
5871
5153550a
AD		 5872 if (ASIC_IS_NODCE(rdev))
5873 return;
5874
25a857fb
AD		 5875 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5876 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5877 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5878 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5879 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5880 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5881 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5882 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5883 if (rdev->num_crtc >= 4) {
5884 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5885 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5886 }
5887 if (rdev->num_crtc >= 6) {
5888 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5889 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5890 }
5891
5892 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5893 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5894 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5895 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5896 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5897 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5898 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5899 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5900 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5901 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5902 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5903 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5904
5905 if (rdev->num_crtc >= 4) {
5906 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5907 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5908 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5909 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5910 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5911 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5912 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5913 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5914 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5915 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5916 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5917 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5918 }
5919
5920 if (rdev->num_crtc >= 6) {
5921 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5922 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5923 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5924 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5925 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5926 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5927 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5928 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5929 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5930 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5931 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5932 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5933 }
5934
5935 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5936 tmp = RREG32(DC_HPD1_INT_CONTROL);
5937 tmp |= DC_HPDx_INT_ACK;
5938 WREG32(DC_HPD1_INT_CONTROL, tmp);
5939 }
5940 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5941 tmp = RREG32(DC_HPD2_INT_CONTROL);
5942 tmp |= DC_HPDx_INT_ACK;
5943 WREG32(DC_HPD2_INT_CONTROL, tmp);
5944 }
5945 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5946 tmp = RREG32(DC_HPD3_INT_CONTROL);
5947 tmp |= DC_HPDx_INT_ACK;
5948 WREG32(DC_HPD3_INT_CONTROL, tmp);
5949 }
5950 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5951 tmp = RREG32(DC_HPD4_INT_CONTROL);
5952 tmp |= DC_HPDx_INT_ACK;
5953 WREG32(DC_HPD4_INT_CONTROL, tmp);
5954 }
5955 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5956 tmp = RREG32(DC_HPD5_INT_CONTROL);
5957 tmp |= DC_HPDx_INT_ACK;
5958 WREG32(DC_HPD5_INT_CONTROL, tmp);
5959 }
5960 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5961 tmp = RREG32(DC_HPD5_INT_CONTROL);
5962 tmp |= DC_HPDx_INT_ACK;
5963 WREG32(DC_HPD6_INT_CONTROL, tmp);
5964 }
5965}
5966
5967static void si_irq_disable(struct radeon_device *rdev)
5968{
5969 si_disable_interrupts(rdev);
5970 /* Wait and acknowledge irq */
5971 mdelay(1);
5972 si_irq_ack(rdev);
5973 si_disable_interrupt_state(rdev);
5974}
5975
5976static void si_irq_suspend(struct radeon_device *rdev)
5977{
5978 si_irq_disable(rdev);
5979 si_rlc_stop(rdev);
5980}
5981
9b136d51
AD		 5982static void si_irq_fini(struct radeon_device *rdev)
5983{
5984 si_irq_suspend(rdev);
5985 r600_ih_ring_fini(rdev);
5986}
5987
25a857fb
AD		 5988static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
5989{
5990 u32 wptr, tmp;
5991
5992 if (rdev->wb.enabled)
5993 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
5994 else
5995 wptr = RREG32(IH_RB_WPTR);
5996
5997 if (wptr & RB_OVERFLOW) {
5998 /* When a ring buffer overflow happen start parsing interrupt
5999 * from the last not overwritten vector (wptr + 16). Hopefully
6000 * this should allow us to catchup.
6001 */
6002 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6003 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6004 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6005 tmp = RREG32(IH_RB_CNTL);
6006 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6007 WREG32(IH_RB_CNTL, tmp);
6008 }
6009 return (wptr & rdev->ih.ptr_mask);
6010}
6011
6012/* SI IV Ring
6013 * Each IV ring entry is 128 bits:
6014 * [7:0] - interrupt source id
6015 * [31:8] - reserved
6016 * [59:32] - interrupt source data
6017 * [63:60] - reserved
6018 * [71:64] - RINGID
6019 * [79:72] - VMID
6020 * [127:80] - reserved
6021 */
6022int si_irq_process(struct radeon_device *rdev)
6023{
6024 u32 wptr;
6025 u32 rptr;
6026 u32 src_id, src_data, ring_id;
6027 u32 ring_index;
25a857fb 6028 bool queue_hotplug = false;
a9e61410 6029 bool queue_thermal = false;
fbf6dc7a 6030 u32 status, addr;
25a857fb
AD		 6031
6032 if (!rdev->ih.enabled || rdev->shutdown)
6033 return IRQ_NONE;
6034
6035 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 6036
6037restart_ih:
6038 /* is somebody else already processing irqs? */
6039 if (atomic_xchg(&rdev->ih.lock, 1))
6040 return IRQ_NONE;
6041
25a857fb
AD		 6042 rptr = rdev->ih.rptr;
6043 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6044
25a857fb
AD		 6045 /* Order reading of wptr vs. reading of IH ring data */
6046 rmb();
6047
6048 /* display interrupts */
6049 si_irq_ack(rdev);
6050
25a857fb
AD		 6051 while (rptr != wptr) {
6052 /* wptr/rptr are in bytes! */
6053 ring_index = rptr / 4;
6054 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6055 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6056 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6057
6058 switch (src_id) {
6059 case 1: /* D1 vblank/vline */
6060 switch (src_data) {
6061 case 0: /* D1 vblank */
6062 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6063 if (rdev->irq.crtc_vblank_int[0]) {
6064 drm_handle_vblank(rdev->ddev, 0);
6065 rdev->pm.vblank_sync = true;
6066 wake_up(&rdev->irq.vblank_queue);
6067 }
736fc37f 6068 if (atomic_read(&rdev->irq.pflip[0]))
25a857fb
AD		 6069 radeon_crtc_handle_flip(rdev, 0);
6070 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6071 DRM_DEBUG("IH: D1 vblank\n");
6072 }
6073 break;
6074 case 1: /* D1 vline */
6075 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6076 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6077 DRM_DEBUG("IH: D1 vline\n");
6078 }
6079 break;
6080 default:
6081 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6082 break;
6083 }
6084 break;
6085 case 2: /* D2 vblank/vline */
6086 switch (src_data) {
6087 case 0: /* D2 vblank */
6088 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6089 if (rdev->irq.crtc_vblank_int[1]) {
6090 drm_handle_vblank(rdev->ddev, 1);
6091 rdev->pm.vblank_sync = true;
6092 wake_up(&rdev->irq.vblank_queue);
6093 }
736fc37f 6094 if (atomic_read(&rdev->irq.pflip[1]))
25a857fb
AD		 6095 radeon_crtc_handle_flip(rdev, 1);
6096 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6097 DRM_DEBUG("IH: D2 vblank\n");
6098 }
6099 break;
6100 case 1: /* D2 vline */
6101 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6102 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6103 DRM_DEBUG("IH: D2 vline\n");
6104 }
6105 break;
6106 default:
6107 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6108 break;
6109 }
6110 break;
6111 case 3: /* D3 vblank/vline */
6112 switch (src_data) {
6113 case 0: /* D3 vblank */
6114 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6115 if (rdev->irq.crtc_vblank_int[2]) {
6116 drm_handle_vblank(rdev->ddev, 2);
6117 rdev->pm.vblank_sync = true;
6118 wake_up(&rdev->irq.vblank_queue);
6119 }
736fc37f 6120 if (atomic_read(&rdev->irq.pflip[2]))
25a857fb
AD		 6121 radeon_crtc_handle_flip(rdev, 2);
6122 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6123 DRM_DEBUG("IH: D3 vblank\n");
6124 }
6125 break;
6126 case 1: /* D3 vline */
6127 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6128 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6129 DRM_DEBUG("IH: D3 vline\n");
6130 }
6131 break;
6132 default:
6133 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6134 break;
6135 }
6136 break;
6137 case 4: /* D4 vblank/vline */
6138 switch (src_data) {
6139 case 0: /* D4 vblank */
6140 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6141 if (rdev->irq.crtc_vblank_int[3]) {
6142 drm_handle_vblank(rdev->ddev, 3);
6143 rdev->pm.vblank_sync = true;
6144 wake_up(&rdev->irq.vblank_queue);
6145 }
736fc37f 6146 if (atomic_read(&rdev->irq.pflip[3]))
25a857fb
AD		 6147 radeon_crtc_handle_flip(rdev, 3);
6148 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6149 DRM_DEBUG("IH: D4 vblank\n");
6150 }
6151 break;
6152 case 1: /* D4 vline */
6153 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6154 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6155 DRM_DEBUG("IH: D4 vline\n");
6156 }
6157 break;
6158 default:
6159 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6160 break;
6161 }
6162 break;
6163 case 5: /* D5 vblank/vline */
6164 switch (src_data) {
6165 case 0: /* D5 vblank */
6166 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6167 if (rdev->irq.crtc_vblank_int[4]) {
6168 drm_handle_vblank(rdev->ddev, 4);
6169 rdev->pm.vblank_sync = true;
6170 wake_up(&rdev->irq.vblank_queue);
6171 }
736fc37f 6172 if (atomic_read(&rdev->irq.pflip[4]))
25a857fb
AD		 6173 radeon_crtc_handle_flip(rdev, 4);
6174 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6175 DRM_DEBUG("IH: D5 vblank\n");
6176 }
6177 break;
6178 case 1: /* D5 vline */
6179 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6180 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6181 DRM_DEBUG("IH: D5 vline\n");
6182 }
6183 break;
6184 default:
6185 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6186 break;
6187 }
6188 break;
6189 case 6: /* D6 vblank/vline */
6190 switch (src_data) {
6191 case 0: /* D6 vblank */
6192 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6193 if (rdev->irq.crtc_vblank_int[5]) {
6194 drm_handle_vblank(rdev->ddev, 5);
6195 rdev->pm.vblank_sync = true;
6196 wake_up(&rdev->irq.vblank_queue);
6197 }
736fc37f 6198 if (atomic_read(&rdev->irq.pflip[5]))
25a857fb
AD		 6199 radeon_crtc_handle_flip(rdev, 5);
6200 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6201 DRM_DEBUG("IH: D6 vblank\n");
6202 }
6203 break;
6204 case 1: /* D6 vline */
6205 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6206 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6207 DRM_DEBUG("IH: D6 vline\n");
6208 }
6209 break;
6210 default:
6211 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6212 break;
6213 }
6214 break;
6215 case 42: /* HPD hotplug */
6216 switch (src_data) {
6217 case 0:
6218 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6219 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6220 queue_hotplug = true;
6221 DRM_DEBUG("IH: HPD1\n");
6222 }
6223 break;
6224 case 1:
6225 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6226 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6227 queue_hotplug = true;
6228 DRM_DEBUG("IH: HPD2\n");
6229 }
6230 break;
6231 case 2:
6232 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6233 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6234 queue_hotplug = true;
6235 DRM_DEBUG("IH: HPD3\n");
6236 }
6237 break;
6238 case 3:
6239 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6240 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6241 queue_hotplug = true;
6242 DRM_DEBUG("IH: HPD4\n");
6243 }
6244 break;
6245 case 4:
6246 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6247 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6248 queue_hotplug = true;
6249 DRM_DEBUG("IH: HPD5\n");
6250 }
6251 break;
6252 case 5:
6253 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6254 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6255 queue_hotplug = true;
6256 DRM_DEBUG("IH: HPD6\n");
6257 }
6258 break;
6259 default:
6260 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6261 break;
6262 }
6263 break;
ae133a11
CK		 6264 case 146:
6265 case 147:
fbf6dc7a
AD		 6266 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6267 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
ae133a11
CK		 6268 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6269 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
fbf6dc7a 6270 addr);
ae133a11 6271 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
fbf6dc7a
AD		 6272 status);
6273 si_vm_decode_fault(rdev, status, addr);
ae133a11
CK		 6274 /* reset addr and status */
6275 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6276 break;
25a857fb
AD		 6277 case 176: /* RINGID0 CP_INT */
6278 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6279 break;
6280 case 177: /* RINGID1 CP_INT */
6281 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6282 break;
6283 case 178: /* RINGID2 CP_INT */
6284 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6285 break;
6286 case 181: /* CP EOP event */
6287 DRM_DEBUG("IH: CP EOP\n");
6288 switch (ring_id) {
6289 case 0:
6290 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6291 break;
6292 case 1:
6293 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6294 break;
6295 case 2:
6296 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6297 break;
6298 }
6299 break;
8c5fd7ef
AD		 6300 case 224: /* DMA trap event */
6301 DRM_DEBUG("IH: DMA trap\n");
6302 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6303 break;
a9e61410
AD		 6304 case 230: /* thermal low to high */
6305 DRM_DEBUG("IH: thermal low to high\n");
6306 rdev->pm.dpm.thermal.high_to_low = false;
6307 queue_thermal = true;
6308 break;
6309 case 231: /* thermal high to low */
6310 DRM_DEBUG("IH: thermal high to low\n");
6311 rdev->pm.dpm.thermal.high_to_low = true;
6312 queue_thermal = true;
6313 break;
25a857fb
AD		 6314 case 233: /* GUI IDLE */
6315 DRM_DEBUG("IH: GUI idle\n");
25a857fb 6316 break;
8c5fd7ef
AD		 6317 case 244: /* DMA trap event */
6318 DRM_DEBUG("IH: DMA1 trap\n");
6319 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6320 break;
25a857fb
AD		 6321 default:
6322 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6323 break;
6324 }
6325
6326 /* wptr/rptr are in bytes! */
6327 rptr += 16;
6328 rptr &= rdev->ih.ptr_mask;
6329 }
25a857fb
AD		 6330 if (queue_hotplug)
6331 schedule_work(&rdev->hotplug_work);
a9e61410
AD		 6332 if (queue_thermal && rdev->pm.dpm_enabled)
6333 schedule_work(&rdev->pm.dpm.thermal.work);
25a857fb
AD		 6334 rdev->ih.rptr = rptr;
6335 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 6336 atomic_set(&rdev->ih.lock, 0);
6337
6338 /* make sure wptr hasn't changed while processing */
6339 wptr = si_get_ih_wptr(rdev);
6340 if (wptr != rptr)
6341 goto restart_ih;
6342
25a857fb
AD		 6343 return IRQ_HANDLED;
6344}
6345
9b136d51
AD		 6346/*
6347 * startup/shutdown callbacks
6348 */
6349static int si_startup(struct radeon_device *rdev)
6350{
6351 struct radeon_ring *ring;
6352 int r;
6353
b9d305df
AD		 6354 /* enable pcie gen2/3 link */
6355 si_pcie_gen3_enable(rdev);
e0bcf165
AD		 6356 /* enable aspm */
6357 si_program_aspm(rdev);
b9d305df 6358
e5903d39
AD		 6359 /* scratch needs to be initialized before MC */
6360 r = r600_vram_scratch_init(rdev);
6361 if (r)
6362 return r;
6363
6fab3feb
AD		 6364 si_mc_program(rdev);
6365
9b136d51
AD		 6366 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6367 !rdev->rlc_fw || !rdev->mc_fw) {
6368 r = si_init_microcode(rdev);
6369 if (r) {
6370 DRM_ERROR("Failed to load firmware!\n");
6371 return r;
6372 }
6373 }
6374
6375 r = si_mc_load_microcode(rdev);
6376 if (r) {
6377 DRM_ERROR("Failed to load MC firmware!\n");
6378 return r;
6379 }
6380
9b136d51
AD		 6381 r = si_pcie_gart_enable(rdev);
6382 if (r)
6383 return r;
6384 si_gpu_init(rdev);
6385
9b136d51 6386 /* allocate rlc buffers */
1fd11777
AD		 6387 if (rdev->family == CHIP_VERDE) {
6388 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6389 rdev->rlc.reg_list_size =
6390 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6391 }
6392 rdev->rlc.cs_data = si_cs_data;
6393 r = sumo_rlc_init(rdev);
9b136d51
AD		 6394 if (r) {
6395 DRM_ERROR("Failed to init rlc BOs!\n");
6396 return r;
6397 }
6398
6399 /* allocate wb buffer */
6400 r = radeon_wb_init(rdev);
6401 if (r)
6402 return r;
6403
6404 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6405 if (r) {
6406 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6407 return r;
6408 }
6409
6410 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6411 if (r) {
6412 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6413 return r;
6414 }
6415
6416 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6417 if (r) {
6418 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6419 return r;
6420 }
6421
8c5fd7ef
AD		 6422 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6423 if (r) {
6424 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6425 return r;
6426 }
6427
6428 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6429 if (r) {
6430 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6431 return r;
6432 }
6433
1df0d523 6434 if (rdev->has_uvd) {
e409b128 6435 r = uvd_v2_2_resume(rdev);
1df0d523
AD		 6436 if (!r) {
6437 r = radeon_fence_driver_start_ring(rdev,
6438 R600_RING_TYPE_UVD_INDEX);
6439 if (r)
6440 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6441 }
f2ba57b5 6442 if (r)
1df0d523 6443 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
f2ba57b5 6444 }
f2ba57b5 6445
9b136d51 6446 /* Enable IRQ */
e49f3959
AH		 6447 if (!rdev->irq.installed) {
6448 r = radeon_irq_kms_init(rdev);
6449 if (r)
6450 return r;
6451 }
6452
9b136d51
AD		 6453 r = si_irq_init(rdev);
6454 if (r) {
6455 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6456 radeon_irq_kms_fini(rdev);
6457 return r;
6458 }
6459 si_irq_set(rdev);
6460
6461 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6462 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6463 CP_RB0_RPTR, CP_RB0_WPTR,
2e1e6dad 6464 RADEON_CP_PACKET2);
9b136d51
AD		 6465 if (r)
6466 return r;
6467
6468 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6469 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6470 CP_RB1_RPTR, CP_RB1_WPTR,
2e1e6dad 6471 RADEON_CP_PACKET2);
9b136d51
AD		 6472 if (r)
6473 return r;
6474
6475 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6476 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6477 CP_RB2_RPTR, CP_RB2_WPTR,
2e1e6dad 6478 RADEON_CP_PACKET2);
9b136d51
AD		 6479 if (r)
6480 return r;
6481
8c5fd7ef
AD		 6482 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6483 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6484 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
6485 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
2e1e6dad 6486 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6487 if (r)
6488 return r;
6489
6490 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6491 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6492 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
6493 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
2e1e6dad 6494 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6495 if (r)
6496 return r;
6497
9b136d51
AD		 6498 r = si_cp_load_microcode(rdev);
6499 if (r)
6500 return r;
6501 r = si_cp_resume(rdev);
6502 if (r)
6503 return r;
6504
8c5fd7ef
AD		 6505 r = cayman_dma_resume(rdev);
6506 if (r)
6507 return r;
6508
1df0d523
AD		 6509 if (rdev->has_uvd) {
6510 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6511 if (ring->ring_size) {
02c9f7fa 6512 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
1df0d523 6513 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2e1e6dad 6514 RADEON_CP_PACKET2);
1df0d523 6515 if (!r)
e409b128 6516 r = uvd_v1_0_init(rdev);
1df0d523
AD		 6517 if (r)
6518 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6519 }
f2ba57b5
CK		 6520 }
6521
2898c348
CK		 6522 r = radeon_ib_pool_init(rdev);
6523 if (r) {
6524 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 6525 return r;
2898c348 6526 }
9b136d51 6527
c6105f24
CK		 6528 r = radeon_vm_manager_init(rdev);
6529 if (r) {
6530 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 6531 return r;
c6105f24 6532 }
9b136d51 6533
b530602f
AD		 6534 r = dce6_audio_init(rdev);
6535 if (r)
6536 return r;
6537
9b136d51
AD		 6538 return 0;
6539}
6540
6541int si_resume(struct radeon_device *rdev)
6542{
6543 int r;
6544
6545 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6546 * posting will perform necessary task to bring back GPU into good
6547 * shape.
6548 */
6549 /* post card */
6550 atom_asic_init(rdev->mode_info.atom_context);
6551
205996c0
AD		 6552 /* init golden registers */
6553 si_init_golden_registers(rdev);
6554
9b136d51
AD		 6555 rdev->accel_working = true;
6556 r = si_startup(rdev);
6557 if (r) {
6558 DRM_ERROR("si startup failed on resume\n");
6559 rdev->accel_working = false;
6560 return r;
6561 }
6562
6563 return r;
6564
6565}
6566
6567int si_suspend(struct radeon_device *rdev)
6568{
b530602f 6569 dce6_audio_fini(rdev);
fa3daf9a 6570 radeon_vm_manager_fini(rdev);
9b136d51 6571 si_cp_enable(rdev, false);
8c5fd7ef 6572 cayman_dma_stop(rdev);
1df0d523 6573 if (rdev->has_uvd) {
e409b128 6574 uvd_v1_0_fini(rdev);
1df0d523
AD		 6575 radeon_uvd_suspend(rdev);
6576 }
e16866ec
AD		 6577 si_fini_pg(rdev);
6578 si_fini_cg(rdev);
9b136d51
AD		 6579 si_irq_suspend(rdev);
6580 radeon_wb_disable(rdev);
6581 si_pcie_gart_disable(rdev);
6582 return 0;
6583}
6584
6585/* Plan is to move initialization in that function and use
6586 * helper function so that radeon_device_init pretty much
6587 * do nothing more than calling asic specific function. This
6588 * should also allow to remove a bunch of callback function
6589 * like vram_info.
6590 */
6591int si_init(struct radeon_device *rdev)
6592{
6593 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6594 int r;
6595
9b136d51
AD		 6596 /* Read BIOS */
6597 if (!radeon_get_bios(rdev)) {
6598 if (ASIC_IS_AVIVO(rdev))
6599 return -EINVAL;
6600 }
6601 /* Must be an ATOMBIOS */
6602 if (!rdev->is_atom_bios) {
6603 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6604 return -EINVAL;
6605 }
6606 r = radeon_atombios_init(rdev);
6607 if (r)
6608 return r;
6609
6610 /* Post card if necessary */
6611 if (!radeon_card_posted(rdev)) {
6612 if (!rdev->bios) {
6613 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6614 return -EINVAL;
6615 }
6616 DRM_INFO("GPU not posted. posting now...\n");
6617 atom_asic_init(rdev->mode_info.atom_context);
6618 }
205996c0
AD		 6619 /* init golden registers */
6620 si_init_golden_registers(rdev);
9b136d51
AD		 6621 /* Initialize scratch registers */
6622 si_scratch_init(rdev);
6623 /* Initialize surface registers */
6624 radeon_surface_init(rdev);
6625 /* Initialize clocks */
6626 radeon_get_clock_info(rdev->ddev);
6627
6628 /* Fence driver */
6629 r = radeon_fence_driver_init(rdev);
6630 if (r)
6631 return r;
6632
6633 /* initialize memory controller */
6634 r = si_mc_init(rdev);
6635 if (r)
6636 return r;
6637 /* Memory manager */
6638 r = radeon_bo_init(rdev);
6639 if (r)
6640 return r;
6641
9b136d51
AD		 6642 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6643 ring->ring_obj = NULL;
6644 r600_ring_init(rdev, ring, 1024 * 1024);
6645
6646 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6647 ring->ring_obj = NULL;
6648 r600_ring_init(rdev, ring, 1024 * 1024);
6649
6650 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6651 ring->ring_obj = NULL;
6652 r600_ring_init(rdev, ring, 1024 * 1024);
6653
8c5fd7ef
AD		 6654 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6655 ring->ring_obj = NULL;
6656 r600_ring_init(rdev, ring, 64 * 1024);
6657
6658 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6659 ring->ring_obj = NULL;
6660 r600_ring_init(rdev, ring, 64 * 1024);
6661
1df0d523
AD		 6662 if (rdev->has_uvd) {
6663 r = radeon_uvd_init(rdev);
6664 if (!r) {
6665 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6666 ring->ring_obj = NULL;
6667 r600_ring_init(rdev, ring, 4096);
6668 }
f2ba57b5
CK		 6669 }
6670
9b136d51
AD		 6671 rdev->ih.ring_obj = NULL;
6672 r600_ih_ring_init(rdev, 64 * 1024);
6673
6674 r = r600_pcie_gart_init(rdev);
6675 if (r)
6676 return r;
6677
9b136d51 6678 rdev->accel_working = true;
9b136d51
AD		 6679 r = si_startup(rdev);
6680 if (r) {
6681 dev_err(rdev->dev, "disabling GPU acceleration\n");
6682 si_cp_fini(rdev);
8c5fd7ef 6683 cayman_dma_fini(rdev);
9b136d51 6684 si_irq_fini(rdev);
1fd11777 6685 sumo_rlc_fini(rdev);
9b136d51 6686 radeon_wb_fini(rdev);
2898c348 6687 radeon_ib_pool_fini(rdev);
9b136d51
AD		 6688 radeon_vm_manager_fini(rdev);
6689 radeon_irq_kms_fini(rdev);
6690 si_pcie_gart_fini(rdev);
6691 rdev->accel_working = false;
6692 }
6693
6694 /* Don't start up if the MC ucode is missing.
6695 * The default clocks and voltages before the MC ucode
6696 * is loaded are not suffient for advanced operations.
6697 */
6698 if (!rdev->mc_fw) {
6699 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6700 return -EINVAL;
6701 }
6702
6703 return 0;
6704}
6705
6706void si_fini(struct radeon_device *rdev)
6707{
9b136d51 6708 si_cp_fini(rdev);
8c5fd7ef 6709 cayman_dma_fini(rdev);
f8f84ac5 6710 si_fini_pg(rdev);
e16866ec 6711 si_fini_cg(rdev);
9b136d51 6712 si_irq_fini(rdev);
1fd11777 6713 sumo_rlc_fini(rdev);
9b136d51
AD		 6714 radeon_wb_fini(rdev);
6715 radeon_vm_manager_fini(rdev);
2898c348 6716 radeon_ib_pool_fini(rdev);
9b136d51 6717 radeon_irq_kms_fini(rdev);
2858c00d 6718 if (rdev->has_uvd) {
e409b128 6719 uvd_v1_0_fini(rdev);
1df0d523 6720 radeon_uvd_fini(rdev);
2858c00d 6721 }
9b136d51
AD		 6722 si_pcie_gart_fini(rdev);
6723 r600_vram_scratch_fini(rdev);
6724 radeon_gem_fini(rdev);
9b136d51
AD		 6725 radeon_fence_driver_fini(rdev);
6726 radeon_bo_fini(rdev);
6727 radeon_atombios_fini(rdev);
6728 kfree(rdev->bios);
6729 rdev->bios = NULL;
6730}
6731
6759a0a7 6732/**
d0418894 6733 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 6734 *
6735 * @rdev: radeon_device pointer
6736 *
6737 * Fetches a GPU clock counter snapshot (SI).
6738 * Returns the 64 bit clock counter snapshot.
6739 */
d0418894 6740uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 6741{
6742 uint64_t clock;
6743
6744 mutex_lock(&rdev->gpu_clock_mutex);
6745 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6746 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6747 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6748 mutex_unlock(&rdev->gpu_clock_mutex);
6749 return clock;
6750}
2539eb02 6751
2539eb02
CK		 6752int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6753{
facd112d 6754 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 6755 int r;
6756
4ed10835
CK		 6757 /* bypass vclk and dclk with bclk */
6758 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6759 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6760 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6761
6762 /* put PLL in bypass mode */
6763 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6764
6765 if (!vclk || !dclk) {
6766 /* keep the Bypass mode, put PLL to sleep */
6767 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6768 return 0;
6769 }
6770
facd112d
CK		 6771 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6772 16384, 0x03FFFFFF, 0, 128, 5,
6773 &fb_div, &vclk_div, &dclk_div);
6774 if (r)
6775 return r;
2539eb02
CK		 6776
6777 /* set RESET_ANTI_MUX to 0 */
6778 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6779
6780 /* set VCO_MODE to 1 */
6781 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6782
6783 /* toggle UPLL_SLEEP to 1 then back to 0 */
6784 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6785 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6786
6787 /* deassert UPLL_RESET */
6788 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6789
6790 mdelay(1);
6791
facd112d 6792 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6793 if (r)
6794 return r;
6795
6796 /* assert UPLL_RESET again */
6797 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6798
6799 /* disable spread spectrum. */
6800 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6801
6802 /* set feedback divider */
facd112d 6803 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 6804
6805 /* set ref divider to 0 */
6806 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6807
facd112d 6808 if (fb_div < 307200)
2539eb02
CK		 6809 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6810 else
6811 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6812
6813 /* set PDIV_A and PDIV_B */
6814 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 6815 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 6816 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6817
6818 /* give the PLL some time to settle */
6819 mdelay(15);
6820
6821 /* deassert PLL_RESET */
6822 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6823
6824 mdelay(15);
6825
6826 /* switch from bypass mode to normal mode */
6827 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6828
facd112d 6829 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6830 if (r)
6831 return r;
6832
6833 /* switch VCLK and DCLK selection */
6834 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6835 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6836 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6837
6838 mdelay(100);
6839
6840 return 0;
6841}
b9d305df
AD		 6842
6843static void si_pcie_gen3_enable(struct radeon_device *rdev)
6844{
6845 struct pci_dev *root = rdev->pdev->bus->self;
6846 int bridge_pos, gpu_pos;
6847 u32 speed_cntl, mask, current_data_rate;
6848 int ret, i;
6849 u16 tmp16;
6850
6851 if (radeon_pcie_gen2 == 0)
6852 return;
6853
6854 if (rdev->flags & RADEON_IS_IGP)
6855 return;
6856
6857 if (!(rdev->flags & RADEON_IS_PCIE))
6858 return;
6859
6860 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6861 if (ret != 0)
6862 return;
6863
6864 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6865 return;
6866
6867 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6868 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6869 LC_CURRENT_DATA_RATE_SHIFT;
6870 if (mask & DRM_PCIE_SPEED_80) {
6871 if (current_data_rate == 2) {
6872 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6873 return;
6874 }
6875 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6876 } else if (mask & DRM_PCIE_SPEED_50) {
6877 if (current_data_rate == 1) {
6878 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6879 return;
6880 }
6881 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6882 }
6883
6884 bridge_pos = pci_pcie_cap(root);
6885 if (!bridge_pos)
6886 return;
6887
6888 gpu_pos = pci_pcie_cap(rdev->pdev);
6889 if (!gpu_pos)
6890 return;
6891
6892 if (mask & DRM_PCIE_SPEED_80) {
6893 /* re-try equalization if gen3 is not already enabled */
6894 if (current_data_rate != 2) {
6895 u16 bridge_cfg, gpu_cfg;
6896 u16 bridge_cfg2, gpu_cfg2;
6897 u32 max_lw, current_lw, tmp;
6898
6899 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6900 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6901
6902 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6903 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6904
6905 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6906 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6907
6908 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6909 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6910 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6911
6912 if (current_lw < max_lw) {
6913 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6914 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6915 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6916 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6917 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6918 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6919 }
6920 }
6921
6922 for (i = 0; i < 10; i++) {
6923 /* check status */
6924 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6925 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6926 break;
6927
6928 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6929 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6930
6931 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6932 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6933
6934 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6935 tmp |= LC_SET_QUIESCE;
6936 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6937
6938 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6939 tmp |= LC_REDO_EQ;
6940 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6941
6942 mdelay(100);
6943
6944 /* linkctl */
6945 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6946 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6947 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6948 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6949
6950 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
6951 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6952 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
6953 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6954
6955 /* linkctl2 */
6956 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
6957 tmp16 &= ~((1 << 4) | (7 << 9));
6958 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
6959 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
6960
6961 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6962 tmp16 &= ~((1 << 4) | (7 << 9));
6963 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
6964 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6965
6966 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6967 tmp &= ~LC_SET_QUIESCE;
6968 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6969 }
6970 }
6971 }
6972
6973 /* set the link speed */
6974 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
6975 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
6976 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6977
6978 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6979 tmp16 &= ~0xf;
6980 if (mask & DRM_PCIE_SPEED_80)
6981 tmp16 |= 3; /* gen3 */
6982 else if (mask & DRM_PCIE_SPEED_50)
6983 tmp16 |= 2; /* gen2 */
6984 else
6985 tmp16 |= 1; /* gen1 */
6986 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6987
6988 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6989 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
6990 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6991
6992 for (i = 0; i < rdev->usec_timeout; i++) {
6993 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6994 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
6995 break;
6996 udelay(1);
6997 }
6998}
6999
e0bcf165
AD		 7000static void si_program_aspm(struct radeon_device *rdev)
7001{
7002 u32 data, orig;
7003 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7004 bool disable_clkreq = false;
7005
1294d4a3
AD		 7006 if (radeon_aspm == 0)
7007 return;
7008
e0bcf165
AD		 7009 if (!(rdev->flags & RADEON_IS_PCIE))
7010 return;
7011
7012 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7013 data &= ~LC_XMIT_N_FTS_MASK;
7014 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7015 if (orig != data)
7016 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7017
7018 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7019 data |= LC_GO_TO_RECOVERY;
7020 if (orig != data)
7021 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7022
7023 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7024 data |= P_IGNORE_EDB_ERR;
7025 if (orig != data)
7026 WREG32_PCIE(PCIE_P_CNTL, data);
7027
7028 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7029 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7030 data |= LC_PMI_TO_L1_DIS;
7031 if (!disable_l0s)
7032 data |= LC_L0S_INACTIVITY(7);
7033
7034 if (!disable_l1) {
7035 data |= LC_L1_INACTIVITY(7);
7036 data &= ~LC_PMI_TO_L1_DIS;
7037 if (orig != data)
7038 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7039
7040 if (!disable_plloff_in_l1) {
7041 bool clk_req_support;
7042
7043 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7044 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7045 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7046 if (orig != data)
7047 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7048
7049 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7050 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7051 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7052 if (orig != data)
7053 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7054
7055 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7056 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7057 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7058 if (orig != data)
7059 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7060
7061 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7062 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7063 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7064 if (orig != data)
7065 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7066
7067 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7068 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7069 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7070 if (orig != data)
7071 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7072
7073 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7074 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7075 if (orig != data)
7076 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7077
7078 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7079 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7080 if (orig != data)
7081 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7082
7083 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7084 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7085 if (orig != data)
7086 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7087
7088 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7089 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7090 if (orig != data)
7091 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7092
7093 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7094 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7095 if (orig != data)
7096 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7097
7098 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7099 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7100 if (orig != data)
7101 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7102
7103 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7104 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7105 if (orig != data)
7106 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7107 }
7108 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7109 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7110 data |= LC_DYN_LANES_PWR_STATE(3);
7111 if (orig != data)
7112 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7113
7114 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7115 data &= ~LS2_EXIT_TIME_MASK;
7116 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7117 data |= LS2_EXIT_TIME(5);
7118 if (orig != data)
7119 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7120
7121 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7122 data &= ~LS2_EXIT_TIME_MASK;
7123 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7124 data |= LS2_EXIT_TIME(5);
7125 if (orig != data)
7126 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7127
7128 if (!disable_clkreq) {
7129 struct pci_dev *root = rdev->pdev->bus->self;
7130 u32 lnkcap;
7131
7132 clk_req_support = false;
7133 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7134 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7135 clk_req_support = true;
7136 } else {
7137 clk_req_support = false;
7138 }
7139
7140 if (clk_req_support) {
7141 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7142 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7143 if (orig != data)
7144 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7145
7146 orig = data = RREG32(THM_CLK_CNTL);
7147 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7148 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7149 if (orig != data)
7150 WREG32(THM_CLK_CNTL, data);
7151
7152 orig = data = RREG32(MISC_CLK_CNTL);
7153 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7154 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7155 if (orig != data)
7156 WREG32(MISC_CLK_CNTL, data);
7157
7158 orig = data = RREG32(CG_CLKPIN_CNTL);
7159 data &= ~BCLK_AS_XCLK;
7160 if (orig != data)
7161 WREG32(CG_CLKPIN_CNTL, data);
7162
7163 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7164 data &= ~FORCE_BIF_REFCLK_EN;
7165 if (orig != data)
7166 WREG32(CG_CLKPIN_CNTL_2, data);
7167
7168 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7169 data &= ~MPLL_CLKOUT_SEL_MASK;
7170 data |= MPLL_CLKOUT_SEL(4);
7171 if (orig != data)
7172 WREG32(MPLL_BYPASSCLK_SEL, data);
7173
7174 orig = data = RREG32(SPLL_CNTL_MODE);
7175 data &= ~SPLL_REFCLK_SEL_MASK;
7176 if (orig != data)
7177 WREG32(SPLL_CNTL_MODE, data);
7178 }
7179 }
7180 } else {
7181 if (orig != data)
7182 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7183 }
7184
7185 orig = data = RREG32_PCIE(PCIE_CNTL2);
7186 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7187 if (orig != data)
7188 WREG32_PCIE(PCIE_CNTL2, data);
7189
7190 if (!disable_l0s) {
7191 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7192 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7193 data = RREG32_PCIE(PCIE_LC_STATUS1);
7194 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7195 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7196 data &= ~LC_L0S_INACTIVITY_MASK;
7197 if (orig != data)
7198 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7199 }
7200 }
7201 }
7202}
Linux 6.x block layer and io_uring tree(s)