projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: skip async dma init on r6xx
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
0f0de06c 24#include <linux/firmware.h>
0f0de06c
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
760285e7 27#include <drm/drmP.h>
43b3cd99
AD		 28#include "radeon.h"
29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
43b3cd99
AD		 31#include "sid.h"
32#include "atom.h"
48c0c902 33#include "si_blit_shaders.h"
bd8cd539 34#include "clearstate_si.h"
a0ceada6 35#include "radeon_ucode.h"
43b3cd99 36
0f0de06c
AD		 37
38MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
42MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
a9e61410 43MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
0f0de06c
AD		 44MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
45MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
46MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
47MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
48MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
a9e61410 49MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
0f0de06c
AD		 50MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
51MODULE_FIRMWARE("radeon/VERDE_me.bin");
52MODULE_FIRMWARE("radeon/VERDE_ce.bin");
53MODULE_FIRMWARE("radeon/VERDE_mc.bin");
54MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
a9e61410 55MODULE_FIRMWARE("radeon/VERDE_smc.bin");
bcc7f5d2
AD		 56MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
57MODULE_FIRMWARE("radeon/OLAND_me.bin");
58MODULE_FIRMWARE("radeon/OLAND_ce.bin");
59MODULE_FIRMWARE("radeon/OLAND_mc.bin");
60MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
a9e61410 61MODULE_FIRMWARE("radeon/OLAND_smc.bin");
c04c00b4
AD		 62MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
63MODULE_FIRMWARE("radeon/HAINAN_me.bin");
64MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
65MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
66MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
a9e61410 67MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
0f0de06c 68
b9d305df 69static void si_pcie_gen3_enable(struct radeon_device *rdev);
e0bcf165 70static void si_program_aspm(struct radeon_device *rdev);
1fd11777
AD		 71extern void sumo_rlc_fini(struct radeon_device *rdev);
72extern int sumo_rlc_init(struct radeon_device *rdev);
25a857fb
AD		 73extern int r600_ih_ring_alloc(struct radeon_device *rdev);
74extern void r600_ih_ring_fini(struct radeon_device *rdev);
0a96d72b 75extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
c476dde2
AD		 76extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
77extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
ca7db22b 78extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
1c534671 79extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
014bb209 80extern bool evergreen_is_display_hung(struct radeon_device *rdev);
811e4d58
AD		 81static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
82 bool enable);
4a5c8ea5
AD		 83static void si_init_pg(struct radeon_device *rdev);
84static void si_init_cg(struct radeon_device *rdev);
a6f4ae8d
AD		 85static void si_fini_pg(struct radeon_device *rdev);
86static void si_fini_cg(struct radeon_device *rdev);
87static void si_rlc_stop(struct radeon_device *rdev);
0a96d72b 88
6d8cf000
AD		 89static const u32 verde_rlc_save_restore_register_list[] =
90{
91 (0x8000 << 16) | (0x98f4 >> 2),
92 0x00000000,
93 (0x8040 << 16) | (0x98f4 >> 2),
94 0x00000000,
95 (0x8000 << 16) | (0xe80 >> 2),
96 0x00000000,
97 (0x8040 << 16) | (0xe80 >> 2),
98 0x00000000,
99 (0x8000 << 16) | (0x89bc >> 2),
100 0x00000000,
101 (0x8040 << 16) | (0x89bc >> 2),
102 0x00000000,
103 (0x8000 << 16) | (0x8c1c >> 2),
104 0x00000000,
105 (0x8040 << 16) | (0x8c1c >> 2),
106 0x00000000,
107 (0x9c00 << 16) | (0x98f0 >> 2),
108 0x00000000,
109 (0x9c00 << 16) | (0xe7c >> 2),
110 0x00000000,
111 (0x8000 << 16) | (0x9148 >> 2),
112 0x00000000,
113 (0x8040 << 16) | (0x9148 >> 2),
114 0x00000000,
115 (0x9c00 << 16) | (0x9150 >> 2),
116 0x00000000,
117 (0x9c00 << 16) | (0x897c >> 2),
118 0x00000000,
119 (0x9c00 << 16) | (0x8d8c >> 2),
120 0x00000000,
121 (0x9c00 << 16) | (0xac54 >> 2),
122 0X00000000,
123 0x3,
124 (0x9c00 << 16) | (0x98f8 >> 2),
125 0x00000000,
126 (0x9c00 << 16) | (0x9910 >> 2),
127 0x00000000,
128 (0x9c00 << 16) | (0x9914 >> 2),
129 0x00000000,
130 (0x9c00 << 16) | (0x9918 >> 2),
131 0x00000000,
132 (0x9c00 << 16) | (0x991c >> 2),
133 0x00000000,
134 (0x9c00 << 16) | (0x9920 >> 2),
135 0x00000000,
136 (0x9c00 << 16) | (0x9924 >> 2),
137 0x00000000,
138 (0x9c00 << 16) | (0x9928 >> 2),
139 0x00000000,
140 (0x9c00 << 16) | (0x992c >> 2),
141 0x00000000,
142 (0x9c00 << 16) | (0x9930 >> 2),
143 0x00000000,
144 (0x9c00 << 16) | (0x9934 >> 2),
145 0x00000000,
146 (0x9c00 << 16) | (0x9938 >> 2),
147 0x00000000,
148 (0x9c00 << 16) | (0x993c >> 2),
149 0x00000000,
150 (0x9c00 << 16) | (0x9940 >> 2),
151 0x00000000,
152 (0x9c00 << 16) | (0x9944 >> 2),
153 0x00000000,
154 (0x9c00 << 16) | (0x9948 >> 2),
155 0x00000000,
156 (0x9c00 << 16) | (0x994c >> 2),
157 0x00000000,
158 (0x9c00 << 16) | (0x9950 >> 2),
159 0x00000000,
160 (0x9c00 << 16) | (0x9954 >> 2),
161 0x00000000,
162 (0x9c00 << 16) | (0x9958 >> 2),
163 0x00000000,
164 (0x9c00 << 16) | (0x995c >> 2),
165 0x00000000,
166 (0x9c00 << 16) | (0x9960 >> 2),
167 0x00000000,
168 (0x9c00 << 16) | (0x9964 >> 2),
169 0x00000000,
170 (0x9c00 << 16) | (0x9968 >> 2),
171 0x00000000,
172 (0x9c00 << 16) | (0x996c >> 2),
173 0x00000000,
174 (0x9c00 << 16) | (0x9970 >> 2),
175 0x00000000,
176 (0x9c00 << 16) | (0x9974 >> 2),
177 0x00000000,
178 (0x9c00 << 16) | (0x9978 >> 2),
179 0x00000000,
180 (0x9c00 << 16) | (0x997c >> 2),
181 0x00000000,
182 (0x9c00 << 16) | (0x9980 >> 2),
183 0x00000000,
184 (0x9c00 << 16) | (0x9984 >> 2),
185 0x00000000,
186 (0x9c00 << 16) | (0x9988 >> 2),
187 0x00000000,
188 (0x9c00 << 16) | (0x998c >> 2),
189 0x00000000,
190 (0x9c00 << 16) | (0x8c00 >> 2),
191 0x00000000,
192 (0x9c00 << 16) | (0x8c14 >> 2),
193 0x00000000,
194 (0x9c00 << 16) | (0x8c04 >> 2),
195 0x00000000,
196 (0x9c00 << 16) | (0x8c08 >> 2),
197 0x00000000,
198 (0x8000 << 16) | (0x9b7c >> 2),
199 0x00000000,
200 (0x8040 << 16) | (0x9b7c >> 2),
201 0x00000000,
202 (0x8000 << 16) | (0xe84 >> 2),
203 0x00000000,
204 (0x8040 << 16) | (0xe84 >> 2),
205 0x00000000,
206 (0x8000 << 16) | (0x89c0 >> 2),
207 0x00000000,
208 (0x8040 << 16) | (0x89c0 >> 2),
209 0x00000000,
210 (0x8000 << 16) | (0x914c >> 2),
211 0x00000000,
212 (0x8040 << 16) | (0x914c >> 2),
213 0x00000000,
214 (0x8000 << 16) | (0x8c20 >> 2),
215 0x00000000,
216 (0x8040 << 16) | (0x8c20 >> 2),
217 0x00000000,
218 (0x8000 << 16) | (0x9354 >> 2),
219 0x00000000,
220 (0x8040 << 16) | (0x9354 >> 2),
221 0x00000000,
222 (0x9c00 << 16) | (0x9060 >> 2),
223 0x00000000,
224 (0x9c00 << 16) | (0x9364 >> 2),
225 0x00000000,
226 (0x9c00 << 16) | (0x9100 >> 2),
227 0x00000000,
228 (0x9c00 << 16) | (0x913c >> 2),
229 0x00000000,
230 (0x8000 << 16) | (0x90e0 >> 2),
231 0x00000000,
232 (0x8000 << 16) | (0x90e4 >> 2),
233 0x00000000,
234 (0x8000 << 16) | (0x90e8 >> 2),
235 0x00000000,
236 (0x8040 << 16) | (0x90e0 >> 2),
237 0x00000000,
238 (0x8040 << 16) | (0x90e4 >> 2),
239 0x00000000,
240 (0x8040 << 16) | (0x90e8 >> 2),
241 0x00000000,
242 (0x9c00 << 16) | (0x8bcc >> 2),
243 0x00000000,
244 (0x9c00 << 16) | (0x8b24 >> 2),
245 0x00000000,
246 (0x9c00 << 16) | (0x88c4 >> 2),
247 0x00000000,
248 (0x9c00 << 16) | (0x8e50 >> 2),
249 0x00000000,
250 (0x9c00 << 16) | (0x8c0c >> 2),
251 0x00000000,
252 (0x9c00 << 16) | (0x8e58 >> 2),
253 0x00000000,
254 (0x9c00 << 16) | (0x8e5c >> 2),
255 0x00000000,
256 (0x9c00 << 16) | (0x9508 >> 2),
257 0x00000000,
258 (0x9c00 << 16) | (0x950c >> 2),
259 0x00000000,
260 (0x9c00 << 16) | (0x9494 >> 2),
261 0x00000000,
262 (0x9c00 << 16) | (0xac0c >> 2),
263 0x00000000,
264 (0x9c00 << 16) | (0xac10 >> 2),
265 0x00000000,
266 (0x9c00 << 16) | (0xac14 >> 2),
267 0x00000000,
268 (0x9c00 << 16) | (0xae00 >> 2),
269 0x00000000,
270 (0x9c00 << 16) | (0xac08 >> 2),
271 0x00000000,
272 (0x9c00 << 16) | (0x88d4 >> 2),
273 0x00000000,
274 (0x9c00 << 16) | (0x88c8 >> 2),
275 0x00000000,
276 (0x9c00 << 16) | (0x88cc >> 2),
277 0x00000000,
278 (0x9c00 << 16) | (0x89b0 >> 2),
279 0x00000000,
280 (0x9c00 << 16) | (0x8b10 >> 2),
281 0x00000000,
282 (0x9c00 << 16) | (0x8a14 >> 2),
283 0x00000000,
284 (0x9c00 << 16) | (0x9830 >> 2),
285 0x00000000,
286 (0x9c00 << 16) | (0x9834 >> 2),
287 0x00000000,
288 (0x9c00 << 16) | (0x9838 >> 2),
289 0x00000000,
290 (0x9c00 << 16) | (0x9a10 >> 2),
291 0x00000000,
292 (0x8000 << 16) | (0x9870 >> 2),
293 0x00000000,
294 (0x8000 << 16) | (0x9874 >> 2),
295 0x00000000,
296 (0x8001 << 16) | (0x9870 >> 2),
297 0x00000000,
298 (0x8001 << 16) | (0x9874 >> 2),
299 0x00000000,
300 (0x8040 << 16) | (0x9870 >> 2),
301 0x00000000,
302 (0x8040 << 16) | (0x9874 >> 2),
303 0x00000000,
304 (0x8041 << 16) | (0x9870 >> 2),
305 0x00000000,
306 (0x8041 << 16) | (0x9874 >> 2),
307 0x00000000,
308 0x00000000
309};
310
205996c0
AD		 311static const u32 tahiti_golden_rlc_registers[] =
312{
313 0xc424, 0xffffffff, 0x00601005,
314 0xc47c, 0xffffffff, 0x10104040,
315 0xc488, 0xffffffff, 0x0100000a,
316 0xc314, 0xffffffff, 0x00000800,
317 0xc30c, 0xffffffff, 0x800000f4,
318 0xf4a8, 0xffffffff, 0x00000000
319};
320
321static const u32 tahiti_golden_registers[] =
322{
323 0x9a10, 0x00010000, 0x00018208,
324 0x9830, 0xffffffff, 0x00000000,
325 0x9834, 0xf00fffff, 0x00000400,
326 0x9838, 0x0002021c, 0x00020200,
327 0xc78, 0x00000080, 0x00000000,
328 0xd030, 0x000300c0, 0x00800040,
329 0xd830, 0x000300c0, 0x00800040,
330 0x5bb0, 0x000000f0, 0x00000070,
331 0x5bc0, 0x00200000, 0x50100000,
332 0x7030, 0x31000311, 0x00000011,
333 0x277c, 0x00000003, 0x000007ff,
334 0x240c, 0x000007ff, 0x00000000,
335 0x8a14, 0xf000001f, 0x00000007,
336 0x8b24, 0xffffffff, 0x00ffffff,
337 0x8b10, 0x0000ff0f, 0x00000000,
338 0x28a4c, 0x07ffffff, 0x4e000000,
339 0x28350, 0x3f3f3fff, 0x2a00126a,
340 0x30, 0x000000ff, 0x0040,
341 0x34, 0x00000040, 0x00004040,
342 0x9100, 0x07ffffff, 0x03000000,
343 0x8e88, 0x01ff1f3f, 0x00000000,
344 0x8e84, 0x01ff1f3f, 0x00000000,
345 0x9060, 0x0000007f, 0x00000020,
346 0x9508, 0x00010000, 0x00010000,
347 0xac14, 0x00000200, 0x000002fb,
348 0xac10, 0xffffffff, 0x0000543b,
349 0xac0c, 0xffffffff, 0xa9210876,
350 0x88d0, 0xffffffff, 0x000fff40,
351 0x88d4, 0x0000001f, 0x00000010,
352 0x1410, 0x20000000, 0x20fffed8,
353 0x15c0, 0x000c0fc0, 0x000c0400
354};
355
356static const u32 tahiti_golden_registers2[] =
357{
358 0xc64, 0x00000001, 0x00000001
359};
360
361static const u32 pitcairn_golden_rlc_registers[] =
362{
363 0xc424, 0xffffffff, 0x00601004,
364 0xc47c, 0xffffffff, 0x10102020,
365 0xc488, 0xffffffff, 0x01000020,
366 0xc314, 0xffffffff, 0x00000800,
367 0xc30c, 0xffffffff, 0x800000a4
368};
369
370static const u32 pitcairn_golden_registers[] =
371{
372 0x9a10, 0x00010000, 0x00018208,
373 0x9830, 0xffffffff, 0x00000000,
374 0x9834, 0xf00fffff, 0x00000400,
375 0x9838, 0x0002021c, 0x00020200,
376 0xc78, 0x00000080, 0x00000000,
377 0xd030, 0x000300c0, 0x00800040,
378 0xd830, 0x000300c0, 0x00800040,
379 0x5bb0, 0x000000f0, 0x00000070,
380 0x5bc0, 0x00200000, 0x50100000,
381 0x7030, 0x31000311, 0x00000011,
382 0x2ae4, 0x00073ffe, 0x000022a2,
383 0x240c, 0x000007ff, 0x00000000,
384 0x8a14, 0xf000001f, 0x00000007,
385 0x8b24, 0xffffffff, 0x00ffffff,
386 0x8b10, 0x0000ff0f, 0x00000000,
387 0x28a4c, 0x07ffffff, 0x4e000000,
388 0x28350, 0x3f3f3fff, 0x2a00126a,
389 0x30, 0x000000ff, 0x0040,
390 0x34, 0x00000040, 0x00004040,
391 0x9100, 0x07ffffff, 0x03000000,
392 0x9060, 0x0000007f, 0x00000020,
393 0x9508, 0x00010000, 0x00010000,
394 0xac14, 0x000003ff, 0x000000f7,
395 0xac10, 0xffffffff, 0x00000000,
396 0xac0c, 0xffffffff, 0x32761054,
397 0x88d4, 0x0000001f, 0x00000010,
398 0x15c0, 0x000c0fc0, 0x000c0400
399};
400
401static const u32 verde_golden_rlc_registers[] =
402{
403 0xc424, 0xffffffff, 0x033f1005,
404 0xc47c, 0xffffffff, 0x10808020,
405 0xc488, 0xffffffff, 0x00800008,
406 0xc314, 0xffffffff, 0x00001000,
407 0xc30c, 0xffffffff, 0x80010014
408};
409
410static const u32 verde_golden_registers[] =
411{
412 0x9a10, 0x00010000, 0x00018208,
413 0x9830, 0xffffffff, 0x00000000,
414 0x9834, 0xf00fffff, 0x00000400,
415 0x9838, 0x0002021c, 0x00020200,
416 0xc78, 0x00000080, 0x00000000,
417 0xd030, 0x000300c0, 0x00800040,
418 0xd030, 0x000300c0, 0x00800040,
419 0xd830, 0x000300c0, 0x00800040,
420 0xd830, 0x000300c0, 0x00800040,
421 0x5bb0, 0x000000f0, 0x00000070,
422 0x5bc0, 0x00200000, 0x50100000,
423 0x7030, 0x31000311, 0x00000011,
424 0x2ae4, 0x00073ffe, 0x000022a2,
425 0x2ae4, 0x00073ffe, 0x000022a2,
426 0x2ae4, 0x00073ffe, 0x000022a2,
427 0x240c, 0x000007ff, 0x00000000,
428 0x240c, 0x000007ff, 0x00000000,
429 0x240c, 0x000007ff, 0x00000000,
430 0x8a14, 0xf000001f, 0x00000007,
431 0x8a14, 0xf000001f, 0x00000007,
432 0x8a14, 0xf000001f, 0x00000007,
433 0x8b24, 0xffffffff, 0x00ffffff,
434 0x8b10, 0x0000ff0f, 0x00000000,
435 0x28a4c, 0x07ffffff, 0x4e000000,
436 0x28350, 0x3f3f3fff, 0x0000124a,
437 0x28350, 0x3f3f3fff, 0x0000124a,
438 0x28350, 0x3f3f3fff, 0x0000124a,
439 0x30, 0x000000ff, 0x0040,
440 0x34, 0x00000040, 0x00004040,
441 0x9100, 0x07ffffff, 0x03000000,
442 0x9100, 0x07ffffff, 0x03000000,
443 0x8e88, 0x01ff1f3f, 0x00000000,
444 0x8e88, 0x01ff1f3f, 0x00000000,
445 0x8e88, 0x01ff1f3f, 0x00000000,
446 0x8e84, 0x01ff1f3f, 0x00000000,
447 0x8e84, 0x01ff1f3f, 0x00000000,
448 0x8e84, 0x01ff1f3f, 0x00000000,
449 0x9060, 0x0000007f, 0x00000020,
450 0x9508, 0x00010000, 0x00010000,
451 0xac14, 0x000003ff, 0x00000003,
452 0xac14, 0x000003ff, 0x00000003,
453 0xac14, 0x000003ff, 0x00000003,
454 0xac10, 0xffffffff, 0x00000000,
455 0xac10, 0xffffffff, 0x00000000,
456 0xac10, 0xffffffff, 0x00000000,
457 0xac0c, 0xffffffff, 0x00001032,
458 0xac0c, 0xffffffff, 0x00001032,
459 0xac0c, 0xffffffff, 0x00001032,
460 0x88d4, 0x0000001f, 0x00000010,
461 0x88d4, 0x0000001f, 0x00000010,
462 0x88d4, 0x0000001f, 0x00000010,
463 0x15c0, 0x000c0fc0, 0x000c0400
464};
465
466static const u32 oland_golden_rlc_registers[] =
467{
468 0xc424, 0xffffffff, 0x00601005,
469 0xc47c, 0xffffffff, 0x10104040,
470 0xc488, 0xffffffff, 0x0100000a,
471 0xc314, 0xffffffff, 0x00000800,
472 0xc30c, 0xffffffff, 0x800000f4
473};
474
475static const u32 oland_golden_registers[] =
476{
477 0x9a10, 0x00010000, 0x00018208,
478 0x9830, 0xffffffff, 0x00000000,
479 0x9834, 0xf00fffff, 0x00000400,
480 0x9838, 0x0002021c, 0x00020200,
481 0xc78, 0x00000080, 0x00000000,
482 0xd030, 0x000300c0, 0x00800040,
483 0xd830, 0x000300c0, 0x00800040,
484 0x5bb0, 0x000000f0, 0x00000070,
485 0x5bc0, 0x00200000, 0x50100000,
486 0x7030, 0x31000311, 0x00000011,
487 0x2ae4, 0x00073ffe, 0x000022a2,
488 0x240c, 0x000007ff, 0x00000000,
489 0x8a14, 0xf000001f, 0x00000007,
490 0x8b24, 0xffffffff, 0x00ffffff,
491 0x8b10, 0x0000ff0f, 0x00000000,
492 0x28a4c, 0x07ffffff, 0x4e000000,
493 0x28350, 0x3f3f3fff, 0x00000082,
494 0x30, 0x000000ff, 0x0040,
495 0x34, 0x00000040, 0x00004040,
496 0x9100, 0x07ffffff, 0x03000000,
497 0x9060, 0x0000007f, 0x00000020,
498 0x9508, 0x00010000, 0x00010000,
499 0xac14, 0x000003ff, 0x000000f3,
500 0xac10, 0xffffffff, 0x00000000,
501 0xac0c, 0xffffffff, 0x00003210,
502 0x88d4, 0x0000001f, 0x00000010,
503 0x15c0, 0x000c0fc0, 0x000c0400
504};
505
fffbdda4
AD		 506static const u32 hainan_golden_registers[] =
507{
508 0x9a10, 0x00010000, 0x00018208,
509 0x9830, 0xffffffff, 0x00000000,
510 0x9834, 0xf00fffff, 0x00000400,
511 0x9838, 0x0002021c, 0x00020200,
512 0xd0c0, 0xff000fff, 0x00000100,
513 0xd030, 0x000300c0, 0x00800040,
514 0xd8c0, 0xff000fff, 0x00000100,
515 0xd830, 0x000300c0, 0x00800040,
516 0x2ae4, 0x00073ffe, 0x000022a2,
517 0x240c, 0x000007ff, 0x00000000,
518 0x8a14, 0xf000001f, 0x00000007,
519 0x8b24, 0xffffffff, 0x00ffffff,
520 0x8b10, 0x0000ff0f, 0x00000000,
521 0x28a4c, 0x07ffffff, 0x4e000000,
522 0x28350, 0x3f3f3fff, 0x00000000,
523 0x30, 0x000000ff, 0x0040,
524 0x34, 0x00000040, 0x00004040,
525 0x9100, 0x03e00000, 0x03600000,
526 0x9060, 0x0000007f, 0x00000020,
527 0x9508, 0x00010000, 0x00010000,
528 0xac14, 0x000003ff, 0x000000f1,
529 0xac10, 0xffffffff, 0x00000000,
530 0xac0c, 0xffffffff, 0x00003210,
531 0x88d4, 0x0000001f, 0x00000010,
532 0x15c0, 0x000c0fc0, 0x000c0400
533};
534
535static const u32 hainan_golden_registers2[] =
536{
537 0x98f8, 0xffffffff, 0x02010001
538};
539
205996c0
AD		 540static const u32 tahiti_mgcg_cgcg_init[] =
541{
542 0xc400, 0xffffffff, 0xfffffffc,
543 0x802c, 0xffffffff, 0xe0000000,
544 0x9a60, 0xffffffff, 0x00000100,
545 0x92a4, 0xffffffff, 0x00000100,
546 0xc164, 0xffffffff, 0x00000100,
547 0x9774, 0xffffffff, 0x00000100,
548 0x8984, 0xffffffff, 0x06000100,
549 0x8a18, 0xffffffff, 0x00000100,
550 0x92a0, 0xffffffff, 0x00000100,
551 0xc380, 0xffffffff, 0x00000100,
552 0x8b28, 0xffffffff, 0x00000100,
553 0x9144, 0xffffffff, 0x00000100,
554 0x8d88, 0xffffffff, 0x00000100,
555 0x8d8c, 0xffffffff, 0x00000100,
556 0x9030, 0xffffffff, 0x00000100,
557 0x9034, 0xffffffff, 0x00000100,
558 0x9038, 0xffffffff, 0x00000100,
559 0x903c, 0xffffffff, 0x00000100,
560 0xad80, 0xffffffff, 0x00000100,
561 0xac54, 0xffffffff, 0x00000100,
562 0x897c, 0xffffffff, 0x06000100,
563 0x9868, 0xffffffff, 0x00000100,
564 0x9510, 0xffffffff, 0x00000100,
565 0xaf04, 0xffffffff, 0x00000100,
566 0xae04, 0xffffffff, 0x00000100,
567 0x949c, 0xffffffff, 0x00000100,
568 0x802c, 0xffffffff, 0xe0000000,
569 0x9160, 0xffffffff, 0x00010000,
570 0x9164, 0xffffffff, 0x00030002,
571 0x9168, 0xffffffff, 0x00040007,
572 0x916c, 0xffffffff, 0x00060005,
573 0x9170, 0xffffffff, 0x00090008,
574 0x9174, 0xffffffff, 0x00020001,
575 0x9178, 0xffffffff, 0x00040003,
576 0x917c, 0xffffffff, 0x00000007,
577 0x9180, 0xffffffff, 0x00060005,
578 0x9184, 0xffffffff, 0x00090008,
579 0x9188, 0xffffffff, 0x00030002,
580 0x918c, 0xffffffff, 0x00050004,
581 0x9190, 0xffffffff, 0x00000008,
582 0x9194, 0xffffffff, 0x00070006,
583 0x9198, 0xffffffff, 0x000a0009,
584 0x919c, 0xffffffff, 0x00040003,
585 0x91a0, 0xffffffff, 0x00060005,
586 0x91a4, 0xffffffff, 0x00000009,
587 0x91a8, 0xffffffff, 0x00080007,
588 0x91ac, 0xffffffff, 0x000b000a,
589 0x91b0, 0xffffffff, 0x00050004,
590 0x91b4, 0xffffffff, 0x00070006,
591 0x91b8, 0xffffffff, 0x0008000b,
592 0x91bc, 0xffffffff, 0x000a0009,
593 0x91c0, 0xffffffff, 0x000d000c,
594 0x91c4, 0xffffffff, 0x00060005,
595 0x91c8, 0xffffffff, 0x00080007,
596 0x91cc, 0xffffffff, 0x0000000b,
597 0x91d0, 0xffffffff, 0x000a0009,
598 0x91d4, 0xffffffff, 0x000d000c,
599 0x91d8, 0xffffffff, 0x00070006,
600 0x91dc, 0xffffffff, 0x00090008,
601 0x91e0, 0xffffffff, 0x0000000c,
602 0x91e4, 0xffffffff, 0x000b000a,
603 0x91e8, 0xffffffff, 0x000e000d,
604 0x91ec, 0xffffffff, 0x00080007,
605 0x91f0, 0xffffffff, 0x000a0009,
606 0x91f4, 0xffffffff, 0x0000000d,
607 0x91f8, 0xffffffff, 0x000c000b,
608 0x91fc, 0xffffffff, 0x000f000e,
609 0x9200, 0xffffffff, 0x00090008,
610 0x9204, 0xffffffff, 0x000b000a,
611 0x9208, 0xffffffff, 0x000c000f,
612 0x920c, 0xffffffff, 0x000e000d,
613 0x9210, 0xffffffff, 0x00110010,
614 0x9214, 0xffffffff, 0x000a0009,
615 0x9218, 0xffffffff, 0x000c000b,
616 0x921c, 0xffffffff, 0x0000000f,
617 0x9220, 0xffffffff, 0x000e000d,
618 0x9224, 0xffffffff, 0x00110010,
619 0x9228, 0xffffffff, 0x000b000a,
620 0x922c, 0xffffffff, 0x000d000c,
621 0x9230, 0xffffffff, 0x00000010,
622 0x9234, 0xffffffff, 0x000f000e,
623 0x9238, 0xffffffff, 0x00120011,
624 0x923c, 0xffffffff, 0x000c000b,
625 0x9240, 0xffffffff, 0x000e000d,
626 0x9244, 0xffffffff, 0x00000011,
627 0x9248, 0xffffffff, 0x0010000f,
628 0x924c, 0xffffffff, 0x00130012,
629 0x9250, 0xffffffff, 0x000d000c,
630 0x9254, 0xffffffff, 0x000f000e,
631 0x9258, 0xffffffff, 0x00100013,
632 0x925c, 0xffffffff, 0x00120011,
633 0x9260, 0xffffffff, 0x00150014,
634 0x9264, 0xffffffff, 0x000e000d,
635 0x9268, 0xffffffff, 0x0010000f,
636 0x926c, 0xffffffff, 0x00000013,
637 0x9270, 0xffffffff, 0x00120011,
638 0x9274, 0xffffffff, 0x00150014,
639 0x9278, 0xffffffff, 0x000f000e,
640 0x927c, 0xffffffff, 0x00110010,
641 0x9280, 0xffffffff, 0x00000014,
642 0x9284, 0xffffffff, 0x00130012,
643 0x9288, 0xffffffff, 0x00160015,
644 0x928c, 0xffffffff, 0x0010000f,
645 0x9290, 0xffffffff, 0x00120011,
646 0x9294, 0xffffffff, 0x00000015,
647 0x9298, 0xffffffff, 0x00140013,
648 0x929c, 0xffffffff, 0x00170016,
649 0x9150, 0xffffffff, 0x96940200,
650 0x8708, 0xffffffff, 0x00900100,
651 0xc478, 0xffffffff, 0x00000080,
652 0xc404, 0xffffffff, 0x0020003f,
653 0x30, 0xffffffff, 0x0000001c,
654 0x34, 0x000f0000, 0x000f0000,
655 0x160c, 0xffffffff, 0x00000100,
656 0x1024, 0xffffffff, 0x00000100,
657 0x102c, 0x00000101, 0x00000000,
658 0x20a8, 0xffffffff, 0x00000104,
659 0x264c, 0x000c0000, 0x000c0000,
660 0x2648, 0x000c0000, 0x000c0000,
661 0x55e4, 0xff000fff, 0x00000100,
662 0x55e8, 0x00000001, 0x00000001,
663 0x2f50, 0x00000001, 0x00000001,
664 0x30cc, 0xc0000fff, 0x00000104,
665 0xc1e4, 0x00000001, 0x00000001,
666 0xd0c0, 0xfffffff0, 0x00000100,
667 0xd8c0, 0xfffffff0, 0x00000100
668};
669
670static const u32 pitcairn_mgcg_cgcg_init[] =
671{
672 0xc400, 0xffffffff, 0xfffffffc,
673 0x802c, 0xffffffff, 0xe0000000,
674 0x9a60, 0xffffffff, 0x00000100,
675 0x92a4, 0xffffffff, 0x00000100,
676 0xc164, 0xffffffff, 0x00000100,
677 0x9774, 0xffffffff, 0x00000100,
678 0x8984, 0xffffffff, 0x06000100,
679 0x8a18, 0xffffffff, 0x00000100,
680 0x92a0, 0xffffffff, 0x00000100,
681 0xc380, 0xffffffff, 0x00000100,
682 0x8b28, 0xffffffff, 0x00000100,
683 0x9144, 0xffffffff, 0x00000100,
684 0x8d88, 0xffffffff, 0x00000100,
685 0x8d8c, 0xffffffff, 0x00000100,
686 0x9030, 0xffffffff, 0x00000100,
687 0x9034, 0xffffffff, 0x00000100,
688 0x9038, 0xffffffff, 0x00000100,
689 0x903c, 0xffffffff, 0x00000100,
690 0xad80, 0xffffffff, 0x00000100,
691 0xac54, 0xffffffff, 0x00000100,
692 0x897c, 0xffffffff, 0x06000100,
693 0x9868, 0xffffffff, 0x00000100,
694 0x9510, 0xffffffff, 0x00000100,
695 0xaf04, 0xffffffff, 0x00000100,
696 0xae04, 0xffffffff, 0x00000100,
697 0x949c, 0xffffffff, 0x00000100,
698 0x802c, 0xffffffff, 0xe0000000,
699 0x9160, 0xffffffff, 0x00010000,
700 0x9164, 0xffffffff, 0x00030002,
701 0x9168, 0xffffffff, 0x00040007,
702 0x916c, 0xffffffff, 0x00060005,
703 0x9170, 0xffffffff, 0x00090008,
704 0x9174, 0xffffffff, 0x00020001,
705 0x9178, 0xffffffff, 0x00040003,
706 0x917c, 0xffffffff, 0x00000007,
707 0x9180, 0xffffffff, 0x00060005,
708 0x9184, 0xffffffff, 0x00090008,
709 0x9188, 0xffffffff, 0x00030002,
710 0x918c, 0xffffffff, 0x00050004,
711 0x9190, 0xffffffff, 0x00000008,
712 0x9194, 0xffffffff, 0x00070006,
713 0x9198, 0xffffffff, 0x000a0009,
714 0x919c, 0xffffffff, 0x00040003,
715 0x91a0, 0xffffffff, 0x00060005,
716 0x91a4, 0xffffffff, 0x00000009,
717 0x91a8, 0xffffffff, 0x00080007,
718 0x91ac, 0xffffffff, 0x000b000a,
719 0x91b0, 0xffffffff, 0x00050004,
720 0x91b4, 0xffffffff, 0x00070006,
721 0x91b8, 0xffffffff, 0x0008000b,
722 0x91bc, 0xffffffff, 0x000a0009,
723 0x91c0, 0xffffffff, 0x000d000c,
724 0x9200, 0xffffffff, 0x00090008,
725 0x9204, 0xffffffff, 0x000b000a,
726 0x9208, 0xffffffff, 0x000c000f,
727 0x920c, 0xffffffff, 0x000e000d,
728 0x9210, 0xffffffff, 0x00110010,
729 0x9214, 0xffffffff, 0x000a0009,
730 0x9218, 0xffffffff, 0x000c000b,
731 0x921c, 0xffffffff, 0x0000000f,
732 0x9220, 0xffffffff, 0x000e000d,
733 0x9224, 0xffffffff, 0x00110010,
734 0x9228, 0xffffffff, 0x000b000a,
735 0x922c, 0xffffffff, 0x000d000c,
736 0x9230, 0xffffffff, 0x00000010,
737 0x9234, 0xffffffff, 0x000f000e,
738 0x9238, 0xffffffff, 0x00120011,
739 0x923c, 0xffffffff, 0x000c000b,
740 0x9240, 0xffffffff, 0x000e000d,
741 0x9244, 0xffffffff, 0x00000011,
742 0x9248, 0xffffffff, 0x0010000f,
743 0x924c, 0xffffffff, 0x00130012,
744 0x9250, 0xffffffff, 0x000d000c,
745 0x9254, 0xffffffff, 0x000f000e,
746 0x9258, 0xffffffff, 0x00100013,
747 0x925c, 0xffffffff, 0x00120011,
748 0x9260, 0xffffffff, 0x00150014,
749 0x9150, 0xffffffff, 0x96940200,
750 0x8708, 0xffffffff, 0x00900100,
751 0xc478, 0xffffffff, 0x00000080,
752 0xc404, 0xffffffff, 0x0020003f,
753 0x30, 0xffffffff, 0x0000001c,
754 0x34, 0x000f0000, 0x000f0000,
755 0x160c, 0xffffffff, 0x00000100,
756 0x1024, 0xffffffff, 0x00000100,
757 0x102c, 0x00000101, 0x00000000,
758 0x20a8, 0xffffffff, 0x00000104,
759 0x55e4, 0xff000fff, 0x00000100,
760 0x55e8, 0x00000001, 0x00000001,
761 0x2f50, 0x00000001, 0x00000001,
762 0x30cc, 0xc0000fff, 0x00000104,
763 0xc1e4, 0x00000001, 0x00000001,
764 0xd0c0, 0xfffffff0, 0x00000100,
765 0xd8c0, 0xfffffff0, 0x00000100
766};
767
768static const u32 verde_mgcg_cgcg_init[] =
769{
770 0xc400, 0xffffffff, 0xfffffffc,
771 0x802c, 0xffffffff, 0xe0000000,
772 0x9a60, 0xffffffff, 0x00000100,
773 0x92a4, 0xffffffff, 0x00000100,
774 0xc164, 0xffffffff, 0x00000100,
775 0x9774, 0xffffffff, 0x00000100,
776 0x8984, 0xffffffff, 0x06000100,
777 0x8a18, 0xffffffff, 0x00000100,
778 0x92a0, 0xffffffff, 0x00000100,
779 0xc380, 0xffffffff, 0x00000100,
780 0x8b28, 0xffffffff, 0x00000100,
781 0x9144, 0xffffffff, 0x00000100,
782 0x8d88, 0xffffffff, 0x00000100,
783 0x8d8c, 0xffffffff, 0x00000100,
784 0x9030, 0xffffffff, 0x00000100,
785 0x9034, 0xffffffff, 0x00000100,
786 0x9038, 0xffffffff, 0x00000100,
787 0x903c, 0xffffffff, 0x00000100,
788 0xad80, 0xffffffff, 0x00000100,
789 0xac54, 0xffffffff, 0x00000100,
790 0x897c, 0xffffffff, 0x06000100,
791 0x9868, 0xffffffff, 0x00000100,
792 0x9510, 0xffffffff, 0x00000100,
793 0xaf04, 0xffffffff, 0x00000100,
794 0xae04, 0xffffffff, 0x00000100,
795 0x949c, 0xffffffff, 0x00000100,
796 0x802c, 0xffffffff, 0xe0000000,
797 0x9160, 0xffffffff, 0x00010000,
798 0x9164, 0xffffffff, 0x00030002,
799 0x9168, 0xffffffff, 0x00040007,
800 0x916c, 0xffffffff, 0x00060005,
801 0x9170, 0xffffffff, 0x00090008,
802 0x9174, 0xffffffff, 0x00020001,
803 0x9178, 0xffffffff, 0x00040003,
804 0x917c, 0xffffffff, 0x00000007,
805 0x9180, 0xffffffff, 0x00060005,
806 0x9184, 0xffffffff, 0x00090008,
807 0x9188, 0xffffffff, 0x00030002,
808 0x918c, 0xffffffff, 0x00050004,
809 0x9190, 0xffffffff, 0x00000008,
810 0x9194, 0xffffffff, 0x00070006,
811 0x9198, 0xffffffff, 0x000a0009,
812 0x919c, 0xffffffff, 0x00040003,
813 0x91a0, 0xffffffff, 0x00060005,
814 0x91a4, 0xffffffff, 0x00000009,
815 0x91a8, 0xffffffff, 0x00080007,
816 0x91ac, 0xffffffff, 0x000b000a,
817 0x91b0, 0xffffffff, 0x00050004,
818 0x91b4, 0xffffffff, 0x00070006,
819 0x91b8, 0xffffffff, 0x0008000b,
820 0x91bc, 0xffffffff, 0x000a0009,
821 0x91c0, 0xffffffff, 0x000d000c,
822 0x9200, 0xffffffff, 0x00090008,
823 0x9204, 0xffffffff, 0x000b000a,
824 0x9208, 0xffffffff, 0x000c000f,
825 0x920c, 0xffffffff, 0x000e000d,
826 0x9210, 0xffffffff, 0x00110010,
827 0x9214, 0xffffffff, 0x000a0009,
828 0x9218, 0xffffffff, 0x000c000b,
829 0x921c, 0xffffffff, 0x0000000f,
830 0x9220, 0xffffffff, 0x000e000d,
831 0x9224, 0xffffffff, 0x00110010,
832 0x9228, 0xffffffff, 0x000b000a,
833 0x922c, 0xffffffff, 0x000d000c,
834 0x9230, 0xffffffff, 0x00000010,
835 0x9234, 0xffffffff, 0x000f000e,
836 0x9238, 0xffffffff, 0x00120011,
837 0x923c, 0xffffffff, 0x000c000b,
838 0x9240, 0xffffffff, 0x000e000d,
839 0x9244, 0xffffffff, 0x00000011,
840 0x9248, 0xffffffff, 0x0010000f,
841 0x924c, 0xffffffff, 0x00130012,
842 0x9250, 0xffffffff, 0x000d000c,
843 0x9254, 0xffffffff, 0x000f000e,
844 0x9258, 0xffffffff, 0x00100013,
845 0x925c, 0xffffffff, 0x00120011,
846 0x9260, 0xffffffff, 0x00150014,
847 0x9150, 0xffffffff, 0x96940200,
848 0x8708, 0xffffffff, 0x00900100,
849 0xc478, 0xffffffff, 0x00000080,
850 0xc404, 0xffffffff, 0x0020003f,
851 0x30, 0xffffffff, 0x0000001c,
852 0x34, 0x000f0000, 0x000f0000,
853 0x160c, 0xffffffff, 0x00000100,
854 0x1024, 0xffffffff, 0x00000100,
855 0x102c, 0x00000101, 0x00000000,
856 0x20a8, 0xffffffff, 0x00000104,
857 0x264c, 0x000c0000, 0x000c0000,
858 0x2648, 0x000c0000, 0x000c0000,
859 0x55e4, 0xff000fff, 0x00000100,
860 0x55e8, 0x00000001, 0x00000001,
861 0x2f50, 0x00000001, 0x00000001,
862 0x30cc, 0xc0000fff, 0x00000104,
863 0xc1e4, 0x00000001, 0x00000001,
864 0xd0c0, 0xfffffff0, 0x00000100,
865 0xd8c0, 0xfffffff0, 0x00000100
866};
867
868static const u32 oland_mgcg_cgcg_init[] =
869{
870 0xc400, 0xffffffff, 0xfffffffc,
871 0x802c, 0xffffffff, 0xe0000000,
872 0x9a60, 0xffffffff, 0x00000100,
873 0x92a4, 0xffffffff, 0x00000100,
874 0xc164, 0xffffffff, 0x00000100,
875 0x9774, 0xffffffff, 0x00000100,
876 0x8984, 0xffffffff, 0x06000100,
877 0x8a18, 0xffffffff, 0x00000100,
878 0x92a0, 0xffffffff, 0x00000100,
879 0xc380, 0xffffffff, 0x00000100,
880 0x8b28, 0xffffffff, 0x00000100,
881 0x9144, 0xffffffff, 0x00000100,
882 0x8d88, 0xffffffff, 0x00000100,
883 0x8d8c, 0xffffffff, 0x00000100,
884 0x9030, 0xffffffff, 0x00000100,
885 0x9034, 0xffffffff, 0x00000100,
886 0x9038, 0xffffffff, 0x00000100,
887 0x903c, 0xffffffff, 0x00000100,
888 0xad80, 0xffffffff, 0x00000100,
889 0xac54, 0xffffffff, 0x00000100,
890 0x897c, 0xffffffff, 0x06000100,
891 0x9868, 0xffffffff, 0x00000100,
892 0x9510, 0xffffffff, 0x00000100,
893 0xaf04, 0xffffffff, 0x00000100,
894 0xae04, 0xffffffff, 0x00000100,
895 0x949c, 0xffffffff, 0x00000100,
896 0x802c, 0xffffffff, 0xe0000000,
897 0x9160, 0xffffffff, 0x00010000,
898 0x9164, 0xffffffff, 0x00030002,
899 0x9168, 0xffffffff, 0x00040007,
900 0x916c, 0xffffffff, 0x00060005,
901 0x9170, 0xffffffff, 0x00090008,
902 0x9174, 0xffffffff, 0x00020001,
903 0x9178, 0xffffffff, 0x00040003,
904 0x917c, 0xffffffff, 0x00000007,
905 0x9180, 0xffffffff, 0x00060005,
906 0x9184, 0xffffffff, 0x00090008,
907 0x9188, 0xffffffff, 0x00030002,
908 0x918c, 0xffffffff, 0x00050004,
909 0x9190, 0xffffffff, 0x00000008,
910 0x9194, 0xffffffff, 0x00070006,
911 0x9198, 0xffffffff, 0x000a0009,
912 0x919c, 0xffffffff, 0x00040003,
913 0x91a0, 0xffffffff, 0x00060005,
914 0x91a4, 0xffffffff, 0x00000009,
915 0x91a8, 0xffffffff, 0x00080007,
916 0x91ac, 0xffffffff, 0x000b000a,
917 0x91b0, 0xffffffff, 0x00050004,
918 0x91b4, 0xffffffff, 0x00070006,
919 0x91b8, 0xffffffff, 0x0008000b,
920 0x91bc, 0xffffffff, 0x000a0009,
921 0x91c0, 0xffffffff, 0x000d000c,
922 0x91c4, 0xffffffff, 0x00060005,
923 0x91c8, 0xffffffff, 0x00080007,
924 0x91cc, 0xffffffff, 0x0000000b,
925 0x91d0, 0xffffffff, 0x000a0009,
926 0x91d4, 0xffffffff, 0x000d000c,
927 0x9150, 0xffffffff, 0x96940200,
928 0x8708, 0xffffffff, 0x00900100,
929 0xc478, 0xffffffff, 0x00000080,
930 0xc404, 0xffffffff, 0x0020003f,
931 0x30, 0xffffffff, 0x0000001c,
932 0x34, 0x000f0000, 0x000f0000,
933 0x160c, 0xffffffff, 0x00000100,
934 0x1024, 0xffffffff, 0x00000100,
935 0x102c, 0x00000101, 0x00000000,
936 0x20a8, 0xffffffff, 0x00000104,
937 0x264c, 0x000c0000, 0x000c0000,
938 0x2648, 0x000c0000, 0x000c0000,
939 0x55e4, 0xff000fff, 0x00000100,
940 0x55e8, 0x00000001, 0x00000001,
941 0x2f50, 0x00000001, 0x00000001,
942 0x30cc, 0xc0000fff, 0x00000104,
943 0xc1e4, 0x00000001, 0x00000001,
944 0xd0c0, 0xfffffff0, 0x00000100,
945 0xd8c0, 0xfffffff0, 0x00000100
946};
947
fffbdda4
AD		 948static const u32 hainan_mgcg_cgcg_init[] =
949{
950 0xc400, 0xffffffff, 0xfffffffc,
951 0x802c, 0xffffffff, 0xe0000000,
952 0x9a60, 0xffffffff, 0x00000100,
953 0x92a4, 0xffffffff, 0x00000100,
954 0xc164, 0xffffffff, 0x00000100,
955 0x9774, 0xffffffff, 0x00000100,
956 0x8984, 0xffffffff, 0x06000100,
957 0x8a18, 0xffffffff, 0x00000100,
958 0x92a0, 0xffffffff, 0x00000100,
959 0xc380, 0xffffffff, 0x00000100,
960 0x8b28, 0xffffffff, 0x00000100,
961 0x9144, 0xffffffff, 0x00000100,
962 0x8d88, 0xffffffff, 0x00000100,
963 0x8d8c, 0xffffffff, 0x00000100,
964 0x9030, 0xffffffff, 0x00000100,
965 0x9034, 0xffffffff, 0x00000100,
966 0x9038, 0xffffffff, 0x00000100,
967 0x903c, 0xffffffff, 0x00000100,
968 0xad80, 0xffffffff, 0x00000100,
969 0xac54, 0xffffffff, 0x00000100,
970 0x897c, 0xffffffff, 0x06000100,
971 0x9868, 0xffffffff, 0x00000100,
972 0x9510, 0xffffffff, 0x00000100,
973 0xaf04, 0xffffffff, 0x00000100,
974 0xae04, 0xffffffff, 0x00000100,
975 0x949c, 0xffffffff, 0x00000100,
976 0x802c, 0xffffffff, 0xe0000000,
977 0x9160, 0xffffffff, 0x00010000,
978 0x9164, 0xffffffff, 0x00030002,
979 0x9168, 0xffffffff, 0x00040007,
980 0x916c, 0xffffffff, 0x00060005,
981 0x9170, 0xffffffff, 0x00090008,
982 0x9174, 0xffffffff, 0x00020001,
983 0x9178, 0xffffffff, 0x00040003,
984 0x917c, 0xffffffff, 0x00000007,
985 0x9180, 0xffffffff, 0x00060005,
986 0x9184, 0xffffffff, 0x00090008,
987 0x9188, 0xffffffff, 0x00030002,
988 0x918c, 0xffffffff, 0x00050004,
989 0x9190, 0xffffffff, 0x00000008,
990 0x9194, 0xffffffff, 0x00070006,
991 0x9198, 0xffffffff, 0x000a0009,
992 0x919c, 0xffffffff, 0x00040003,
993 0x91a0, 0xffffffff, 0x00060005,
994 0x91a4, 0xffffffff, 0x00000009,
995 0x91a8, 0xffffffff, 0x00080007,
996 0x91ac, 0xffffffff, 0x000b000a,
997 0x91b0, 0xffffffff, 0x00050004,
998 0x91b4, 0xffffffff, 0x00070006,
999 0x91b8, 0xffffffff, 0x0008000b,
1000 0x91bc, 0xffffffff, 0x000a0009,
1001 0x91c0, 0xffffffff, 0x000d000c,
1002 0x91c4, 0xffffffff, 0x00060005,
1003 0x91c8, 0xffffffff, 0x00080007,
1004 0x91cc, 0xffffffff, 0x0000000b,
1005 0x91d0, 0xffffffff, 0x000a0009,
1006 0x91d4, 0xffffffff, 0x000d000c,
1007 0x9150, 0xffffffff, 0x96940200,
1008 0x8708, 0xffffffff, 0x00900100,
1009 0xc478, 0xffffffff, 0x00000080,
1010 0xc404, 0xffffffff, 0x0020003f,
1011 0x30, 0xffffffff, 0x0000001c,
1012 0x34, 0x000f0000, 0x000f0000,
1013 0x160c, 0xffffffff, 0x00000100,
1014 0x1024, 0xffffffff, 0x00000100,
1015 0x20a8, 0xffffffff, 0x00000104,
1016 0x264c, 0x000c0000, 0x000c0000,
1017 0x2648, 0x000c0000, 0x000c0000,
1018 0x2f50, 0x00000001, 0x00000001,
1019 0x30cc, 0xc0000fff, 0x00000104,
1020 0xc1e4, 0x00000001, 0x00000001,
1021 0xd0c0, 0xfffffff0, 0x00000100,
1022 0xd8c0, 0xfffffff0, 0x00000100
1023};
1024
205996c0
AD		 1025static u32 verde_pg_init[] =
1026{
1027 0x353c, 0xffffffff, 0x40000,
1028 0x3538, 0xffffffff, 0x200010ff,
1029 0x353c, 0xffffffff, 0x0,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x0,
1034 0x353c, 0xffffffff, 0x7007,
1035 0x3538, 0xffffffff, 0x300010ff,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x0,
1041 0x353c, 0xffffffff, 0x400000,
1042 0x3538, 0xffffffff, 0x100010ff,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x0,
1048 0x353c, 0xffffffff, 0x120200,
1049 0x3538, 0xffffffff, 0x500010ff,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x0,
1055 0x353c, 0xffffffff, 0x1e1e16,
1056 0x3538, 0xffffffff, 0x600010ff,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x0,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x0,
1061 0x353c, 0xffffffff, 0x0,
1062 0x353c, 0xffffffff, 0x171f1e,
1063 0x3538, 0xffffffff, 0x700010ff,
1064 0x353c, 0xffffffff, 0x0,
1065 0x353c, 0xffffffff, 0x0,
1066 0x353c, 0xffffffff, 0x0,
1067 0x353c, 0xffffffff, 0x0,
1068 0x353c, 0xffffffff, 0x0,
1069 0x353c, 0xffffffff, 0x0,
1070 0x3538, 0xffffffff, 0x9ff,
1071 0x3500, 0xffffffff, 0x0,
1072 0x3504, 0xffffffff, 0x10000800,
1073 0x3504, 0xffffffff, 0xf,
1074 0x3504, 0xffffffff, 0xf,
1075 0x3500, 0xffffffff, 0x4,
1076 0x3504, 0xffffffff, 0x1000051e,
1077 0x3504, 0xffffffff, 0xffff,
1078 0x3504, 0xffffffff, 0xffff,
1079 0x3500, 0xffffffff, 0x8,
1080 0x3504, 0xffffffff, 0x80500,
1081 0x3500, 0xffffffff, 0x12,
1082 0x3504, 0xffffffff, 0x9050c,
1083 0x3500, 0xffffffff, 0x1d,
1084 0x3504, 0xffffffff, 0xb052c,
1085 0x3500, 0xffffffff, 0x2a,
1086 0x3504, 0xffffffff, 0x1053e,
1087 0x3500, 0xffffffff, 0x2d,
1088 0x3504, 0xffffffff, 0x10546,
1089 0x3500, 0xffffffff, 0x30,
1090 0x3504, 0xffffffff, 0xa054e,
1091 0x3500, 0xffffffff, 0x3c,
1092 0x3504, 0xffffffff, 0x1055f,
1093 0x3500, 0xffffffff, 0x3f,
1094 0x3504, 0xffffffff, 0x10567,
1095 0x3500, 0xffffffff, 0x42,
1096 0x3504, 0xffffffff, 0x1056f,
1097 0x3500, 0xffffffff, 0x45,
1098 0x3504, 0xffffffff, 0x10572,
1099 0x3500, 0xffffffff, 0x48,
1100 0x3504, 0xffffffff, 0x20575,
1101 0x3500, 0xffffffff, 0x4c,
1102 0x3504, 0xffffffff, 0x190801,
1103 0x3500, 0xffffffff, 0x67,
1104 0x3504, 0xffffffff, 0x1082a,
1105 0x3500, 0xffffffff, 0x6a,
1106 0x3504, 0xffffffff, 0x1b082d,
1107 0x3500, 0xffffffff, 0x87,
1108 0x3504, 0xffffffff, 0x310851,
1109 0x3500, 0xffffffff, 0xba,
1110 0x3504, 0xffffffff, 0x891,
1111 0x3500, 0xffffffff, 0xbc,
1112 0x3504, 0xffffffff, 0x893,
1113 0x3500, 0xffffffff, 0xbe,
1114 0x3504, 0xffffffff, 0x20895,
1115 0x3500, 0xffffffff, 0xc2,
1116 0x3504, 0xffffffff, 0x20899,
1117 0x3500, 0xffffffff, 0xc6,
1118 0x3504, 0xffffffff, 0x2089d,
1119 0x3500, 0xffffffff, 0xca,
1120 0x3504, 0xffffffff, 0x8a1,
1121 0x3500, 0xffffffff, 0xcc,
1122 0x3504, 0xffffffff, 0x8a3,
1123 0x3500, 0xffffffff, 0xce,
1124 0x3504, 0xffffffff, 0x308a5,
1125 0x3500, 0xffffffff, 0xd3,
1126 0x3504, 0xffffffff, 0x6d08cd,
1127 0x3500, 0xffffffff, 0x142,
1128 0x3504, 0xffffffff, 0x2000095a,
1129 0x3504, 0xffffffff, 0x1,
1130 0x3500, 0xffffffff, 0x144,
1131 0x3504, 0xffffffff, 0x301f095b,
1132 0x3500, 0xffffffff, 0x165,
1133 0x3504, 0xffffffff, 0xc094d,
1134 0x3500, 0xffffffff, 0x173,
1135 0x3504, 0xffffffff, 0xf096d,
1136 0x3500, 0xffffffff, 0x184,
1137 0x3504, 0xffffffff, 0x15097f,
1138 0x3500, 0xffffffff, 0x19b,
1139 0x3504, 0xffffffff, 0xc0998,
1140 0x3500, 0xffffffff, 0x1a9,
1141 0x3504, 0xffffffff, 0x409a7,
1142 0x3500, 0xffffffff, 0x1af,
1143 0x3504, 0xffffffff, 0xcdc,
1144 0x3500, 0xffffffff, 0x1b1,
1145 0x3504, 0xffffffff, 0x800,
1146 0x3508, 0xffffffff, 0x6c9b2000,
1147 0x3510, 0xfc00, 0x2000,
1148 0x3544, 0xffffffff, 0xfc0,
1149 0x28d4, 0x00000100, 0x100
1150};
1151
1152static void si_init_golden_registers(struct radeon_device *rdev)
1153{
1154 switch (rdev->family) {
1155 case CHIP_TAHITI:
1156 radeon_program_register_sequence(rdev,
1157 tahiti_golden_registers,
1158 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1159 radeon_program_register_sequence(rdev,
1160 tahiti_golden_rlc_registers,
1161 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1162 radeon_program_register_sequence(rdev,
1163 tahiti_mgcg_cgcg_init,
1164 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1165 radeon_program_register_sequence(rdev,
1166 tahiti_golden_registers2,
1167 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1168 break;
1169 case CHIP_PITCAIRN:
1170 radeon_program_register_sequence(rdev,
1171 pitcairn_golden_registers,
1172 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1173 radeon_program_register_sequence(rdev,
1174 pitcairn_golden_rlc_registers,
1175 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1176 radeon_program_register_sequence(rdev,
1177 pitcairn_mgcg_cgcg_init,
1178 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1179 break;
1180 case CHIP_VERDE:
1181 radeon_program_register_sequence(rdev,
1182 verde_golden_registers,
1183 (const u32)ARRAY_SIZE(verde_golden_registers));
1184 radeon_program_register_sequence(rdev,
1185 verde_golden_rlc_registers,
1186 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1187 radeon_program_register_sequence(rdev,
1188 verde_mgcg_cgcg_init,
1189 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1190 radeon_program_register_sequence(rdev,
1191 verde_pg_init,
1192 (const u32)ARRAY_SIZE(verde_pg_init));
1193 break;
1194 case CHIP_OLAND:
1195 radeon_program_register_sequence(rdev,
1196 oland_golden_registers,
1197 (const u32)ARRAY_SIZE(oland_golden_registers));
1198 radeon_program_register_sequence(rdev,
1199 oland_golden_rlc_registers,
1200 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1201 radeon_program_register_sequence(rdev,
1202 oland_mgcg_cgcg_init,
1203 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1204 break;
fffbdda4
AD		 1205 case CHIP_HAINAN:
1206 radeon_program_register_sequence(rdev,
1207 hainan_golden_registers,
1208 (const u32)ARRAY_SIZE(hainan_golden_registers));
1209 radeon_program_register_sequence(rdev,
1210 hainan_golden_registers2,
1211 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1212 radeon_program_register_sequence(rdev,
1213 hainan_mgcg_cgcg_init,
1214 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1215 break;
205996c0
AD		 1216 default:
1217 break;
1218 }
1219}
1220
454d2e2a
AD		 1221#define PCIE_BUS_CLK 10000
1222#define TCLK (PCIE_BUS_CLK / 10)
1223
1224/**
1225 * si_get_xclk - get the xclk
1226 *
1227 * @rdev: radeon_device pointer
1228 *
1229 * Returns the reference clock used by the gfx engine
1230 * (SI).
1231 */
1232u32 si_get_xclk(struct radeon_device *rdev)
1233{
1234 u32 reference_clock = rdev->clock.spll.reference_freq;
1235 u32 tmp;
1236
1237 tmp = RREG32(CG_CLKPIN_CNTL_2);
1238 if (tmp & MUX_TCLK_TO_XCLK)
1239 return TCLK;
1240
1241 tmp = RREG32(CG_CLKPIN_CNTL);
1242 if (tmp & XTALIN_DIVIDE)
1243 return reference_clock / 4;
1244
1245 return reference_clock;
1246}
1247
1bd47d2e
AD		 1248/* get temperature in millidegrees */
1249int si_get_temp(struct radeon_device *rdev)
1250{
1251 u32 temp;
1252 int actual_temp = 0;
1253
1254 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1255 CTF_TEMP_SHIFT;
1256
1257 if (temp & 0x200)
1258 actual_temp = 255;
1259 else
1260 actual_temp = temp & 0x1ff;
1261
1262 actual_temp = (actual_temp * 1000);
1263
1264 return actual_temp;
1265}
1266
8b074dd6
AD		 1267#define TAHITI_IO_MC_REGS_SIZE 36
1268
1269static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1270 {0x0000006f, 0x03044000},
1271 {0x00000070, 0x0480c018},
1272 {0x00000071, 0x00000040},
1273 {0x00000072, 0x01000000},
1274 {0x00000074, 0x000000ff},
1275 {0x00000075, 0x00143400},
1276 {0x00000076, 0x08ec0800},
1277 {0x00000077, 0x040000cc},
1278 {0x00000079, 0x00000000},
1279 {0x0000007a, 0x21000409},
1280 {0x0000007c, 0x00000000},
1281 {0x0000007d, 0xe8000000},
1282 {0x0000007e, 0x044408a8},
1283 {0x0000007f, 0x00000003},
1284 {0x00000080, 0x00000000},
1285 {0x00000081, 0x01000000},
1286 {0x00000082, 0x02000000},
1287 {0x00000083, 0x00000000},
1288 {0x00000084, 0xe3f3e4f4},
1289 {0x00000085, 0x00052024},
1290 {0x00000087, 0x00000000},
1291 {0x00000088, 0x66036603},
1292 {0x00000089, 0x01000000},
1293 {0x0000008b, 0x1c0a0000},
1294 {0x0000008c, 0xff010000},
1295 {0x0000008e, 0xffffefff},
1296 {0x0000008f, 0xfff3efff},
1297 {0x00000090, 0xfff3efbf},
1298 {0x00000094, 0x00101101},
1299 {0x00000095, 0x00000fff},
1300 {0x00000096, 0x00116fff},
1301 {0x00000097, 0x60010000},
1302 {0x00000098, 0x10010000},
1303 {0x00000099, 0x00006000},
1304 {0x0000009a, 0x00001000},
1305 {0x0000009f, 0x00a77400}
1306};
1307
1308static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1309 {0x0000006f, 0x03044000},
1310 {0x00000070, 0x0480c018},
1311 {0x00000071, 0x00000040},
1312 {0x00000072, 0x01000000},
1313 {0x00000074, 0x000000ff},
1314 {0x00000075, 0x00143400},
1315 {0x00000076, 0x08ec0800},
1316 {0x00000077, 0x040000cc},
1317 {0x00000079, 0x00000000},
1318 {0x0000007a, 0x21000409},
1319 {0x0000007c, 0x00000000},
1320 {0x0000007d, 0xe8000000},
1321 {0x0000007e, 0x044408a8},
1322 {0x0000007f, 0x00000003},
1323 {0x00000080, 0x00000000},
1324 {0x00000081, 0x01000000},
1325 {0x00000082, 0x02000000},
1326 {0x00000083, 0x00000000},
1327 {0x00000084, 0xe3f3e4f4},
1328 {0x00000085, 0x00052024},
1329 {0x00000087, 0x00000000},
1330 {0x00000088, 0x66036603},
1331 {0x00000089, 0x01000000},
1332 {0x0000008b, 0x1c0a0000},
1333 {0x0000008c, 0xff010000},
1334 {0x0000008e, 0xffffefff},
1335 {0x0000008f, 0xfff3efff},
1336 {0x00000090, 0xfff3efbf},
1337 {0x00000094, 0x00101101},
1338 {0x00000095, 0x00000fff},
1339 {0x00000096, 0x00116fff},
1340 {0x00000097, 0x60010000},
1341 {0x00000098, 0x10010000},
1342 {0x00000099, 0x00006000},
1343 {0x0000009a, 0x00001000},
1344 {0x0000009f, 0x00a47400}
1345};
1346
1347static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1348 {0x0000006f, 0x03044000},
1349 {0x00000070, 0x0480c018},
1350 {0x00000071, 0x00000040},
1351 {0x00000072, 0x01000000},
1352 {0x00000074, 0x000000ff},
1353 {0x00000075, 0x00143400},
1354 {0x00000076, 0x08ec0800},
1355 {0x00000077, 0x040000cc},
1356 {0x00000079, 0x00000000},
1357 {0x0000007a, 0x21000409},
1358 {0x0000007c, 0x00000000},
1359 {0x0000007d, 0xe8000000},
1360 {0x0000007e, 0x044408a8},
1361 {0x0000007f, 0x00000003},
1362 {0x00000080, 0x00000000},
1363 {0x00000081, 0x01000000},
1364 {0x00000082, 0x02000000},
1365 {0x00000083, 0x00000000},
1366 {0x00000084, 0xe3f3e4f4},
1367 {0x00000085, 0x00052024},
1368 {0x00000087, 0x00000000},
1369 {0x00000088, 0x66036603},
1370 {0x00000089, 0x01000000},
1371 {0x0000008b, 0x1c0a0000},
1372 {0x0000008c, 0xff010000},
1373 {0x0000008e, 0xffffefff},
1374 {0x0000008f, 0xfff3efff},
1375 {0x00000090, 0xfff3efbf},
1376 {0x00000094, 0x00101101},
1377 {0x00000095, 0x00000fff},
1378 {0x00000096, 0x00116fff},
1379 {0x00000097, 0x60010000},
1380 {0x00000098, 0x10010000},
1381 {0x00000099, 0x00006000},
1382 {0x0000009a, 0x00001000},
1383 {0x0000009f, 0x00a37400}
1384};
1385
bcc7f5d2
AD		 1386static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1387 {0x0000006f, 0x03044000},
1388 {0x00000070, 0x0480c018},
1389 {0x00000071, 0x00000040},
1390 {0x00000072, 0x01000000},
1391 {0x00000074, 0x000000ff},
1392 {0x00000075, 0x00143400},
1393 {0x00000076, 0x08ec0800},
1394 {0x00000077, 0x040000cc},
1395 {0x00000079, 0x00000000},
1396 {0x0000007a, 0x21000409},
1397 {0x0000007c, 0x00000000},
1398 {0x0000007d, 0xe8000000},
1399 {0x0000007e, 0x044408a8},
1400 {0x0000007f, 0x00000003},
1401 {0x00000080, 0x00000000},
1402 {0x00000081, 0x01000000},
1403 {0x00000082, 0x02000000},
1404 {0x00000083, 0x00000000},
1405 {0x00000084, 0xe3f3e4f4},
1406 {0x00000085, 0x00052024},
1407 {0x00000087, 0x00000000},
1408 {0x00000088, 0x66036603},
1409 {0x00000089, 0x01000000},
1410 {0x0000008b, 0x1c0a0000},
1411 {0x0000008c, 0xff010000},
1412 {0x0000008e, 0xffffefff},
1413 {0x0000008f, 0xfff3efff},
1414 {0x00000090, 0xfff3efbf},
1415 {0x00000094, 0x00101101},
1416 {0x00000095, 0x00000fff},
1417 {0x00000096, 0x00116fff},
1418 {0x00000097, 0x60010000},
1419 {0x00000098, 0x10010000},
1420 {0x00000099, 0x00006000},
1421 {0x0000009a, 0x00001000},
1422 {0x0000009f, 0x00a17730}
1423};
1424
c04c00b4
AD		 1425static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1426 {0x0000006f, 0x03044000},
1427 {0x00000070, 0x0480c018},
1428 {0x00000071, 0x00000040},
1429 {0x00000072, 0x01000000},
1430 {0x00000074, 0x000000ff},
1431 {0x00000075, 0x00143400},
1432 {0x00000076, 0x08ec0800},
1433 {0x00000077, 0x040000cc},
1434 {0x00000079, 0x00000000},
1435 {0x0000007a, 0x21000409},
1436 {0x0000007c, 0x00000000},
1437 {0x0000007d, 0xe8000000},
1438 {0x0000007e, 0x044408a8},
1439 {0x0000007f, 0x00000003},
1440 {0x00000080, 0x00000000},
1441 {0x00000081, 0x01000000},
1442 {0x00000082, 0x02000000},
1443 {0x00000083, 0x00000000},
1444 {0x00000084, 0xe3f3e4f4},
1445 {0x00000085, 0x00052024},
1446 {0x00000087, 0x00000000},
1447 {0x00000088, 0x66036603},
1448 {0x00000089, 0x01000000},
1449 {0x0000008b, 0x1c0a0000},
1450 {0x0000008c, 0xff010000},
1451 {0x0000008e, 0xffffefff},
1452 {0x0000008f, 0xfff3efff},
1453 {0x00000090, 0xfff3efbf},
1454 {0x00000094, 0x00101101},
1455 {0x00000095, 0x00000fff},
1456 {0x00000096, 0x00116fff},
1457 {0x00000097, 0x60010000},
1458 {0x00000098, 0x10010000},
1459 {0x00000099, 0x00006000},
1460 {0x0000009a, 0x00001000},
1461 {0x0000009f, 0x00a07730}
1462};
1463
8b074dd6 1464/* ucode loading */
6c7bccea 1465int si_mc_load_microcode(struct radeon_device *rdev)
8b074dd6
AD		 1466{
1467 const __be32 *fw_data;
1468 u32 running, blackout = 0;
1469 u32 *io_mc_regs;
1470 int i, ucode_size, regs_size;
1471
1472 if (!rdev->mc_fw)
1473 return -EINVAL;
1474
1475 switch (rdev->family) {
1476 case CHIP_TAHITI:
1477 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1478 ucode_size = SI_MC_UCODE_SIZE;
1479 regs_size = TAHITI_IO_MC_REGS_SIZE;
1480 break;
1481 case CHIP_PITCAIRN:
1482 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1483 ucode_size = SI_MC_UCODE_SIZE;
1484 regs_size = TAHITI_IO_MC_REGS_SIZE;
1485 break;
1486 case CHIP_VERDE:
1487 default:
1488 io_mc_regs = (u32 *)&verde_io_mc_regs;
1489 ucode_size = SI_MC_UCODE_SIZE;
1490 regs_size = TAHITI_IO_MC_REGS_SIZE;
1491 break;
bcc7f5d2
AD		 1492 case CHIP_OLAND:
1493 io_mc_regs = (u32 *)&oland_io_mc_regs;
1494 ucode_size = OLAND_MC_UCODE_SIZE;
1495 regs_size = TAHITI_IO_MC_REGS_SIZE;
1496 break;
c04c00b4
AD		 1497 case CHIP_HAINAN:
1498 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1499 ucode_size = OLAND_MC_UCODE_SIZE;
1500 regs_size = TAHITI_IO_MC_REGS_SIZE;
1501 break;
8b074dd6
AD		 1502 }
1503
1504 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1505
1506 if (running == 0) {
1507 if (running) {
1508 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1509 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1510 }
1511
1512 /* reset the engine and set to writable */
1513 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1514 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1515
1516 /* load mc io regs */
1517 for (i = 0; i < regs_size; i++) {
1518 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1519 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1520 }
1521 /* load the MC ucode */
1522 fw_data = (const __be32 *)rdev->mc_fw->data;
1523 for (i = 0; i < ucode_size; i++)
1524 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1525
1526 /* put the engine back into the active state */
1527 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1528 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1529 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1530
1531 /* wait for training to complete */
1532 for (i = 0; i < rdev->usec_timeout; i++) {
1533 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1534 break;
1535 udelay(1);
1536 }
1537 for (i = 0; i < rdev->usec_timeout; i++) {
1538 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1539 break;
1540 udelay(1);
1541 }
1542
1543 if (running)
1544 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1545 }
1546
1547 return 0;
1548}
1549
0f0de06c
AD		 1550static int si_init_microcode(struct radeon_device *rdev)
1551{
0f0de06c
AD		 1552 const char *chip_name;
1553 const char *rlc_chip_name;
1554 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
a9e61410 1555 size_t smc_req_size;
0f0de06c
AD		 1556 char fw_name[30];
1557 int err;
1558
1559 DRM_DEBUG("\n");
1560
0f0de06c
AD		 1561 switch (rdev->family) {
1562 case CHIP_TAHITI:
1563 chip_name = "TAHITI";
1564 rlc_chip_name = "TAHITI";
1565 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1566 me_req_size = SI_PM4_UCODE_SIZE * 4;
1567 ce_req_size = SI_CE_UCODE_SIZE * 4;
1568 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1569 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1570 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1571 break;
1572 case CHIP_PITCAIRN:
1573 chip_name = "PITCAIRN";
1574 rlc_chip_name = "PITCAIRN";
1575 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1576 me_req_size = SI_PM4_UCODE_SIZE * 4;
1577 ce_req_size = SI_CE_UCODE_SIZE * 4;
1578 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1579 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1580 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1581 break;
1582 case CHIP_VERDE:
1583 chip_name = "VERDE";
1584 rlc_chip_name = "VERDE";
1585 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1586 me_req_size = SI_PM4_UCODE_SIZE * 4;
1587 ce_req_size = SI_CE_UCODE_SIZE * 4;
1588 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1589 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1590 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
0f0de06c 1591 break;
bcc7f5d2
AD		 1592 case CHIP_OLAND:
1593 chip_name = "OLAND";
1594 rlc_chip_name = "OLAND";
1595 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1596 me_req_size = SI_PM4_UCODE_SIZE * 4;
1597 ce_req_size = SI_CE_UCODE_SIZE * 4;
1598 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1599 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1600 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
bcc7f5d2 1601 break;
c04c00b4
AD		 1602 case CHIP_HAINAN:
1603 chip_name = "HAINAN";
1604 rlc_chip_name = "HAINAN";
1605 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1606 me_req_size = SI_PM4_UCODE_SIZE * 4;
1607 ce_req_size = SI_CE_UCODE_SIZE * 4;
1608 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1609 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1610 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
c04c00b4 1611 break;
0f0de06c
AD		 1612 default: BUG();
1613 }
1614
1615 DRM_INFO("Loading %s Microcode\n", chip_name);
1616
1617 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
0a168933 1618 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
0f0de06c
AD		 1619 if (err)
1620 goto out;
1621 if (rdev->pfp_fw->size != pfp_req_size) {
1622 printk(KERN_ERR
1623 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1624 rdev->pfp_fw->size, fw_name);
1625 err = -EINVAL;
1626 goto out;
1627 }
1628
1629 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
0a168933 1630 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
0f0de06c
AD		 1631 if (err)
1632 goto out;
1633 if (rdev->me_fw->size != me_req_size) {
1634 printk(KERN_ERR
1635 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1636 rdev->me_fw->size, fw_name);
1637 err = -EINVAL;
1638 }
1639
1640 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
0a168933 1641 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
0f0de06c
AD		 1642 if (err)
1643 goto out;
1644 if (rdev->ce_fw->size != ce_req_size) {
1645 printk(KERN_ERR
1646 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1647 rdev->ce_fw->size, fw_name);
1648 err = -EINVAL;
1649 }
1650
1651 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
0a168933 1652 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
0f0de06c
AD		 1653 if (err)
1654 goto out;
1655 if (rdev->rlc_fw->size != rlc_req_size) {
1656 printk(KERN_ERR
1657 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1658 rdev->rlc_fw->size, fw_name);
1659 err = -EINVAL;
1660 }
1661
1662 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
0a168933 1663 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
0f0de06c
AD		 1664 if (err)
1665 goto out;
1666 if (rdev->mc_fw->size != mc_req_size) {
1667 printk(KERN_ERR
1668 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1669 rdev->mc_fw->size, fw_name);
1670 err = -EINVAL;
1671 }
1672
a9e61410 1673 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
0a168933 1674 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
8a53fa23
AD		 1675 if (err) {
1676 printk(KERN_ERR
1677 "smc: error loading firmware \"%s\"\n",
1678 fw_name);
1679 release_firmware(rdev->smc_fw);
1680 rdev->smc_fw = NULL;
d8367112 1681 err = 0;
8a53fa23 1682 } else if (rdev->smc_fw->size != smc_req_size) {
a9e61410
AD		 1683 printk(KERN_ERR
1684 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1685 rdev->smc_fw->size, fw_name);
1686 err = -EINVAL;
1687 }
1688
0f0de06c 1689out:
0f0de06c
AD		 1690 if (err) {
1691 if (err != -EINVAL)
1692 printk(KERN_ERR
1693 "si_cp: Failed to load firmware \"%s\"\n",
1694 fw_name);
1695 release_firmware(rdev->pfp_fw);
1696 rdev->pfp_fw = NULL;
1697 release_firmware(rdev->me_fw);
1698 rdev->me_fw = NULL;
1699 release_firmware(rdev->ce_fw);
1700 rdev->ce_fw = NULL;
1701 release_firmware(rdev->rlc_fw);
1702 rdev->rlc_fw = NULL;
1703 release_firmware(rdev->mc_fw);
1704 rdev->mc_fw = NULL;
a9e61410
AD		 1705 release_firmware(rdev->smc_fw);
1706 rdev->smc_fw = NULL;
0f0de06c
AD		 1707 }
1708 return err;
1709}
1710
43b3cd99
AD		 1711/* watermark setup */
1712static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1713 struct radeon_crtc *radeon_crtc,
1714 struct drm_display_mode *mode,
1715 struct drm_display_mode *other_mode)
1716{
290d2457
AD		 1717 u32 tmp, buffer_alloc, i;
1718 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
43b3cd99
AD		 1719 /*
1720 * Line Buffer Setup
1721 * There are 3 line buffers, each one shared by 2 display controllers.
1722 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1723 * the display controllers. The paritioning is done via one of four
1724 * preset allocations specified in bits 21:20:
1725 * 0 - half lb
1726 * 2 - whole lb, other crtc must be disabled
1727 */
1728 /* this can get tricky if we have two large displays on a paired group
1729 * of crtcs. Ideally for multiple large displays we'd assign them to
1730 * non-linked crtcs for maximum line buffer allocation.
1731 */
1732 if (radeon_crtc->base.enabled && mode) {
290d2457 1733 if (other_mode) {
43b3cd99 1734 tmp = 0; /* 1/2 */
290d2457
AD		 1735 buffer_alloc = 1;
1736 } else {
43b3cd99 1737 tmp = 2; /* whole */
290d2457
AD		 1738 buffer_alloc = 2;
1739 }
1740 } else {
43b3cd99 1741 tmp = 0;
290d2457
AD		 1742 buffer_alloc = 0;
1743 }
43b3cd99
AD		 1744
1745 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1746 DC_LB_MEMORY_CONFIG(tmp));
1747
290d2457
AD		 1748 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1749 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1750 for (i = 0; i < rdev->usec_timeout; i++) {
1751 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1752 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1753 break;
1754 udelay(1);
1755 }
1756
43b3cd99
AD		 1757 if (radeon_crtc->base.enabled && mode) {
1758 switch (tmp) {
1759 case 0:
1760 default:
1761 return 4096 * 2;
1762 case 2:
1763 return 8192 * 2;
1764 }
1765 }
1766
1767 /* controller not enabled, so no lb used */
1768 return 0;
1769}
1770
ca7db22b 1771static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 1772{
1773 u32 tmp = RREG32(MC_SHARED_CHMAP);
1774
1775 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1776 case 0:
1777 default:
1778 return 1;
1779 case 1:
1780 return 2;
1781 case 2:
1782 return 4;
1783 case 3:
1784 return 8;
1785 case 4:
1786 return 3;
1787 case 5:
1788 return 6;
1789 case 6:
1790 return 10;
1791 case 7:
1792 return 12;
1793 case 8:
1794 return 16;
1795 }
1796}
1797
1798struct dce6_wm_params {
1799 u32 dram_channels; /* number of dram channels */
1800 u32 yclk; /* bandwidth per dram data pin in kHz */
1801 u32 sclk; /* engine clock in kHz */
1802 u32 disp_clk; /* display clock in kHz */
1803 u32 src_width; /* viewport width */
1804 u32 active_time; /* active display time in ns */
1805 u32 blank_time; /* blank time in ns */
1806 bool interlaced; /* mode is interlaced */
1807 fixed20_12 vsc; /* vertical scale ratio */
1808 u32 num_heads; /* number of active crtcs */
1809 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1810 u32 lb_size; /* line buffer allocated to pipe */
1811 u32 vtaps; /* vertical scaler taps */
1812};
1813
1814static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1815{
1816 /* Calculate raw DRAM Bandwidth */
1817 fixed20_12 dram_efficiency; /* 0.7 */
1818 fixed20_12 yclk, dram_channels, bandwidth;
1819 fixed20_12 a;
1820
1821 a.full = dfixed_const(1000);
1822 yclk.full = dfixed_const(wm->yclk);
1823 yclk.full = dfixed_div(yclk, a);
1824 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1825 a.full = dfixed_const(10);
1826 dram_efficiency.full = dfixed_const(7);
1827 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1828 bandwidth.full = dfixed_mul(dram_channels, yclk);
1829 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1830
1831 return dfixed_trunc(bandwidth);
1832}
1833
1834static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1835{
1836 /* Calculate DRAM Bandwidth and the part allocated to display. */
1837 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1838 fixed20_12 yclk, dram_channels, bandwidth;
1839 fixed20_12 a;
1840
1841 a.full = dfixed_const(1000);
1842 yclk.full = dfixed_const(wm->yclk);
1843 yclk.full = dfixed_div(yclk, a);
1844 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1845 a.full = dfixed_const(10);
1846 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1847 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1848 bandwidth.full = dfixed_mul(dram_channels, yclk);
1849 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1850
1851 return dfixed_trunc(bandwidth);
1852}
1853
1854static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1855{
1856 /* Calculate the display Data return Bandwidth */
1857 fixed20_12 return_efficiency; /* 0.8 */
1858 fixed20_12 sclk, bandwidth;
1859 fixed20_12 a;
1860
1861 a.full = dfixed_const(1000);
1862 sclk.full = dfixed_const(wm->sclk);
1863 sclk.full = dfixed_div(sclk, a);
1864 a.full = dfixed_const(10);
1865 return_efficiency.full = dfixed_const(8);
1866 return_efficiency.full = dfixed_div(return_efficiency, a);
1867 a.full = dfixed_const(32);
1868 bandwidth.full = dfixed_mul(a, sclk);
1869 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1870
1871 return dfixed_trunc(bandwidth);
1872}
1873
1874static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1875{
1876 return 32;
1877}
1878
1879static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1880{
1881 /* Calculate the DMIF Request Bandwidth */
1882 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1883 fixed20_12 disp_clk, sclk, bandwidth;
1884 fixed20_12 a, b1, b2;
1885 u32 min_bandwidth;
1886
1887 a.full = dfixed_const(1000);
1888 disp_clk.full = dfixed_const(wm->disp_clk);
1889 disp_clk.full = dfixed_div(disp_clk, a);
1890 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1891 b1.full = dfixed_mul(a, disp_clk);
1892
1893 a.full = dfixed_const(1000);
1894 sclk.full = dfixed_const(wm->sclk);
1895 sclk.full = dfixed_div(sclk, a);
1896 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1897 b2.full = dfixed_mul(a, sclk);
1898
1899 a.full = dfixed_const(10);
1900 disp_clk_request_efficiency.full = dfixed_const(8);
1901 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1902
1903 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1904
1905 a.full = dfixed_const(min_bandwidth);
1906 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1907
1908 return dfixed_trunc(bandwidth);
1909}
1910
1911static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1912{
1913 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1914 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1915 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1916 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1917
1918 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1919}
1920
1921static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1922{
1923 /* Calculate the display mode Average Bandwidth
1924 * DisplayMode should contain the source and destination dimensions,
1925 * timing, etc.
1926 */
1927 fixed20_12 bpp;
1928 fixed20_12 line_time;
1929 fixed20_12 src_width;
1930 fixed20_12 bandwidth;
1931 fixed20_12 a;
1932
1933 a.full = dfixed_const(1000);
1934 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1935 line_time.full = dfixed_div(line_time, a);
1936 bpp.full = dfixed_const(wm->bytes_per_pixel);
1937 src_width.full = dfixed_const(wm->src_width);
1938 bandwidth.full = dfixed_mul(src_width, bpp);
1939 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1940 bandwidth.full = dfixed_div(bandwidth, line_time);
1941
1942 return dfixed_trunc(bandwidth);
1943}
1944
1945static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1946{
1947 /* First calcualte the latency in ns */
1948 u32 mc_latency = 2000; /* 2000 ns. */
1949 u32 available_bandwidth = dce6_available_bandwidth(wm);
1950 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1951 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1952 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1953 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1954 (wm->num_heads * cursor_line_pair_return_time);
1955 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1956 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1957 u32 tmp, dmif_size = 12288;
1958 fixed20_12 a, b, c;
1959
1960 if (wm->num_heads == 0)
1961 return 0;
1962
1963 a.full = dfixed_const(2);
1964 b.full = dfixed_const(1);
1965 if ((wm->vsc.full > a.full) ||
1966 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1967 (wm->vtaps >= 5) ||
1968 ((wm->vsc.full >= a.full) && wm->interlaced))
1969 max_src_lines_per_dst_line = 4;
1970 else
1971 max_src_lines_per_dst_line = 2;
1972
1973 a.full = dfixed_const(available_bandwidth);
1974 b.full = dfixed_const(wm->num_heads);
1975 a.full = dfixed_div(a, b);
1976
1977 b.full = dfixed_const(mc_latency + 512);
1978 c.full = dfixed_const(wm->disp_clk);
1979 b.full = dfixed_div(b, c);
1980
1981 c.full = dfixed_const(dmif_size);
1982 b.full = dfixed_div(c, b);
1983
1984 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1985
1986 b.full = dfixed_const(1000);
1987 c.full = dfixed_const(wm->disp_clk);
1988 b.full = dfixed_div(c, b);
1989 c.full = dfixed_const(wm->bytes_per_pixel);
1990 b.full = dfixed_mul(b, c);
1991
1992 lb_fill_bw = min(tmp, dfixed_trunc(b));
1993
1994 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1995 b.full = dfixed_const(1000);
1996 c.full = dfixed_const(lb_fill_bw);
1997 b.full = dfixed_div(c, b);
1998 a.full = dfixed_div(a, b);
1999 line_fill_time = dfixed_trunc(a);
2000
2001 if (line_fill_time < wm->active_time)
2002 return latency;
2003 else
2004 return latency + (line_fill_time - wm->active_time);
2005
2006}
2007
2008static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2009{
2010 if (dce6_average_bandwidth(wm) <=
2011 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2012 return true;
2013 else
2014 return false;
2015};
2016
2017static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2018{
2019 if (dce6_average_bandwidth(wm) <=
2020 (dce6_available_bandwidth(wm) / wm->num_heads))
2021 return true;
2022 else
2023 return false;
2024};
2025
2026static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2027{
2028 u32 lb_partitions = wm->lb_size / wm->src_width;
2029 u32 line_time = wm->active_time + wm->blank_time;
2030 u32 latency_tolerant_lines;
2031 u32 latency_hiding;
2032 fixed20_12 a;
2033
2034 a.full = dfixed_const(1);
2035 if (wm->vsc.full > a.full)
2036 latency_tolerant_lines = 1;
2037 else {
2038 if (lb_partitions <= (wm->vtaps + 1))
2039 latency_tolerant_lines = 1;
2040 else
2041 latency_tolerant_lines = 2;
2042 }
2043
2044 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2045
2046 if (dce6_latency_watermark(wm) <= latency_hiding)
2047 return true;
2048 else
2049 return false;
2050}
2051
2052static void dce6_program_watermarks(struct radeon_device *rdev,
2053 struct radeon_crtc *radeon_crtc,
2054 u32 lb_size, u32 num_heads)
2055{
2056 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 2057 struct dce6_wm_params wm_low, wm_high;
2058 u32 dram_channels;
43b3cd99
AD		 2059 u32 pixel_period;
2060 u32 line_time = 0;
2061 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2062 u32 priority_a_mark = 0, priority_b_mark = 0;
2063 u32 priority_a_cnt = PRIORITY_OFF;
2064 u32 priority_b_cnt = PRIORITY_OFF;
2065 u32 tmp, arb_control3;
2066 fixed20_12 a, b, c;
2067
2068 if (radeon_crtc->base.enabled && num_heads && mode) {
2069 pixel_period = 1000000 / (u32)mode->clock;
2070 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2071 priority_a_cnt = 0;
2072 priority_b_cnt = 0;
2073
ca7db22b 2074 if (rdev->family == CHIP_ARUBA)
c696e53f 2075 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 2076 else
c696e53f
AD		 2077 dram_channels = si_get_number_of_dram_channels(rdev);
2078
2079 /* watermark for high clocks */
2080 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2081 wm_high.yclk =
2082 radeon_dpm_get_mclk(rdev, false) * 10;
2083 wm_high.sclk =
2084 radeon_dpm_get_sclk(rdev, false) * 10;
2085 } else {
2086 wm_high.yclk = rdev->pm.current_mclk * 10;
2087 wm_high.sclk = rdev->pm.current_sclk * 10;
2088 }
2089
2090 wm_high.disp_clk = mode->clock;
2091 wm_high.src_width = mode->crtc_hdisplay;
2092 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2093 wm_high.blank_time = line_time - wm_high.active_time;
2094 wm_high.interlaced = false;
2095 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2096 wm_high.interlaced = true;
2097 wm_high.vsc = radeon_crtc->vsc;
2098 wm_high.vtaps = 1;
2099 if (radeon_crtc->rmx_type != RMX_OFF)
2100 wm_high.vtaps = 2;
2101 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2102 wm_high.lb_size = lb_size;
2103 wm_high.dram_channels = dram_channels;
2104 wm_high.num_heads = num_heads;
2105
2106 /* watermark for low clocks */
2107 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2108 wm_low.yclk =
2109 radeon_dpm_get_mclk(rdev, true) * 10;
2110 wm_low.sclk =
2111 radeon_dpm_get_sclk(rdev, true) * 10;
2112 } else {
2113 wm_low.yclk = rdev->pm.current_mclk * 10;
2114 wm_low.sclk = rdev->pm.current_sclk * 10;
2115 }
2116
2117 wm_low.disp_clk = mode->clock;
2118 wm_low.src_width = mode->crtc_hdisplay;
2119 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2120 wm_low.blank_time = line_time - wm_low.active_time;
2121 wm_low.interlaced = false;
2122 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2123 wm_low.interlaced = true;
2124 wm_low.vsc = radeon_crtc->vsc;
2125 wm_low.vtaps = 1;
2126 if (radeon_crtc->rmx_type != RMX_OFF)
2127 wm_low.vtaps = 2;
2128 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2129 wm_low.lb_size = lb_size;
2130 wm_low.dram_channels = dram_channels;
2131 wm_low.num_heads = num_heads;
43b3cd99
AD		 2132
2133 /* set for high clocks */
c696e53f 2134 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 2135 /* set for low clocks */
c696e53f 2136 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 2137
2138 /* possibly force display priority to high */
2139 /* should really do this at mode validation time... */
c696e53f
AD		 2140 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2141 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2142 !dce6_check_latency_hiding(&wm_high) ||
2143 (rdev->disp_priority == 2)) {
2144 DRM_DEBUG_KMS("force priority to high\n");
2145 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2146 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2147 }
2148 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2149 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2150 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 2151 (rdev->disp_priority == 2)) {
2152 DRM_DEBUG_KMS("force priority to high\n");
2153 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2154 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2155 }
2156
2157 a.full = dfixed_const(1000);
2158 b.full = dfixed_const(mode->clock);
2159 b.full = dfixed_div(b, a);
2160 c.full = dfixed_const(latency_watermark_a);
2161 c.full = dfixed_mul(c, b);
2162 c.full = dfixed_mul(c, radeon_crtc->hsc);
2163 c.full = dfixed_div(c, a);
2164 a.full = dfixed_const(16);
2165 c.full = dfixed_div(c, a);
2166 priority_a_mark = dfixed_trunc(c);
2167 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2168
2169 a.full = dfixed_const(1000);
2170 b.full = dfixed_const(mode->clock);
2171 b.full = dfixed_div(b, a);
2172 c.full = dfixed_const(latency_watermark_b);
2173 c.full = dfixed_mul(c, b);
2174 c.full = dfixed_mul(c, radeon_crtc->hsc);
2175 c.full = dfixed_div(c, a);
2176 a.full = dfixed_const(16);
2177 c.full = dfixed_div(c, a);
2178 priority_b_mark = dfixed_trunc(c);
2179 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2180 }
2181
2182 /* select wm A */
2183 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2184 tmp = arb_control3;
2185 tmp &= ~LATENCY_WATERMARK_MASK(3);
2186 tmp |= LATENCY_WATERMARK_MASK(1);
2187 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2188 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2189 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2190 LATENCY_HIGH_WATERMARK(line_time)));
2191 /* select wm B */
2192 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2193 tmp &= ~LATENCY_WATERMARK_MASK(3);
2194 tmp |= LATENCY_WATERMARK_MASK(2);
2195 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2196 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2197 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2198 LATENCY_HIGH_WATERMARK(line_time)));
2199 /* restore original selection */
2200 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2201
2202 /* write the priority marks */
2203 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2204 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2205
7178d2a6
AD		 2206 /* save values for DPM */
2207 radeon_crtc->line_time = line_time;
2208 radeon_crtc->wm_high = latency_watermark_a;
2209 radeon_crtc->wm_low = latency_watermark_b;
43b3cd99
AD		 2210}
2211
2212void dce6_bandwidth_update(struct radeon_device *rdev)
2213{
2214 struct drm_display_mode *mode0 = NULL;
2215 struct drm_display_mode *mode1 = NULL;
2216 u32 num_heads = 0, lb_size;
2217 int i;
2218
2219 radeon_update_display_priority(rdev);
2220
2221 for (i = 0; i < rdev->num_crtc; i++) {
2222 if (rdev->mode_info.crtcs[i]->base.enabled)
2223 num_heads++;
2224 }
2225 for (i = 0; i < rdev->num_crtc; i += 2) {
2226 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2227 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2228 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2229 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2230 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2231 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2232 }
2233}
2234
0a96d72b
AD		 2235/*
2236 * Core functions
2237 */
0a96d72b
AD		 2238static void si_tiling_mode_table_init(struct radeon_device *rdev)
2239{
2240 const u32 num_tile_mode_states = 32;
2241 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2242
2243 switch (rdev->config.si.mem_row_size_in_kb) {
2244 case 1:
2245 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2246 break;
2247 case 2:
2248 default:
2249 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2250 break;
2251 case 4:
2252 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2253 break;
2254 }
2255
2256 if ((rdev->family == CHIP_TAHITI) ||
2257 (rdev->family == CHIP_PITCAIRN)) {
2258 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2259 switch (reg_offset) {
2260 case 0: /* non-AA compressed depth or any compressed stencil */
2261 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2262 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2263 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2264 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2265 NUM_BANKS(ADDR_SURF_16_BANK) |
2266 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2267 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2268 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2269 break;
2270 case 1: /* 2xAA/4xAA compressed depth only */
2271 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2272 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2273 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2274 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2275 NUM_BANKS(ADDR_SURF_16_BANK) |
2276 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2279 break;
2280 case 2: /* 8xAA compressed depth only */
2281 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2282 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2283 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2284 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2285 NUM_BANKS(ADDR_SURF_16_BANK) |
2286 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2287 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2288 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2289 break;
2290 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2291 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2292 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2293 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2294 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2295 NUM_BANKS(ADDR_SURF_16_BANK) |
2296 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2297 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2298 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2299 break;
2300 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2301 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2302 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2303 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2304 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2305 NUM_BANKS(ADDR_SURF_16_BANK) |
2306 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2309 break;
2310 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2311 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2312 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2313 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2314 TILE_SPLIT(split_equal_to_row_size) |
2315 NUM_BANKS(ADDR_SURF_16_BANK) |
2316 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2317 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2318 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2319 break;
2320 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2321 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2322 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2323 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2324 TILE_SPLIT(split_equal_to_row_size) |
2325 NUM_BANKS(ADDR_SURF_16_BANK) |
2326 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2327 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2328 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2329 break;
2330 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2331 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2332 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2333 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2334 TILE_SPLIT(split_equal_to_row_size) |
2335 NUM_BANKS(ADDR_SURF_16_BANK) |
2336 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2337 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2338 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2339 break;
2340 case 8: /* 1D and 1D Array Surfaces */
2341 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2342 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2343 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2344 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2345 NUM_BANKS(ADDR_SURF_16_BANK) |
2346 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2347 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2348 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2349 break;
2350 case 9: /* Displayable maps. */
2351 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2352 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2353 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2355 NUM_BANKS(ADDR_SURF_16_BANK) |
2356 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2357 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2358 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2359 break;
2360 case 10: /* Display 8bpp. */
2361 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2362 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2363 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2364 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2365 NUM_BANKS(ADDR_SURF_16_BANK) |
2366 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2367 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2368 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2369 break;
2370 case 11: /* Display 16bpp. */
2371 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2372 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2373 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2374 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2375 NUM_BANKS(ADDR_SURF_16_BANK) |
2376 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2377 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2378 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2379 break;
2380 case 12: /* Display 32bpp. */
2381 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2383 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2384 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2385 NUM_BANKS(ADDR_SURF_16_BANK) |
2386 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2387 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2388 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2389 break;
2390 case 13: /* Thin. */
2391 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2392 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2393 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2394 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2395 NUM_BANKS(ADDR_SURF_16_BANK) |
2396 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2397 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2398 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2399 break;
2400 case 14: /* Thin 8 bpp. */
2401 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2402 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2403 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2404 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2405 NUM_BANKS(ADDR_SURF_16_BANK) |
2406 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2407 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2408 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2409 break;
2410 case 15: /* Thin 16 bpp. */
2411 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2412 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2413 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2414 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2415 NUM_BANKS(ADDR_SURF_16_BANK) |
2416 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2417 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2418 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2419 break;
2420 case 16: /* Thin 32 bpp. */
2421 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2422 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2423 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2424 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2425 NUM_BANKS(ADDR_SURF_16_BANK) |
2426 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2427 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2428 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2429 break;
2430 case 17: /* Thin 64 bpp. */
2431 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2432 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2433 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2434 TILE_SPLIT(split_equal_to_row_size) |
2435 NUM_BANKS(ADDR_SURF_16_BANK) |
2436 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2437 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2438 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2439 break;
2440 case 21: /* 8 bpp PRT. */
2441 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2442 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2443 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2444 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2445 NUM_BANKS(ADDR_SURF_16_BANK) |
2446 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2447 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2448 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2449 break;
2450 case 22: /* 16 bpp PRT */
2451 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2452 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2453 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2454 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2455 NUM_BANKS(ADDR_SURF_16_BANK) |
2456 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2457 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2458 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2459 break;
2460 case 23: /* 32 bpp PRT */
2461 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2462 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2463 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2464 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2465 NUM_BANKS(ADDR_SURF_16_BANK) |
2466 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2467 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2468 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2469 break;
2470 case 24: /* 64 bpp PRT */
2471 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2472 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2473 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2474 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2475 NUM_BANKS(ADDR_SURF_16_BANK) |
2476 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2478 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2479 break;
2480 case 25: /* 128 bpp PRT */
2481 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2482 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2483 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2484 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2485 NUM_BANKS(ADDR_SURF_8_BANK) |
2486 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2487 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2488 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2489 break;
2490 default:
2491 gb_tile_moden = 0;
2492 break;
2493 }
64d7b8be 2494 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2495 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2496 }
d0ae7fcc 2497 } else if ((rdev->family == CHIP_VERDE) ||
8b02859d
AD		 2498 (rdev->family == CHIP_OLAND) ||
2499 (rdev->family == CHIP_HAINAN)) {
0a96d72b
AD		 2500 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2501 switch (reg_offset) {
2502 case 0: /* non-AA compressed depth or any compressed stencil */
2503 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2504 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2505 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2506 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2507 NUM_BANKS(ADDR_SURF_16_BANK) |
2508 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2509 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2510 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2511 break;
2512 case 1: /* 2xAA/4xAA compressed depth only */
2513 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2515 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2516 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2517 NUM_BANKS(ADDR_SURF_16_BANK) |
2518 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2519 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2520 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2521 break;
2522 case 2: /* 8xAA compressed depth only */
2523 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2524 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2525 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2526 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2527 NUM_BANKS(ADDR_SURF_16_BANK) |
2528 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2529 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2530 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2531 break;
2532 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2533 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2534 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2535 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2536 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2537 NUM_BANKS(ADDR_SURF_16_BANK) |
2538 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2539 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2540 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2541 break;
2542 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2543 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2544 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2545 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2546 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2547 NUM_BANKS(ADDR_SURF_16_BANK) |
2548 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2549 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2550 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2551 break;
2552 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2553 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2554 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2555 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2556 TILE_SPLIT(split_equal_to_row_size) |
2557 NUM_BANKS(ADDR_SURF_16_BANK) |
2558 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2559 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2560 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2561 break;
2562 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2566 TILE_SPLIT(split_equal_to_row_size) |
2567 NUM_BANKS(ADDR_SURF_16_BANK) |
2568 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2569 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2570 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2571 break;
2572 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2573 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2574 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2575 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2576 TILE_SPLIT(split_equal_to_row_size) |
2577 NUM_BANKS(ADDR_SURF_16_BANK) |
2578 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2579 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2580 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2581 break;
2582 case 8: /* 1D and 1D Array Surfaces */
2583 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2584 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2585 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2586 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2587 NUM_BANKS(ADDR_SURF_16_BANK) |
2588 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2589 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2590 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2591 break;
2592 case 9: /* Displayable maps. */
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2594 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2595 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2596 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2597 NUM_BANKS(ADDR_SURF_16_BANK) |
2598 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2599 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2600 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2601 break;
2602 case 10: /* Display 8bpp. */
2603 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2604 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2605 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2606 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2607 NUM_BANKS(ADDR_SURF_16_BANK) |
2608 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2609 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2610 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2611 break;
2612 case 11: /* Display 16bpp. */
2613 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2614 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2615 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2616 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2617 NUM_BANKS(ADDR_SURF_16_BANK) |
2618 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2619 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2620 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2621 break;
2622 case 12: /* Display 32bpp. */
2623 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2624 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2625 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2626 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2627 NUM_BANKS(ADDR_SURF_16_BANK) |
2628 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2629 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2630 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2631 break;
2632 case 13: /* Thin. */
2633 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2634 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2635 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2636 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2637 NUM_BANKS(ADDR_SURF_16_BANK) |
2638 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2639 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2640 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2641 break;
2642 case 14: /* Thin 8 bpp. */
2643 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2644 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2645 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2646 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2647 NUM_BANKS(ADDR_SURF_16_BANK) |
2648 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2649 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2650 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2651 break;
2652 case 15: /* Thin 16 bpp. */
2653 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2654 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2655 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2656 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2657 NUM_BANKS(ADDR_SURF_16_BANK) |
2658 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2659 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2660 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2661 break;
2662 case 16: /* Thin 32 bpp. */
2663 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2664 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2665 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2666 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2667 NUM_BANKS(ADDR_SURF_16_BANK) |
2668 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2669 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2670 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2671 break;
2672 case 17: /* Thin 64 bpp. */
2673 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2674 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2675 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2676 TILE_SPLIT(split_equal_to_row_size) |
2677 NUM_BANKS(ADDR_SURF_16_BANK) |
2678 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2679 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2680 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2681 break;
2682 case 21: /* 8 bpp PRT. */
2683 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2684 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2685 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2686 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2687 NUM_BANKS(ADDR_SURF_16_BANK) |
2688 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2689 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2690 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2691 break;
2692 case 22: /* 16 bpp PRT */
2693 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2694 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2695 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2696 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2697 NUM_BANKS(ADDR_SURF_16_BANK) |
2698 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2699 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2700 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2701 break;
2702 case 23: /* 32 bpp PRT */
2703 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2704 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2705 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2706 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2707 NUM_BANKS(ADDR_SURF_16_BANK) |
2708 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2711 break;
2712 case 24: /* 64 bpp PRT */
2713 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2714 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2715 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2716 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2717 NUM_BANKS(ADDR_SURF_16_BANK) |
2718 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2719 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2720 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2721 break;
2722 case 25: /* 128 bpp PRT */
2723 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2724 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2725 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2726 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2727 NUM_BANKS(ADDR_SURF_8_BANK) |
2728 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2729 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2730 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2731 break;
2732 default:
2733 gb_tile_moden = 0;
2734 break;
2735 }
64d7b8be 2736 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2737 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2738 }
2739 } else
2740 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2741}
2742
1a8ca750
AD		 2743static void si_select_se_sh(struct radeon_device *rdev,
2744 u32 se_num, u32 sh_num)
2745{
2746 u32 data = INSTANCE_BROADCAST_WRITES;
2747
2748 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2749 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2750 else if (se_num == 0xffffffff)
2751 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2752 else if (sh_num == 0xffffffff)
2753 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2754 else
2755 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2756 WREG32(GRBM_GFX_INDEX, data);
2757}
2758
2759static u32 si_create_bitmask(u32 bit_width)
2760{
2761 u32 i, mask = 0;
2762
2763 for (i = 0; i < bit_width; i++) {
2764 mask <<= 1;
2765 mask |= 1;
2766 }
2767 return mask;
2768}
2769
2770static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2771{
2772 u32 data, mask;
2773
2774 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2775 if (data & 1)
2776 data &= INACTIVE_CUS_MASK;
2777 else
2778 data = 0;
2779 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2780
2781 data >>= INACTIVE_CUS_SHIFT;
2782
2783 mask = si_create_bitmask(cu_per_sh);
2784
2785 return ~data & mask;
2786}
2787
2788static void si_setup_spi(struct radeon_device *rdev,
2789 u32 se_num, u32 sh_per_se,
2790 u32 cu_per_sh)
2791{
2792 int i, j, k;
2793 u32 data, mask, active_cu;
2794
2795 for (i = 0; i < se_num; i++) {
2796 for (j = 0; j < sh_per_se; j++) {
2797 si_select_se_sh(rdev, i, j);
2798 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2799 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2800
2801 mask = 1;
2802 for (k = 0; k < 16; k++) {
2803 mask <<= k;
2804 if (active_cu & mask) {
2805 data &= ~mask;
2806 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2807 break;
2808 }
2809 }
2810 }
2811 }
2812 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2813}
2814
2815static u32 si_get_rb_disabled(struct radeon_device *rdev,
9fadb352 2816 u32 max_rb_num_per_se,
1a8ca750
AD		 2817 u32 sh_per_se)
2818{
2819 u32 data, mask;
2820
2821 data = RREG32(CC_RB_BACKEND_DISABLE);
2822 if (data & 1)
2823 data &= BACKEND_DISABLE_MASK;
2824 else
2825 data = 0;
2826 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2827
2828 data >>= BACKEND_DISABLE_SHIFT;
2829
9fadb352 2830 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
1a8ca750
AD		 2831
2832 return data & mask;
2833}
2834
2835static void si_setup_rb(struct radeon_device *rdev,
2836 u32 se_num, u32 sh_per_se,
9fadb352 2837 u32 max_rb_num_per_se)
1a8ca750
AD		 2838{
2839 int i, j;
2840 u32 data, mask;
2841 u32 disabled_rbs = 0;
2842 u32 enabled_rbs = 0;
2843
2844 for (i = 0; i < se_num; i++) {
2845 for (j = 0; j < sh_per_se; j++) {
2846 si_select_se_sh(rdev, i, j);
9fadb352 2847 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
1a8ca750
AD		 2848 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2849 }
2850 }
2851 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2852
2853 mask = 1;
9fadb352 2854 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
1a8ca750
AD		 2855 if (!(disabled_rbs & mask))
2856 enabled_rbs |= mask;
2857 mask <<= 1;
2858 }
2859
439a1cff
MO		 2860 rdev->config.si.backend_enable_mask = enabled_rbs;
2861
1a8ca750
AD		 2862 for (i = 0; i < se_num; i++) {
2863 si_select_se_sh(rdev, i, 0xffffffff);
2864 data = 0;
2865 for (j = 0; j < sh_per_se; j++) {
2866 switch (enabled_rbs & 3) {
2867 case 1:
2868 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2869 break;
2870 case 2:
2871 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2872 break;
2873 case 3:
2874 default:
2875 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2876 break;
2877 }
2878 enabled_rbs >>= 2;
2879 }
2880 WREG32(PA_SC_RASTER_CONFIG, data);
2881 }
2882 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2883}
2884
0a96d72b
AD		 2885static void si_gpu_init(struct radeon_device *rdev)
2886{
0a96d72b
AD		 2887 u32 gb_addr_config = 0;
2888 u32 mc_shared_chmap, mc_arb_ramcfg;
0a96d72b 2889 u32 sx_debug_1;
0a96d72b
AD		 2890 u32 hdp_host_path_cntl;
2891 u32 tmp;
2892 int i, j;
2893
2894 switch (rdev->family) {
2895 case CHIP_TAHITI:
2896 rdev->config.si.max_shader_engines = 2;
0a96d72b 2897 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 2898 rdev->config.si.max_cu_per_sh = 8;
2899 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2900 rdev->config.si.max_backends_per_se = 4;
2901 rdev->config.si.max_texture_channel_caches = 12;
2902 rdev->config.si.max_gprs = 256;
2903 rdev->config.si.max_gs_threads = 32;
2904 rdev->config.si.max_hw_contexts = 8;
2905
2906 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2907 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2908 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2909 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2910 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2911 break;
2912 case CHIP_PITCAIRN:
2913 rdev->config.si.max_shader_engines = 2;
0a96d72b 2914 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 2915 rdev->config.si.max_cu_per_sh = 5;
2916 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2917 rdev->config.si.max_backends_per_se = 4;
2918 rdev->config.si.max_texture_channel_caches = 8;
2919 rdev->config.si.max_gprs = 256;
2920 rdev->config.si.max_gs_threads = 32;
2921 rdev->config.si.max_hw_contexts = 8;
2922
2923 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2924 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2925 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2926 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2927 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2928 break;
2929 case CHIP_VERDE:
2930 default:
2931 rdev->config.si.max_shader_engines = 1;
0a96d72b 2932 rdev->config.si.max_tile_pipes = 4;
468ef1a5 2933 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 2934 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2935 rdev->config.si.max_backends_per_se = 4;
2936 rdev->config.si.max_texture_channel_caches = 4;
2937 rdev->config.si.max_gprs = 256;
2938 rdev->config.si.max_gs_threads = 32;
2939 rdev->config.si.max_hw_contexts = 8;
2940
d0ae7fcc
AD		 2941 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2942 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2943 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2944 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2945 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2946 break;
2947 case CHIP_OLAND:
2948 rdev->config.si.max_shader_engines = 1;
2949 rdev->config.si.max_tile_pipes = 4;
2950 rdev->config.si.max_cu_per_sh = 6;
2951 rdev->config.si.max_sh_per_se = 1;
2952 rdev->config.si.max_backends_per_se = 2;
2953 rdev->config.si.max_texture_channel_caches = 4;
2954 rdev->config.si.max_gprs = 256;
2955 rdev->config.si.max_gs_threads = 16;
2956 rdev->config.si.max_hw_contexts = 8;
2957
0a96d72b
AD		 2958 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2959 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2960 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2961 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2962 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 2963 break;
8b02859d
AD		 2964 case CHIP_HAINAN:
2965 rdev->config.si.max_shader_engines = 1;
2966 rdev->config.si.max_tile_pipes = 4;
2967 rdev->config.si.max_cu_per_sh = 5;
2968 rdev->config.si.max_sh_per_se = 1;
2969 rdev->config.si.max_backends_per_se = 1;
2970 rdev->config.si.max_texture_channel_caches = 2;
2971 rdev->config.si.max_gprs = 256;
2972 rdev->config.si.max_gs_threads = 16;
2973 rdev->config.si.max_hw_contexts = 8;
2974
2975 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2976 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2977 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2978 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2979 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2980 break;
0a96d72b
AD		 2981 }
2982
2983 /* Initialize HDP */
2984 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2985 WREG32((0x2c14 + j), 0x00000000);
2986 WREG32((0x2c18 + j), 0x00000000);
2987 WREG32((0x2c1c + j), 0x00000000);
2988 WREG32((0x2c20 + j), 0x00000000);
2989 WREG32((0x2c24 + j), 0x00000000);
2990 }
2991
2992 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2993
2994 evergreen_fix_pci_max_read_req_size(rdev);
2995
2996 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2997
2998 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2999 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3000
0a96d72b 3001 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 3002 rdev->config.si.mem_max_burst_length_bytes = 256;
3003 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3004 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3005 if (rdev->config.si.mem_row_size_in_kb > 4)
3006 rdev->config.si.mem_row_size_in_kb = 4;
3007 /* XXX use MC settings? */
3008 rdev->config.si.shader_engine_tile_size = 32;
3009 rdev->config.si.num_gpus = 1;
3010 rdev->config.si.multi_gpu_tile_size = 64;
3011
1a8ca750
AD		 3012 /* fix up row size */
3013 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 3014 switch (rdev->config.si.mem_row_size_in_kb) {
3015 case 1:
3016 default:
3017 gb_addr_config |= ROW_SIZE(0);
3018 break;
3019 case 2:
3020 gb_addr_config |= ROW_SIZE(1);
3021 break;
3022 case 4:
3023 gb_addr_config |= ROW_SIZE(2);
3024 break;
3025 }
3026
0a96d72b
AD		 3027 /* setup tiling info dword. gb_addr_config is not adequate since it does
3028 * not have bank info, so create a custom tiling dword.
3029 * bits 3:0 num_pipes
3030 * bits 7:4 num_banks
3031 * bits 11:8 group_size
3032 * bits 15:12 row_size
3033 */
3034 rdev->config.si.tile_config = 0;
3035 switch (rdev->config.si.num_tile_pipes) {
3036 case 1:
3037 rdev->config.si.tile_config |= (0 << 0);
3038 break;
3039 case 2:
3040 rdev->config.si.tile_config |= (1 << 0);
3041 break;
3042 case 4:
3043 rdev->config.si.tile_config |= (2 << 0);
3044 break;
3045 case 8:
3046 default:
3047 /* XXX what about 12? */
3048 rdev->config.si.tile_config |= (3 << 0);
3049 break;
dca571a6
CK		 3050 }
3051 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3052 case 0: /* four banks */
1a8ca750 3053 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 3054 break;
3055 case 1: /* eight banks */
3056 rdev->config.si.tile_config |= 1 << 4;
3057 break;
3058 case 2: /* sixteen banks */
3059 default:
3060 rdev->config.si.tile_config |= 2 << 4;
3061 break;
3062 }
0a96d72b
AD		 3063 rdev->config.si.tile_config |=
3064 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3065 rdev->config.si.tile_config |=
3066 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3067
0a96d72b
AD		 3068 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3069 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 3070 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 3071 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 3072 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3073 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 3074 if (rdev->has_uvd) {
3075 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3076 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3077 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3078 }
0a96d72b 3079
1a8ca750 3080 si_tiling_mode_table_init(rdev);
0a96d72b 3081
1a8ca750
AD		 3082 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3083 rdev->config.si.max_sh_per_se,
3084 rdev->config.si.max_backends_per_se);
0a96d72b 3085
1a8ca750
AD		 3086 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3087 rdev->config.si.max_sh_per_se,
3088 rdev->config.si.max_cu_per_sh);
0a96d72b 3089
0a96d72b
AD		 3090
3091 /* set HW defaults for 3D engine */
3092 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3093 ROQ_IB2_START(0x2b)));
3094 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3095
3096 sx_debug_1 = RREG32(SX_DEBUG_1);
3097 WREG32(SX_DEBUG_1, sx_debug_1);
3098
3099 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3100
3101 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3102 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3103 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3104 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3105
3106 WREG32(VGT_NUM_INSTANCES, 1);
3107
3108 WREG32(CP_PERFMON_CNTL, 0);
3109
3110 WREG32(SQ_CONFIG, 0);
3111
3112 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3113 FORCE_EOV_MAX_REZ_CNT(255)));
3114
3115 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3116 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3117
3118 WREG32(VGT_GS_VERTEX_REUSE, 16);
3119 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3120
3121 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3122 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3123 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3124 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3125 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3126 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3127 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3128 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3129
3130 tmp = RREG32(HDP_MISC_CNTL);
3131 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3132 WREG32(HDP_MISC_CNTL, tmp);
3133
3134 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3135 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3136
3137 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3138
3139 udelay(50);
3140}
c476dde2 3141
2ece2e8b
AD		 3142/*
3143 * GPU scratch registers helpers function.
3144 */
3145static void si_scratch_init(struct radeon_device *rdev)
3146{
3147 int i;
3148
3149 rdev->scratch.num_reg = 7;
3150 rdev->scratch.reg_base = SCRATCH_REG0;
3151 for (i = 0; i < rdev->scratch.num_reg; i++) {
3152 rdev->scratch.free[i] = true;
3153 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3154 }
3155}
3156
3157void si_fence_ring_emit(struct radeon_device *rdev,
3158 struct radeon_fence *fence)
3159{
3160 struct radeon_ring *ring = &rdev->ring[fence->ring];
3161 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3162
3163 /* flush read cache over gart */
3164 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3165 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3166 radeon_ring_write(ring, 0);
3167 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3168 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3169 PACKET3_TC_ACTION_ENA |
3170 PACKET3_SH_KCACHE_ACTION_ENA |
3171 PACKET3_SH_ICACHE_ACTION_ENA);
3172 radeon_ring_write(ring, 0xFFFFFFFF);
3173 radeon_ring_write(ring, 0);
3174 radeon_ring_write(ring, 10); /* poll interval */
3175 /* EVENT_WRITE_EOP - flush caches, send int */
3176 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3177 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3178 radeon_ring_write(ring, addr & 0xffffffff);
3179 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3180 radeon_ring_write(ring, fence->seq);
3181 radeon_ring_write(ring, 0);
3182}
3183
3184/*
3185 * IB stuff
3186 */
3187void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3188{
876dc9f3 3189 struct radeon_ring *ring = &rdev->ring[ib->ring];
2ece2e8b
AD		 3190 u32 header;
3191
a85a7da4
AD		 3192 if (ib->is_const_ib) {
3193 /* set switch buffer packet before const IB */
3194 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3195 radeon_ring_write(ring, 0);
45df6803 3196
2ece2e8b 3197 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 3198 } else {
89d35807 3199 u32 next_rptr;
a85a7da4 3200 if (ring->rptr_save_reg) {
89d35807 3201 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 3202 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3203 radeon_ring_write(ring, ((ring->rptr_save_reg -
3204 PACKET3_SET_CONFIG_REG_START) >> 2));
3205 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3206 } else if (rdev->wb.enabled) {
3207 next_rptr = ring->wptr + 5 + 4 + 8;
3208 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3209 radeon_ring_write(ring, (1 << 8));
3210 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3211 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3212 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 3213 }
3214
2ece2e8b 3215 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 3216 }
2ece2e8b
AD		 3217
3218 radeon_ring_write(ring, header);
3219 radeon_ring_write(ring,
3220#ifdef __BIG_ENDIAN
3221 (2 << 0) |
3222#endif
3223 (ib->gpu_addr & 0xFFFFFFFC));
3224 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4bf3dd92
CK		 3225 radeon_ring_write(ring, ib->length_dw |
3226 (ib->vm ? (ib->vm->id << 24) : 0));
2ece2e8b 3227
a85a7da4
AD		 3228 if (!ib->is_const_ib) {
3229 /* flush read cache over gart for this vmid */
3230 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3231 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 3232 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
a85a7da4
AD		 3233 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3234 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3235 PACKET3_TC_ACTION_ENA |
3236 PACKET3_SH_KCACHE_ACTION_ENA |
3237 PACKET3_SH_ICACHE_ACTION_ENA);
3238 radeon_ring_write(ring, 0xFFFFFFFF);
3239 radeon_ring_write(ring, 0);
3240 radeon_ring_write(ring, 10); /* poll interval */
3241 }
2ece2e8b
AD		 3242}
3243
48c0c902
AD		 3244/*
3245 * CP.
3246 */
3247static void si_cp_enable(struct radeon_device *rdev, bool enable)
3248{
3249 if (enable)
3250 WREG32(CP_ME_CNTL, 0);
3251 else {
3252 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3253 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3254 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 3255 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3256 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3257 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 3258 }
3259 udelay(50);
3260}
3261
3262static int si_cp_load_microcode(struct radeon_device *rdev)
3263{
3264 const __be32 *fw_data;
3265 int i;
3266
3267 if (!rdev->me_fw || !rdev->pfp_fw)
3268 return -EINVAL;
3269
3270 si_cp_enable(rdev, false);
3271
3272 /* PFP */
3273 fw_data = (const __be32 *)rdev->pfp_fw->data;
3274 WREG32(CP_PFP_UCODE_ADDR, 0);
3275 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3276 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3277 WREG32(CP_PFP_UCODE_ADDR, 0);
3278
3279 /* CE */
3280 fw_data = (const __be32 *)rdev->ce_fw->data;
3281 WREG32(CP_CE_UCODE_ADDR, 0);
3282 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3283 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3284 WREG32(CP_CE_UCODE_ADDR, 0);
3285
3286 /* ME */
3287 fw_data = (const __be32 *)rdev->me_fw->data;
3288 WREG32(CP_ME_RAM_WADDR, 0);
3289 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3290 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3291 WREG32(CP_ME_RAM_WADDR, 0);
3292
3293 WREG32(CP_PFP_UCODE_ADDR, 0);
3294 WREG32(CP_CE_UCODE_ADDR, 0);
3295 WREG32(CP_ME_RAM_WADDR, 0);
3296 WREG32(CP_ME_RAM_RADDR, 0);
3297 return 0;
3298}
3299
3300static int si_cp_start(struct radeon_device *rdev)
3301{
3302 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3303 int r, i;
3304
3305 r = radeon_ring_lock(rdev, ring, 7 + 4);
3306 if (r) {
3307 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3308 return r;
3309 }
3310 /* init the CP */
3311 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3312 radeon_ring_write(ring, 0x1);
3313 radeon_ring_write(ring, 0x0);
3314 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3315 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3316 radeon_ring_write(ring, 0);
3317 radeon_ring_write(ring, 0);
3318
3319 /* init the CE partitions */
3320 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3321 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3322 radeon_ring_write(ring, 0xc000);
3323 radeon_ring_write(ring, 0xe000);
3324 radeon_ring_unlock_commit(rdev, ring);
3325
3326 si_cp_enable(rdev, true);
3327
3328 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3329 if (r) {
3330 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3331 return r;
3332 }
3333
3334 /* setup clear context state */
3335 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3336 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3337
3338 for (i = 0; i < si_default_size; i++)
3339 radeon_ring_write(ring, si_default_state[i]);
3340
3341 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3342 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3343
3344 /* set clear context state */
3345 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3346 radeon_ring_write(ring, 0);
3347
3348 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3349 radeon_ring_write(ring, 0x00000316);
3350 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3351 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3352
3353 radeon_ring_unlock_commit(rdev, ring);
3354
3355 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3356 ring = &rdev->ring[i];
3357 r = radeon_ring_lock(rdev, ring, 2);
3358
3359 /* clear the compute context state */
3360 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3361 radeon_ring_write(ring, 0);
3362
3363 radeon_ring_unlock_commit(rdev, ring);
3364 }
3365
3366 return 0;
3367}
3368
3369static void si_cp_fini(struct radeon_device *rdev)
3370{
45df6803 3371 struct radeon_ring *ring;
48c0c902 3372 si_cp_enable(rdev, false);
45df6803
CK		 3373
3374 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3375 radeon_ring_fini(rdev, ring);
3376 radeon_scratch_free(rdev, ring->rptr_save_reg);
3377
3378 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3379 radeon_ring_fini(rdev, ring);
3380 radeon_scratch_free(rdev, ring->rptr_save_reg);
3381
3382 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3383 radeon_ring_fini(rdev, ring);
3384 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3385}
3386
3387static int si_cp_resume(struct radeon_device *rdev)
3388{
3389 struct radeon_ring *ring;
3390 u32 tmp;
3391 u32 rb_bufsz;
3392 int r;
3393
811e4d58
AD		 3394 si_enable_gui_idle_interrupt(rdev, false);
3395
48c0c902
AD		 3396 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3397 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3398
3399 /* Set the write pointer delay */
3400 WREG32(CP_RB_WPTR_DELAY, 0);
3401
3402 WREG32(CP_DEBUG, 0);
3403 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3404
3405 /* ring 0 - compute and gfx */
3406 /* Set ring buffer size */
3407 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 3408 rb_bufsz = order_base_2(ring->ring_size / 8);
3409 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3410#ifdef __BIG_ENDIAN
3411 tmp |= BUF_SWAP_32BIT;
3412#endif
3413 WREG32(CP_RB0_CNTL, tmp);
3414
3415 /* Initialize the ring buffer's read and write pointers */
3416 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3417 ring->wptr = 0;
3418 WREG32(CP_RB0_WPTR, ring->wptr);
3419
48fc7f7e 3420 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3421 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3422 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3423
3424 if (rdev->wb.enabled)
3425 WREG32(SCRATCH_UMSK, 0xff);
3426 else {
3427 tmp |= RB_NO_UPDATE;
3428 WREG32(SCRATCH_UMSK, 0);
3429 }
3430
3431 mdelay(1);
3432 WREG32(CP_RB0_CNTL, tmp);
3433
3434 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3435
3436 ring->rptr = RREG32(CP_RB0_RPTR);
3437
3438 /* ring1 - compute only */
3439 /* Set ring buffer size */
3440 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
b72a8925
DV		 3441 rb_bufsz = order_base_2(ring->ring_size / 8);
3442 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3443#ifdef __BIG_ENDIAN
3444 tmp |= BUF_SWAP_32BIT;
3445#endif
3446 WREG32(CP_RB1_CNTL, tmp);
3447
3448 /* Initialize the ring buffer's read and write pointers */
3449 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3450 ring->wptr = 0;
3451 WREG32(CP_RB1_WPTR, ring->wptr);
3452
48fc7f7e 3453 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3454 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3455 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3456
3457 mdelay(1);
3458 WREG32(CP_RB1_CNTL, tmp);
3459
3460 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3461
3462 ring->rptr = RREG32(CP_RB1_RPTR);
3463
3464 /* ring2 - compute only */
3465 /* Set ring buffer size */
3466 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
b72a8925
DV		 3467 rb_bufsz = order_base_2(ring->ring_size / 8);
3468 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3469#ifdef __BIG_ENDIAN
3470 tmp |= BUF_SWAP_32BIT;
3471#endif
3472 WREG32(CP_RB2_CNTL, tmp);
3473
3474 /* Initialize the ring buffer's read and write pointers */
3475 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3476 ring->wptr = 0;
3477 WREG32(CP_RB2_WPTR, ring->wptr);
3478
48fc7f7e 3479 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3480 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3481 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3482
3483 mdelay(1);
3484 WREG32(CP_RB2_CNTL, tmp);
3485
3486 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3487
3488 ring->rptr = RREG32(CP_RB2_RPTR);
3489
3490 /* start the rings */
3491 si_cp_start(rdev);
3492 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3493 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3494 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3495 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3496 if (r) {
3497 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3498 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3499 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3500 return r;
3501 }
3502 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3503 if (r) {
3504 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3505 }
3506 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3507 if (r) {
3508 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3509 }
3510
811e4d58
AD		 3511 si_enable_gui_idle_interrupt(rdev, true);
3512
48c0c902
AD		 3513 return 0;
3514}
3515
2483b4ea 3516u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3517{
014bb209 3518 u32 reset_mask = 0;
1c534671 3519 u32 tmp;
06bc6df0 3520
014bb209
AD		 3521 /* GRBM_STATUS */
3522 tmp = RREG32(GRBM_STATUS);
3523 if (tmp & (PA_BUSY | SC_BUSY |
3524 BCI_BUSY | SX_BUSY |
3525 TA_BUSY | VGT_BUSY |
3526 DB_BUSY | CB_BUSY |
3527 GDS_BUSY | SPI_BUSY |
3528 IA_BUSY | IA_BUSY_NO_DMA))
3529 reset_mask |= RADEON_RESET_GFX;
3530
3531 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3532 CP_BUSY | CP_COHERENCY_BUSY))
3533 reset_mask |= RADEON_RESET_CP;
3534
3535 if (tmp & GRBM_EE_BUSY)
3536 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3537
3538 /* GRBM_STATUS2 */
3539 tmp = RREG32(GRBM_STATUS2);
3540 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3541 reset_mask |= RADEON_RESET_RLC;
3542
3543 /* DMA_STATUS_REG 0 */
3544 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3545 if (!(tmp & DMA_IDLE))
3546 reset_mask |= RADEON_RESET_DMA;
3547
3548 /* DMA_STATUS_REG 1 */
3549 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3550 if (!(tmp & DMA_IDLE))
3551 reset_mask |= RADEON_RESET_DMA1;
3552
3553 /* SRBM_STATUS2 */
3554 tmp = RREG32(SRBM_STATUS2);
3555 if (tmp & DMA_BUSY)
3556 reset_mask |= RADEON_RESET_DMA;
3557
3558 if (tmp & DMA1_BUSY)
3559 reset_mask |= RADEON_RESET_DMA1;
3560
3561 /* SRBM_STATUS */
3562 tmp = RREG32(SRBM_STATUS);
3563
3564 if (tmp & IH_BUSY)
3565 reset_mask |= RADEON_RESET_IH;
3566
3567 if (tmp & SEM_BUSY)
3568 reset_mask |= RADEON_RESET_SEM;
3569
3570 if (tmp & GRBM_RQ_PENDING)
3571 reset_mask |= RADEON_RESET_GRBM;
3572
3573 if (tmp & VMC_BUSY)
3574 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3575
014bb209
AD		 3576 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3577 MCC_BUSY | MCD_BUSY))
3578 reset_mask |= RADEON_RESET_MC;
3579
3580 if (evergreen_is_display_hung(rdev))
3581 reset_mask |= RADEON_RESET_DISPLAY;
3582
3583 /* VM_L2_STATUS */
3584 tmp = RREG32(VM_L2_STATUS);
3585 if (tmp & L2_BUSY)
3586 reset_mask |= RADEON_RESET_VMC;
3587
d808fc88
AD		 3588 /* Skip MC reset as it's mostly likely not hung, just busy */
3589 if (reset_mask & RADEON_RESET_MC) {
3590 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3591 reset_mask &= ~RADEON_RESET_MC;
3592 }
3593
014bb209
AD		 3594 return reset_mask;
3595}
3596
3597static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3598{
3599 struct evergreen_mc_save save;
3600 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3601 u32 tmp;
19fc42ed 3602
06bc6df0 3603 if (reset_mask == 0)
014bb209 3604 return;
06bc6df0
AD		 3605
3606 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3607
1c534671 3608 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3609 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3610 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3611 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3612 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3613
a6f4ae8d
AD		 3614 /* disable PG/CG */
3615 si_fini_pg(rdev);
3616 si_fini_cg(rdev);
3617
3618 /* stop the rlc */
3619 si_rlc_stop(rdev);
3620
1c534671
AD		 3621 /* Disable CP parsing/prefetching */
3622 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3623
3624 if (reset_mask & RADEON_RESET_DMA) {
3625 /* dma0 */
3626 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3627 tmp &= ~DMA_RB_ENABLE;
3628 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3629 }
3630 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3631 /* dma1 */
3632 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3633 tmp &= ~DMA_RB_ENABLE;
3634 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3635 }
3636
f770d78a
AD		 3637 udelay(50);
3638
3639 evergreen_mc_stop(rdev, &save);
3640 if (evergreen_mc_wait_for_idle(rdev)) {
3641 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3642 }
3643
1c534671
AD		 3644 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3645 grbm_soft_reset = SOFT_RESET_CB |
3646 SOFT_RESET_DB |
3647 SOFT_RESET_GDS |
3648 SOFT_RESET_PA |
3649 SOFT_RESET_SC |
3650 SOFT_RESET_BCI |
3651 SOFT_RESET_SPI |
3652 SOFT_RESET_SX |
3653 SOFT_RESET_TC |
3654 SOFT_RESET_TA |
3655 SOFT_RESET_VGT |
3656 SOFT_RESET_IA;
3657 }
3658
3659 if (reset_mask & RADEON_RESET_CP) {
3660 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3661
3662 srbm_soft_reset |= SOFT_RESET_GRBM;
3663 }
06bc6df0
AD		 3664
3665 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3666 srbm_soft_reset |= SOFT_RESET_DMA;
3667
3668 if (reset_mask & RADEON_RESET_DMA1)
3669 srbm_soft_reset |= SOFT_RESET_DMA1;
3670
3671 if (reset_mask & RADEON_RESET_DISPLAY)
3672 srbm_soft_reset |= SOFT_RESET_DC;
3673
3674 if (reset_mask & RADEON_RESET_RLC)
3675 grbm_soft_reset |= SOFT_RESET_RLC;
3676
3677 if (reset_mask & RADEON_RESET_SEM)
3678 srbm_soft_reset |= SOFT_RESET_SEM;
3679
3680 if (reset_mask & RADEON_RESET_IH)
3681 srbm_soft_reset |= SOFT_RESET_IH;
3682
3683 if (reset_mask & RADEON_RESET_GRBM)
3684 srbm_soft_reset |= SOFT_RESET_GRBM;
3685
3686 if (reset_mask & RADEON_RESET_VMC)
3687 srbm_soft_reset |= SOFT_RESET_VMC;
3688
3689 if (reset_mask & RADEON_RESET_MC)
3690 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3691
3692 if (grbm_soft_reset) {
3693 tmp = RREG32(GRBM_SOFT_RESET);
3694 tmp |= grbm_soft_reset;
3695 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3696 WREG32(GRBM_SOFT_RESET, tmp);
3697 tmp = RREG32(GRBM_SOFT_RESET);
3698
3699 udelay(50);
3700
3701 tmp &= ~grbm_soft_reset;
3702 WREG32(GRBM_SOFT_RESET, tmp);
3703 tmp = RREG32(GRBM_SOFT_RESET);
3704 }
3705
3706 if (srbm_soft_reset) {
3707 tmp = RREG32(SRBM_SOFT_RESET);
3708 tmp |= srbm_soft_reset;
3709 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3710 WREG32(SRBM_SOFT_RESET, tmp);
3711 tmp = RREG32(SRBM_SOFT_RESET);
3712
3713 udelay(50);
3714
3715 tmp &= ~srbm_soft_reset;
3716 WREG32(SRBM_SOFT_RESET, tmp);
3717 tmp = RREG32(SRBM_SOFT_RESET);
3718 }
06bc6df0
AD		 3719
3720 /* Wait a little for things to settle down */
3721 udelay(50);
3722
c476dde2 3723 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3724 udelay(50);
3725
1c534671 3726 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3727}
3728
4a5c8ea5
AD		 3729static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3730{
3731 u32 tmp, i;
3732
3733 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3734 tmp |= SPLL_BYPASS_EN;
3735 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3736
3737 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3738 tmp |= SPLL_CTLREQ_CHG;
3739 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3740
3741 for (i = 0; i < rdev->usec_timeout; i++) {
3742 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3743 break;
3744 udelay(1);
3745 }
3746
3747 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3748 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3749 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3750
3751 tmp = RREG32(MPLL_CNTL_MODE);
3752 tmp &= ~MPLL_MCLK_SEL;
3753 WREG32(MPLL_CNTL_MODE, tmp);
3754}
3755
3756static void si_spll_powerdown(struct radeon_device *rdev)
3757{
3758 u32 tmp;
3759
3760 tmp = RREG32(SPLL_CNTL_MODE);
3761 tmp |= SPLL_SW_DIR_CONTROL;
3762 WREG32(SPLL_CNTL_MODE, tmp);
3763
3764 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3765 tmp |= SPLL_RESET;
3766 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3767
3768 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3769 tmp |= SPLL_SLEEP;
3770 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3771
3772 tmp = RREG32(SPLL_CNTL_MODE);
3773 tmp &= ~SPLL_SW_DIR_CONTROL;
3774 WREG32(SPLL_CNTL_MODE, tmp);
3775}
3776
3777static void si_gpu_pci_config_reset(struct radeon_device *rdev)
3778{
3779 struct evergreen_mc_save save;
3780 u32 tmp, i;
3781
3782 dev_info(rdev->dev, "GPU pci config reset\n");
3783
3784 /* disable dpm? */
3785
3786 /* disable cg/pg */
3787 si_fini_pg(rdev);
3788 si_fini_cg(rdev);
3789
3790 /* Disable CP parsing/prefetching */
3791 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3792 /* dma0 */
3793 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3794 tmp &= ~DMA_RB_ENABLE;
3795 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3796 /* dma1 */
3797 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3798 tmp &= ~DMA_RB_ENABLE;
3799 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3800 /* XXX other engines? */
3801
3802 /* halt the rlc, disable cp internal ints */
3803 si_rlc_stop(rdev);
3804
3805 udelay(50);
3806
3807 /* disable mem access */
3808 evergreen_mc_stop(rdev, &save);
3809 if (evergreen_mc_wait_for_idle(rdev)) {
3810 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3811 }
3812
3813 /* set mclk/sclk to bypass */
3814 si_set_clk_bypass_mode(rdev);
3815 /* powerdown spll */
3816 si_spll_powerdown(rdev);
3817 /* disable BM */
3818 pci_clear_master(rdev->pdev);
3819 /* reset */
3820 radeon_pci_config_reset(rdev);
3821 /* wait for asic to come out of reset */
3822 for (i = 0; i < rdev->usec_timeout; i++) {
3823 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3824 break;
3825 udelay(1);
3826 }
3827}
3828
c476dde2
AD		 3829int si_asic_reset(struct radeon_device *rdev)
3830{
014bb209
AD		 3831 u32 reset_mask;
3832
3833 reset_mask = si_gpu_check_soft_reset(rdev);
3834
3835 if (reset_mask)
3836 r600_set_bios_scratch_engine_hung(rdev, true);
3837
4a5c8ea5 3838 /* try soft reset */
014bb209
AD		 3839 si_gpu_soft_reset(rdev, reset_mask);
3840
3841 reset_mask = si_gpu_check_soft_reset(rdev);
3842
4a5c8ea5
AD		 3843 /* try pci config reset */
3844 if (reset_mask && radeon_hard_reset)
3845 si_gpu_pci_config_reset(rdev);
3846
3847 reset_mask = si_gpu_check_soft_reset(rdev);
3848
014bb209
AD		 3849 if (!reset_mask)
3850 r600_set_bios_scratch_engine_hung(rdev, false);
3851
3852 return 0;
c476dde2
AD		 3853}
3854
123bc183
AD		 3855/**
3856 * si_gfx_is_lockup - Check if the GFX engine is locked up
3857 *
3858 * @rdev: radeon_device pointer
3859 * @ring: radeon_ring structure holding ring information
3860 *
3861 * Check if the GFX engine is locked up.
3862 * Returns true if the engine appears to be locked up, false if not.
3863 */
3864bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3865{
3866 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3867
3868 if (!(reset_mask & (RADEON_RESET_GFX |
3869 RADEON_RESET_COMPUTE |
3870 RADEON_RESET_CP))) {
3871 radeon_ring_lockup_update(ring);
3872 return false;
3873 }
3874 /* force CP activities */
3875 radeon_ring_force_activity(rdev, ring);
3876 return radeon_ring_test_lockup(rdev, ring);
3877}
3878
d2800ee5
AD		 3879/* MC */
3880static void si_mc_program(struct radeon_device *rdev)
3881{
3882 struct evergreen_mc_save save;
3883 u32 tmp;
3884 int i, j;
3885
3886 /* Initialize HDP */
3887 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3888 WREG32((0x2c14 + j), 0x00000000);
3889 WREG32((0x2c18 + j), 0x00000000);
3890 WREG32((0x2c1c + j), 0x00000000);
3891 WREG32((0x2c20 + j), 0x00000000);
3892 WREG32((0x2c24 + j), 0x00000000);
3893 }
3894 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3895
3896 evergreen_mc_stop(rdev, &save);
3897 if (radeon_mc_wait_for_idle(rdev)) {
3898 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3899 }
5153550a
AD		 3900 if (!ASIC_IS_NODCE(rdev))
3901 /* Lockout access through VGA aperture*/
3902 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 3903 /* Update configuration */
3904 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3905 rdev->mc.vram_start >> 12);
3906 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3907 rdev->mc.vram_end >> 12);
3908 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3909 rdev->vram_scratch.gpu_addr >> 12);
3910 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3911 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3912 WREG32(MC_VM_FB_LOCATION, tmp);
3913 /* XXX double check these! */
3914 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3915 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3916 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3917 WREG32(MC_VM_AGP_BASE, 0);
3918 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3919 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3920 if (radeon_mc_wait_for_idle(rdev)) {
3921 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3922 }
3923 evergreen_mc_resume(rdev, &save);
5153550a
AD		 3924 if (!ASIC_IS_NODCE(rdev)) {
3925 /* we need to own VRAM, so turn off the VGA renderer here
3926 * to stop it overwriting our objects */
3927 rv515_vga_render_disable(rdev);
3928 }
d2800ee5
AD		 3929}
3930
1c49165d
AD		 3931void si_vram_gtt_location(struct radeon_device *rdev,
3932 struct radeon_mc *mc)
d2800ee5
AD		 3933{
3934 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3935 /* leave room for at least 1024M GTT */
3936 dev_warn(rdev->dev, "limiting VRAM\n");
3937 mc->real_vram_size = 0xFFC0000000ULL;
3938 mc->mc_vram_size = 0xFFC0000000ULL;
3939 }
9ed8b1f9 3940 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 3941 rdev->mc.gtt_base_align = 0;
9ed8b1f9 3942 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 3943}
3944
3945static int si_mc_init(struct radeon_device *rdev)
3946{
3947 u32 tmp;
3948 int chansize, numchan;
3949
3950 /* Get VRAM informations */
3951 rdev->mc.vram_is_ddr = true;
3952 tmp = RREG32(MC_ARB_RAMCFG);
3953 if (tmp & CHANSIZE_OVERRIDE) {
3954 chansize = 16;
3955 } else if (tmp & CHANSIZE_MASK) {
3956 chansize = 64;
3957 } else {
3958 chansize = 32;
3959 }
3960 tmp = RREG32(MC_SHARED_CHMAP);
3961 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3962 case 0:
3963 default:
3964 numchan = 1;
3965 break;
3966 case 1:
3967 numchan = 2;
3968 break;
3969 case 2:
3970 numchan = 4;
3971 break;
3972 case 3:
3973 numchan = 8;
3974 break;
3975 case 4:
3976 numchan = 3;
3977 break;
3978 case 5:
3979 numchan = 6;
3980 break;
3981 case 6:
3982 numchan = 10;
3983 break;
3984 case 7:
3985 numchan = 12;
3986 break;
3987 case 8:
3988 numchan = 16;
3989 break;
3990 }
3991 rdev->mc.vram_width = numchan * chansize;
3992 /* Could aper size report 0 ? */
3993 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3994 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3995 /* size in MB on si */
0ca223b0
AD		 3996 tmp = RREG32(CONFIG_MEMSIZE);
3997 /* some boards may have garbage in the upper 16 bits */
3998 if (tmp & 0xffff0000) {
3999 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4000 if (tmp & 0xffff)
4001 tmp &= 0xffff;
4002 }
4003 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4004 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
d2800ee5
AD		 4005 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4006 si_vram_gtt_location(rdev, &rdev->mc);
4007 radeon_update_bandwidth_info(rdev);
4008
4009 return 0;
4010}
4011
4012/*
4013 * GART
4014 */
4015void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4016{
4017 /* flush hdp cache */
4018 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4019
4020 /* bits 0-15 are the VM contexts0-15 */
4021 WREG32(VM_INVALIDATE_REQUEST, 1);
4022}
4023
1109ca09 4024static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 4025{
4026 int r, i;
4027
4028 if (rdev->gart.robj == NULL) {
4029 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4030 return -EINVAL;
4031 }
4032 r = radeon_gart_table_vram_pin(rdev);
4033 if (r)
4034 return r;
4035 radeon_gart_restore(rdev);
4036 /* Setup TLB control */
4037 WREG32(MC_VM_MX_L1_TLB_CNTL,
4038 (0xA << 7) |
4039 ENABLE_L1_TLB |
4040 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4041 ENABLE_ADVANCED_DRIVER_MODEL |
4042 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4043 /* Setup L2 cache */
4044 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4045 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4046 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4047 EFFECTIVE_L2_QUEUE_SIZE(7) |
4048 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4049 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4050 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4051 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4052 /* setup context0 */
4053 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4054 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4055 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4056 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4057 (u32)(rdev->dummy_page.addr >> 12));
4058 WREG32(VM_CONTEXT0_CNTL2, 0);
4059 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4060 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4061
4062 WREG32(0x15D4, 0);
4063 WREG32(0x15D8, 0);
4064 WREG32(0x15DC, 0);
4065
4066 /* empty context1-15 */
d2800ee5
AD		 4067 /* set vm size, must be a multiple of 4 */
4068 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
c21b328e 4069 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
23d4f1f2
AD		 4070 /* Assign the pt base to something valid for now; the pts used for
4071 * the VMs are determined by the application and setup and assigned
4072 * on the fly in the vm part of radeon_gart.c
4073 */
d2800ee5
AD		 4074 for (i = 1; i < 16; i++) {
4075 if (i < 8)
4076 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4077 rdev->gart.table_addr >> 12);
4078 else
4079 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4080 rdev->gart.table_addr >> 12);
4081 }
4082
4083 /* enable context1-15 */
4084 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4085 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 4086 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 4087 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
ae133a11
CK		 4088 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4089 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4090 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4091 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4092 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4093 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4094 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4095 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4096 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4097 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4098 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4099 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 4100
4101 si_pcie_gart_tlb_flush(rdev);
4102 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4103 (unsigned)(rdev->mc.gtt_size >> 20),
4104 (unsigned long long)rdev->gart.table_addr);
4105 rdev->gart.ready = true;
4106 return 0;
4107}
4108
1109ca09 4109static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5
AD		 4110{
4111 /* Disable all tables */
4112 WREG32(VM_CONTEXT0_CNTL, 0);
4113 WREG32(VM_CONTEXT1_CNTL, 0);
4114 /* Setup TLB control */
4115 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4116 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4117 /* Setup L2 cache */
4118 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4119 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4120 EFFECTIVE_L2_QUEUE_SIZE(7) |
4121 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4122 WREG32(VM_L2_CNTL2, 0);
4123 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4124 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4125 radeon_gart_table_vram_unpin(rdev);
4126}
4127
1109ca09 4128static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 4129{
4130 si_pcie_gart_disable(rdev);
4131 radeon_gart_table_vram_free(rdev);
4132 radeon_gart_fini(rdev);
4133}
4134
498dd8b3
AD		 4135/* vm parser */
4136static bool si_vm_reg_valid(u32 reg)
4137{
4138 /* context regs are fine */
4139 if (reg >= 0x28000)
4140 return true;
4141
4142 /* check config regs */
4143 switch (reg) {
4144 case GRBM_GFX_INDEX:
f418b88a 4145 case CP_STRMOUT_CNTL:
498dd8b3
AD		 4146 case VGT_VTX_VECT_EJECT_REG:
4147 case VGT_CACHE_INVALIDATION:
4148 case VGT_ESGS_RING_SIZE:
4149 case VGT_GSVS_RING_SIZE:
4150 case VGT_GS_VERTEX_REUSE:
4151 case VGT_PRIMITIVE_TYPE:
4152 case VGT_INDEX_TYPE:
4153 case VGT_NUM_INDICES:
4154 case VGT_NUM_INSTANCES:
4155 case VGT_TF_RING_SIZE:
4156 case VGT_HS_OFFCHIP_PARAM:
4157 case VGT_TF_MEMORY_BASE:
4158 case PA_CL_ENHANCE:
4159 case PA_SU_LINE_STIPPLE_VALUE:
4160 case PA_SC_LINE_STIPPLE_STATE:
4161 case PA_SC_ENHANCE:
4162 case SQC_CACHES:
4163 case SPI_STATIC_THREAD_MGMT_1:
4164 case SPI_STATIC_THREAD_MGMT_2:
4165 case SPI_STATIC_THREAD_MGMT_3:
4166 case SPI_PS_MAX_WAVE_ID:
4167 case SPI_CONFIG_CNTL:
4168 case SPI_CONFIG_CNTL_1:
4169 case TA_CNTL_AUX:
4170 return true;
4171 default:
4172 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4173 return false;
4174 }
4175}
4176
4177static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4178 u32 *ib, struct radeon_cs_packet *pkt)
4179{
4180 switch (pkt->opcode) {
4181 case PACKET3_NOP:
4182 case PACKET3_SET_BASE:
4183 case PACKET3_SET_CE_DE_COUNTERS:
4184 case PACKET3_LOAD_CONST_RAM:
4185 case PACKET3_WRITE_CONST_RAM:
4186 case PACKET3_WRITE_CONST_RAM_OFFSET:
4187 case PACKET3_DUMP_CONST_RAM:
4188 case PACKET3_INCREMENT_CE_COUNTER:
4189 case PACKET3_WAIT_ON_DE_COUNTER:
4190 case PACKET3_CE_WRITE:
4191 break;
4192 default:
4193 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4194 return -EINVAL;
4195 }
4196 return 0;
4197}
4198
e5b9e750
TS		 4199static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4200{
4201 u32 start_reg, reg, i;
4202 u32 command = ib[idx + 4];
4203 u32 info = ib[idx + 1];
4204 u32 idx_value = ib[idx];
4205 if (command & PACKET3_CP_DMA_CMD_SAS) {
4206 /* src address space is register */
4207 if (((info & 0x60000000) >> 29) == 0) {
4208 start_reg = idx_value << 2;
4209 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4210 reg = start_reg;
4211 if (!si_vm_reg_valid(reg)) {
4212 DRM_ERROR("CP DMA Bad SRC register\n");
4213 return -EINVAL;
4214 }
4215 } else {
4216 for (i = 0; i < (command & 0x1fffff); i++) {
4217 reg = start_reg + (4 * i);
4218 if (!si_vm_reg_valid(reg)) {
4219 DRM_ERROR("CP DMA Bad SRC register\n");
4220 return -EINVAL;
4221 }
4222 }
4223 }
4224 }
4225 }
4226 if (command & PACKET3_CP_DMA_CMD_DAS) {
4227 /* dst address space is register */
4228 if (((info & 0x00300000) >> 20) == 0) {
4229 start_reg = ib[idx + 2];
4230 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4231 reg = start_reg;
4232 if (!si_vm_reg_valid(reg)) {
4233 DRM_ERROR("CP DMA Bad DST register\n");
4234 return -EINVAL;
4235 }
4236 } else {
4237 for (i = 0; i < (command & 0x1fffff); i++) {
4238 reg = start_reg + (4 * i);
4239 if (!si_vm_reg_valid(reg)) {
4240 DRM_ERROR("CP DMA Bad DST register\n");
4241 return -EINVAL;
4242 }
4243 }
4244 }
4245 }
4246 }
4247 return 0;
4248}
4249
498dd8b3
AD		 4250static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4251 u32 *ib, struct radeon_cs_packet *pkt)
4252{
e5b9e750 4253 int r;
498dd8b3
AD		 4254 u32 idx = pkt->idx + 1;
4255 u32 idx_value = ib[idx];
4256 u32 start_reg, end_reg, reg, i;
4257
4258 switch (pkt->opcode) {
4259 case PACKET3_NOP:
4260 case PACKET3_SET_BASE:
4261 case PACKET3_CLEAR_STATE:
4262 case PACKET3_INDEX_BUFFER_SIZE:
4263 case PACKET3_DISPATCH_DIRECT:
4264 case PACKET3_DISPATCH_INDIRECT:
4265 case PACKET3_ALLOC_GDS:
4266 case PACKET3_WRITE_GDS_RAM:
4267 case PACKET3_ATOMIC_GDS:
4268 case PACKET3_ATOMIC:
4269 case PACKET3_OCCLUSION_QUERY:
4270 case PACKET3_SET_PREDICATION:
4271 case PACKET3_COND_EXEC:
4272 case PACKET3_PRED_EXEC:
4273 case PACKET3_DRAW_INDIRECT:
4274 case PACKET3_DRAW_INDEX_INDIRECT:
4275 case PACKET3_INDEX_BASE:
4276 case PACKET3_DRAW_INDEX_2:
4277 case PACKET3_CONTEXT_CONTROL:
4278 case PACKET3_INDEX_TYPE:
4279 case PACKET3_DRAW_INDIRECT_MULTI:
4280 case PACKET3_DRAW_INDEX_AUTO:
4281 case PACKET3_DRAW_INDEX_IMMD:
4282 case PACKET3_NUM_INSTANCES:
4283 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4284 case PACKET3_STRMOUT_BUFFER_UPDATE:
4285 case PACKET3_DRAW_INDEX_OFFSET_2:
4286 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4287 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4288 case PACKET3_MPEG_INDEX:
4289 case PACKET3_WAIT_REG_MEM:
4290 case PACKET3_MEM_WRITE:
4291 case PACKET3_PFP_SYNC_ME:
4292 case PACKET3_SURFACE_SYNC:
4293 case PACKET3_EVENT_WRITE:
4294 case PACKET3_EVENT_WRITE_EOP:
4295 case PACKET3_EVENT_WRITE_EOS:
4296 case PACKET3_SET_CONTEXT_REG:
4297 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4298 case PACKET3_SET_SH_REG:
4299 case PACKET3_SET_SH_REG_OFFSET:
4300 case PACKET3_INCREMENT_DE_COUNTER:
4301 case PACKET3_WAIT_ON_CE_COUNTER:
4302 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4303 case PACKET3_ME_WRITE:
4304 break;
4305 case PACKET3_COPY_DATA:
4306 if ((idx_value & 0xf00) == 0) {
4307 reg = ib[idx + 3] * 4;
4308 if (!si_vm_reg_valid(reg))
4309 return -EINVAL;
4310 }
4311 break;
4312 case PACKET3_WRITE_DATA:
4313 if ((idx_value & 0xf00) == 0) {
4314 start_reg = ib[idx + 1] * 4;
4315 if (idx_value & 0x10000) {
4316 if (!si_vm_reg_valid(start_reg))
4317 return -EINVAL;
4318 } else {
4319 for (i = 0; i < (pkt->count - 2); i++) {
4320 reg = start_reg + (4 * i);
4321 if (!si_vm_reg_valid(reg))
4322 return -EINVAL;
4323 }
4324 }
4325 }
4326 break;
4327 case PACKET3_COND_WRITE:
4328 if (idx_value & 0x100) {
4329 reg = ib[idx + 5] * 4;
4330 if (!si_vm_reg_valid(reg))
4331 return -EINVAL;
4332 }
4333 break;
4334 case PACKET3_COPY_DW:
4335 if (idx_value & 0x2) {
4336 reg = ib[idx + 3] * 4;
4337 if (!si_vm_reg_valid(reg))
4338 return -EINVAL;
4339 }
4340 break;
4341 case PACKET3_SET_CONFIG_REG:
4342 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4343 end_reg = 4 * pkt->count + start_reg - 4;
4344 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4345 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4346 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4347 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4348 return -EINVAL;
4349 }
4350 for (i = 0; i < pkt->count; i++) {
4351 reg = start_reg + (4 * i);
4352 if (!si_vm_reg_valid(reg))
4353 return -EINVAL;
4354 }
4355 break;
5aa709be 4356 case PACKET3_CP_DMA:
e5b9e750
TS		 4357 r = si_vm_packet3_cp_dma_check(ib, idx);
4358 if (r)
4359 return r;
5aa709be 4360 break;
498dd8b3
AD		 4361 default:
4362 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4363 return -EINVAL;
4364 }
4365 return 0;
4366}
4367
4368static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4369 u32 *ib, struct radeon_cs_packet *pkt)
4370{
e5b9e750 4371 int r;
498dd8b3
AD		 4372 u32 idx = pkt->idx + 1;
4373 u32 idx_value = ib[idx];
4374 u32 start_reg, reg, i;
4375
4376 switch (pkt->opcode) {
4377 case PACKET3_NOP:
4378 case PACKET3_SET_BASE:
4379 case PACKET3_CLEAR_STATE:
4380 case PACKET3_DISPATCH_DIRECT:
4381 case PACKET3_DISPATCH_INDIRECT:
4382 case PACKET3_ALLOC_GDS:
4383 case PACKET3_WRITE_GDS_RAM:
4384 case PACKET3_ATOMIC_GDS:
4385 case PACKET3_ATOMIC:
4386 case PACKET3_OCCLUSION_QUERY:
4387 case PACKET3_SET_PREDICATION:
4388 case PACKET3_COND_EXEC:
4389 case PACKET3_PRED_EXEC:
4390 case PACKET3_CONTEXT_CONTROL:
4391 case PACKET3_STRMOUT_BUFFER_UPDATE:
4392 case PACKET3_WAIT_REG_MEM:
4393 case PACKET3_MEM_WRITE:
4394 case PACKET3_PFP_SYNC_ME:
4395 case PACKET3_SURFACE_SYNC:
4396 case PACKET3_EVENT_WRITE:
4397 case PACKET3_EVENT_WRITE_EOP:
4398 case PACKET3_EVENT_WRITE_EOS:
4399 case PACKET3_SET_CONTEXT_REG:
4400 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4401 case PACKET3_SET_SH_REG:
4402 case PACKET3_SET_SH_REG_OFFSET:
4403 case PACKET3_INCREMENT_DE_COUNTER:
4404 case PACKET3_WAIT_ON_CE_COUNTER:
4405 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4406 case PACKET3_ME_WRITE:
4407 break;
4408 case PACKET3_COPY_DATA:
4409 if ((idx_value & 0xf00) == 0) {
4410 reg = ib[idx + 3] * 4;
4411 if (!si_vm_reg_valid(reg))
4412 return -EINVAL;
4413 }
4414 break;
4415 case PACKET3_WRITE_DATA:
4416 if ((idx_value & 0xf00) == 0) {
4417 start_reg = ib[idx + 1] * 4;
4418 if (idx_value & 0x10000) {
4419 if (!si_vm_reg_valid(start_reg))
4420 return -EINVAL;
4421 } else {
4422 for (i = 0; i < (pkt->count - 2); i++) {
4423 reg = start_reg + (4 * i);
4424 if (!si_vm_reg_valid(reg))
4425 return -EINVAL;
4426 }
4427 }
4428 }
4429 break;
4430 case PACKET3_COND_WRITE:
4431 if (idx_value & 0x100) {
4432 reg = ib[idx + 5] * 4;
4433 if (!si_vm_reg_valid(reg))
4434 return -EINVAL;
4435 }
4436 break;
4437 case PACKET3_COPY_DW:
4438 if (idx_value & 0x2) {
4439 reg = ib[idx + 3] * 4;
4440 if (!si_vm_reg_valid(reg))
4441 return -EINVAL;
4442 }
4443 break;
e5b9e750
TS		 4444 case PACKET3_CP_DMA:
4445 r = si_vm_packet3_cp_dma_check(ib, idx);
4446 if (r)
4447 return r;
4448 break;
498dd8b3
AD		 4449 default:
4450 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4451 return -EINVAL;
4452 }
4453 return 0;
4454}
4455
4456int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4457{
4458 int ret = 0;
4459 u32 idx = 0;
4460 struct radeon_cs_packet pkt;
4461
4462 do {
4463 pkt.idx = idx;
4e872ae2
IH		 4464 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4465 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4466 pkt.one_reg_wr = 0;
4467 switch (pkt.type) {
4e872ae2 4468 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 4469 dev_err(rdev->dev, "Packet0 not allowed!\n");
4470 ret = -EINVAL;
4471 break;
4e872ae2 4472 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4473 idx += 1;
4474 break;
4e872ae2
IH		 4475 case RADEON_PACKET_TYPE3:
4476 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4477 if (ib->is_const_ib)
4478 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4479 else {
876dc9f3 4480 switch (ib->ring) {
498dd8b3
AD		 4481 case RADEON_RING_TYPE_GFX_INDEX:
4482 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4483 break;
4484 case CAYMAN_RING_TYPE_CP1_INDEX:
4485 case CAYMAN_RING_TYPE_CP2_INDEX:
4486 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4487 break;
4488 default:
876dc9f3 4489 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4490 ret = -EINVAL;
4491 break;
4492 }
4493 }
4494 idx += pkt.count + 2;
4495 break;
4496 default:
4497 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4498 ret = -EINVAL;
4499 break;
4500 }
4501 if (ret)
4502 break;
4503 } while (idx < ib->length_dw);
4504
4505 return ret;
4506}
4507
d2800ee5
AD		 4508/*
4509 * vm
4510 */
4511int si_vm_init(struct radeon_device *rdev)
4512{
4513 /* number of VMs */
4514 rdev->vm_manager.nvm = 16;
4515 /* base offset of vram pages */
4516 rdev->vm_manager.vram_base_offset = 0;
4517
4518 return 0;
4519}
4520
4521void si_vm_fini(struct radeon_device *rdev)
4522{
4523}
4524
fbf6dc7a
AD		 4525/**
4526 * si_vm_decode_fault - print human readable fault info
4527 *
4528 * @rdev: radeon_device pointer
4529 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4530 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4531 *
4532 * Print human readable fault information (SI).
4533 */
4534static void si_vm_decode_fault(struct radeon_device *rdev,
4535 u32 status, u32 addr)
4536{
4537 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4538 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4539 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4540 char *block;
4541
4542 if (rdev->family == CHIP_TAHITI) {
4543 switch (mc_id) {
4544 case 160:
4545 case 144:
4546 case 96:
4547 case 80:
4548 case 224:
4549 case 208:
4550 case 32:
4551 case 16:
4552 block = "CB";
4553 break;
4554 case 161:
4555 case 145:
4556 case 97:
4557 case 81:
4558 case 225:
4559 case 209:
4560 case 33:
4561 case 17:
4562 block = "CB_FMASK";
4563 break;
4564 case 162:
4565 case 146:
4566 case 98:
4567 case 82:
4568 case 226:
4569 case 210:
4570 case 34:
4571 case 18:
4572 block = "CB_CMASK";
4573 break;
4574 case 163:
4575 case 147:
4576 case 99:
4577 case 83:
4578 case 227:
4579 case 211:
4580 case 35:
4581 case 19:
4582 block = "CB_IMMED";
4583 break;
4584 case 164:
4585 case 148:
4586 case 100:
4587 case 84:
4588 case 228:
4589 case 212:
4590 case 36:
4591 case 20:
4592 block = "DB";
4593 break;
4594 case 165:
4595 case 149:
4596 case 101:
4597 case 85:
4598 case 229:
4599 case 213:
4600 case 37:
4601 case 21:
4602 block = "DB_HTILE";
4603 break;
4604 case 167:
4605 case 151:
4606 case 103:
4607 case 87:
4608 case 231:
4609 case 215:
4610 case 39:
4611 case 23:
4612 block = "DB_STEN";
4613 break;
4614 case 72:
4615 case 68:
4616 case 64:
4617 case 8:
4618 case 4:
4619 case 0:
4620 case 136:
4621 case 132:
4622 case 128:
4623 case 200:
4624 case 196:
4625 case 192:
4626 block = "TC";
4627 break;
4628 case 112:
4629 case 48:
4630 block = "CP";
4631 break;
4632 case 49:
4633 case 177:
4634 case 50:
4635 case 178:
4636 block = "SH";
4637 break;
4638 case 53:
4639 case 190:
4640 block = "VGT";
4641 break;
4642 case 117:
4643 block = "IH";
4644 break;
4645 case 51:
4646 case 115:
4647 block = "RLC";
4648 break;
4649 case 119:
4650 case 183:
4651 block = "DMA0";
4652 break;
4653 case 61:
4654 block = "DMA1";
4655 break;
4656 case 248:
4657 case 120:
4658 block = "HDP";
4659 break;
4660 default:
4661 block = "unknown";
4662 break;
4663 }
4664 } else {
4665 switch (mc_id) {
4666 case 32:
4667 case 16:
4668 case 96:
4669 case 80:
4670 case 160:
4671 case 144:
4672 case 224:
4673 case 208:
4674 block = "CB";
4675 break;
4676 case 33:
4677 case 17:
4678 case 97:
4679 case 81:
4680 case 161:
4681 case 145:
4682 case 225:
4683 case 209:
4684 block = "CB_FMASK";
4685 break;
4686 case 34:
4687 case 18:
4688 case 98:
4689 case 82:
4690 case 162:
4691 case 146:
4692 case 226:
4693 case 210:
4694 block = "CB_CMASK";
4695 break;
4696 case 35:
4697 case 19:
4698 case 99:
4699 case 83:
4700 case 163:
4701 case 147:
4702 case 227:
4703 case 211:
4704 block = "CB_IMMED";
4705 break;
4706 case 36:
4707 case 20:
4708 case 100:
4709 case 84:
4710 case 164:
4711 case 148:
4712 case 228:
4713 case 212:
4714 block = "DB";
4715 break;
4716 case 37:
4717 case 21:
4718 case 101:
4719 case 85:
4720 case 165:
4721 case 149:
4722 case 229:
4723 case 213:
4724 block = "DB_HTILE";
4725 break;
4726 case 39:
4727 case 23:
4728 case 103:
4729 case 87:
4730 case 167:
4731 case 151:
4732 case 231:
4733 case 215:
4734 block = "DB_STEN";
4735 break;
4736 case 72:
4737 case 68:
4738 case 8:
4739 case 4:
4740 case 136:
4741 case 132:
4742 case 200:
4743 case 196:
4744 block = "TC";
4745 break;
4746 case 112:
4747 case 48:
4748 block = "CP";
4749 break;
4750 case 49:
4751 case 177:
4752 case 50:
4753 case 178:
4754 block = "SH";
4755 break;
4756 case 53:
4757 block = "VGT";
4758 break;
4759 case 117:
4760 block = "IH";
4761 break;
4762 case 51:
4763 case 115:
4764 block = "RLC";
4765 break;
4766 case 119:
4767 case 183:
4768 block = "DMA0";
4769 break;
4770 case 61:
4771 block = "DMA1";
4772 break;
4773 case 248:
4774 case 120:
4775 block = "HDP";
4776 break;
4777 default:
4778 block = "unknown";
4779 break;
4780 }
4781 }
4782
4783 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4784 protections, vmid, addr,
4785 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4786 block, mc_id);
4787}
4788
498522b4 4789void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
d2800ee5 4790{
498522b4 4791 struct radeon_ring *ring = &rdev->ring[ridx];
d2800ee5 4792
ee60e29f 4793 if (vm == NULL)
d2800ee5
AD		 4794 return;
4795
76c44f2c
AD		 4796 /* write new base address */
4797 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4798 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4799 WRITE_DATA_DST_SEL(0)));
4800
ee60e29f 4801 if (vm->id < 8) {
76c44f2c
AD		 4802 radeon_ring_write(ring,
4803 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
ee60e29f 4804 } else {
76c44f2c
AD		 4805 radeon_ring_write(ring,
4806 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
ee60e29f 4807 }
76c44f2c 4808 radeon_ring_write(ring, 0);
fa87e62d 4809 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 4810
d2800ee5 4811 /* flush hdp cache */
76c44f2c
AD		 4812 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4813 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4814 WRITE_DATA_DST_SEL(0)));
4815 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4816 radeon_ring_write(ring, 0);
ee60e29f
CK		 4817 radeon_ring_write(ring, 0x1);
4818
d2800ee5 4819 /* bits 0-15 are the VM contexts0-15 */
76c44f2c
AD		 4820 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4821 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4822 WRITE_DATA_DST_SEL(0)));
4823 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4824 radeon_ring_write(ring, 0);
498522b4 4825 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 4826
4827 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4828 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4829 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 4830}
4831
f8f84ac5
AD		 4832/*
4833 * Power and clock gating
4834 */
4835static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4836{
4837 int i;
4838
4839 for (i = 0; i < rdev->usec_timeout; i++) {
4840 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4841 break;
4842 udelay(1);
4843 }
4844
4845 for (i = 0; i < rdev->usec_timeout; i++) {
4846 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4847 break;
4848 udelay(1);
4849 }
4850}
4851
4852static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4853 bool enable)
4854{
4855 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4856 u32 mask;
4857 int i;
4858
4859 if (enable)
4860 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4861 else
4862 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4863 WREG32(CP_INT_CNTL_RING0, tmp);
4864
4865 if (!enable) {
4866 /* read a gfx register */
4867 tmp = RREG32(DB_DEPTH_INFO);
4868
4869 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4870 for (i = 0; i < rdev->usec_timeout; i++) {
4871 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4872 break;
4873 udelay(1);
4874 }
4875 }
4876}
4877
4878static void si_set_uvd_dcm(struct radeon_device *rdev,
4879 bool sw_mode)
4880{
4881 u32 tmp, tmp2;
4882
4883 tmp = RREG32(UVD_CGC_CTRL);
4884 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4885 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4886
4887 if (sw_mode) {
4888 tmp &= ~0x7ffff800;
4889 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4890 } else {
4891 tmp |= 0x7ffff800;
4892 tmp2 = 0;
4893 }
4894
4895 WREG32(UVD_CGC_CTRL, tmp);
4896 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4897}
4898
22c775ce 4899void si_init_uvd_internal_cg(struct radeon_device *rdev)
f8f84ac5
AD		 4900{
4901 bool hw_mode = true;
4902
4903 if (hw_mode) {
4904 si_set_uvd_dcm(rdev, false);
4905 } else {
4906 u32 tmp = RREG32(UVD_CGC_CTRL);
4907 tmp &= ~DCM;
4908 WREG32(UVD_CGC_CTRL, tmp);
4909 }
4910}
4911
4912static u32 si_halt_rlc(struct radeon_device *rdev)
4913{
4914 u32 data, orig;
4915
4916 orig = data = RREG32(RLC_CNTL);
4917
4918 if (data & RLC_ENABLE) {
4919 data &= ~RLC_ENABLE;
4920 WREG32(RLC_CNTL, data);
4921
4922 si_wait_for_rlc_serdes(rdev);
4923 }
4924
4925 return orig;
4926}
4927
4928static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4929{
4930 u32 tmp;
4931
4932 tmp = RREG32(RLC_CNTL);
4933 if (tmp != rlc)
4934 WREG32(RLC_CNTL, rlc);
4935}
4936
4937static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4938{
4939 u32 data, orig;
4940
4941 orig = data = RREG32(DMA_PG);
e16866ec 4942 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
f8f84ac5
AD		 4943 data |= PG_CNTL_ENABLE;
4944 else
4945 data &= ~PG_CNTL_ENABLE;
4946 if (orig != data)
4947 WREG32(DMA_PG, data);
4948}
4949
4950static void si_init_dma_pg(struct radeon_device *rdev)
4951{
4952 u32 tmp;
4953
4954 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4955 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4956
4957 for (tmp = 0; tmp < 5; tmp++)
4958 WREG32(DMA_PGFSM_WRITE, 0);
4959}
4960
4961static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4962 bool enable)
4963{
4964 u32 tmp;
4965
2b19d17f 4966 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
f8f84ac5
AD		 4967 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4968 WREG32(RLC_TTOP_D, tmp);
4969
4970 tmp = RREG32(RLC_PG_CNTL);
4971 tmp |= GFX_PG_ENABLE;
4972 WREG32(RLC_PG_CNTL, tmp);
4973
4974 tmp = RREG32(RLC_AUTO_PG_CTRL);
4975 tmp |= AUTO_PG_EN;
4976 WREG32(RLC_AUTO_PG_CTRL, tmp);
4977 } else {
4978 tmp = RREG32(RLC_AUTO_PG_CTRL);
4979 tmp &= ~AUTO_PG_EN;
4980 WREG32(RLC_AUTO_PG_CTRL, tmp);
4981
4982 tmp = RREG32(DB_RENDER_CONTROL);
4983 }
4984}
4985
4986static void si_init_gfx_cgpg(struct radeon_device *rdev)
4987{
4988 u32 tmp;
4989
4990 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4991
4992 tmp = RREG32(RLC_PG_CNTL);
4993 tmp |= GFX_PG_SRC;
4994 WREG32(RLC_PG_CNTL, tmp);
4995
4996 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4997
4998 tmp = RREG32(RLC_AUTO_PG_CTRL);
4999
5000 tmp &= ~GRBM_REG_SGIT_MASK;
5001 tmp |= GRBM_REG_SGIT(0x700);
5002 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5003 WREG32(RLC_AUTO_PG_CTRL, tmp);
5004}
5005
ba19031a 5006static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
f8f84ac5
AD		 5007{
5008 u32 mask = 0, tmp, tmp1;
5009 int i;
5010
5011 si_select_se_sh(rdev, se, sh);
5012 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5013 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5014 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5015
5016 tmp &= 0xffff0000;
5017
5018 tmp |= tmp1;
5019 tmp >>= 16;
5020
5021 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5022 mask <<= 1;
5023 mask |= 1;
5024 }
5025
5026 return (~tmp) & mask;
5027}
5028
5029static void si_init_ao_cu_mask(struct radeon_device *rdev)
5030{
5031 u32 i, j, k, active_cu_number = 0;
5032 u32 mask, counter, cu_bitmap;
5033 u32 tmp = 0;
5034
5035 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5036 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5037 mask = 1;
5038 cu_bitmap = 0;
5039 counter = 0;
5040 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
ba19031a 5041 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
f8f84ac5
AD		 5042 if (counter < 2)
5043 cu_bitmap |= mask;
5044 counter++;
5045 }
5046 mask <<= 1;
5047 }
5048
5049 active_cu_number += counter;
5050 tmp |= (cu_bitmap << (i * 16 + j * 8));
5051 }
5052 }
5053
5054 WREG32(RLC_PG_AO_CU_MASK, tmp);
5055
5056 tmp = RREG32(RLC_MAX_PG_CU);
5057 tmp &= ~MAX_PU_CU_MASK;
5058 tmp |= MAX_PU_CU(active_cu_number);
5059 WREG32(RLC_MAX_PG_CU, tmp);
5060}
5061
5062static void si_enable_cgcg(struct radeon_device *rdev,
5063 bool enable)
5064{
5065 u32 data, orig, tmp;
5066
5067 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5068
e16866ec 5069 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5594a558 5070 si_enable_gui_idle_interrupt(rdev, true);
f8f84ac5 5071
f8f84ac5
AD		 5072 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5073
5074 tmp = si_halt_rlc(rdev);
5075
5076 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5077 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5078 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5079
5080 si_wait_for_rlc_serdes(rdev);
5081
5082 si_update_rlc(rdev, tmp);
5083
5084 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5085
5086 data |= CGCG_EN | CGLS_EN;
5087 } else {
5594a558
AD		 5088 si_enable_gui_idle_interrupt(rdev, false);
5089
f8f84ac5
AD		 5090 RREG32(CB_CGTT_SCLK_CTRL);
5091 RREG32(CB_CGTT_SCLK_CTRL);
5092 RREG32(CB_CGTT_SCLK_CTRL);
5093 RREG32(CB_CGTT_SCLK_CTRL);
5094
5095 data &= ~(CGCG_EN | CGLS_EN);
5096 }
5097
5098 if (orig != data)
5099 WREG32(RLC_CGCG_CGLS_CTRL, data);
5100}
5101
5102static void si_enable_mgcg(struct radeon_device *rdev,
5103 bool enable)
5104{
5105 u32 data, orig, tmp = 0;
5106
e16866ec 5107 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
f8f84ac5
AD		 5108 orig = data = RREG32(CGTS_SM_CTRL_REG);
5109 data = 0x96940200;
5110 if (orig != data)
5111 WREG32(CGTS_SM_CTRL_REG, data);
5112
e16866ec
AD		 5113 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5114 orig = data = RREG32(CP_MEM_SLP_CNTL);
5115 data |= CP_MEM_LS_EN;
5116 if (orig != data)
5117 WREG32(CP_MEM_SLP_CNTL, data);
5118 }
f8f84ac5
AD		 5119
5120 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5121 data &= 0xffffffc0;
5122 if (orig != data)
5123 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5124
5125 tmp = si_halt_rlc(rdev);
5126
5127 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5128 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5129 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5130
5131 si_update_rlc(rdev, tmp);
5132 } else {
5133 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5134 data |= 0x00000003;
5135 if (orig != data)
5136 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5137
5138 data = RREG32(CP_MEM_SLP_CNTL);
5139 if (data & CP_MEM_LS_EN) {
5140 data &= ~CP_MEM_LS_EN;
5141 WREG32(CP_MEM_SLP_CNTL, data);
5142 }
5143 orig = data = RREG32(CGTS_SM_CTRL_REG);
5144 data |= LS_OVERRIDE | OVERRIDE;
5145 if (orig != data)
5146 WREG32(CGTS_SM_CTRL_REG, data);
5147
5148 tmp = si_halt_rlc(rdev);
5149
5150 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5151 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5152 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5153
5154 si_update_rlc(rdev, tmp);
5155 }
5156}
5157
5158static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5159 bool enable)
5160{
5161 u32 orig, data, tmp;
5162
e16866ec 5163 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
f8f84ac5
AD		 5164 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5165 tmp |= 0x3fff;
5166 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5167
5168 orig = data = RREG32(UVD_CGC_CTRL);
5169 data |= DCM;
5170 if (orig != data)
5171 WREG32(UVD_CGC_CTRL, data);
5172
5173 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5174 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5175 } else {
5176 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5177 tmp &= ~0x3fff;
5178 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5179
5180 orig = data = RREG32(UVD_CGC_CTRL);
5181 data &= ~DCM;
5182 if (orig != data)
5183 WREG32(UVD_CGC_CTRL, data);
5184
5185 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5186 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5187 }
5188}
5189
5190static const u32 mc_cg_registers[] =
5191{
5192 MC_HUB_MISC_HUB_CG,
5193 MC_HUB_MISC_SIP_CG,
5194 MC_HUB_MISC_VM_CG,
5195 MC_XPB_CLK_GAT,
5196 ATC_MISC_CG,
5197 MC_CITF_MISC_WR_CG,
5198 MC_CITF_MISC_RD_CG,
5199 MC_CITF_MISC_VM_CG,
5200 VM_L2_CG,
5201};
5202
5203static void si_enable_mc_ls(struct radeon_device *rdev,
5204 bool enable)
5205{
5206 int i;
5207 u32 orig, data;
5208
5209 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5210 orig = data = RREG32(mc_cg_registers[i]);
e16866ec 5211 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
f8f84ac5
AD		 5212 data |= MC_LS_ENABLE;
5213 else
5214 data &= ~MC_LS_ENABLE;
5215 if (data != orig)
5216 WREG32(mc_cg_registers[i], data);
5217 }
5218}
5219
e16866ec
AD		 5220static void si_enable_mc_mgcg(struct radeon_device *rdev,
5221 bool enable)
f8f84ac5 5222{
e16866ec
AD		 5223 int i;
5224 u32 orig, data;
5225
5226 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5227 orig = data = RREG32(mc_cg_registers[i]);
5228 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5229 data |= MC_CG_ENABLE;
5230 else
5231 data &= ~MC_CG_ENABLE;
5232 if (data != orig)
5233 WREG32(mc_cg_registers[i], data);
f8f84ac5
AD		 5234 }
5235}
5236
e16866ec
AD		 5237static void si_enable_dma_mgcg(struct radeon_device *rdev,
5238 bool enable)
f8f84ac5 5239{
e16866ec
AD		 5240 u32 orig, data, offset;
5241 int i;
f8f84ac5 5242
e16866ec
AD		 5243 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5244 for (i = 0; i < 2; i++) {
5245 if (i == 0)
5246 offset = DMA0_REGISTER_OFFSET;
5247 else
5248 offset = DMA1_REGISTER_OFFSET;
5249 orig = data = RREG32(DMA_POWER_CNTL + offset);
5250 data &= ~MEM_POWER_OVERRIDE;
5251 if (data != orig)
5252 WREG32(DMA_POWER_CNTL + offset, data);
5253 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5254 }
5255 } else {
5256 for (i = 0; i < 2; i++) {
5257 if (i == 0)
5258 offset = DMA0_REGISTER_OFFSET;
5259 else
5260 offset = DMA1_REGISTER_OFFSET;
5261 orig = data = RREG32(DMA_POWER_CNTL + offset);
5262 data |= MEM_POWER_OVERRIDE;
5263 if (data != orig)
5264 WREG32(DMA_POWER_CNTL + offset, data);
5265
5266 orig = data = RREG32(DMA_CLK_CTRL + offset);
5267 data = 0xff000000;
5268 if (data != orig)
5269 WREG32(DMA_CLK_CTRL + offset, data);
5270 }
5271 }
f8f84ac5
AD		 5272}
5273
e16866ec
AD		 5274static void si_enable_bif_mgls(struct radeon_device *rdev,
5275 bool enable)
f8f84ac5 5276{
e16866ec 5277 u32 orig, data;
f8f84ac5 5278
e16866ec 5279 orig = data = RREG32_PCIE(PCIE_CNTL2);
f8f84ac5 5280
e16866ec
AD		 5281 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5282 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5283 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5284 else
5285 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5286 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5287
5288 if (orig != data)
5289 WREG32_PCIE(PCIE_CNTL2, data);
f8f84ac5
AD		 5290}
5291
e16866ec
AD		 5292static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5293 bool enable)
f8f84ac5 5294{
e16866ec 5295 u32 orig, data;
f8f84ac5 5296
e16866ec 5297 orig = data = RREG32(HDP_HOST_PATH_CNTL);
f8f84ac5 5298
e16866ec
AD		 5299 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5300 data &= ~CLOCK_GATING_DIS;
5301 else
5302 data |= CLOCK_GATING_DIS;
5303
5304 if (orig != data)
5305 WREG32(HDP_HOST_PATH_CNTL, data);
f8f84ac5
AD		 5306}
5307
e16866ec
AD		 5308static void si_enable_hdp_ls(struct radeon_device *rdev,
5309 bool enable)
347e7592 5310{
e16866ec 5311 u32 orig, data;
347e7592 5312
e16866ec 5313 orig = data = RREG32(HDP_MEM_POWER_LS);
347e7592 5314
e16866ec
AD		 5315 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5316 data |= HDP_LS_ENABLE;
5317 else
5318 data &= ~HDP_LS_ENABLE;
347e7592 5319
e16866ec
AD		 5320 if (orig != data)
5321 WREG32(HDP_MEM_POWER_LS, data);
347e7592
AD		 5322}
5323
68e3a092
AD		 5324static void si_update_cg(struct radeon_device *rdev,
5325 u32 block, bool enable)
347e7592 5326{
e16866ec 5327 if (block & RADEON_CG_BLOCK_GFX) {
811e4d58 5328 si_enable_gui_idle_interrupt(rdev, false);
e16866ec
AD		 5329 /* order matters! */
5330 if (enable) {
5331 si_enable_mgcg(rdev, true);
5332 si_enable_cgcg(rdev, true);
5333 } else {
5334 si_enable_cgcg(rdev, false);
5335 si_enable_mgcg(rdev, false);
347e7592 5336 }
811e4d58 5337 si_enable_gui_idle_interrupt(rdev, true);
347e7592
AD		 5338 }
5339
e16866ec
AD		 5340 if (block & RADEON_CG_BLOCK_MC) {
5341 si_enable_mc_mgcg(rdev, enable);
5342 si_enable_mc_ls(rdev, enable);
347e7592 5343 }
e16866ec
AD		 5344
5345 if (block & RADEON_CG_BLOCK_SDMA) {
5346 si_enable_dma_mgcg(rdev, enable);
347e7592
AD		 5347 }
5348
e16866ec
AD		 5349 if (block & RADEON_CG_BLOCK_BIF) {
5350 si_enable_bif_mgls(rdev, enable);
6d8cf000 5351 }
6d8cf000 5352
e16866ec
AD		 5353 if (block & RADEON_CG_BLOCK_UVD) {
5354 if (rdev->has_uvd) {
5355 si_enable_uvd_mgcg(rdev, enable);
bd8cd539
AD		 5356 }
5357 }
bd8cd539 5358
e16866ec
AD		 5359 if (block & RADEON_CG_BLOCK_HDP) {
5360 si_enable_hdp_mgcg(rdev, enable);
5361 si_enable_hdp_ls(rdev, enable);
347e7592 5362 }
e16866ec 5363}
f8f84ac5
AD		 5364
5365static void si_init_cg(struct radeon_device *rdev)
5366{
e16866ec
AD		 5367 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5368 RADEON_CG_BLOCK_MC |
5369 RADEON_CG_BLOCK_SDMA |
5370 RADEON_CG_BLOCK_BIF |
5371 RADEON_CG_BLOCK_HDP), true);
b2d70917 5372 if (rdev->has_uvd) {
e16866ec 5373 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
f8f84ac5 5374 si_init_uvd_internal_cg(rdev);
347e7592 5375 }
f8f84ac5 5376}
bd8cd539 5377
f8f84ac5
AD		 5378static void si_fini_cg(struct radeon_device *rdev)
5379{
0116e1ef 5380 if (rdev->has_uvd) {
e16866ec 5381 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
347e7592 5382 }
e16866ec
AD		 5383 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5384 RADEON_CG_BLOCK_MC |
5385 RADEON_CG_BLOCK_SDMA |
5386 RADEON_CG_BLOCK_BIF |
5387 RADEON_CG_BLOCK_HDP), false);
5388}
5389
59a82d0e
AD		 5390u32 si_get_csb_size(struct radeon_device *rdev)
5391{
5392 u32 count = 0;
5393 const struct cs_section_def *sect = NULL;
5394 const struct cs_extent_def *ext = NULL;
5395
5396 if (rdev->rlc.cs_data == NULL)
5397 return 0;
5398
5399 /* begin clear state */
5400 count += 2;
5401 /* context control state */
5402 count += 3;
5403
5404 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5405 for (ext = sect->section; ext->extent != NULL; ++ext) {
5406 if (sect->id == SECT_CONTEXT)
5407 count += 2 + ext->reg_count;
5408 else
5409 return 0;
5410 }
bd8cd539 5411 }
59a82d0e
AD		 5412 /* pa_sc_raster_config */
5413 count += 3;
5414 /* end clear state */
5415 count += 2;
5416 /* clear state */
5417 count += 2;
5418
5419 return count;
5420}
5421
5422void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5423{
5424 u32 count = 0, i;
5425 const struct cs_section_def *sect = NULL;
5426 const struct cs_extent_def *ext = NULL;
5427
5428 if (rdev->rlc.cs_data == NULL)
5429 return;
5430 if (buffer == NULL)
5431 return;
5432
6ba81e53
AD		 5433 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5434 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
59a82d0e 5435
6ba81e53
AD		 5436 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5437 buffer[count++] = cpu_to_le32(0x80000000);
5438 buffer[count++] = cpu_to_le32(0x80000000);
59a82d0e
AD		 5439
5440 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5441 for (ext = sect->section; ext->extent != NULL; ++ext) {
5442 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 5443 buffer[count++] =
5444 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5445 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
59a82d0e 5446 for (i = 0; i < ext->reg_count; i++)
6ba81e53 5447 buffer[count++] = cpu_to_le32(ext->extent[i]);
59a82d0e
AD		 5448 } else {
5449 return;
bd8cd539 5450 }
bd8cd539
AD		 5451 }
5452 }
bd8cd539 5453
6ba81e53
AD		 5454 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5455 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
59a82d0e
AD		 5456 switch (rdev->family) {
5457 case CHIP_TAHITI:
5458 case CHIP_PITCAIRN:
6ba81e53 5459 buffer[count++] = cpu_to_le32(0x2a00126a);
59a82d0e
AD		 5460 break;
5461 case CHIP_VERDE:
6ba81e53 5462 buffer[count++] = cpu_to_le32(0x0000124a);
59a82d0e
AD		 5463 break;
5464 case CHIP_OLAND:
6ba81e53 5465 buffer[count++] = cpu_to_le32(0x00000082);
59a82d0e
AD		 5466 break;
5467 case CHIP_HAINAN:
6ba81e53 5468 buffer[count++] = cpu_to_le32(0x00000000);
59a82d0e
AD		 5469 break;
5470 default:
6ba81e53 5471 buffer[count++] = cpu_to_le32(0x00000000);
59a82d0e
AD		 5472 break;
5473 }
5474
6ba81e53
AD		 5475 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5476 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
347e7592 5477
6ba81e53
AD		 5478 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5479 buffer[count++] = cpu_to_le32(0);
59a82d0e
AD		 5480}
5481
f8f84ac5
AD		 5482static void si_init_pg(struct radeon_device *rdev)
5483{
0116e1ef
AD		 5484 if (rdev->pg_flags) {
5485 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5486 si_init_dma_pg(rdev);
0116e1ef 5487 }
f8f84ac5 5488 si_init_ao_cu_mask(rdev);
2b19d17f 5489 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
0116e1ef 5490 si_init_gfx_cgpg(rdev);
aa34dba8
AD		 5491 } else {
5492 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5493 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
0116e1ef 5494 }
ca6ebb39
AD		 5495 si_enable_dma_pg(rdev, true);
5496 si_enable_gfx_cgpg(rdev, true);
f8f84ac5
AD		 5497 } else {
5498 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5499 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5500 }
5501}
5502
5503static void si_fini_pg(struct radeon_device *rdev)
5504{
0116e1ef 5505 if (rdev->pg_flags) {
ca6ebb39
AD		 5506 si_enable_dma_pg(rdev, false);
5507 si_enable_gfx_cgpg(rdev, false);
f8f84ac5 5508 }
347e7592
AD		 5509}
5510
347e7592
AD		 5511/*
5512 * RLC
5513 */
866d83de 5514void si_rlc_reset(struct radeon_device *rdev)
d719cef3 5515{
f8f84ac5 5516 u32 tmp = RREG32(GRBM_SOFT_RESET);
d719cef3 5517
f8f84ac5
AD		 5518 tmp |= SOFT_RESET_RLC;
5519 WREG32(GRBM_SOFT_RESET, tmp);
5520 udelay(50);
5521 tmp &= ~SOFT_RESET_RLC;
5522 WREG32(GRBM_SOFT_RESET, tmp);
5523 udelay(50);
d719cef3
AD		 5524}
5525
347e7592
AD		 5526static void si_rlc_stop(struct radeon_device *rdev)
5527{
5528 WREG32(RLC_CNTL, 0);
d719cef3
AD		 5529
5530 si_enable_gui_idle_interrupt(rdev, false);
5531
5532 si_wait_for_rlc_serdes(rdev);
347e7592
AD		 5533}
5534
5535static void si_rlc_start(struct radeon_device *rdev)
5536{
5537 WREG32(RLC_CNTL, RLC_ENABLE);
d719cef3
AD		 5538
5539 si_enable_gui_idle_interrupt(rdev, true);
5540
5541 udelay(50);
5542}
5543
5544static bool si_lbpw_supported(struct radeon_device *rdev)
5545{
5546 u32 tmp;
5547
5548 /* Enable LBPW only for DDR3 */
5549 tmp = RREG32(MC_SEQ_MISC0);
5550 if ((tmp & 0xF0000000) == 0xB0000000)
5551 return true;
5552 return false;
5553}
5554
5555static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5556{
5557 u32 tmp;
5558
5559 tmp = RREG32(RLC_LB_CNTL);
5560 if (enable)
5561 tmp |= LOAD_BALANCE_ENABLE;
5562 else
5563 tmp &= ~LOAD_BALANCE_ENABLE;
5564 WREG32(RLC_LB_CNTL, tmp);
5565
5566 if (!enable) {
5567 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5568 WREG32(SPI_LB_CU_MASK, 0x00ff);
5569 }
347e7592
AD		 5570}
5571
5572static int si_rlc_resume(struct radeon_device *rdev)
5573{
5574 u32 i;
5575 const __be32 *fw_data;
5576
5577 if (!rdev->rlc_fw)
5578 return -EINVAL;
5579
5580 si_rlc_stop(rdev);
5581
f8f84ac5
AD		 5582 si_rlc_reset(rdev);
5583
5584 si_init_pg(rdev);
5585
5586 si_init_cg(rdev);
5587
347e7592
AD		 5588 WREG32(RLC_RL_BASE, 0);
5589 WREG32(RLC_RL_SIZE, 0);
5590 WREG32(RLC_LB_CNTL, 0);
5591 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5592 WREG32(RLC_LB_CNTR_INIT, 0);
d719cef3 5593 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
347e7592 5594
347e7592
AD		 5595 WREG32(RLC_MC_CNTL, 0);
5596 WREG32(RLC_UCODE_CNTL, 0);
5597
5598 fw_data = (const __be32 *)rdev->rlc_fw->data;
5599 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5600 WREG32(RLC_UCODE_ADDR, i);
5601 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5602 }
5603 WREG32(RLC_UCODE_ADDR, 0);
5604
d719cef3
AD		 5605 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5606
347e7592
AD		 5607 si_rlc_start(rdev);
5608
5609 return 0;
5610}
5611
25a857fb
AD		 5612static void si_enable_interrupts(struct radeon_device *rdev)
5613{
5614 u32 ih_cntl = RREG32(IH_CNTL);
5615 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5616
5617 ih_cntl |= ENABLE_INTR;
5618 ih_rb_cntl |= IH_RB_ENABLE;
5619 WREG32(IH_CNTL, ih_cntl);
5620 WREG32(IH_RB_CNTL, ih_rb_cntl);
5621 rdev->ih.enabled = true;
5622}
5623
5624static void si_disable_interrupts(struct radeon_device *rdev)
5625{
5626 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5627 u32 ih_cntl = RREG32(IH_CNTL);
5628
5629 ih_rb_cntl &= ~IH_RB_ENABLE;
5630 ih_cntl &= ~ENABLE_INTR;
5631 WREG32(IH_RB_CNTL, ih_rb_cntl);
5632 WREG32(IH_CNTL, ih_cntl);
5633 /* set rptr, wptr to 0 */
5634 WREG32(IH_RB_RPTR, 0);
5635 WREG32(IH_RB_WPTR, 0);
5636 rdev->ih.enabled = false;
25a857fb
AD		 5637 rdev->ih.rptr = 0;
5638}
5639
5640static void si_disable_interrupt_state(struct radeon_device *rdev)
5641{
5642 u32 tmp;
5643
811e4d58
AD		 5644 tmp = RREG32(CP_INT_CNTL_RING0) &
5645 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5646 WREG32(CP_INT_CNTL_RING0, tmp);
25a857fb
AD		 5647 WREG32(CP_INT_CNTL_RING1, 0);
5648 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 5649 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5650 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5651 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5652 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 5653 WREG32(GRBM_INT_CNTL, 0);
5153550a
AD		 5654 if (rdev->num_crtc >= 2) {
5655 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5656 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5657 }
25a857fb
AD		 5658 if (rdev->num_crtc >= 4) {
5659 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5660 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5661 }
5662 if (rdev->num_crtc >= 6) {
5663 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5664 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5665 }
5666
5153550a
AD		 5667 if (rdev->num_crtc >= 2) {
5668 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5669 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5670 }
25a857fb
AD		 5671 if (rdev->num_crtc >= 4) {
5672 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5673 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5674 }
5675 if (rdev->num_crtc >= 6) {
5676 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5677 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5678 }
5679
5153550a
AD		 5680 if (!ASIC_IS_NODCE(rdev)) {
5681 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
5682
5683 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5684 WREG32(DC_HPD1_INT_CONTROL, tmp);
5685 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5686 WREG32(DC_HPD2_INT_CONTROL, tmp);
5687 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5688 WREG32(DC_HPD3_INT_CONTROL, tmp);
5689 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5690 WREG32(DC_HPD4_INT_CONTROL, tmp);
5691 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5692 WREG32(DC_HPD5_INT_CONTROL, tmp);
5693 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5694 WREG32(DC_HPD6_INT_CONTROL, tmp);
5695 }
25a857fb
AD		 5696}
5697
5698static int si_irq_init(struct radeon_device *rdev)
5699{
5700 int ret = 0;
5701 int rb_bufsz;
5702 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5703
5704 /* allocate ring */
5705 ret = r600_ih_ring_alloc(rdev);
5706 if (ret)
5707 return ret;
5708
5709 /* disable irqs */
5710 si_disable_interrupts(rdev);
5711
5712 /* init rlc */
5713 ret = si_rlc_resume(rdev);
5714 if (ret) {
5715 r600_ih_ring_fini(rdev);
5716 return ret;
5717 }
5718
5719 /* setup interrupt control */
5720 /* set dummy read address to ring address */
5721 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5722 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5723 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5724 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5725 */
5726 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5727 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5728 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5729 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5730
5731 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 5732 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
25a857fb
AD		 5733
5734 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5735 IH_WPTR_OVERFLOW_CLEAR |
5736 (rb_bufsz << 1));
5737
5738 if (rdev->wb.enabled)
5739 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5740
5741 /* set the writeback address whether it's enabled or not */
5742 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5743 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5744
5745 WREG32(IH_RB_CNTL, ih_rb_cntl);
5746
5747 /* set rptr, wptr to 0 */
5748 WREG32(IH_RB_RPTR, 0);
5749 WREG32(IH_RB_WPTR, 0);
5750
5751 /* Default settings for IH_CNTL (disabled at first) */
5752 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5753 /* RPTR_REARM only works if msi's are enabled */
5754 if (rdev->msi_enabled)
5755 ih_cntl |= RPTR_REARM;
5756 WREG32(IH_CNTL, ih_cntl);
5757
5758 /* force the active interrupt state to all disabled */
5759 si_disable_interrupt_state(rdev);
5760
2099810f
DA		 5761 pci_set_master(rdev->pdev);
5762
25a857fb
AD		 5763 /* enable irqs */
5764 si_enable_interrupts(rdev);
5765
5766 return ret;
5767}
5768
5769int si_irq_set(struct radeon_device *rdev)
5770{
811e4d58 5771 u32 cp_int_cntl;
25a857fb
AD		 5772 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5773 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5153550a 5774 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
25a857fb
AD		 5775 u32 grbm_int_cntl = 0;
5776 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
8c5fd7ef 5777 u32 dma_cntl, dma_cntl1;
a9e61410 5778 u32 thermal_int = 0;
25a857fb
AD		 5779
5780 if (!rdev->irq.installed) {
5781 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5782 return -EINVAL;
5783 }
5784 /* don't enable anything if the ih is disabled */
5785 if (!rdev->ih.enabled) {
5786 si_disable_interrupts(rdev);
5787 /* force the active interrupt state to all disabled */
5788 si_disable_interrupt_state(rdev);
5789 return 0;
5790 }
5791
811e4d58
AD		 5792 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
5793 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5794
5153550a
AD		 5795 if (!ASIC_IS_NODCE(rdev)) {
5796 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5797 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5798 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5799 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5800 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5801 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5802 }
25a857fb 5803
8c5fd7ef
AD		 5804 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5805 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5806
a9e61410
AD		 5807 thermal_int = RREG32(CG_THERMAL_INT) &
5808 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5809
25a857fb 5810 /* enable CP interrupts on all rings */
736fc37f 5811 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 5812 DRM_DEBUG("si_irq_set: sw int gfx\n");
5813 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5814 }
736fc37f 5815 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 5816 DRM_DEBUG("si_irq_set: sw int cp1\n");
5817 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5818 }
736fc37f 5819 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 5820 DRM_DEBUG("si_irq_set: sw int cp2\n");
5821 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5822 }
8c5fd7ef
AD		 5823 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5824 DRM_DEBUG("si_irq_set: sw int dma\n");
5825 dma_cntl |= TRAP_ENABLE;
5826 }
5827
5828 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5829 DRM_DEBUG("si_irq_set: sw int dma1\n");
5830 dma_cntl1 |= TRAP_ENABLE;
5831 }
25a857fb 5832 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 5833 atomic_read(&rdev->irq.pflip[0])) {
25a857fb
AD		 5834 DRM_DEBUG("si_irq_set: vblank 0\n");
5835 crtc1 |= VBLANK_INT_MASK;
5836 }
5837 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 5838 atomic_read(&rdev->irq.pflip[1])) {
25a857fb
AD		 5839 DRM_DEBUG("si_irq_set: vblank 1\n");
5840 crtc2 |= VBLANK_INT_MASK;
5841 }
5842 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 5843 atomic_read(&rdev->irq.pflip[2])) {
25a857fb
AD		 5844 DRM_DEBUG("si_irq_set: vblank 2\n");
5845 crtc3 |= VBLANK_INT_MASK;
5846 }
5847 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 5848 atomic_read(&rdev->irq.pflip[3])) {
25a857fb
AD		 5849 DRM_DEBUG("si_irq_set: vblank 3\n");
5850 crtc4 |= VBLANK_INT_MASK;
5851 }
5852 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 5853 atomic_read(&rdev->irq.pflip[4])) {
25a857fb
AD		 5854 DRM_DEBUG("si_irq_set: vblank 4\n");
5855 crtc5 |= VBLANK_INT_MASK;
5856 }
5857 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 5858 atomic_read(&rdev->irq.pflip[5])) {
25a857fb
AD		 5859 DRM_DEBUG("si_irq_set: vblank 5\n");
5860 crtc6 |= VBLANK_INT_MASK;
5861 }
5862 if (rdev->irq.hpd[0]) {
5863 DRM_DEBUG("si_irq_set: hpd 1\n");
5864 hpd1 |= DC_HPDx_INT_EN;
5865 }
5866 if (rdev->irq.hpd[1]) {
5867 DRM_DEBUG("si_irq_set: hpd 2\n");
5868 hpd2 |= DC_HPDx_INT_EN;
5869 }
5870 if (rdev->irq.hpd[2]) {
5871 DRM_DEBUG("si_irq_set: hpd 3\n");
5872 hpd3 |= DC_HPDx_INT_EN;
5873 }
5874 if (rdev->irq.hpd[3]) {
5875 DRM_DEBUG("si_irq_set: hpd 4\n");
5876 hpd4 |= DC_HPDx_INT_EN;
5877 }
5878 if (rdev->irq.hpd[4]) {
5879 DRM_DEBUG("si_irq_set: hpd 5\n");
5880 hpd5 |= DC_HPDx_INT_EN;
5881 }
5882 if (rdev->irq.hpd[5]) {
5883 DRM_DEBUG("si_irq_set: hpd 6\n");
5884 hpd6 |= DC_HPDx_INT_EN;
5885 }
25a857fb
AD		 5886
5887 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5888 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5889 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5890
8c5fd7ef
AD		 5891 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5892 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5893
25a857fb
AD		 5894 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5895
a9e61410
AD		 5896 if (rdev->irq.dpm_thermal) {
5897 DRM_DEBUG("dpm thermal\n");
5898 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5899 }
5900
5153550a
AD		 5901 if (rdev->num_crtc >= 2) {
5902 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5903 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5904 }
25a857fb
AD		 5905 if (rdev->num_crtc >= 4) {
5906 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5907 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5908 }
5909 if (rdev->num_crtc >= 6) {
5910 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5911 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5912 }
5913
5153550a
AD		 5914 if (rdev->num_crtc >= 2) {
5915 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5916 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5917 }
25a857fb
AD		 5918 if (rdev->num_crtc >= 4) {
5919 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5920 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5921 }
5922 if (rdev->num_crtc >= 6) {
5923 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5924 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5925 }
5926
5153550a
AD		 5927 if (!ASIC_IS_NODCE(rdev)) {
5928 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5929 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5930 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5931 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5932 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5933 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5934 }
25a857fb 5935
a9e61410
AD		 5936 WREG32(CG_THERMAL_INT, thermal_int);
5937
25a857fb
AD		 5938 return 0;
5939}
5940
5941static inline void si_irq_ack(struct radeon_device *rdev)
5942{
5943 u32 tmp;
5944
5153550a
AD		 5945 if (ASIC_IS_NODCE(rdev))
5946 return;
5947
25a857fb
AD		 5948 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5949 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5950 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5951 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5952 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5953 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5954 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5955 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5956 if (rdev->num_crtc >= 4) {
5957 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5958 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5959 }
5960 if (rdev->num_crtc >= 6) {
5961 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5962 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5963 }
5964
5965 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5966 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5967 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5968 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5969 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5970 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5971 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5972 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5973 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5974 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5975 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5976 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5977
5978 if (rdev->num_crtc >= 4) {
5979 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5980 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5981 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5982 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5983 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5984 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5985 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5986 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5987 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5988 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5989 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5990 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5991 }
5992
5993 if (rdev->num_crtc >= 6) {
5994 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5995 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5996 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5997 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5998 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5999 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6000 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6001 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6002 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6003 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6004 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6005 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6006 }
6007
6008 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6009 tmp = RREG32(DC_HPD1_INT_CONTROL);
6010 tmp |= DC_HPDx_INT_ACK;
6011 WREG32(DC_HPD1_INT_CONTROL, tmp);
6012 }
6013 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6014 tmp = RREG32(DC_HPD2_INT_CONTROL);
6015 tmp |= DC_HPDx_INT_ACK;
6016 WREG32(DC_HPD2_INT_CONTROL, tmp);
6017 }
6018 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6019 tmp = RREG32(DC_HPD3_INT_CONTROL);
6020 tmp |= DC_HPDx_INT_ACK;
6021 WREG32(DC_HPD3_INT_CONTROL, tmp);
6022 }
6023 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6024 tmp = RREG32(DC_HPD4_INT_CONTROL);
6025 tmp |= DC_HPDx_INT_ACK;
6026 WREG32(DC_HPD4_INT_CONTROL, tmp);
6027 }
6028 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6029 tmp = RREG32(DC_HPD5_INT_CONTROL);
6030 tmp |= DC_HPDx_INT_ACK;
6031 WREG32(DC_HPD5_INT_CONTROL, tmp);
6032 }
6033 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6034 tmp = RREG32(DC_HPD5_INT_CONTROL);
6035 tmp |= DC_HPDx_INT_ACK;
6036 WREG32(DC_HPD6_INT_CONTROL, tmp);
6037 }
6038}
6039
6040static void si_irq_disable(struct radeon_device *rdev)
6041{
6042 si_disable_interrupts(rdev);
6043 /* Wait and acknowledge irq */
6044 mdelay(1);
6045 si_irq_ack(rdev);
6046 si_disable_interrupt_state(rdev);
6047}
6048
6049static void si_irq_suspend(struct radeon_device *rdev)
6050{
6051 si_irq_disable(rdev);
6052 si_rlc_stop(rdev);
6053}
6054
9b136d51
AD		 6055static void si_irq_fini(struct radeon_device *rdev)
6056{
6057 si_irq_suspend(rdev);
6058 r600_ih_ring_fini(rdev);
6059}
6060
25a857fb
AD		 6061static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6062{
6063 u32 wptr, tmp;
6064
6065 if (rdev->wb.enabled)
6066 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6067 else
6068 wptr = RREG32(IH_RB_WPTR);
6069
6070 if (wptr & RB_OVERFLOW) {
6071 /* When a ring buffer overflow happen start parsing interrupt
6072 * from the last not overwritten vector (wptr + 16). Hopefully
6073 * this should allow us to catchup.
6074 */
6075 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6076 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6077 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6078 tmp = RREG32(IH_RB_CNTL);
6079 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6080 WREG32(IH_RB_CNTL, tmp);
6081 }
6082 return (wptr & rdev->ih.ptr_mask);
6083}
6084
6085/* SI IV Ring
6086 * Each IV ring entry is 128 bits:
6087 * [7:0] - interrupt source id
6088 * [31:8] - reserved
6089 * [59:32] - interrupt source data
6090 * [63:60] - reserved
6091 * [71:64] - RINGID
6092 * [79:72] - VMID
6093 * [127:80] - reserved
6094 */
6095int si_irq_process(struct radeon_device *rdev)
6096{
6097 u32 wptr;
6098 u32 rptr;
6099 u32 src_id, src_data, ring_id;
6100 u32 ring_index;
25a857fb 6101 bool queue_hotplug = false;
a9e61410 6102 bool queue_thermal = false;
fbf6dc7a 6103 u32 status, addr;
25a857fb
AD		 6104
6105 if (!rdev->ih.enabled || rdev->shutdown)
6106 return IRQ_NONE;
6107
6108 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 6109
6110restart_ih:
6111 /* is somebody else already processing irqs? */
6112 if (atomic_xchg(&rdev->ih.lock, 1))
6113 return IRQ_NONE;
6114
25a857fb
AD		 6115 rptr = rdev->ih.rptr;
6116 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6117
25a857fb
AD		 6118 /* Order reading of wptr vs. reading of IH ring data */
6119 rmb();
6120
6121 /* display interrupts */
6122 si_irq_ack(rdev);
6123
25a857fb
AD		 6124 while (rptr != wptr) {
6125 /* wptr/rptr are in bytes! */
6126 ring_index = rptr / 4;
6127 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6128 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6129 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6130
6131 switch (src_id) {
6132 case 1: /* D1 vblank/vline */
6133 switch (src_data) {
6134 case 0: /* D1 vblank */
6135 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6136 if (rdev->irq.crtc_vblank_int[0]) {
6137 drm_handle_vblank(rdev->ddev, 0);
6138 rdev->pm.vblank_sync = true;
6139 wake_up(&rdev->irq.vblank_queue);
6140 }
736fc37f 6141 if (atomic_read(&rdev->irq.pflip[0]))
25a857fb
AD		 6142 radeon_crtc_handle_flip(rdev, 0);
6143 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6144 DRM_DEBUG("IH: D1 vblank\n");
6145 }
6146 break;
6147 case 1: /* D1 vline */
6148 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6149 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6150 DRM_DEBUG("IH: D1 vline\n");
6151 }
6152 break;
6153 default:
6154 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6155 break;
6156 }
6157 break;
6158 case 2: /* D2 vblank/vline */
6159 switch (src_data) {
6160 case 0: /* D2 vblank */
6161 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6162 if (rdev->irq.crtc_vblank_int[1]) {
6163 drm_handle_vblank(rdev->ddev, 1);
6164 rdev->pm.vblank_sync = true;
6165 wake_up(&rdev->irq.vblank_queue);
6166 }
736fc37f 6167 if (atomic_read(&rdev->irq.pflip[1]))
25a857fb
AD		 6168 radeon_crtc_handle_flip(rdev, 1);
6169 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6170 DRM_DEBUG("IH: D2 vblank\n");
6171 }
6172 break;
6173 case 1: /* D2 vline */
6174 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6175 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6176 DRM_DEBUG("IH: D2 vline\n");
6177 }
6178 break;
6179 default:
6180 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6181 break;
6182 }
6183 break;
6184 case 3: /* D3 vblank/vline */
6185 switch (src_data) {
6186 case 0: /* D3 vblank */
6187 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6188 if (rdev->irq.crtc_vblank_int[2]) {
6189 drm_handle_vblank(rdev->ddev, 2);
6190 rdev->pm.vblank_sync = true;
6191 wake_up(&rdev->irq.vblank_queue);
6192 }
736fc37f 6193 if (atomic_read(&rdev->irq.pflip[2]))
25a857fb
AD		 6194 radeon_crtc_handle_flip(rdev, 2);
6195 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6196 DRM_DEBUG("IH: D3 vblank\n");
6197 }
6198 break;
6199 case 1: /* D3 vline */
6200 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6201 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6202 DRM_DEBUG("IH: D3 vline\n");
6203 }
6204 break;
6205 default:
6206 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6207 break;
6208 }
6209 break;
6210 case 4: /* D4 vblank/vline */
6211 switch (src_data) {
6212 case 0: /* D4 vblank */
6213 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6214 if (rdev->irq.crtc_vblank_int[3]) {
6215 drm_handle_vblank(rdev->ddev, 3);
6216 rdev->pm.vblank_sync = true;
6217 wake_up(&rdev->irq.vblank_queue);
6218 }
736fc37f 6219 if (atomic_read(&rdev->irq.pflip[3]))
25a857fb
AD		 6220 radeon_crtc_handle_flip(rdev, 3);
6221 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6222 DRM_DEBUG("IH: D4 vblank\n");
6223 }
6224 break;
6225 case 1: /* D4 vline */
6226 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6227 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6228 DRM_DEBUG("IH: D4 vline\n");
6229 }
6230 break;
6231 default:
6232 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6233 break;
6234 }
6235 break;
6236 case 5: /* D5 vblank/vline */
6237 switch (src_data) {
6238 case 0: /* D5 vblank */
6239 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6240 if (rdev->irq.crtc_vblank_int[4]) {
6241 drm_handle_vblank(rdev->ddev, 4);
6242 rdev->pm.vblank_sync = true;
6243 wake_up(&rdev->irq.vblank_queue);
6244 }
736fc37f 6245 if (atomic_read(&rdev->irq.pflip[4]))
25a857fb
AD		 6246 radeon_crtc_handle_flip(rdev, 4);
6247 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6248 DRM_DEBUG("IH: D5 vblank\n");
6249 }
6250 break;
6251 case 1: /* D5 vline */
6252 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6253 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6254 DRM_DEBUG("IH: D5 vline\n");
6255 }
6256 break;
6257 default:
6258 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6259 break;
6260 }
6261 break;
6262 case 6: /* D6 vblank/vline */
6263 switch (src_data) {
6264 case 0: /* D6 vblank */
6265 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6266 if (rdev->irq.crtc_vblank_int[5]) {
6267 drm_handle_vblank(rdev->ddev, 5);
6268 rdev->pm.vblank_sync = true;
6269 wake_up(&rdev->irq.vblank_queue);
6270 }
736fc37f 6271 if (atomic_read(&rdev->irq.pflip[5]))
25a857fb
AD		 6272 radeon_crtc_handle_flip(rdev, 5);
6273 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6274 DRM_DEBUG("IH: D6 vblank\n");
6275 }
6276 break;
6277 case 1: /* D6 vline */
6278 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6279 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6280 DRM_DEBUG("IH: D6 vline\n");
6281 }
6282 break;
6283 default:
6284 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6285 break;
6286 }
6287 break;
6288 case 42: /* HPD hotplug */
6289 switch (src_data) {
6290 case 0:
6291 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6292 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6293 queue_hotplug = true;
6294 DRM_DEBUG("IH: HPD1\n");
6295 }
6296 break;
6297 case 1:
6298 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6299 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6300 queue_hotplug = true;
6301 DRM_DEBUG("IH: HPD2\n");
6302 }
6303 break;
6304 case 2:
6305 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6306 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6307 queue_hotplug = true;
6308 DRM_DEBUG("IH: HPD3\n");
6309 }
6310 break;
6311 case 3:
6312 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6313 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6314 queue_hotplug = true;
6315 DRM_DEBUG("IH: HPD4\n");
6316 }
6317 break;
6318 case 4:
6319 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6320 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6321 queue_hotplug = true;
6322 DRM_DEBUG("IH: HPD5\n");
6323 }
6324 break;
6325 case 5:
6326 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6327 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6328 queue_hotplug = true;
6329 DRM_DEBUG("IH: HPD6\n");
6330 }
6331 break;
6332 default:
6333 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6334 break;
6335 }
6336 break;
ae133a11
CK		 6337 case 146:
6338 case 147:
fbf6dc7a
AD		 6339 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6340 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
ae133a11
CK		 6341 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6342 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
fbf6dc7a 6343 addr);
ae133a11 6344 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
fbf6dc7a
AD		 6345 status);
6346 si_vm_decode_fault(rdev, status, addr);
ae133a11
CK		 6347 /* reset addr and status */
6348 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6349 break;
25a857fb
AD		 6350 case 176: /* RINGID0 CP_INT */
6351 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6352 break;
6353 case 177: /* RINGID1 CP_INT */
6354 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6355 break;
6356 case 178: /* RINGID2 CP_INT */
6357 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6358 break;
6359 case 181: /* CP EOP event */
6360 DRM_DEBUG("IH: CP EOP\n");
6361 switch (ring_id) {
6362 case 0:
6363 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6364 break;
6365 case 1:
6366 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6367 break;
6368 case 2:
6369 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6370 break;
6371 }
6372 break;
8c5fd7ef
AD		 6373 case 224: /* DMA trap event */
6374 DRM_DEBUG("IH: DMA trap\n");
6375 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6376 break;
a9e61410
AD		 6377 case 230: /* thermal low to high */
6378 DRM_DEBUG("IH: thermal low to high\n");
6379 rdev->pm.dpm.thermal.high_to_low = false;
6380 queue_thermal = true;
6381 break;
6382 case 231: /* thermal high to low */
6383 DRM_DEBUG("IH: thermal high to low\n");
6384 rdev->pm.dpm.thermal.high_to_low = true;
6385 queue_thermal = true;
6386 break;
25a857fb
AD		 6387 case 233: /* GUI IDLE */
6388 DRM_DEBUG("IH: GUI idle\n");
25a857fb 6389 break;
8c5fd7ef
AD		 6390 case 244: /* DMA trap event */
6391 DRM_DEBUG("IH: DMA1 trap\n");
6392 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6393 break;
25a857fb
AD		 6394 default:
6395 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6396 break;
6397 }
6398
6399 /* wptr/rptr are in bytes! */
6400 rptr += 16;
6401 rptr &= rdev->ih.ptr_mask;
6402 }
25a857fb
AD		 6403 if (queue_hotplug)
6404 schedule_work(&rdev->hotplug_work);
a9e61410
AD		 6405 if (queue_thermal && rdev->pm.dpm_enabled)
6406 schedule_work(&rdev->pm.dpm.thermal.work);
25a857fb
AD		 6407 rdev->ih.rptr = rptr;
6408 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 6409 atomic_set(&rdev->ih.lock, 0);
6410
6411 /* make sure wptr hasn't changed while processing */
6412 wptr = si_get_ih_wptr(rdev);
6413 if (wptr != rptr)
6414 goto restart_ih;
6415
25a857fb
AD		 6416 return IRQ_HANDLED;
6417}
6418
9b136d51
AD		 6419/*
6420 * startup/shutdown callbacks
6421 */
6422static int si_startup(struct radeon_device *rdev)
6423{
6424 struct radeon_ring *ring;
6425 int r;
6426
b9d305df
AD		 6427 /* enable pcie gen2/3 link */
6428 si_pcie_gen3_enable(rdev);
e0bcf165
AD		 6429 /* enable aspm */
6430 si_program_aspm(rdev);
b9d305df 6431
e5903d39
AD		 6432 /* scratch needs to be initialized before MC */
6433 r = r600_vram_scratch_init(rdev);
6434 if (r)
6435 return r;
6436
6fab3feb
AD		 6437 si_mc_program(rdev);
6438
6c7bccea
AD		 6439 if (!rdev->pm.dpm_enabled) {
6440 r = si_mc_load_microcode(rdev);
6441 if (r) {
6442 DRM_ERROR("Failed to load MC firmware!\n");
6443 return r;
6444 }
9b136d51
AD		 6445 }
6446
9b136d51
AD		 6447 r = si_pcie_gart_enable(rdev);
6448 if (r)
6449 return r;
6450 si_gpu_init(rdev);
6451
9b136d51 6452 /* allocate rlc buffers */
1fd11777
AD		 6453 if (rdev->family == CHIP_VERDE) {
6454 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6455 rdev->rlc.reg_list_size =
6456 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6457 }
6458 rdev->rlc.cs_data = si_cs_data;
6459 r = sumo_rlc_init(rdev);
9b136d51
AD		 6460 if (r) {
6461 DRM_ERROR("Failed to init rlc BOs!\n");
6462 return r;
6463 }
6464
6465 /* allocate wb buffer */
6466 r = radeon_wb_init(rdev);
6467 if (r)
6468 return r;
6469
6470 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6471 if (r) {
6472 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6473 return r;
6474 }
6475
6476 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6477 if (r) {
6478 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6479 return r;
6480 }
6481
6482 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6483 if (r) {
6484 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6485 return r;
6486 }
6487
8c5fd7ef
AD		 6488 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6489 if (r) {
6490 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6491 return r;
6492 }
6493
6494 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6495 if (r) {
6496 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6497 return r;
6498 }
6499
1df0d523 6500 if (rdev->has_uvd) {
e409b128 6501 r = uvd_v2_2_resume(rdev);
1df0d523
AD		 6502 if (!r) {
6503 r = radeon_fence_driver_start_ring(rdev,
6504 R600_RING_TYPE_UVD_INDEX);
6505 if (r)
6506 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6507 }
f2ba57b5 6508 if (r)
1df0d523 6509 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
f2ba57b5 6510 }
f2ba57b5 6511
9b136d51 6512 /* Enable IRQ */
e49f3959
AH		 6513 if (!rdev->irq.installed) {
6514 r = radeon_irq_kms_init(rdev);
6515 if (r)
6516 return r;
6517 }
6518
9b136d51
AD		 6519 r = si_irq_init(rdev);
6520 if (r) {
6521 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6522 radeon_irq_kms_fini(rdev);
6523 return r;
6524 }
6525 si_irq_set(rdev);
6526
6527 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6528 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2e1e6dad 6529 RADEON_CP_PACKET2);
9b136d51
AD		 6530 if (r)
6531 return r;
6532
6533 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6534 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
2e1e6dad 6535 RADEON_CP_PACKET2);
9b136d51
AD		 6536 if (r)
6537 return r;
6538
6539 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6540 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
2e1e6dad 6541 RADEON_CP_PACKET2);
9b136d51
AD		 6542 if (r)
6543 return r;
6544
8c5fd7ef
AD		 6545 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6546 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 6547 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6548 if (r)
6549 return r;
6550
6551 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6552 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 6553 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6554 if (r)
6555 return r;
6556
9b136d51
AD		 6557 r = si_cp_load_microcode(rdev);
6558 if (r)
6559 return r;
6560 r = si_cp_resume(rdev);
6561 if (r)
6562 return r;
6563
8c5fd7ef
AD		 6564 r = cayman_dma_resume(rdev);
6565 if (r)
6566 return r;
6567
1df0d523
AD		 6568 if (rdev->has_uvd) {
6569 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6570 if (ring->ring_size) {
02c9f7fa 6571 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 6572 RADEON_CP_PACKET2);
1df0d523 6573 if (!r)
e409b128 6574 r = uvd_v1_0_init(rdev);
1df0d523
AD		 6575 if (r)
6576 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6577 }
f2ba57b5
CK		 6578 }
6579
2898c348
CK		 6580 r = radeon_ib_pool_init(rdev);
6581 if (r) {
6582 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 6583 return r;
2898c348 6584 }
9b136d51 6585
c6105f24
CK		 6586 r = radeon_vm_manager_init(rdev);
6587 if (r) {
6588 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 6589 return r;
c6105f24 6590 }
9b136d51 6591
b530602f
AD		 6592 r = dce6_audio_init(rdev);
6593 if (r)
6594 return r;
6595
9b136d51
AD		 6596 return 0;
6597}
6598
6599int si_resume(struct radeon_device *rdev)
6600{
6601 int r;
6602
6603 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6604 * posting will perform necessary task to bring back GPU into good
6605 * shape.
6606 */
6607 /* post card */
6608 atom_asic_init(rdev->mode_info.atom_context);
6609
205996c0
AD		 6610 /* init golden registers */
6611 si_init_golden_registers(rdev);
6612
6c7bccea
AD		 6613 radeon_pm_resume(rdev);
6614
9b136d51
AD		 6615 rdev->accel_working = true;
6616 r = si_startup(rdev);
6617 if (r) {
6618 DRM_ERROR("si startup failed on resume\n");
6619 rdev->accel_working = false;
6620 return r;
6621 }
6622
6623 return r;
6624
6625}
6626
6627int si_suspend(struct radeon_device *rdev)
6628{
6c7bccea 6629 radeon_pm_suspend(rdev);
b530602f 6630 dce6_audio_fini(rdev);
fa3daf9a 6631 radeon_vm_manager_fini(rdev);
9b136d51 6632 si_cp_enable(rdev, false);
8c5fd7ef 6633 cayman_dma_stop(rdev);
1df0d523 6634 if (rdev->has_uvd) {
e409b128 6635 uvd_v1_0_fini(rdev);
1df0d523
AD		 6636 radeon_uvd_suspend(rdev);
6637 }
e16866ec
AD		 6638 si_fini_pg(rdev);
6639 si_fini_cg(rdev);
9b136d51
AD		 6640 si_irq_suspend(rdev);
6641 radeon_wb_disable(rdev);
6642 si_pcie_gart_disable(rdev);
6643 return 0;
6644}
6645
6646/* Plan is to move initialization in that function and use
6647 * helper function so that radeon_device_init pretty much
6648 * do nothing more than calling asic specific function. This
6649 * should also allow to remove a bunch of callback function
6650 * like vram_info.
6651 */
6652int si_init(struct radeon_device *rdev)
6653{
6654 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6655 int r;
6656
9b136d51
AD		 6657 /* Read BIOS */
6658 if (!radeon_get_bios(rdev)) {
6659 if (ASIC_IS_AVIVO(rdev))
6660 return -EINVAL;
6661 }
6662 /* Must be an ATOMBIOS */
6663 if (!rdev->is_atom_bios) {
6664 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6665 return -EINVAL;
6666 }
6667 r = radeon_atombios_init(rdev);
6668 if (r)
6669 return r;
6670
6671 /* Post card if necessary */
6672 if (!radeon_card_posted(rdev)) {
6673 if (!rdev->bios) {
6674 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6675 return -EINVAL;
6676 }
6677 DRM_INFO("GPU not posted. posting now...\n");
6678 atom_asic_init(rdev->mode_info.atom_context);
6679 }
205996c0
AD		 6680 /* init golden registers */
6681 si_init_golden_registers(rdev);
9b136d51
AD		 6682 /* Initialize scratch registers */
6683 si_scratch_init(rdev);
6684 /* Initialize surface registers */
6685 radeon_surface_init(rdev);
6686 /* Initialize clocks */
6687 radeon_get_clock_info(rdev->ddev);
6688
6689 /* Fence driver */
6690 r = radeon_fence_driver_init(rdev);
6691 if (r)
6692 return r;
6693
6694 /* initialize memory controller */
6695 r = si_mc_init(rdev);
6696 if (r)
6697 return r;
6698 /* Memory manager */
6699 r = radeon_bo_init(rdev);
6700 if (r)
6701 return r;
6702
01ac8794
AD		 6703 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6704 !rdev->rlc_fw || !rdev->mc_fw) {
6705 r = si_init_microcode(rdev);
6706 if (r) {
6707 DRM_ERROR("Failed to load firmware!\n");
6708 return r;
6709 }
6710 }
6711
6c7bccea
AD		 6712 /* Initialize power management */
6713 radeon_pm_init(rdev);
6714
9b136d51
AD		 6715 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6716 ring->ring_obj = NULL;
6717 r600_ring_init(rdev, ring, 1024 * 1024);
6718
6719 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6720 ring->ring_obj = NULL;
6721 r600_ring_init(rdev, ring, 1024 * 1024);
6722
6723 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6724 ring->ring_obj = NULL;
6725 r600_ring_init(rdev, ring, 1024 * 1024);
6726
8c5fd7ef
AD		 6727 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6728 ring->ring_obj = NULL;
6729 r600_ring_init(rdev, ring, 64 * 1024);
6730
6731 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6732 ring->ring_obj = NULL;
6733 r600_ring_init(rdev, ring, 64 * 1024);
6734
1df0d523
AD		 6735 if (rdev->has_uvd) {
6736 r = radeon_uvd_init(rdev);
6737 if (!r) {
6738 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6739 ring->ring_obj = NULL;
6740 r600_ring_init(rdev, ring, 4096);
6741 }
f2ba57b5
CK		 6742 }
6743
9b136d51
AD		 6744 rdev->ih.ring_obj = NULL;
6745 r600_ih_ring_init(rdev, 64 * 1024);
6746
6747 r = r600_pcie_gart_init(rdev);
6748 if (r)
6749 return r;
6750
9b136d51 6751 rdev->accel_working = true;
9b136d51
AD		 6752 r = si_startup(rdev);
6753 if (r) {
6754 dev_err(rdev->dev, "disabling GPU acceleration\n");
6755 si_cp_fini(rdev);
8c5fd7ef 6756 cayman_dma_fini(rdev);
9b136d51 6757 si_irq_fini(rdev);
1fd11777 6758 sumo_rlc_fini(rdev);
9b136d51 6759 radeon_wb_fini(rdev);
2898c348 6760 radeon_ib_pool_fini(rdev);
9b136d51
AD		 6761 radeon_vm_manager_fini(rdev);
6762 radeon_irq_kms_fini(rdev);
6763 si_pcie_gart_fini(rdev);
6764 rdev->accel_working = false;
6765 }
6766
6767 /* Don't start up if the MC ucode is missing.
6768 * The default clocks and voltages before the MC ucode
6769 * is loaded are not suffient for advanced operations.
6770 */
6771 if (!rdev->mc_fw) {
6772 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6773 return -EINVAL;
6774 }
6775
6776 return 0;
6777}
6778
6779void si_fini(struct radeon_device *rdev)
6780{
6c7bccea 6781 radeon_pm_fini(rdev);
9b136d51 6782 si_cp_fini(rdev);
8c5fd7ef 6783 cayman_dma_fini(rdev);
f8f84ac5 6784 si_fini_pg(rdev);
e16866ec 6785 si_fini_cg(rdev);
9b136d51 6786 si_irq_fini(rdev);
1fd11777 6787 sumo_rlc_fini(rdev);
9b136d51
AD		 6788 radeon_wb_fini(rdev);
6789 radeon_vm_manager_fini(rdev);
2898c348 6790 radeon_ib_pool_fini(rdev);
9b136d51 6791 radeon_irq_kms_fini(rdev);
2858c00d 6792 if (rdev->has_uvd) {
e409b128 6793 uvd_v1_0_fini(rdev);
1df0d523 6794 radeon_uvd_fini(rdev);
2858c00d 6795 }
9b136d51
AD		 6796 si_pcie_gart_fini(rdev);
6797 r600_vram_scratch_fini(rdev);
6798 radeon_gem_fini(rdev);
9b136d51
AD		 6799 radeon_fence_driver_fini(rdev);
6800 radeon_bo_fini(rdev);
6801 radeon_atombios_fini(rdev);
6802 kfree(rdev->bios);
6803 rdev->bios = NULL;
6804}
6805
6759a0a7 6806/**
d0418894 6807 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 6808 *
6809 * @rdev: radeon_device pointer
6810 *
6811 * Fetches a GPU clock counter snapshot (SI).
6812 * Returns the 64 bit clock counter snapshot.
6813 */
d0418894 6814uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 6815{
6816 uint64_t clock;
6817
6818 mutex_lock(&rdev->gpu_clock_mutex);
6819 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6820 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6821 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6822 mutex_unlock(&rdev->gpu_clock_mutex);
6823 return clock;
6824}
2539eb02 6825
2539eb02
CK		 6826int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6827{
facd112d 6828 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 6829 int r;
6830
4ed10835
CK		 6831 /* bypass vclk and dclk with bclk */
6832 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6833 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6834 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6835
6836 /* put PLL in bypass mode */
6837 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6838
6839 if (!vclk || !dclk) {
6840 /* keep the Bypass mode, put PLL to sleep */
6841 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6842 return 0;
6843 }
6844
facd112d
CK		 6845 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6846 16384, 0x03FFFFFF, 0, 128, 5,
6847 &fb_div, &vclk_div, &dclk_div);
6848 if (r)
6849 return r;
2539eb02
CK		 6850
6851 /* set RESET_ANTI_MUX to 0 */
6852 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6853
6854 /* set VCO_MODE to 1 */
6855 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6856
6857 /* toggle UPLL_SLEEP to 1 then back to 0 */
6858 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6859 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6860
6861 /* deassert UPLL_RESET */
6862 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6863
6864 mdelay(1);
6865
facd112d 6866 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6867 if (r)
6868 return r;
6869
6870 /* assert UPLL_RESET again */
6871 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6872
6873 /* disable spread spectrum. */
6874 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6875
6876 /* set feedback divider */
facd112d 6877 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 6878
6879 /* set ref divider to 0 */
6880 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6881
facd112d 6882 if (fb_div < 307200)
2539eb02
CK		 6883 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6884 else
6885 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6886
6887 /* set PDIV_A and PDIV_B */
6888 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 6889 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 6890 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6891
6892 /* give the PLL some time to settle */
6893 mdelay(15);
6894
6895 /* deassert PLL_RESET */
6896 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6897
6898 mdelay(15);
6899
6900 /* switch from bypass mode to normal mode */
6901 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6902
facd112d 6903 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6904 if (r)
6905 return r;
6906
6907 /* switch VCLK and DCLK selection */
6908 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6909 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6910 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6911
6912 mdelay(100);
6913
6914 return 0;
6915}
b9d305df
AD		 6916
6917static void si_pcie_gen3_enable(struct radeon_device *rdev)
6918{
6919 struct pci_dev *root = rdev->pdev->bus->self;
6920 int bridge_pos, gpu_pos;
6921 u32 speed_cntl, mask, current_data_rate;
6922 int ret, i;
6923 u16 tmp16;
6924
6925 if (radeon_pcie_gen2 == 0)
6926 return;
6927
6928 if (rdev->flags & RADEON_IS_IGP)
6929 return;
6930
6931 if (!(rdev->flags & RADEON_IS_PCIE))
6932 return;
6933
6934 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6935 if (ret != 0)
6936 return;
6937
6938 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6939 return;
6940
6941 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6942 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6943 LC_CURRENT_DATA_RATE_SHIFT;
6944 if (mask & DRM_PCIE_SPEED_80) {
6945 if (current_data_rate == 2) {
6946 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6947 return;
6948 }
6949 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6950 } else if (mask & DRM_PCIE_SPEED_50) {
6951 if (current_data_rate == 1) {
6952 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6953 return;
6954 }
6955 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6956 }
6957
6958 bridge_pos = pci_pcie_cap(root);
6959 if (!bridge_pos)
6960 return;
6961
6962 gpu_pos = pci_pcie_cap(rdev->pdev);
6963 if (!gpu_pos)
6964 return;
6965
6966 if (mask & DRM_PCIE_SPEED_80) {
6967 /* re-try equalization if gen3 is not already enabled */
6968 if (current_data_rate != 2) {
6969 u16 bridge_cfg, gpu_cfg;
6970 u16 bridge_cfg2, gpu_cfg2;
6971 u32 max_lw, current_lw, tmp;
6972
6973 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6974 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6975
6976 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6977 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6978
6979 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6980 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6981
6982 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6983 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6984 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6985
6986 if (current_lw < max_lw) {
6987 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6988 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6989 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6990 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6991 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6992 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6993 }
6994 }
6995
6996 for (i = 0; i < 10; i++) {
6997 /* check status */
6998 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6999 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7000 break;
7001
7002 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7003 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7004
7005 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7006 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7007
7008 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7009 tmp |= LC_SET_QUIESCE;
7010 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7011
7012 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7013 tmp |= LC_REDO_EQ;
7014 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7015
7016 mdelay(100);
7017
7018 /* linkctl */
7019 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7020 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7021 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7022 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7023
7024 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7025 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7026 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7027 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7028
7029 /* linkctl2 */
7030 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7031 tmp16 &= ~((1 << 4) | (7 << 9));
7032 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7033 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7034
7035 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7036 tmp16 &= ~((1 << 4) | (7 << 9));
7037 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7038 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7039
7040 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7041 tmp &= ~LC_SET_QUIESCE;
7042 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7043 }
7044 }
7045 }
7046
7047 /* set the link speed */
7048 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7049 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7050 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7051
7052 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7053 tmp16 &= ~0xf;
7054 if (mask & DRM_PCIE_SPEED_80)
7055 tmp16 |= 3; /* gen3 */
7056 else if (mask & DRM_PCIE_SPEED_50)
7057 tmp16 |= 2; /* gen2 */
7058 else
7059 tmp16 |= 1; /* gen1 */
7060 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7061
7062 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7063 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7064 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7065
7066 for (i = 0; i < rdev->usec_timeout; i++) {
7067 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7068 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7069 break;
7070 udelay(1);
7071 }
7072}
7073
e0bcf165
AD		 7074static void si_program_aspm(struct radeon_device *rdev)
7075{
7076 u32 data, orig;
7077 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7078 bool disable_clkreq = false;
7079
1294d4a3
AD		 7080 if (radeon_aspm == 0)
7081 return;
7082
e0bcf165
AD		 7083 if (!(rdev->flags & RADEON_IS_PCIE))
7084 return;
7085
7086 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7087 data &= ~LC_XMIT_N_FTS_MASK;
7088 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7089 if (orig != data)
7090 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7091
7092 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7093 data |= LC_GO_TO_RECOVERY;
7094 if (orig != data)
7095 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7096
7097 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7098 data |= P_IGNORE_EDB_ERR;
7099 if (orig != data)
7100 WREG32_PCIE(PCIE_P_CNTL, data);
7101
7102 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7103 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7104 data |= LC_PMI_TO_L1_DIS;
7105 if (!disable_l0s)
7106 data |= LC_L0S_INACTIVITY(7);
7107
7108 if (!disable_l1) {
7109 data |= LC_L1_INACTIVITY(7);
7110 data &= ~LC_PMI_TO_L1_DIS;
7111 if (orig != data)
7112 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7113
7114 if (!disable_plloff_in_l1) {
7115 bool clk_req_support;
7116
7117 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7118 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7119 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7120 if (orig != data)
7121 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7122
7123 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7124 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7125 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7126 if (orig != data)
7127 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7128
7129 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7130 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7131 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7132 if (orig != data)
7133 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7134
7135 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7136 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7137 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7138 if (orig != data)
7139 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7140
7141 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7142 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7143 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7144 if (orig != data)
7145 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7146
7147 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7148 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7149 if (orig != data)
7150 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7151
7152 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7153 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7154 if (orig != data)
7155 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7156
7157 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7158 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7159 if (orig != data)
7160 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7161
7162 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7163 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7164 if (orig != data)
7165 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7166
7167 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7168 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7169 if (orig != data)
7170 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7171
7172 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7173 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7174 if (orig != data)
7175 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7176
7177 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7178 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7179 if (orig != data)
7180 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7181 }
7182 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7183 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7184 data |= LC_DYN_LANES_PWR_STATE(3);
7185 if (orig != data)
7186 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7187
7188 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7189 data &= ~LS2_EXIT_TIME_MASK;
7190 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7191 data |= LS2_EXIT_TIME(5);
7192 if (orig != data)
7193 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7194
7195 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7196 data &= ~LS2_EXIT_TIME_MASK;
7197 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7198 data |= LS2_EXIT_TIME(5);
7199 if (orig != data)
7200 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7201
7202 if (!disable_clkreq) {
7203 struct pci_dev *root = rdev->pdev->bus->self;
7204 u32 lnkcap;
7205
7206 clk_req_support = false;
7207 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7208 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7209 clk_req_support = true;
7210 } else {
7211 clk_req_support = false;
7212 }
7213
7214 if (clk_req_support) {
7215 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7216 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7217 if (orig != data)
7218 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7219
7220 orig = data = RREG32(THM_CLK_CNTL);
7221 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7222 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7223 if (orig != data)
7224 WREG32(THM_CLK_CNTL, data);
7225
7226 orig = data = RREG32(MISC_CLK_CNTL);
7227 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7228 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7229 if (orig != data)
7230 WREG32(MISC_CLK_CNTL, data);
7231
7232 orig = data = RREG32(CG_CLKPIN_CNTL);
7233 data &= ~BCLK_AS_XCLK;
7234 if (orig != data)
7235 WREG32(CG_CLKPIN_CNTL, data);
7236
7237 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7238 data &= ~FORCE_BIF_REFCLK_EN;
7239 if (orig != data)
7240 WREG32(CG_CLKPIN_CNTL_2, data);
7241
7242 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7243 data &= ~MPLL_CLKOUT_SEL_MASK;
7244 data |= MPLL_CLKOUT_SEL(4);
7245 if (orig != data)
7246 WREG32(MPLL_BYPASSCLK_SEL, data);
7247
7248 orig = data = RREG32(SPLL_CNTL_MODE);
7249 data &= ~SPLL_REFCLK_SEL_MASK;
7250 if (orig != data)
7251 WREG32(SPLL_CNTL_MODE, data);
7252 }
7253 }
7254 } else {
7255 if (orig != data)
7256 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7257 }
7258
7259 orig = data = RREG32_PCIE(PCIE_CNTL2);
7260 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7261 if (orig != data)
7262 WREG32_PCIE(PCIE_CNTL2, data);
7263
7264 if (!disable_l0s) {
7265 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7266 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7267 data = RREG32_PCIE(PCIE_LC_STATUS1);
7268 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7269 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7270 data &= ~LC_L0S_INACTIVITY_MASK;
7271 if (orig != data)
7272 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7273 }
7274 }
7275 }
7276}
Linux 6.x block layer and io_uring tree(s)