projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: apply more strict limits for PLL params v2
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
0f0de06c 24#include <linux/firmware.h>
0f0de06c
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
760285e7 27#include <drm/drmP.h>
43b3cd99
AD		 28#include "radeon.h"
29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
43b3cd99
AD		 31#include "sid.h"
32#include "atom.h"
48c0c902 33#include "si_blit_shaders.h"
bd8cd539 34#include "clearstate_si.h"
a0ceada6 35#include "radeon_ucode.h"
43b3cd99 36
0f0de06c
AD		 37
38MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
42MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
a9e61410 43MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
0f0de06c
AD		 44MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
45MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
46MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
47MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
48MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
a9e61410 49MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
0f0de06c
AD		 50MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
51MODULE_FIRMWARE("radeon/VERDE_me.bin");
52MODULE_FIRMWARE("radeon/VERDE_ce.bin");
53MODULE_FIRMWARE("radeon/VERDE_mc.bin");
54MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
a9e61410 55MODULE_FIRMWARE("radeon/VERDE_smc.bin");
bcc7f5d2
AD		 56MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
57MODULE_FIRMWARE("radeon/OLAND_me.bin");
58MODULE_FIRMWARE("radeon/OLAND_ce.bin");
59MODULE_FIRMWARE("radeon/OLAND_mc.bin");
60MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
a9e61410 61MODULE_FIRMWARE("radeon/OLAND_smc.bin");
c04c00b4
AD		 62MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
63MODULE_FIRMWARE("radeon/HAINAN_me.bin");
64MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
65MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
66MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
a9e61410 67MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
0f0de06c 68
b9d305df 69static void si_pcie_gen3_enable(struct radeon_device *rdev);
e0bcf165 70static void si_program_aspm(struct radeon_device *rdev);
1fd11777
AD		 71extern void sumo_rlc_fini(struct radeon_device *rdev);
72extern int sumo_rlc_init(struct radeon_device *rdev);
25a857fb
AD		 73extern int r600_ih_ring_alloc(struct radeon_device *rdev);
74extern void r600_ih_ring_fini(struct radeon_device *rdev);
0a96d72b 75extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
c476dde2
AD		 76extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
77extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
ca7db22b 78extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
1c534671 79extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
014bb209 80extern bool evergreen_is_display_hung(struct radeon_device *rdev);
811e4d58
AD		 81static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
82 bool enable);
4a5c8ea5
AD		 83static void si_init_pg(struct radeon_device *rdev);
84static void si_init_cg(struct radeon_device *rdev);
a6f4ae8d
AD		 85static void si_fini_pg(struct radeon_device *rdev);
86static void si_fini_cg(struct radeon_device *rdev);
87static void si_rlc_stop(struct radeon_device *rdev);
0a96d72b 88
6d8cf000
AD		 89static const u32 verde_rlc_save_restore_register_list[] =
90{
91 (0x8000 << 16) | (0x98f4 >> 2),
92 0x00000000,
93 (0x8040 << 16) | (0x98f4 >> 2),
94 0x00000000,
95 (0x8000 << 16) | (0xe80 >> 2),
96 0x00000000,
97 (0x8040 << 16) | (0xe80 >> 2),
98 0x00000000,
99 (0x8000 << 16) | (0x89bc >> 2),
100 0x00000000,
101 (0x8040 << 16) | (0x89bc >> 2),
102 0x00000000,
103 (0x8000 << 16) | (0x8c1c >> 2),
104 0x00000000,
105 (0x8040 << 16) | (0x8c1c >> 2),
106 0x00000000,
107 (0x9c00 << 16) | (0x98f0 >> 2),
108 0x00000000,
109 (0x9c00 << 16) | (0xe7c >> 2),
110 0x00000000,
111 (0x8000 << 16) | (0x9148 >> 2),
112 0x00000000,
113 (0x8040 << 16) | (0x9148 >> 2),
114 0x00000000,
115 (0x9c00 << 16) | (0x9150 >> 2),
116 0x00000000,
117 (0x9c00 << 16) | (0x897c >> 2),
118 0x00000000,
119 (0x9c00 << 16) | (0x8d8c >> 2),
120 0x00000000,
121 (0x9c00 << 16) | (0xac54 >> 2),
122 0X00000000,
123 0x3,
124 (0x9c00 << 16) | (0x98f8 >> 2),
125 0x00000000,
126 (0x9c00 << 16) | (0x9910 >> 2),
127 0x00000000,
128 (0x9c00 << 16) | (0x9914 >> 2),
129 0x00000000,
130 (0x9c00 << 16) | (0x9918 >> 2),
131 0x00000000,
132 (0x9c00 << 16) | (0x991c >> 2),
133 0x00000000,
134 (0x9c00 << 16) | (0x9920 >> 2),
135 0x00000000,
136 (0x9c00 << 16) | (0x9924 >> 2),
137 0x00000000,
138 (0x9c00 << 16) | (0x9928 >> 2),
139 0x00000000,
140 (0x9c00 << 16) | (0x992c >> 2),
141 0x00000000,
142 (0x9c00 << 16) | (0x9930 >> 2),
143 0x00000000,
144 (0x9c00 << 16) | (0x9934 >> 2),
145 0x00000000,
146 (0x9c00 << 16) | (0x9938 >> 2),
147 0x00000000,
148 (0x9c00 << 16) | (0x993c >> 2),
149 0x00000000,
150 (0x9c00 << 16) | (0x9940 >> 2),
151 0x00000000,
152 (0x9c00 << 16) | (0x9944 >> 2),
153 0x00000000,
154 (0x9c00 << 16) | (0x9948 >> 2),
155 0x00000000,
156 (0x9c00 << 16) | (0x994c >> 2),
157 0x00000000,
158 (0x9c00 << 16) | (0x9950 >> 2),
159 0x00000000,
160 (0x9c00 << 16) | (0x9954 >> 2),
161 0x00000000,
162 (0x9c00 << 16) | (0x9958 >> 2),
163 0x00000000,
164 (0x9c00 << 16) | (0x995c >> 2),
165 0x00000000,
166 (0x9c00 << 16) | (0x9960 >> 2),
167 0x00000000,
168 (0x9c00 << 16) | (0x9964 >> 2),
169 0x00000000,
170 (0x9c00 << 16) | (0x9968 >> 2),
171 0x00000000,
172 (0x9c00 << 16) | (0x996c >> 2),
173 0x00000000,
174 (0x9c00 << 16) | (0x9970 >> 2),
175 0x00000000,
176 (0x9c00 << 16) | (0x9974 >> 2),
177 0x00000000,
178 (0x9c00 << 16) | (0x9978 >> 2),
179 0x00000000,
180 (0x9c00 << 16) | (0x997c >> 2),
181 0x00000000,
182 (0x9c00 << 16) | (0x9980 >> 2),
183 0x00000000,
184 (0x9c00 << 16) | (0x9984 >> 2),
185 0x00000000,
186 (0x9c00 << 16) | (0x9988 >> 2),
187 0x00000000,
188 (0x9c00 << 16) | (0x998c >> 2),
189 0x00000000,
190 (0x9c00 << 16) | (0x8c00 >> 2),
191 0x00000000,
192 (0x9c00 << 16) | (0x8c14 >> 2),
193 0x00000000,
194 (0x9c00 << 16) | (0x8c04 >> 2),
195 0x00000000,
196 (0x9c00 << 16) | (0x8c08 >> 2),
197 0x00000000,
198 (0x8000 << 16) | (0x9b7c >> 2),
199 0x00000000,
200 (0x8040 << 16) | (0x9b7c >> 2),
201 0x00000000,
202 (0x8000 << 16) | (0xe84 >> 2),
203 0x00000000,
204 (0x8040 << 16) | (0xe84 >> 2),
205 0x00000000,
206 (0x8000 << 16) | (0x89c0 >> 2),
207 0x00000000,
208 (0x8040 << 16) | (0x89c0 >> 2),
209 0x00000000,
210 (0x8000 << 16) | (0x914c >> 2),
211 0x00000000,
212 (0x8040 << 16) | (0x914c >> 2),
213 0x00000000,
214 (0x8000 << 16) | (0x8c20 >> 2),
215 0x00000000,
216 (0x8040 << 16) | (0x8c20 >> 2),
217 0x00000000,
218 (0x8000 << 16) | (0x9354 >> 2),
219 0x00000000,
220 (0x8040 << 16) | (0x9354 >> 2),
221 0x00000000,
222 (0x9c00 << 16) | (0x9060 >> 2),
223 0x00000000,
224 (0x9c00 << 16) | (0x9364 >> 2),
225 0x00000000,
226 (0x9c00 << 16) | (0x9100 >> 2),
227 0x00000000,
228 (0x9c00 << 16) | (0x913c >> 2),
229 0x00000000,
230 (0x8000 << 16) | (0x90e0 >> 2),
231 0x00000000,
232 (0x8000 << 16) | (0x90e4 >> 2),
233 0x00000000,
234 (0x8000 << 16) | (0x90e8 >> 2),
235 0x00000000,
236 (0x8040 << 16) | (0x90e0 >> 2),
237 0x00000000,
238 (0x8040 << 16) | (0x90e4 >> 2),
239 0x00000000,
240 (0x8040 << 16) | (0x90e8 >> 2),
241 0x00000000,
242 (0x9c00 << 16) | (0x8bcc >> 2),
243 0x00000000,
244 (0x9c00 << 16) | (0x8b24 >> 2),
245 0x00000000,
246 (0x9c00 << 16) | (0x88c4 >> 2),
247 0x00000000,
248 (0x9c00 << 16) | (0x8e50 >> 2),
249 0x00000000,
250 (0x9c00 << 16) | (0x8c0c >> 2),
251 0x00000000,
252 (0x9c00 << 16) | (0x8e58 >> 2),
253 0x00000000,
254 (0x9c00 << 16) | (0x8e5c >> 2),
255 0x00000000,
256 (0x9c00 << 16) | (0x9508 >> 2),
257 0x00000000,
258 (0x9c00 << 16) | (0x950c >> 2),
259 0x00000000,
260 (0x9c00 << 16) | (0x9494 >> 2),
261 0x00000000,
262 (0x9c00 << 16) | (0xac0c >> 2),
263 0x00000000,
264 (0x9c00 << 16) | (0xac10 >> 2),
265 0x00000000,
266 (0x9c00 << 16) | (0xac14 >> 2),
267 0x00000000,
268 (0x9c00 << 16) | (0xae00 >> 2),
269 0x00000000,
270 (0x9c00 << 16) | (0xac08 >> 2),
271 0x00000000,
272 (0x9c00 << 16) | (0x88d4 >> 2),
273 0x00000000,
274 (0x9c00 << 16) | (0x88c8 >> 2),
275 0x00000000,
276 (0x9c00 << 16) | (0x88cc >> 2),
277 0x00000000,
278 (0x9c00 << 16) | (0x89b0 >> 2),
279 0x00000000,
280 (0x9c00 << 16) | (0x8b10 >> 2),
281 0x00000000,
282 (0x9c00 << 16) | (0x8a14 >> 2),
283 0x00000000,
284 (0x9c00 << 16) | (0x9830 >> 2),
285 0x00000000,
286 (0x9c00 << 16) | (0x9834 >> 2),
287 0x00000000,
288 (0x9c00 << 16) | (0x9838 >> 2),
289 0x00000000,
290 (0x9c00 << 16) | (0x9a10 >> 2),
291 0x00000000,
292 (0x8000 << 16) | (0x9870 >> 2),
293 0x00000000,
294 (0x8000 << 16) | (0x9874 >> 2),
295 0x00000000,
296 (0x8001 << 16) | (0x9870 >> 2),
297 0x00000000,
298 (0x8001 << 16) | (0x9874 >> 2),
299 0x00000000,
300 (0x8040 << 16) | (0x9870 >> 2),
301 0x00000000,
302 (0x8040 << 16) | (0x9874 >> 2),
303 0x00000000,
304 (0x8041 << 16) | (0x9870 >> 2),
305 0x00000000,
306 (0x8041 << 16) | (0x9874 >> 2),
307 0x00000000,
308 0x00000000
309};
310
205996c0
AD		 311static const u32 tahiti_golden_rlc_registers[] =
312{
313 0xc424, 0xffffffff, 0x00601005,
314 0xc47c, 0xffffffff, 0x10104040,
315 0xc488, 0xffffffff, 0x0100000a,
316 0xc314, 0xffffffff, 0x00000800,
317 0xc30c, 0xffffffff, 0x800000f4,
318 0xf4a8, 0xffffffff, 0x00000000
319};
320
321static const u32 tahiti_golden_registers[] =
322{
323 0x9a10, 0x00010000, 0x00018208,
324 0x9830, 0xffffffff, 0x00000000,
325 0x9834, 0xf00fffff, 0x00000400,
326 0x9838, 0x0002021c, 0x00020200,
327 0xc78, 0x00000080, 0x00000000,
328 0xd030, 0x000300c0, 0x00800040,
329 0xd830, 0x000300c0, 0x00800040,
330 0x5bb0, 0x000000f0, 0x00000070,
331 0x5bc0, 0x00200000, 0x50100000,
332 0x7030, 0x31000311, 0x00000011,
333 0x277c, 0x00000003, 0x000007ff,
334 0x240c, 0x000007ff, 0x00000000,
335 0x8a14, 0xf000001f, 0x00000007,
336 0x8b24, 0xffffffff, 0x00ffffff,
337 0x8b10, 0x0000ff0f, 0x00000000,
338 0x28a4c, 0x07ffffff, 0x4e000000,
339 0x28350, 0x3f3f3fff, 0x2a00126a,
340 0x30, 0x000000ff, 0x0040,
341 0x34, 0x00000040, 0x00004040,
342 0x9100, 0x07ffffff, 0x03000000,
343 0x8e88, 0x01ff1f3f, 0x00000000,
344 0x8e84, 0x01ff1f3f, 0x00000000,
345 0x9060, 0x0000007f, 0x00000020,
346 0x9508, 0x00010000, 0x00010000,
347 0xac14, 0x00000200, 0x000002fb,
348 0xac10, 0xffffffff, 0x0000543b,
349 0xac0c, 0xffffffff, 0xa9210876,
350 0x88d0, 0xffffffff, 0x000fff40,
351 0x88d4, 0x0000001f, 0x00000010,
352 0x1410, 0x20000000, 0x20fffed8,
353 0x15c0, 0x000c0fc0, 0x000c0400
354};
355
356static const u32 tahiti_golden_registers2[] =
357{
358 0xc64, 0x00000001, 0x00000001
359};
360
361static const u32 pitcairn_golden_rlc_registers[] =
362{
363 0xc424, 0xffffffff, 0x00601004,
364 0xc47c, 0xffffffff, 0x10102020,
365 0xc488, 0xffffffff, 0x01000020,
366 0xc314, 0xffffffff, 0x00000800,
367 0xc30c, 0xffffffff, 0x800000a4
368};
369
370static const u32 pitcairn_golden_registers[] =
371{
372 0x9a10, 0x00010000, 0x00018208,
373 0x9830, 0xffffffff, 0x00000000,
374 0x9834, 0xf00fffff, 0x00000400,
375 0x9838, 0x0002021c, 0x00020200,
376 0xc78, 0x00000080, 0x00000000,
377 0xd030, 0x000300c0, 0x00800040,
378 0xd830, 0x000300c0, 0x00800040,
379 0x5bb0, 0x000000f0, 0x00000070,
380 0x5bc0, 0x00200000, 0x50100000,
381 0x7030, 0x31000311, 0x00000011,
382 0x2ae4, 0x00073ffe, 0x000022a2,
383 0x240c, 0x000007ff, 0x00000000,
384 0x8a14, 0xf000001f, 0x00000007,
385 0x8b24, 0xffffffff, 0x00ffffff,
386 0x8b10, 0x0000ff0f, 0x00000000,
387 0x28a4c, 0x07ffffff, 0x4e000000,
388 0x28350, 0x3f3f3fff, 0x2a00126a,
389 0x30, 0x000000ff, 0x0040,
390 0x34, 0x00000040, 0x00004040,
391 0x9100, 0x07ffffff, 0x03000000,
392 0x9060, 0x0000007f, 0x00000020,
393 0x9508, 0x00010000, 0x00010000,
394 0xac14, 0x000003ff, 0x000000f7,
395 0xac10, 0xffffffff, 0x00000000,
396 0xac0c, 0xffffffff, 0x32761054,
397 0x88d4, 0x0000001f, 0x00000010,
398 0x15c0, 0x000c0fc0, 0x000c0400
399};
400
401static const u32 verde_golden_rlc_registers[] =
402{
403 0xc424, 0xffffffff, 0x033f1005,
404 0xc47c, 0xffffffff, 0x10808020,
405 0xc488, 0xffffffff, 0x00800008,
406 0xc314, 0xffffffff, 0x00001000,
407 0xc30c, 0xffffffff, 0x80010014
408};
409
410static const u32 verde_golden_registers[] =
411{
412 0x9a10, 0x00010000, 0x00018208,
413 0x9830, 0xffffffff, 0x00000000,
414 0x9834, 0xf00fffff, 0x00000400,
415 0x9838, 0x0002021c, 0x00020200,
416 0xc78, 0x00000080, 0x00000000,
417 0xd030, 0x000300c0, 0x00800040,
418 0xd030, 0x000300c0, 0x00800040,
419 0xd830, 0x000300c0, 0x00800040,
420 0xd830, 0x000300c0, 0x00800040,
421 0x5bb0, 0x000000f0, 0x00000070,
422 0x5bc0, 0x00200000, 0x50100000,
423 0x7030, 0x31000311, 0x00000011,
424 0x2ae4, 0x00073ffe, 0x000022a2,
425 0x2ae4, 0x00073ffe, 0x000022a2,
426 0x2ae4, 0x00073ffe, 0x000022a2,
427 0x240c, 0x000007ff, 0x00000000,
428 0x240c, 0x000007ff, 0x00000000,
429 0x240c, 0x000007ff, 0x00000000,
430 0x8a14, 0xf000001f, 0x00000007,
431 0x8a14, 0xf000001f, 0x00000007,
432 0x8a14, 0xf000001f, 0x00000007,
433 0x8b24, 0xffffffff, 0x00ffffff,
434 0x8b10, 0x0000ff0f, 0x00000000,
435 0x28a4c, 0x07ffffff, 0x4e000000,
436 0x28350, 0x3f3f3fff, 0x0000124a,
437 0x28350, 0x3f3f3fff, 0x0000124a,
438 0x28350, 0x3f3f3fff, 0x0000124a,
439 0x30, 0x000000ff, 0x0040,
440 0x34, 0x00000040, 0x00004040,
441 0x9100, 0x07ffffff, 0x03000000,
442 0x9100, 0x07ffffff, 0x03000000,
443 0x8e88, 0x01ff1f3f, 0x00000000,
444 0x8e88, 0x01ff1f3f, 0x00000000,
445 0x8e88, 0x01ff1f3f, 0x00000000,
446 0x8e84, 0x01ff1f3f, 0x00000000,
447 0x8e84, 0x01ff1f3f, 0x00000000,
448 0x8e84, 0x01ff1f3f, 0x00000000,
449 0x9060, 0x0000007f, 0x00000020,
450 0x9508, 0x00010000, 0x00010000,
451 0xac14, 0x000003ff, 0x00000003,
452 0xac14, 0x000003ff, 0x00000003,
453 0xac14, 0x000003ff, 0x00000003,
454 0xac10, 0xffffffff, 0x00000000,
455 0xac10, 0xffffffff, 0x00000000,
456 0xac10, 0xffffffff, 0x00000000,
457 0xac0c, 0xffffffff, 0x00001032,
458 0xac0c, 0xffffffff, 0x00001032,
459 0xac0c, 0xffffffff, 0x00001032,
460 0x88d4, 0x0000001f, 0x00000010,
461 0x88d4, 0x0000001f, 0x00000010,
462 0x88d4, 0x0000001f, 0x00000010,
463 0x15c0, 0x000c0fc0, 0x000c0400
464};
465
466static const u32 oland_golden_rlc_registers[] =
467{
468 0xc424, 0xffffffff, 0x00601005,
469 0xc47c, 0xffffffff, 0x10104040,
470 0xc488, 0xffffffff, 0x0100000a,
471 0xc314, 0xffffffff, 0x00000800,
472 0xc30c, 0xffffffff, 0x800000f4
473};
474
475static const u32 oland_golden_registers[] =
476{
477 0x9a10, 0x00010000, 0x00018208,
478 0x9830, 0xffffffff, 0x00000000,
479 0x9834, 0xf00fffff, 0x00000400,
480 0x9838, 0x0002021c, 0x00020200,
481 0xc78, 0x00000080, 0x00000000,
482 0xd030, 0x000300c0, 0x00800040,
483 0xd830, 0x000300c0, 0x00800040,
484 0x5bb0, 0x000000f0, 0x00000070,
485 0x5bc0, 0x00200000, 0x50100000,
486 0x7030, 0x31000311, 0x00000011,
487 0x2ae4, 0x00073ffe, 0x000022a2,
488 0x240c, 0x000007ff, 0x00000000,
489 0x8a14, 0xf000001f, 0x00000007,
490 0x8b24, 0xffffffff, 0x00ffffff,
491 0x8b10, 0x0000ff0f, 0x00000000,
492 0x28a4c, 0x07ffffff, 0x4e000000,
493 0x28350, 0x3f3f3fff, 0x00000082,
494 0x30, 0x000000ff, 0x0040,
495 0x34, 0x00000040, 0x00004040,
496 0x9100, 0x07ffffff, 0x03000000,
497 0x9060, 0x0000007f, 0x00000020,
498 0x9508, 0x00010000, 0x00010000,
499 0xac14, 0x000003ff, 0x000000f3,
500 0xac10, 0xffffffff, 0x00000000,
501 0xac0c, 0xffffffff, 0x00003210,
502 0x88d4, 0x0000001f, 0x00000010,
503 0x15c0, 0x000c0fc0, 0x000c0400
504};
505
fffbdda4
AD		 506static const u32 hainan_golden_registers[] =
507{
508 0x9a10, 0x00010000, 0x00018208,
509 0x9830, 0xffffffff, 0x00000000,
510 0x9834, 0xf00fffff, 0x00000400,
511 0x9838, 0x0002021c, 0x00020200,
512 0xd0c0, 0xff000fff, 0x00000100,
513 0xd030, 0x000300c0, 0x00800040,
514 0xd8c0, 0xff000fff, 0x00000100,
515 0xd830, 0x000300c0, 0x00800040,
516 0x2ae4, 0x00073ffe, 0x000022a2,
517 0x240c, 0x000007ff, 0x00000000,
518 0x8a14, 0xf000001f, 0x00000007,
519 0x8b24, 0xffffffff, 0x00ffffff,
520 0x8b10, 0x0000ff0f, 0x00000000,
521 0x28a4c, 0x07ffffff, 0x4e000000,
522 0x28350, 0x3f3f3fff, 0x00000000,
523 0x30, 0x000000ff, 0x0040,
524 0x34, 0x00000040, 0x00004040,
525 0x9100, 0x03e00000, 0x03600000,
526 0x9060, 0x0000007f, 0x00000020,
527 0x9508, 0x00010000, 0x00010000,
528 0xac14, 0x000003ff, 0x000000f1,
529 0xac10, 0xffffffff, 0x00000000,
530 0xac0c, 0xffffffff, 0x00003210,
531 0x88d4, 0x0000001f, 0x00000010,
532 0x15c0, 0x000c0fc0, 0x000c0400
533};
534
535static const u32 hainan_golden_registers2[] =
536{
537 0x98f8, 0xffffffff, 0x02010001
538};
539
205996c0
AD		 540static const u32 tahiti_mgcg_cgcg_init[] =
541{
542 0xc400, 0xffffffff, 0xfffffffc,
543 0x802c, 0xffffffff, 0xe0000000,
544 0x9a60, 0xffffffff, 0x00000100,
545 0x92a4, 0xffffffff, 0x00000100,
546 0xc164, 0xffffffff, 0x00000100,
547 0x9774, 0xffffffff, 0x00000100,
548 0x8984, 0xffffffff, 0x06000100,
549 0x8a18, 0xffffffff, 0x00000100,
550 0x92a0, 0xffffffff, 0x00000100,
551 0xc380, 0xffffffff, 0x00000100,
552 0x8b28, 0xffffffff, 0x00000100,
553 0x9144, 0xffffffff, 0x00000100,
554 0x8d88, 0xffffffff, 0x00000100,
555 0x8d8c, 0xffffffff, 0x00000100,
556 0x9030, 0xffffffff, 0x00000100,
557 0x9034, 0xffffffff, 0x00000100,
558 0x9038, 0xffffffff, 0x00000100,
559 0x903c, 0xffffffff, 0x00000100,
560 0xad80, 0xffffffff, 0x00000100,
561 0xac54, 0xffffffff, 0x00000100,
562 0x897c, 0xffffffff, 0x06000100,
563 0x9868, 0xffffffff, 0x00000100,
564 0x9510, 0xffffffff, 0x00000100,
565 0xaf04, 0xffffffff, 0x00000100,
566 0xae04, 0xffffffff, 0x00000100,
567 0x949c, 0xffffffff, 0x00000100,
568 0x802c, 0xffffffff, 0xe0000000,
569 0x9160, 0xffffffff, 0x00010000,
570 0x9164, 0xffffffff, 0x00030002,
571 0x9168, 0xffffffff, 0x00040007,
572 0x916c, 0xffffffff, 0x00060005,
573 0x9170, 0xffffffff, 0x00090008,
574 0x9174, 0xffffffff, 0x00020001,
575 0x9178, 0xffffffff, 0x00040003,
576 0x917c, 0xffffffff, 0x00000007,
577 0x9180, 0xffffffff, 0x00060005,
578 0x9184, 0xffffffff, 0x00090008,
579 0x9188, 0xffffffff, 0x00030002,
580 0x918c, 0xffffffff, 0x00050004,
581 0x9190, 0xffffffff, 0x00000008,
582 0x9194, 0xffffffff, 0x00070006,
583 0x9198, 0xffffffff, 0x000a0009,
584 0x919c, 0xffffffff, 0x00040003,
585 0x91a0, 0xffffffff, 0x00060005,
586 0x91a4, 0xffffffff, 0x00000009,
587 0x91a8, 0xffffffff, 0x00080007,
588 0x91ac, 0xffffffff, 0x000b000a,
589 0x91b0, 0xffffffff, 0x00050004,
590 0x91b4, 0xffffffff, 0x00070006,
591 0x91b8, 0xffffffff, 0x0008000b,
592 0x91bc, 0xffffffff, 0x000a0009,
593 0x91c0, 0xffffffff, 0x000d000c,
594 0x91c4, 0xffffffff, 0x00060005,
595 0x91c8, 0xffffffff, 0x00080007,
596 0x91cc, 0xffffffff, 0x0000000b,
597 0x91d0, 0xffffffff, 0x000a0009,
598 0x91d4, 0xffffffff, 0x000d000c,
599 0x91d8, 0xffffffff, 0x00070006,
600 0x91dc, 0xffffffff, 0x00090008,
601 0x91e0, 0xffffffff, 0x0000000c,
602 0x91e4, 0xffffffff, 0x000b000a,
603 0x91e8, 0xffffffff, 0x000e000d,
604 0x91ec, 0xffffffff, 0x00080007,
605 0x91f0, 0xffffffff, 0x000a0009,
606 0x91f4, 0xffffffff, 0x0000000d,
607 0x91f8, 0xffffffff, 0x000c000b,
608 0x91fc, 0xffffffff, 0x000f000e,
609 0x9200, 0xffffffff, 0x00090008,
610 0x9204, 0xffffffff, 0x000b000a,
611 0x9208, 0xffffffff, 0x000c000f,
612 0x920c, 0xffffffff, 0x000e000d,
613 0x9210, 0xffffffff, 0x00110010,
614 0x9214, 0xffffffff, 0x000a0009,
615 0x9218, 0xffffffff, 0x000c000b,
616 0x921c, 0xffffffff, 0x0000000f,
617 0x9220, 0xffffffff, 0x000e000d,
618 0x9224, 0xffffffff, 0x00110010,
619 0x9228, 0xffffffff, 0x000b000a,
620 0x922c, 0xffffffff, 0x000d000c,
621 0x9230, 0xffffffff, 0x00000010,
622 0x9234, 0xffffffff, 0x000f000e,
623 0x9238, 0xffffffff, 0x00120011,
624 0x923c, 0xffffffff, 0x000c000b,
625 0x9240, 0xffffffff, 0x000e000d,
626 0x9244, 0xffffffff, 0x00000011,
627 0x9248, 0xffffffff, 0x0010000f,
628 0x924c, 0xffffffff, 0x00130012,
629 0x9250, 0xffffffff, 0x000d000c,
630 0x9254, 0xffffffff, 0x000f000e,
631 0x9258, 0xffffffff, 0x00100013,
632 0x925c, 0xffffffff, 0x00120011,
633 0x9260, 0xffffffff, 0x00150014,
634 0x9264, 0xffffffff, 0x000e000d,
635 0x9268, 0xffffffff, 0x0010000f,
636 0x926c, 0xffffffff, 0x00000013,
637 0x9270, 0xffffffff, 0x00120011,
638 0x9274, 0xffffffff, 0x00150014,
639 0x9278, 0xffffffff, 0x000f000e,
640 0x927c, 0xffffffff, 0x00110010,
641 0x9280, 0xffffffff, 0x00000014,
642 0x9284, 0xffffffff, 0x00130012,
643 0x9288, 0xffffffff, 0x00160015,
644 0x928c, 0xffffffff, 0x0010000f,
645 0x9290, 0xffffffff, 0x00120011,
646 0x9294, 0xffffffff, 0x00000015,
647 0x9298, 0xffffffff, 0x00140013,
648 0x929c, 0xffffffff, 0x00170016,
649 0x9150, 0xffffffff, 0x96940200,
650 0x8708, 0xffffffff, 0x00900100,
651 0xc478, 0xffffffff, 0x00000080,
652 0xc404, 0xffffffff, 0x0020003f,
653 0x30, 0xffffffff, 0x0000001c,
654 0x34, 0x000f0000, 0x000f0000,
655 0x160c, 0xffffffff, 0x00000100,
656 0x1024, 0xffffffff, 0x00000100,
657 0x102c, 0x00000101, 0x00000000,
658 0x20a8, 0xffffffff, 0x00000104,
659 0x264c, 0x000c0000, 0x000c0000,
660 0x2648, 0x000c0000, 0x000c0000,
661 0x55e4, 0xff000fff, 0x00000100,
662 0x55e8, 0x00000001, 0x00000001,
663 0x2f50, 0x00000001, 0x00000001,
664 0x30cc, 0xc0000fff, 0x00000104,
665 0xc1e4, 0x00000001, 0x00000001,
666 0xd0c0, 0xfffffff0, 0x00000100,
667 0xd8c0, 0xfffffff0, 0x00000100
668};
669
670static const u32 pitcairn_mgcg_cgcg_init[] =
671{
672 0xc400, 0xffffffff, 0xfffffffc,
673 0x802c, 0xffffffff, 0xe0000000,
674 0x9a60, 0xffffffff, 0x00000100,
675 0x92a4, 0xffffffff, 0x00000100,
676 0xc164, 0xffffffff, 0x00000100,
677 0x9774, 0xffffffff, 0x00000100,
678 0x8984, 0xffffffff, 0x06000100,
679 0x8a18, 0xffffffff, 0x00000100,
680 0x92a0, 0xffffffff, 0x00000100,
681 0xc380, 0xffffffff, 0x00000100,
682 0x8b28, 0xffffffff, 0x00000100,
683 0x9144, 0xffffffff, 0x00000100,
684 0x8d88, 0xffffffff, 0x00000100,
685 0x8d8c, 0xffffffff, 0x00000100,
686 0x9030, 0xffffffff, 0x00000100,
687 0x9034, 0xffffffff, 0x00000100,
688 0x9038, 0xffffffff, 0x00000100,
689 0x903c, 0xffffffff, 0x00000100,
690 0xad80, 0xffffffff, 0x00000100,
691 0xac54, 0xffffffff, 0x00000100,
692 0x897c, 0xffffffff, 0x06000100,
693 0x9868, 0xffffffff, 0x00000100,
694 0x9510, 0xffffffff, 0x00000100,
695 0xaf04, 0xffffffff, 0x00000100,
696 0xae04, 0xffffffff, 0x00000100,
697 0x949c, 0xffffffff, 0x00000100,
698 0x802c, 0xffffffff, 0xe0000000,
699 0x9160, 0xffffffff, 0x00010000,
700 0x9164, 0xffffffff, 0x00030002,
701 0x9168, 0xffffffff, 0x00040007,
702 0x916c, 0xffffffff, 0x00060005,
703 0x9170, 0xffffffff, 0x00090008,
704 0x9174, 0xffffffff, 0x00020001,
705 0x9178, 0xffffffff, 0x00040003,
706 0x917c, 0xffffffff, 0x00000007,
707 0x9180, 0xffffffff, 0x00060005,
708 0x9184, 0xffffffff, 0x00090008,
709 0x9188, 0xffffffff, 0x00030002,
710 0x918c, 0xffffffff, 0x00050004,
711 0x9190, 0xffffffff, 0x00000008,
712 0x9194, 0xffffffff, 0x00070006,
713 0x9198, 0xffffffff, 0x000a0009,
714 0x919c, 0xffffffff, 0x00040003,
715 0x91a0, 0xffffffff, 0x00060005,
716 0x91a4, 0xffffffff, 0x00000009,
717 0x91a8, 0xffffffff, 0x00080007,
718 0x91ac, 0xffffffff, 0x000b000a,
719 0x91b0, 0xffffffff, 0x00050004,
720 0x91b4, 0xffffffff, 0x00070006,
721 0x91b8, 0xffffffff, 0x0008000b,
722 0x91bc, 0xffffffff, 0x000a0009,
723 0x91c0, 0xffffffff, 0x000d000c,
724 0x9200, 0xffffffff, 0x00090008,
725 0x9204, 0xffffffff, 0x000b000a,
726 0x9208, 0xffffffff, 0x000c000f,
727 0x920c, 0xffffffff, 0x000e000d,
728 0x9210, 0xffffffff, 0x00110010,
729 0x9214, 0xffffffff, 0x000a0009,
730 0x9218, 0xffffffff, 0x000c000b,
731 0x921c, 0xffffffff, 0x0000000f,
732 0x9220, 0xffffffff, 0x000e000d,
733 0x9224, 0xffffffff, 0x00110010,
734 0x9228, 0xffffffff, 0x000b000a,
735 0x922c, 0xffffffff, 0x000d000c,
736 0x9230, 0xffffffff, 0x00000010,
737 0x9234, 0xffffffff, 0x000f000e,
738 0x9238, 0xffffffff, 0x00120011,
739 0x923c, 0xffffffff, 0x000c000b,
740 0x9240, 0xffffffff, 0x000e000d,
741 0x9244, 0xffffffff, 0x00000011,
742 0x9248, 0xffffffff, 0x0010000f,
743 0x924c, 0xffffffff, 0x00130012,
744 0x9250, 0xffffffff, 0x000d000c,
745 0x9254, 0xffffffff, 0x000f000e,
746 0x9258, 0xffffffff, 0x00100013,
747 0x925c, 0xffffffff, 0x00120011,
748 0x9260, 0xffffffff, 0x00150014,
749 0x9150, 0xffffffff, 0x96940200,
750 0x8708, 0xffffffff, 0x00900100,
751 0xc478, 0xffffffff, 0x00000080,
752 0xc404, 0xffffffff, 0x0020003f,
753 0x30, 0xffffffff, 0x0000001c,
754 0x34, 0x000f0000, 0x000f0000,
755 0x160c, 0xffffffff, 0x00000100,
756 0x1024, 0xffffffff, 0x00000100,
757 0x102c, 0x00000101, 0x00000000,
758 0x20a8, 0xffffffff, 0x00000104,
759 0x55e4, 0xff000fff, 0x00000100,
760 0x55e8, 0x00000001, 0x00000001,
761 0x2f50, 0x00000001, 0x00000001,
762 0x30cc, 0xc0000fff, 0x00000104,
763 0xc1e4, 0x00000001, 0x00000001,
764 0xd0c0, 0xfffffff0, 0x00000100,
765 0xd8c0, 0xfffffff0, 0x00000100
766};
767
768static const u32 verde_mgcg_cgcg_init[] =
769{
770 0xc400, 0xffffffff, 0xfffffffc,
771 0x802c, 0xffffffff, 0xe0000000,
772 0x9a60, 0xffffffff, 0x00000100,
773 0x92a4, 0xffffffff, 0x00000100,
774 0xc164, 0xffffffff, 0x00000100,
775 0x9774, 0xffffffff, 0x00000100,
776 0x8984, 0xffffffff, 0x06000100,
777 0x8a18, 0xffffffff, 0x00000100,
778 0x92a0, 0xffffffff, 0x00000100,
779 0xc380, 0xffffffff, 0x00000100,
780 0x8b28, 0xffffffff, 0x00000100,
781 0x9144, 0xffffffff, 0x00000100,
782 0x8d88, 0xffffffff, 0x00000100,
783 0x8d8c, 0xffffffff, 0x00000100,
784 0x9030, 0xffffffff, 0x00000100,
785 0x9034, 0xffffffff, 0x00000100,
786 0x9038, 0xffffffff, 0x00000100,
787 0x903c, 0xffffffff, 0x00000100,
788 0xad80, 0xffffffff, 0x00000100,
789 0xac54, 0xffffffff, 0x00000100,
790 0x897c, 0xffffffff, 0x06000100,
791 0x9868, 0xffffffff, 0x00000100,
792 0x9510, 0xffffffff, 0x00000100,
793 0xaf04, 0xffffffff, 0x00000100,
794 0xae04, 0xffffffff, 0x00000100,
795 0x949c, 0xffffffff, 0x00000100,
796 0x802c, 0xffffffff, 0xe0000000,
797 0x9160, 0xffffffff, 0x00010000,
798 0x9164, 0xffffffff, 0x00030002,
799 0x9168, 0xffffffff, 0x00040007,
800 0x916c, 0xffffffff, 0x00060005,
801 0x9170, 0xffffffff, 0x00090008,
802 0x9174, 0xffffffff, 0x00020001,
803 0x9178, 0xffffffff, 0x00040003,
804 0x917c, 0xffffffff, 0x00000007,
805 0x9180, 0xffffffff, 0x00060005,
806 0x9184, 0xffffffff, 0x00090008,
807 0x9188, 0xffffffff, 0x00030002,
808 0x918c, 0xffffffff, 0x00050004,
809 0x9190, 0xffffffff, 0x00000008,
810 0x9194, 0xffffffff, 0x00070006,
811 0x9198, 0xffffffff, 0x000a0009,
812 0x919c, 0xffffffff, 0x00040003,
813 0x91a0, 0xffffffff, 0x00060005,
814 0x91a4, 0xffffffff, 0x00000009,
815 0x91a8, 0xffffffff, 0x00080007,
816 0x91ac, 0xffffffff, 0x000b000a,
817 0x91b0, 0xffffffff, 0x00050004,
818 0x91b4, 0xffffffff, 0x00070006,
819 0x91b8, 0xffffffff, 0x0008000b,
820 0x91bc, 0xffffffff, 0x000a0009,
821 0x91c0, 0xffffffff, 0x000d000c,
822 0x9200, 0xffffffff, 0x00090008,
823 0x9204, 0xffffffff, 0x000b000a,
824 0x9208, 0xffffffff, 0x000c000f,
825 0x920c, 0xffffffff, 0x000e000d,
826 0x9210, 0xffffffff, 0x00110010,
827 0x9214, 0xffffffff, 0x000a0009,
828 0x9218, 0xffffffff, 0x000c000b,
829 0x921c, 0xffffffff, 0x0000000f,
830 0x9220, 0xffffffff, 0x000e000d,
831 0x9224, 0xffffffff, 0x00110010,
832 0x9228, 0xffffffff, 0x000b000a,
833 0x922c, 0xffffffff, 0x000d000c,
834 0x9230, 0xffffffff, 0x00000010,
835 0x9234, 0xffffffff, 0x000f000e,
836 0x9238, 0xffffffff, 0x00120011,
837 0x923c, 0xffffffff, 0x000c000b,
838 0x9240, 0xffffffff, 0x000e000d,
839 0x9244, 0xffffffff, 0x00000011,
840 0x9248, 0xffffffff, 0x0010000f,
841 0x924c, 0xffffffff, 0x00130012,
842 0x9250, 0xffffffff, 0x000d000c,
843 0x9254, 0xffffffff, 0x000f000e,
844 0x9258, 0xffffffff, 0x00100013,
845 0x925c, 0xffffffff, 0x00120011,
846 0x9260, 0xffffffff, 0x00150014,
847 0x9150, 0xffffffff, 0x96940200,
848 0x8708, 0xffffffff, 0x00900100,
849 0xc478, 0xffffffff, 0x00000080,
850 0xc404, 0xffffffff, 0x0020003f,
851 0x30, 0xffffffff, 0x0000001c,
852 0x34, 0x000f0000, 0x000f0000,
853 0x160c, 0xffffffff, 0x00000100,
854 0x1024, 0xffffffff, 0x00000100,
855 0x102c, 0x00000101, 0x00000000,
856 0x20a8, 0xffffffff, 0x00000104,
857 0x264c, 0x000c0000, 0x000c0000,
858 0x2648, 0x000c0000, 0x000c0000,
859 0x55e4, 0xff000fff, 0x00000100,
860 0x55e8, 0x00000001, 0x00000001,
861 0x2f50, 0x00000001, 0x00000001,
862 0x30cc, 0xc0000fff, 0x00000104,
863 0xc1e4, 0x00000001, 0x00000001,
864 0xd0c0, 0xfffffff0, 0x00000100,
865 0xd8c0, 0xfffffff0, 0x00000100
866};
867
868static const u32 oland_mgcg_cgcg_init[] =
869{
870 0xc400, 0xffffffff, 0xfffffffc,
871 0x802c, 0xffffffff, 0xe0000000,
872 0x9a60, 0xffffffff, 0x00000100,
873 0x92a4, 0xffffffff, 0x00000100,
874 0xc164, 0xffffffff, 0x00000100,
875 0x9774, 0xffffffff, 0x00000100,
876 0x8984, 0xffffffff, 0x06000100,
877 0x8a18, 0xffffffff, 0x00000100,
878 0x92a0, 0xffffffff, 0x00000100,
879 0xc380, 0xffffffff, 0x00000100,
880 0x8b28, 0xffffffff, 0x00000100,
881 0x9144, 0xffffffff, 0x00000100,
882 0x8d88, 0xffffffff, 0x00000100,
883 0x8d8c, 0xffffffff, 0x00000100,
884 0x9030, 0xffffffff, 0x00000100,
885 0x9034, 0xffffffff, 0x00000100,
886 0x9038, 0xffffffff, 0x00000100,
887 0x903c, 0xffffffff, 0x00000100,
888 0xad80, 0xffffffff, 0x00000100,
889 0xac54, 0xffffffff, 0x00000100,
890 0x897c, 0xffffffff, 0x06000100,
891 0x9868, 0xffffffff, 0x00000100,
892 0x9510, 0xffffffff, 0x00000100,
893 0xaf04, 0xffffffff, 0x00000100,
894 0xae04, 0xffffffff, 0x00000100,
895 0x949c, 0xffffffff, 0x00000100,
896 0x802c, 0xffffffff, 0xe0000000,
897 0x9160, 0xffffffff, 0x00010000,
898 0x9164, 0xffffffff, 0x00030002,
899 0x9168, 0xffffffff, 0x00040007,
900 0x916c, 0xffffffff, 0x00060005,
901 0x9170, 0xffffffff, 0x00090008,
902 0x9174, 0xffffffff, 0x00020001,
903 0x9178, 0xffffffff, 0x00040003,
904 0x917c, 0xffffffff, 0x00000007,
905 0x9180, 0xffffffff, 0x00060005,
906 0x9184, 0xffffffff, 0x00090008,
907 0x9188, 0xffffffff, 0x00030002,
908 0x918c, 0xffffffff, 0x00050004,
909 0x9190, 0xffffffff, 0x00000008,
910 0x9194, 0xffffffff, 0x00070006,
911 0x9198, 0xffffffff, 0x000a0009,
912 0x919c, 0xffffffff, 0x00040003,
913 0x91a0, 0xffffffff, 0x00060005,
914 0x91a4, 0xffffffff, 0x00000009,
915 0x91a8, 0xffffffff, 0x00080007,
916 0x91ac, 0xffffffff, 0x000b000a,
917 0x91b0, 0xffffffff, 0x00050004,
918 0x91b4, 0xffffffff, 0x00070006,
919 0x91b8, 0xffffffff, 0x0008000b,
920 0x91bc, 0xffffffff, 0x000a0009,
921 0x91c0, 0xffffffff, 0x000d000c,
922 0x91c4, 0xffffffff, 0x00060005,
923 0x91c8, 0xffffffff, 0x00080007,
924 0x91cc, 0xffffffff, 0x0000000b,
925 0x91d0, 0xffffffff, 0x000a0009,
926 0x91d4, 0xffffffff, 0x000d000c,
927 0x9150, 0xffffffff, 0x96940200,
928 0x8708, 0xffffffff, 0x00900100,
929 0xc478, 0xffffffff, 0x00000080,
930 0xc404, 0xffffffff, 0x0020003f,
931 0x30, 0xffffffff, 0x0000001c,
932 0x34, 0x000f0000, 0x000f0000,
933 0x160c, 0xffffffff, 0x00000100,
934 0x1024, 0xffffffff, 0x00000100,
935 0x102c, 0x00000101, 0x00000000,
936 0x20a8, 0xffffffff, 0x00000104,
937 0x264c, 0x000c0000, 0x000c0000,
938 0x2648, 0x000c0000, 0x000c0000,
939 0x55e4, 0xff000fff, 0x00000100,
940 0x55e8, 0x00000001, 0x00000001,
941 0x2f50, 0x00000001, 0x00000001,
942 0x30cc, 0xc0000fff, 0x00000104,
943 0xc1e4, 0x00000001, 0x00000001,
944 0xd0c0, 0xfffffff0, 0x00000100,
945 0xd8c0, 0xfffffff0, 0x00000100
946};
947
fffbdda4
AD		 948static const u32 hainan_mgcg_cgcg_init[] =
949{
950 0xc400, 0xffffffff, 0xfffffffc,
951 0x802c, 0xffffffff, 0xe0000000,
952 0x9a60, 0xffffffff, 0x00000100,
953 0x92a4, 0xffffffff, 0x00000100,
954 0xc164, 0xffffffff, 0x00000100,
955 0x9774, 0xffffffff, 0x00000100,
956 0x8984, 0xffffffff, 0x06000100,
957 0x8a18, 0xffffffff, 0x00000100,
958 0x92a0, 0xffffffff, 0x00000100,
959 0xc380, 0xffffffff, 0x00000100,
960 0x8b28, 0xffffffff, 0x00000100,
961 0x9144, 0xffffffff, 0x00000100,
962 0x8d88, 0xffffffff, 0x00000100,
963 0x8d8c, 0xffffffff, 0x00000100,
964 0x9030, 0xffffffff, 0x00000100,
965 0x9034, 0xffffffff, 0x00000100,
966 0x9038, 0xffffffff, 0x00000100,
967 0x903c, 0xffffffff, 0x00000100,
968 0xad80, 0xffffffff, 0x00000100,
969 0xac54, 0xffffffff, 0x00000100,
970 0x897c, 0xffffffff, 0x06000100,
971 0x9868, 0xffffffff, 0x00000100,
972 0x9510, 0xffffffff, 0x00000100,
973 0xaf04, 0xffffffff, 0x00000100,
974 0xae04, 0xffffffff, 0x00000100,
975 0x949c, 0xffffffff, 0x00000100,
976 0x802c, 0xffffffff, 0xe0000000,
977 0x9160, 0xffffffff, 0x00010000,
978 0x9164, 0xffffffff, 0x00030002,
979 0x9168, 0xffffffff, 0x00040007,
980 0x916c, 0xffffffff, 0x00060005,
981 0x9170, 0xffffffff, 0x00090008,
982 0x9174, 0xffffffff, 0x00020001,
983 0x9178, 0xffffffff, 0x00040003,
984 0x917c, 0xffffffff, 0x00000007,
985 0x9180, 0xffffffff, 0x00060005,
986 0x9184, 0xffffffff, 0x00090008,
987 0x9188, 0xffffffff, 0x00030002,
988 0x918c, 0xffffffff, 0x00050004,
989 0x9190, 0xffffffff, 0x00000008,
990 0x9194, 0xffffffff, 0x00070006,
991 0x9198, 0xffffffff, 0x000a0009,
992 0x919c, 0xffffffff, 0x00040003,
993 0x91a0, 0xffffffff, 0x00060005,
994 0x91a4, 0xffffffff, 0x00000009,
995 0x91a8, 0xffffffff, 0x00080007,
996 0x91ac, 0xffffffff, 0x000b000a,
997 0x91b0, 0xffffffff, 0x00050004,
998 0x91b4, 0xffffffff, 0x00070006,
999 0x91b8, 0xffffffff, 0x0008000b,
1000 0x91bc, 0xffffffff, 0x000a0009,
1001 0x91c0, 0xffffffff, 0x000d000c,
1002 0x91c4, 0xffffffff, 0x00060005,
1003 0x91c8, 0xffffffff, 0x00080007,
1004 0x91cc, 0xffffffff, 0x0000000b,
1005 0x91d0, 0xffffffff, 0x000a0009,
1006 0x91d4, 0xffffffff, 0x000d000c,
1007 0x9150, 0xffffffff, 0x96940200,
1008 0x8708, 0xffffffff, 0x00900100,
1009 0xc478, 0xffffffff, 0x00000080,
1010 0xc404, 0xffffffff, 0x0020003f,
1011 0x30, 0xffffffff, 0x0000001c,
1012 0x34, 0x000f0000, 0x000f0000,
1013 0x160c, 0xffffffff, 0x00000100,
1014 0x1024, 0xffffffff, 0x00000100,
1015 0x20a8, 0xffffffff, 0x00000104,
1016 0x264c, 0x000c0000, 0x000c0000,
1017 0x2648, 0x000c0000, 0x000c0000,
1018 0x2f50, 0x00000001, 0x00000001,
1019 0x30cc, 0xc0000fff, 0x00000104,
1020 0xc1e4, 0x00000001, 0x00000001,
1021 0xd0c0, 0xfffffff0, 0x00000100,
1022 0xd8c0, 0xfffffff0, 0x00000100
1023};
1024
205996c0
AD		 1025static u32 verde_pg_init[] =
1026{
1027 0x353c, 0xffffffff, 0x40000,
1028 0x3538, 0xffffffff, 0x200010ff,
1029 0x353c, 0xffffffff, 0x0,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x0,
1034 0x353c, 0xffffffff, 0x7007,
1035 0x3538, 0xffffffff, 0x300010ff,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x0,
1041 0x353c, 0xffffffff, 0x400000,
1042 0x3538, 0xffffffff, 0x100010ff,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x0,
1048 0x353c, 0xffffffff, 0x120200,
1049 0x3538, 0xffffffff, 0x500010ff,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x0,
1055 0x353c, 0xffffffff, 0x1e1e16,
1056 0x3538, 0xffffffff, 0x600010ff,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x0,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x0,
1061 0x353c, 0xffffffff, 0x0,
1062 0x353c, 0xffffffff, 0x171f1e,
1063 0x3538, 0xffffffff, 0x700010ff,
1064 0x353c, 0xffffffff, 0x0,
1065 0x353c, 0xffffffff, 0x0,
1066 0x353c, 0xffffffff, 0x0,
1067 0x353c, 0xffffffff, 0x0,
1068 0x353c, 0xffffffff, 0x0,
1069 0x353c, 0xffffffff, 0x0,
1070 0x3538, 0xffffffff, 0x9ff,
1071 0x3500, 0xffffffff, 0x0,
1072 0x3504, 0xffffffff, 0x10000800,
1073 0x3504, 0xffffffff, 0xf,
1074 0x3504, 0xffffffff, 0xf,
1075 0x3500, 0xffffffff, 0x4,
1076 0x3504, 0xffffffff, 0x1000051e,
1077 0x3504, 0xffffffff, 0xffff,
1078 0x3504, 0xffffffff, 0xffff,
1079 0x3500, 0xffffffff, 0x8,
1080 0x3504, 0xffffffff, 0x80500,
1081 0x3500, 0xffffffff, 0x12,
1082 0x3504, 0xffffffff, 0x9050c,
1083 0x3500, 0xffffffff, 0x1d,
1084 0x3504, 0xffffffff, 0xb052c,
1085 0x3500, 0xffffffff, 0x2a,
1086 0x3504, 0xffffffff, 0x1053e,
1087 0x3500, 0xffffffff, 0x2d,
1088 0x3504, 0xffffffff, 0x10546,
1089 0x3500, 0xffffffff, 0x30,
1090 0x3504, 0xffffffff, 0xa054e,
1091 0x3500, 0xffffffff, 0x3c,
1092 0x3504, 0xffffffff, 0x1055f,
1093 0x3500, 0xffffffff, 0x3f,
1094 0x3504, 0xffffffff, 0x10567,
1095 0x3500, 0xffffffff, 0x42,
1096 0x3504, 0xffffffff, 0x1056f,
1097 0x3500, 0xffffffff, 0x45,
1098 0x3504, 0xffffffff, 0x10572,
1099 0x3500, 0xffffffff, 0x48,
1100 0x3504, 0xffffffff, 0x20575,
1101 0x3500, 0xffffffff, 0x4c,
1102 0x3504, 0xffffffff, 0x190801,
1103 0x3500, 0xffffffff, 0x67,
1104 0x3504, 0xffffffff, 0x1082a,
1105 0x3500, 0xffffffff, 0x6a,
1106 0x3504, 0xffffffff, 0x1b082d,
1107 0x3500, 0xffffffff, 0x87,
1108 0x3504, 0xffffffff, 0x310851,
1109 0x3500, 0xffffffff, 0xba,
1110 0x3504, 0xffffffff, 0x891,
1111 0x3500, 0xffffffff, 0xbc,
1112 0x3504, 0xffffffff, 0x893,
1113 0x3500, 0xffffffff, 0xbe,
1114 0x3504, 0xffffffff, 0x20895,
1115 0x3500, 0xffffffff, 0xc2,
1116 0x3504, 0xffffffff, 0x20899,
1117 0x3500, 0xffffffff, 0xc6,
1118 0x3504, 0xffffffff, 0x2089d,
1119 0x3500, 0xffffffff, 0xca,
1120 0x3504, 0xffffffff, 0x8a1,
1121 0x3500, 0xffffffff, 0xcc,
1122 0x3504, 0xffffffff, 0x8a3,
1123 0x3500, 0xffffffff, 0xce,
1124 0x3504, 0xffffffff, 0x308a5,
1125 0x3500, 0xffffffff, 0xd3,
1126 0x3504, 0xffffffff, 0x6d08cd,
1127 0x3500, 0xffffffff, 0x142,
1128 0x3504, 0xffffffff, 0x2000095a,
1129 0x3504, 0xffffffff, 0x1,
1130 0x3500, 0xffffffff, 0x144,
1131 0x3504, 0xffffffff, 0x301f095b,
1132 0x3500, 0xffffffff, 0x165,
1133 0x3504, 0xffffffff, 0xc094d,
1134 0x3500, 0xffffffff, 0x173,
1135 0x3504, 0xffffffff, 0xf096d,
1136 0x3500, 0xffffffff, 0x184,
1137 0x3504, 0xffffffff, 0x15097f,
1138 0x3500, 0xffffffff, 0x19b,
1139 0x3504, 0xffffffff, 0xc0998,
1140 0x3500, 0xffffffff, 0x1a9,
1141 0x3504, 0xffffffff, 0x409a7,
1142 0x3500, 0xffffffff, 0x1af,
1143 0x3504, 0xffffffff, 0xcdc,
1144 0x3500, 0xffffffff, 0x1b1,
1145 0x3504, 0xffffffff, 0x800,
1146 0x3508, 0xffffffff, 0x6c9b2000,
1147 0x3510, 0xfc00, 0x2000,
1148 0x3544, 0xffffffff, 0xfc0,
1149 0x28d4, 0x00000100, 0x100
1150};
1151
1152static void si_init_golden_registers(struct radeon_device *rdev)
1153{
1154 switch (rdev->family) {
1155 case CHIP_TAHITI:
1156 radeon_program_register_sequence(rdev,
1157 tahiti_golden_registers,
1158 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1159 radeon_program_register_sequence(rdev,
1160 tahiti_golden_rlc_registers,
1161 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1162 radeon_program_register_sequence(rdev,
1163 tahiti_mgcg_cgcg_init,
1164 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1165 radeon_program_register_sequence(rdev,
1166 tahiti_golden_registers2,
1167 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1168 break;
1169 case CHIP_PITCAIRN:
1170 radeon_program_register_sequence(rdev,
1171 pitcairn_golden_registers,
1172 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1173 radeon_program_register_sequence(rdev,
1174 pitcairn_golden_rlc_registers,
1175 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1176 radeon_program_register_sequence(rdev,
1177 pitcairn_mgcg_cgcg_init,
1178 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1179 break;
1180 case CHIP_VERDE:
1181 radeon_program_register_sequence(rdev,
1182 verde_golden_registers,
1183 (const u32)ARRAY_SIZE(verde_golden_registers));
1184 radeon_program_register_sequence(rdev,
1185 verde_golden_rlc_registers,
1186 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1187 radeon_program_register_sequence(rdev,
1188 verde_mgcg_cgcg_init,
1189 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1190 radeon_program_register_sequence(rdev,
1191 verde_pg_init,
1192 (const u32)ARRAY_SIZE(verde_pg_init));
1193 break;
1194 case CHIP_OLAND:
1195 radeon_program_register_sequence(rdev,
1196 oland_golden_registers,
1197 (const u32)ARRAY_SIZE(oland_golden_registers));
1198 radeon_program_register_sequence(rdev,
1199 oland_golden_rlc_registers,
1200 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1201 radeon_program_register_sequence(rdev,
1202 oland_mgcg_cgcg_init,
1203 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1204 break;
fffbdda4
AD		 1205 case CHIP_HAINAN:
1206 radeon_program_register_sequence(rdev,
1207 hainan_golden_registers,
1208 (const u32)ARRAY_SIZE(hainan_golden_registers));
1209 radeon_program_register_sequence(rdev,
1210 hainan_golden_registers2,
1211 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1212 radeon_program_register_sequence(rdev,
1213 hainan_mgcg_cgcg_init,
1214 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1215 break;
205996c0
AD		 1216 default:
1217 break;
1218 }
1219}
1220
454d2e2a
AD		 1221#define PCIE_BUS_CLK 10000
1222#define TCLK (PCIE_BUS_CLK / 10)
1223
1224/**
1225 * si_get_xclk - get the xclk
1226 *
1227 * @rdev: radeon_device pointer
1228 *
1229 * Returns the reference clock used by the gfx engine
1230 * (SI).
1231 */
1232u32 si_get_xclk(struct radeon_device *rdev)
1233{
1234 u32 reference_clock = rdev->clock.spll.reference_freq;
1235 u32 tmp;
1236
1237 tmp = RREG32(CG_CLKPIN_CNTL_2);
1238 if (tmp & MUX_TCLK_TO_XCLK)
1239 return TCLK;
1240
1241 tmp = RREG32(CG_CLKPIN_CNTL);
1242 if (tmp & XTALIN_DIVIDE)
1243 return reference_clock / 4;
1244
1245 return reference_clock;
1246}
1247
1bd47d2e
AD		 1248/* get temperature in millidegrees */
1249int si_get_temp(struct radeon_device *rdev)
1250{
1251 u32 temp;
1252 int actual_temp = 0;
1253
1254 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1255 CTF_TEMP_SHIFT;
1256
1257 if (temp & 0x200)
1258 actual_temp = 255;
1259 else
1260 actual_temp = temp & 0x1ff;
1261
1262 actual_temp = (actual_temp * 1000);
1263
1264 return actual_temp;
1265}
1266
8b074dd6
AD		 1267#define TAHITI_IO_MC_REGS_SIZE 36
1268
1269static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1270 {0x0000006f, 0x03044000},
1271 {0x00000070, 0x0480c018},
1272 {0x00000071, 0x00000040},
1273 {0x00000072, 0x01000000},
1274 {0x00000074, 0x000000ff},
1275 {0x00000075, 0x00143400},
1276 {0x00000076, 0x08ec0800},
1277 {0x00000077, 0x040000cc},
1278 {0x00000079, 0x00000000},
1279 {0x0000007a, 0x21000409},
1280 {0x0000007c, 0x00000000},
1281 {0x0000007d, 0xe8000000},
1282 {0x0000007e, 0x044408a8},
1283 {0x0000007f, 0x00000003},
1284 {0x00000080, 0x00000000},
1285 {0x00000081, 0x01000000},
1286 {0x00000082, 0x02000000},
1287 {0x00000083, 0x00000000},
1288 {0x00000084, 0xe3f3e4f4},
1289 {0x00000085, 0x00052024},
1290 {0x00000087, 0x00000000},
1291 {0x00000088, 0x66036603},
1292 {0x00000089, 0x01000000},
1293 {0x0000008b, 0x1c0a0000},
1294 {0x0000008c, 0xff010000},
1295 {0x0000008e, 0xffffefff},
1296 {0x0000008f, 0xfff3efff},
1297 {0x00000090, 0xfff3efbf},
1298 {0x00000094, 0x00101101},
1299 {0x00000095, 0x00000fff},
1300 {0x00000096, 0x00116fff},
1301 {0x00000097, 0x60010000},
1302 {0x00000098, 0x10010000},
1303 {0x00000099, 0x00006000},
1304 {0x0000009a, 0x00001000},
1305 {0x0000009f, 0x00a77400}
1306};
1307
1308static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1309 {0x0000006f, 0x03044000},
1310 {0x00000070, 0x0480c018},
1311 {0x00000071, 0x00000040},
1312 {0x00000072, 0x01000000},
1313 {0x00000074, 0x000000ff},
1314 {0x00000075, 0x00143400},
1315 {0x00000076, 0x08ec0800},
1316 {0x00000077, 0x040000cc},
1317 {0x00000079, 0x00000000},
1318 {0x0000007a, 0x21000409},
1319 {0x0000007c, 0x00000000},
1320 {0x0000007d, 0xe8000000},
1321 {0x0000007e, 0x044408a8},
1322 {0x0000007f, 0x00000003},
1323 {0x00000080, 0x00000000},
1324 {0x00000081, 0x01000000},
1325 {0x00000082, 0x02000000},
1326 {0x00000083, 0x00000000},
1327 {0x00000084, 0xe3f3e4f4},
1328 {0x00000085, 0x00052024},
1329 {0x00000087, 0x00000000},
1330 {0x00000088, 0x66036603},
1331 {0x00000089, 0x01000000},
1332 {0x0000008b, 0x1c0a0000},
1333 {0x0000008c, 0xff010000},
1334 {0x0000008e, 0xffffefff},
1335 {0x0000008f, 0xfff3efff},
1336 {0x00000090, 0xfff3efbf},
1337 {0x00000094, 0x00101101},
1338 {0x00000095, 0x00000fff},
1339 {0x00000096, 0x00116fff},
1340 {0x00000097, 0x60010000},
1341 {0x00000098, 0x10010000},
1342 {0x00000099, 0x00006000},
1343 {0x0000009a, 0x00001000},
1344 {0x0000009f, 0x00a47400}
1345};
1346
1347static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1348 {0x0000006f, 0x03044000},
1349 {0x00000070, 0x0480c018},
1350 {0x00000071, 0x00000040},
1351 {0x00000072, 0x01000000},
1352 {0x00000074, 0x000000ff},
1353 {0x00000075, 0x00143400},
1354 {0x00000076, 0x08ec0800},
1355 {0x00000077, 0x040000cc},
1356 {0x00000079, 0x00000000},
1357 {0x0000007a, 0x21000409},
1358 {0x0000007c, 0x00000000},
1359 {0x0000007d, 0xe8000000},
1360 {0x0000007e, 0x044408a8},
1361 {0x0000007f, 0x00000003},
1362 {0x00000080, 0x00000000},
1363 {0x00000081, 0x01000000},
1364 {0x00000082, 0x02000000},
1365 {0x00000083, 0x00000000},
1366 {0x00000084, 0xe3f3e4f4},
1367 {0x00000085, 0x00052024},
1368 {0x00000087, 0x00000000},
1369 {0x00000088, 0x66036603},
1370 {0x00000089, 0x01000000},
1371 {0x0000008b, 0x1c0a0000},
1372 {0x0000008c, 0xff010000},
1373 {0x0000008e, 0xffffefff},
1374 {0x0000008f, 0xfff3efff},
1375 {0x00000090, 0xfff3efbf},
1376 {0x00000094, 0x00101101},
1377 {0x00000095, 0x00000fff},
1378 {0x00000096, 0x00116fff},
1379 {0x00000097, 0x60010000},
1380 {0x00000098, 0x10010000},
1381 {0x00000099, 0x00006000},
1382 {0x0000009a, 0x00001000},
1383 {0x0000009f, 0x00a37400}
1384};
1385
bcc7f5d2
AD		 1386static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1387 {0x0000006f, 0x03044000},
1388 {0x00000070, 0x0480c018},
1389 {0x00000071, 0x00000040},
1390 {0x00000072, 0x01000000},
1391 {0x00000074, 0x000000ff},
1392 {0x00000075, 0x00143400},
1393 {0x00000076, 0x08ec0800},
1394 {0x00000077, 0x040000cc},
1395 {0x00000079, 0x00000000},
1396 {0x0000007a, 0x21000409},
1397 {0x0000007c, 0x00000000},
1398 {0x0000007d, 0xe8000000},
1399 {0x0000007e, 0x044408a8},
1400 {0x0000007f, 0x00000003},
1401 {0x00000080, 0x00000000},
1402 {0x00000081, 0x01000000},
1403 {0x00000082, 0x02000000},
1404 {0x00000083, 0x00000000},
1405 {0x00000084, 0xe3f3e4f4},
1406 {0x00000085, 0x00052024},
1407 {0x00000087, 0x00000000},
1408 {0x00000088, 0x66036603},
1409 {0x00000089, 0x01000000},
1410 {0x0000008b, 0x1c0a0000},
1411 {0x0000008c, 0xff010000},
1412 {0x0000008e, 0xffffefff},
1413 {0x0000008f, 0xfff3efff},
1414 {0x00000090, 0xfff3efbf},
1415 {0x00000094, 0x00101101},
1416 {0x00000095, 0x00000fff},
1417 {0x00000096, 0x00116fff},
1418 {0x00000097, 0x60010000},
1419 {0x00000098, 0x10010000},
1420 {0x00000099, 0x00006000},
1421 {0x0000009a, 0x00001000},
1422 {0x0000009f, 0x00a17730}
1423};
1424
c04c00b4
AD		 1425static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1426 {0x0000006f, 0x03044000},
1427 {0x00000070, 0x0480c018},
1428 {0x00000071, 0x00000040},
1429 {0x00000072, 0x01000000},
1430 {0x00000074, 0x000000ff},
1431 {0x00000075, 0x00143400},
1432 {0x00000076, 0x08ec0800},
1433 {0x00000077, 0x040000cc},
1434 {0x00000079, 0x00000000},
1435 {0x0000007a, 0x21000409},
1436 {0x0000007c, 0x00000000},
1437 {0x0000007d, 0xe8000000},
1438 {0x0000007e, 0x044408a8},
1439 {0x0000007f, 0x00000003},
1440 {0x00000080, 0x00000000},
1441 {0x00000081, 0x01000000},
1442 {0x00000082, 0x02000000},
1443 {0x00000083, 0x00000000},
1444 {0x00000084, 0xe3f3e4f4},
1445 {0x00000085, 0x00052024},
1446 {0x00000087, 0x00000000},
1447 {0x00000088, 0x66036603},
1448 {0x00000089, 0x01000000},
1449 {0x0000008b, 0x1c0a0000},
1450 {0x0000008c, 0xff010000},
1451 {0x0000008e, 0xffffefff},
1452 {0x0000008f, 0xfff3efff},
1453 {0x00000090, 0xfff3efbf},
1454 {0x00000094, 0x00101101},
1455 {0x00000095, 0x00000fff},
1456 {0x00000096, 0x00116fff},
1457 {0x00000097, 0x60010000},
1458 {0x00000098, 0x10010000},
1459 {0x00000099, 0x00006000},
1460 {0x0000009a, 0x00001000},
1461 {0x0000009f, 0x00a07730}
1462};
1463
8b074dd6 1464/* ucode loading */
6c7bccea 1465int si_mc_load_microcode(struct radeon_device *rdev)
8b074dd6
AD		 1466{
1467 const __be32 *fw_data;
1468 u32 running, blackout = 0;
1469 u32 *io_mc_regs;
1470 int i, ucode_size, regs_size;
1471
1472 if (!rdev->mc_fw)
1473 return -EINVAL;
1474
1475 switch (rdev->family) {
1476 case CHIP_TAHITI:
1477 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1478 ucode_size = SI_MC_UCODE_SIZE;
1479 regs_size = TAHITI_IO_MC_REGS_SIZE;
1480 break;
1481 case CHIP_PITCAIRN:
1482 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1483 ucode_size = SI_MC_UCODE_SIZE;
1484 regs_size = TAHITI_IO_MC_REGS_SIZE;
1485 break;
1486 case CHIP_VERDE:
1487 default:
1488 io_mc_regs = (u32 *)&verde_io_mc_regs;
1489 ucode_size = SI_MC_UCODE_SIZE;
1490 regs_size = TAHITI_IO_MC_REGS_SIZE;
1491 break;
bcc7f5d2
AD		 1492 case CHIP_OLAND:
1493 io_mc_regs = (u32 *)&oland_io_mc_regs;
1494 ucode_size = OLAND_MC_UCODE_SIZE;
1495 regs_size = TAHITI_IO_MC_REGS_SIZE;
1496 break;
c04c00b4
AD		 1497 case CHIP_HAINAN:
1498 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1499 ucode_size = OLAND_MC_UCODE_SIZE;
1500 regs_size = TAHITI_IO_MC_REGS_SIZE;
1501 break;
8b074dd6
AD		 1502 }
1503
1504 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1505
1506 if (running == 0) {
1507 if (running) {
1508 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1509 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1510 }
1511
1512 /* reset the engine and set to writable */
1513 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1514 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1515
1516 /* load mc io regs */
1517 for (i = 0; i < regs_size; i++) {
1518 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1519 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1520 }
1521 /* load the MC ucode */
1522 fw_data = (const __be32 *)rdev->mc_fw->data;
1523 for (i = 0; i < ucode_size; i++)
1524 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1525
1526 /* put the engine back into the active state */
1527 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1528 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1529 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1530
1531 /* wait for training to complete */
1532 for (i = 0; i < rdev->usec_timeout; i++) {
1533 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1534 break;
1535 udelay(1);
1536 }
1537 for (i = 0; i < rdev->usec_timeout; i++) {
1538 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1539 break;
1540 udelay(1);
1541 }
1542
1543 if (running)
1544 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1545 }
1546
1547 return 0;
1548}
1549
0f0de06c
AD		 1550static int si_init_microcode(struct radeon_device *rdev)
1551{
0f0de06c
AD		 1552 const char *chip_name;
1553 const char *rlc_chip_name;
1554 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
a9e61410 1555 size_t smc_req_size;
0f0de06c
AD		 1556 char fw_name[30];
1557 int err;
1558
1559 DRM_DEBUG("\n");
1560
0f0de06c
AD		 1561 switch (rdev->family) {
1562 case CHIP_TAHITI:
1563 chip_name = "TAHITI";
1564 rlc_chip_name = "TAHITI";
1565 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1566 me_req_size = SI_PM4_UCODE_SIZE * 4;
1567 ce_req_size = SI_CE_UCODE_SIZE * 4;
1568 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1569 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1570 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1571 break;
1572 case CHIP_PITCAIRN:
1573 chip_name = "PITCAIRN";
1574 rlc_chip_name = "PITCAIRN";
1575 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1576 me_req_size = SI_PM4_UCODE_SIZE * 4;
1577 ce_req_size = SI_CE_UCODE_SIZE * 4;
1578 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1579 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1580 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1581 break;
1582 case CHIP_VERDE:
1583 chip_name = "VERDE";
1584 rlc_chip_name = "VERDE";
1585 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1586 me_req_size = SI_PM4_UCODE_SIZE * 4;
1587 ce_req_size = SI_CE_UCODE_SIZE * 4;
1588 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1589 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1590 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
0f0de06c 1591 break;
bcc7f5d2
AD		 1592 case CHIP_OLAND:
1593 chip_name = "OLAND";
1594 rlc_chip_name = "OLAND";
1595 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1596 me_req_size = SI_PM4_UCODE_SIZE * 4;
1597 ce_req_size = SI_CE_UCODE_SIZE * 4;
1598 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1599 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1600 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
bcc7f5d2 1601 break;
c04c00b4
AD		 1602 case CHIP_HAINAN:
1603 chip_name = "HAINAN";
1604 rlc_chip_name = "HAINAN";
1605 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1606 me_req_size = SI_PM4_UCODE_SIZE * 4;
1607 ce_req_size = SI_CE_UCODE_SIZE * 4;
1608 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1609 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1610 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
c04c00b4 1611 break;
0f0de06c
AD		 1612 default: BUG();
1613 }
1614
1615 DRM_INFO("Loading %s Microcode\n", chip_name);
1616
1617 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
0a168933 1618 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
0f0de06c
AD		 1619 if (err)
1620 goto out;
1621 if (rdev->pfp_fw->size != pfp_req_size) {
1622 printk(KERN_ERR
1623 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1624 rdev->pfp_fw->size, fw_name);
1625 err = -EINVAL;
1626 goto out;
1627 }
1628
1629 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
0a168933 1630 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
0f0de06c
AD		 1631 if (err)
1632 goto out;
1633 if (rdev->me_fw->size != me_req_size) {
1634 printk(KERN_ERR
1635 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1636 rdev->me_fw->size, fw_name);
1637 err = -EINVAL;
1638 }
1639
1640 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
0a168933 1641 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
0f0de06c
AD		 1642 if (err)
1643 goto out;
1644 if (rdev->ce_fw->size != ce_req_size) {
1645 printk(KERN_ERR
1646 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1647 rdev->ce_fw->size, fw_name);
1648 err = -EINVAL;
1649 }
1650
1651 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
0a168933 1652 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
0f0de06c
AD		 1653 if (err)
1654 goto out;
1655 if (rdev->rlc_fw->size != rlc_req_size) {
1656 printk(KERN_ERR
1657 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1658 rdev->rlc_fw->size, fw_name);
1659 err = -EINVAL;
1660 }
1661
1662 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
0a168933 1663 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
0f0de06c
AD		 1664 if (err)
1665 goto out;
1666 if (rdev->mc_fw->size != mc_req_size) {
1667 printk(KERN_ERR
1668 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1669 rdev->mc_fw->size, fw_name);
1670 err = -EINVAL;
1671 }
1672
a9e61410 1673 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
0a168933 1674 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
8a53fa23
AD		 1675 if (err) {
1676 printk(KERN_ERR
1677 "smc: error loading firmware \"%s\"\n",
1678 fw_name);
1679 release_firmware(rdev->smc_fw);
1680 rdev->smc_fw = NULL;
d8367112 1681 err = 0;
8a53fa23 1682 } else if (rdev->smc_fw->size != smc_req_size) {
a9e61410
AD		 1683 printk(KERN_ERR
1684 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1685 rdev->smc_fw->size, fw_name);
1686 err = -EINVAL;
1687 }
1688
0f0de06c 1689out:
0f0de06c
AD		 1690 if (err) {
1691 if (err != -EINVAL)
1692 printk(KERN_ERR
1693 "si_cp: Failed to load firmware \"%s\"\n",
1694 fw_name);
1695 release_firmware(rdev->pfp_fw);
1696 rdev->pfp_fw = NULL;
1697 release_firmware(rdev->me_fw);
1698 rdev->me_fw = NULL;
1699 release_firmware(rdev->ce_fw);
1700 rdev->ce_fw = NULL;
1701 release_firmware(rdev->rlc_fw);
1702 rdev->rlc_fw = NULL;
1703 release_firmware(rdev->mc_fw);
1704 rdev->mc_fw = NULL;
a9e61410
AD		 1705 release_firmware(rdev->smc_fw);
1706 rdev->smc_fw = NULL;
0f0de06c
AD		 1707 }
1708 return err;
1709}
1710
43b3cd99
AD		 1711/* watermark setup */
1712static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1713 struct radeon_crtc *radeon_crtc,
1714 struct drm_display_mode *mode,
1715 struct drm_display_mode *other_mode)
1716{
290d2457
AD		 1717 u32 tmp, buffer_alloc, i;
1718 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
43b3cd99
AD		 1719 /*
1720 * Line Buffer Setup
1721 * There are 3 line buffers, each one shared by 2 display controllers.
1722 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1723 * the display controllers. The paritioning is done via one of four
1724 * preset allocations specified in bits 21:20:
1725 * 0 - half lb
1726 * 2 - whole lb, other crtc must be disabled
1727 */
1728 /* this can get tricky if we have two large displays on a paired group
1729 * of crtcs. Ideally for multiple large displays we'd assign them to
1730 * non-linked crtcs for maximum line buffer allocation.
1731 */
1732 if (radeon_crtc->base.enabled && mode) {
290d2457 1733 if (other_mode) {
43b3cd99 1734 tmp = 0; /* 1/2 */
290d2457
AD		 1735 buffer_alloc = 1;
1736 } else {
43b3cd99 1737 tmp = 2; /* whole */
290d2457
AD		 1738 buffer_alloc = 2;
1739 }
1740 } else {
43b3cd99 1741 tmp = 0;
290d2457
AD		 1742 buffer_alloc = 0;
1743 }
43b3cd99
AD		 1744
1745 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1746 DC_LB_MEMORY_CONFIG(tmp));
1747
290d2457
AD		 1748 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1749 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1750 for (i = 0; i < rdev->usec_timeout; i++) {
1751 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1752 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1753 break;
1754 udelay(1);
1755 }
1756
43b3cd99
AD		 1757 if (radeon_crtc->base.enabled && mode) {
1758 switch (tmp) {
1759 case 0:
1760 default:
1761 return 4096 * 2;
1762 case 2:
1763 return 8192 * 2;
1764 }
1765 }
1766
1767 /* controller not enabled, so no lb used */
1768 return 0;
1769}
1770
ca7db22b 1771static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 1772{
1773 u32 tmp = RREG32(MC_SHARED_CHMAP);
1774
1775 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1776 case 0:
1777 default:
1778 return 1;
1779 case 1:
1780 return 2;
1781 case 2:
1782 return 4;
1783 case 3:
1784 return 8;
1785 case 4:
1786 return 3;
1787 case 5:
1788 return 6;
1789 case 6:
1790 return 10;
1791 case 7:
1792 return 12;
1793 case 8:
1794 return 16;
1795 }
1796}
1797
1798struct dce6_wm_params {
1799 u32 dram_channels; /* number of dram channels */
1800 u32 yclk; /* bandwidth per dram data pin in kHz */
1801 u32 sclk; /* engine clock in kHz */
1802 u32 disp_clk; /* display clock in kHz */
1803 u32 src_width; /* viewport width */
1804 u32 active_time; /* active display time in ns */
1805 u32 blank_time; /* blank time in ns */
1806 bool interlaced; /* mode is interlaced */
1807 fixed20_12 vsc; /* vertical scale ratio */
1808 u32 num_heads; /* number of active crtcs */
1809 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1810 u32 lb_size; /* line buffer allocated to pipe */
1811 u32 vtaps; /* vertical scaler taps */
1812};
1813
1814static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1815{
1816 /* Calculate raw DRAM Bandwidth */
1817 fixed20_12 dram_efficiency; /* 0.7 */
1818 fixed20_12 yclk, dram_channels, bandwidth;
1819 fixed20_12 a;
1820
1821 a.full = dfixed_const(1000);
1822 yclk.full = dfixed_const(wm->yclk);
1823 yclk.full = dfixed_div(yclk, a);
1824 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1825 a.full = dfixed_const(10);
1826 dram_efficiency.full = dfixed_const(7);
1827 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1828 bandwidth.full = dfixed_mul(dram_channels, yclk);
1829 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1830
1831 return dfixed_trunc(bandwidth);
1832}
1833
1834static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1835{
1836 /* Calculate DRAM Bandwidth and the part allocated to display. */
1837 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1838 fixed20_12 yclk, dram_channels, bandwidth;
1839 fixed20_12 a;
1840
1841 a.full = dfixed_const(1000);
1842 yclk.full = dfixed_const(wm->yclk);
1843 yclk.full = dfixed_div(yclk, a);
1844 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1845 a.full = dfixed_const(10);
1846 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1847 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1848 bandwidth.full = dfixed_mul(dram_channels, yclk);
1849 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1850
1851 return dfixed_trunc(bandwidth);
1852}
1853
1854static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1855{
1856 /* Calculate the display Data return Bandwidth */
1857 fixed20_12 return_efficiency; /* 0.8 */
1858 fixed20_12 sclk, bandwidth;
1859 fixed20_12 a;
1860
1861 a.full = dfixed_const(1000);
1862 sclk.full = dfixed_const(wm->sclk);
1863 sclk.full = dfixed_div(sclk, a);
1864 a.full = dfixed_const(10);
1865 return_efficiency.full = dfixed_const(8);
1866 return_efficiency.full = dfixed_div(return_efficiency, a);
1867 a.full = dfixed_const(32);
1868 bandwidth.full = dfixed_mul(a, sclk);
1869 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1870
1871 return dfixed_trunc(bandwidth);
1872}
1873
1874static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1875{
1876 return 32;
1877}
1878
1879static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1880{
1881 /* Calculate the DMIF Request Bandwidth */
1882 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1883 fixed20_12 disp_clk, sclk, bandwidth;
1884 fixed20_12 a, b1, b2;
1885 u32 min_bandwidth;
1886
1887 a.full = dfixed_const(1000);
1888 disp_clk.full = dfixed_const(wm->disp_clk);
1889 disp_clk.full = dfixed_div(disp_clk, a);
1890 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1891 b1.full = dfixed_mul(a, disp_clk);
1892
1893 a.full = dfixed_const(1000);
1894 sclk.full = dfixed_const(wm->sclk);
1895 sclk.full = dfixed_div(sclk, a);
1896 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1897 b2.full = dfixed_mul(a, sclk);
1898
1899 a.full = dfixed_const(10);
1900 disp_clk_request_efficiency.full = dfixed_const(8);
1901 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1902
1903 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1904
1905 a.full = dfixed_const(min_bandwidth);
1906 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1907
1908 return dfixed_trunc(bandwidth);
1909}
1910
1911static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1912{
1913 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1914 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1915 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1916 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1917
1918 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1919}
1920
1921static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1922{
1923 /* Calculate the display mode Average Bandwidth
1924 * DisplayMode should contain the source and destination dimensions,
1925 * timing, etc.
1926 */
1927 fixed20_12 bpp;
1928 fixed20_12 line_time;
1929 fixed20_12 src_width;
1930 fixed20_12 bandwidth;
1931 fixed20_12 a;
1932
1933 a.full = dfixed_const(1000);
1934 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1935 line_time.full = dfixed_div(line_time, a);
1936 bpp.full = dfixed_const(wm->bytes_per_pixel);
1937 src_width.full = dfixed_const(wm->src_width);
1938 bandwidth.full = dfixed_mul(src_width, bpp);
1939 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1940 bandwidth.full = dfixed_div(bandwidth, line_time);
1941
1942 return dfixed_trunc(bandwidth);
1943}
1944
1945static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1946{
1947 /* First calcualte the latency in ns */
1948 u32 mc_latency = 2000; /* 2000 ns. */
1949 u32 available_bandwidth = dce6_available_bandwidth(wm);
1950 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1951 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1952 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1953 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1954 (wm->num_heads * cursor_line_pair_return_time);
1955 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1956 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1957 u32 tmp, dmif_size = 12288;
1958 fixed20_12 a, b, c;
1959
1960 if (wm->num_heads == 0)
1961 return 0;
1962
1963 a.full = dfixed_const(2);
1964 b.full = dfixed_const(1);
1965 if ((wm->vsc.full > a.full) ||
1966 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1967 (wm->vtaps >= 5) ||
1968 ((wm->vsc.full >= a.full) && wm->interlaced))
1969 max_src_lines_per_dst_line = 4;
1970 else
1971 max_src_lines_per_dst_line = 2;
1972
1973 a.full = dfixed_const(available_bandwidth);
1974 b.full = dfixed_const(wm->num_heads);
1975 a.full = dfixed_div(a, b);
1976
1977 b.full = dfixed_const(mc_latency + 512);
1978 c.full = dfixed_const(wm->disp_clk);
1979 b.full = dfixed_div(b, c);
1980
1981 c.full = dfixed_const(dmif_size);
1982 b.full = dfixed_div(c, b);
1983
1984 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1985
1986 b.full = dfixed_const(1000);
1987 c.full = dfixed_const(wm->disp_clk);
1988 b.full = dfixed_div(c, b);
1989 c.full = dfixed_const(wm->bytes_per_pixel);
1990 b.full = dfixed_mul(b, c);
1991
1992 lb_fill_bw = min(tmp, dfixed_trunc(b));
1993
1994 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1995 b.full = dfixed_const(1000);
1996 c.full = dfixed_const(lb_fill_bw);
1997 b.full = dfixed_div(c, b);
1998 a.full = dfixed_div(a, b);
1999 line_fill_time = dfixed_trunc(a);
2000
2001 if (line_fill_time < wm->active_time)
2002 return latency;
2003 else
2004 return latency + (line_fill_time - wm->active_time);
2005
2006}
2007
2008static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2009{
2010 if (dce6_average_bandwidth(wm) <=
2011 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2012 return true;
2013 else
2014 return false;
2015};
2016
2017static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2018{
2019 if (dce6_average_bandwidth(wm) <=
2020 (dce6_available_bandwidth(wm) / wm->num_heads))
2021 return true;
2022 else
2023 return false;
2024};
2025
2026static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2027{
2028 u32 lb_partitions = wm->lb_size / wm->src_width;
2029 u32 line_time = wm->active_time + wm->blank_time;
2030 u32 latency_tolerant_lines;
2031 u32 latency_hiding;
2032 fixed20_12 a;
2033
2034 a.full = dfixed_const(1);
2035 if (wm->vsc.full > a.full)
2036 latency_tolerant_lines = 1;
2037 else {
2038 if (lb_partitions <= (wm->vtaps + 1))
2039 latency_tolerant_lines = 1;
2040 else
2041 latency_tolerant_lines = 2;
2042 }
2043
2044 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2045
2046 if (dce6_latency_watermark(wm) <= latency_hiding)
2047 return true;
2048 else
2049 return false;
2050}
2051
2052static void dce6_program_watermarks(struct radeon_device *rdev,
2053 struct radeon_crtc *radeon_crtc,
2054 u32 lb_size, u32 num_heads)
2055{
2056 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 2057 struct dce6_wm_params wm_low, wm_high;
2058 u32 dram_channels;
43b3cd99
AD		 2059 u32 pixel_period;
2060 u32 line_time = 0;
2061 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2062 u32 priority_a_mark = 0, priority_b_mark = 0;
2063 u32 priority_a_cnt = PRIORITY_OFF;
2064 u32 priority_b_cnt = PRIORITY_OFF;
2065 u32 tmp, arb_control3;
2066 fixed20_12 a, b, c;
2067
2068 if (radeon_crtc->base.enabled && num_heads && mode) {
2069 pixel_period = 1000000 / (u32)mode->clock;
2070 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2071 priority_a_cnt = 0;
2072 priority_b_cnt = 0;
2073
ca7db22b 2074 if (rdev->family == CHIP_ARUBA)
c696e53f 2075 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 2076 else
c696e53f
AD		 2077 dram_channels = si_get_number_of_dram_channels(rdev);
2078
2079 /* watermark for high clocks */
2080 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2081 wm_high.yclk =
2082 radeon_dpm_get_mclk(rdev, false) * 10;
2083 wm_high.sclk =
2084 radeon_dpm_get_sclk(rdev, false) * 10;
2085 } else {
2086 wm_high.yclk = rdev->pm.current_mclk * 10;
2087 wm_high.sclk = rdev->pm.current_sclk * 10;
2088 }
2089
2090 wm_high.disp_clk = mode->clock;
2091 wm_high.src_width = mode->crtc_hdisplay;
2092 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2093 wm_high.blank_time = line_time - wm_high.active_time;
2094 wm_high.interlaced = false;
2095 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2096 wm_high.interlaced = true;
2097 wm_high.vsc = radeon_crtc->vsc;
2098 wm_high.vtaps = 1;
2099 if (radeon_crtc->rmx_type != RMX_OFF)
2100 wm_high.vtaps = 2;
2101 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2102 wm_high.lb_size = lb_size;
2103 wm_high.dram_channels = dram_channels;
2104 wm_high.num_heads = num_heads;
2105
2106 /* watermark for low clocks */
2107 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2108 wm_low.yclk =
2109 radeon_dpm_get_mclk(rdev, true) * 10;
2110 wm_low.sclk =
2111 radeon_dpm_get_sclk(rdev, true) * 10;
2112 } else {
2113 wm_low.yclk = rdev->pm.current_mclk * 10;
2114 wm_low.sclk = rdev->pm.current_sclk * 10;
2115 }
2116
2117 wm_low.disp_clk = mode->clock;
2118 wm_low.src_width = mode->crtc_hdisplay;
2119 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2120 wm_low.blank_time = line_time - wm_low.active_time;
2121 wm_low.interlaced = false;
2122 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2123 wm_low.interlaced = true;
2124 wm_low.vsc = radeon_crtc->vsc;
2125 wm_low.vtaps = 1;
2126 if (radeon_crtc->rmx_type != RMX_OFF)
2127 wm_low.vtaps = 2;
2128 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2129 wm_low.lb_size = lb_size;
2130 wm_low.dram_channels = dram_channels;
2131 wm_low.num_heads = num_heads;
43b3cd99
AD		 2132
2133 /* set for high clocks */
c696e53f 2134 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 2135 /* set for low clocks */
c696e53f 2136 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 2137
2138 /* possibly force display priority to high */
2139 /* should really do this at mode validation time... */
c696e53f
AD		 2140 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2141 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2142 !dce6_check_latency_hiding(&wm_high) ||
2143 (rdev->disp_priority == 2)) {
2144 DRM_DEBUG_KMS("force priority to high\n");
2145 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2146 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2147 }
2148 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2149 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2150 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 2151 (rdev->disp_priority == 2)) {
2152 DRM_DEBUG_KMS("force priority to high\n");
2153 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2154 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2155 }
2156
2157 a.full = dfixed_const(1000);
2158 b.full = dfixed_const(mode->clock);
2159 b.full = dfixed_div(b, a);
2160 c.full = dfixed_const(latency_watermark_a);
2161 c.full = dfixed_mul(c, b);
2162 c.full = dfixed_mul(c, radeon_crtc->hsc);
2163 c.full = dfixed_div(c, a);
2164 a.full = dfixed_const(16);
2165 c.full = dfixed_div(c, a);
2166 priority_a_mark = dfixed_trunc(c);
2167 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2168
2169 a.full = dfixed_const(1000);
2170 b.full = dfixed_const(mode->clock);
2171 b.full = dfixed_div(b, a);
2172 c.full = dfixed_const(latency_watermark_b);
2173 c.full = dfixed_mul(c, b);
2174 c.full = dfixed_mul(c, radeon_crtc->hsc);
2175 c.full = dfixed_div(c, a);
2176 a.full = dfixed_const(16);
2177 c.full = dfixed_div(c, a);
2178 priority_b_mark = dfixed_trunc(c);
2179 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2180 }
2181
2182 /* select wm A */
2183 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2184 tmp = arb_control3;
2185 tmp &= ~LATENCY_WATERMARK_MASK(3);
2186 tmp |= LATENCY_WATERMARK_MASK(1);
2187 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2188 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2189 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2190 LATENCY_HIGH_WATERMARK(line_time)));
2191 /* select wm B */
2192 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2193 tmp &= ~LATENCY_WATERMARK_MASK(3);
2194 tmp |= LATENCY_WATERMARK_MASK(2);
2195 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2196 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2197 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2198 LATENCY_HIGH_WATERMARK(line_time)));
2199 /* restore original selection */
2200 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2201
2202 /* write the priority marks */
2203 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2204 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2205
7178d2a6
AD		 2206 /* save values for DPM */
2207 radeon_crtc->line_time = line_time;
2208 radeon_crtc->wm_high = latency_watermark_a;
2209 radeon_crtc->wm_low = latency_watermark_b;
43b3cd99
AD		 2210}
2211
2212void dce6_bandwidth_update(struct radeon_device *rdev)
2213{
2214 struct drm_display_mode *mode0 = NULL;
2215 struct drm_display_mode *mode1 = NULL;
2216 u32 num_heads = 0, lb_size;
2217 int i;
2218
2219 radeon_update_display_priority(rdev);
2220
2221 for (i = 0; i < rdev->num_crtc; i++) {
2222 if (rdev->mode_info.crtcs[i]->base.enabled)
2223 num_heads++;
2224 }
2225 for (i = 0; i < rdev->num_crtc; i += 2) {
2226 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2227 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2228 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2229 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2230 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2231 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2232 }
2233}
2234
0a96d72b
AD		 2235/*
2236 * Core functions
2237 */
0a96d72b
AD		 2238static void si_tiling_mode_table_init(struct radeon_device *rdev)
2239{
2240 const u32 num_tile_mode_states = 32;
2241 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2242
2243 switch (rdev->config.si.mem_row_size_in_kb) {
2244 case 1:
2245 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2246 break;
2247 case 2:
2248 default:
2249 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2250 break;
2251 case 4:
2252 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2253 break;
2254 }
2255
2256 if ((rdev->family == CHIP_TAHITI) ||
2257 (rdev->family == CHIP_PITCAIRN)) {
2258 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2259 switch (reg_offset) {
2260 case 0: /* non-AA compressed depth or any compressed stencil */
2261 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2262 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2263 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2264 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2265 NUM_BANKS(ADDR_SURF_16_BANK) |
2266 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2267 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2268 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2269 break;
2270 case 1: /* 2xAA/4xAA compressed depth only */
2271 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2272 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2273 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2274 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2275 NUM_BANKS(ADDR_SURF_16_BANK) |
2276 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2279 break;
2280 case 2: /* 8xAA compressed depth only */
2281 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2282 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2283 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2284 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2285 NUM_BANKS(ADDR_SURF_16_BANK) |
2286 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2287 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2288 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2289 break;
2290 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2291 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2292 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2293 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2294 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2295 NUM_BANKS(ADDR_SURF_16_BANK) |
2296 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2297 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2298 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2299 break;
2300 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2301 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2302 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2303 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2304 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2305 NUM_BANKS(ADDR_SURF_16_BANK) |
2306 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2307 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2308 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2309 break;
2310 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2311 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2312 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2313 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2314 TILE_SPLIT(split_equal_to_row_size) |
2315 NUM_BANKS(ADDR_SURF_16_BANK) |
2316 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2317 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2318 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2319 break;
2320 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2321 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2322 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2323 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2324 TILE_SPLIT(split_equal_to_row_size) |
2325 NUM_BANKS(ADDR_SURF_16_BANK) |
2326 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2327 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2328 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2329 break;
2330 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2331 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2332 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2333 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2334 TILE_SPLIT(split_equal_to_row_size) |
2335 NUM_BANKS(ADDR_SURF_16_BANK) |
2336 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2337 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2338 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2339 break;
2340 case 8: /* 1D and 1D Array Surfaces */
2341 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2342 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2343 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2344 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2345 NUM_BANKS(ADDR_SURF_16_BANK) |
2346 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2347 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2348 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2349 break;
2350 case 9: /* Displayable maps. */
2351 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2352 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2353 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2354 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2355 NUM_BANKS(ADDR_SURF_16_BANK) |
2356 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2357 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2358 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2359 break;
2360 case 10: /* Display 8bpp. */
2361 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2362 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2363 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2364 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2365 NUM_BANKS(ADDR_SURF_16_BANK) |
2366 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2367 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2368 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2369 break;
2370 case 11: /* Display 16bpp. */
2371 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2372 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2373 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2374 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2375 NUM_BANKS(ADDR_SURF_16_BANK) |
2376 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2377 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2378 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2379 break;
2380 case 12: /* Display 32bpp. */
2381 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2383 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2384 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2385 NUM_BANKS(ADDR_SURF_16_BANK) |
2386 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2387 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2388 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2389 break;
2390 case 13: /* Thin. */
2391 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2392 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2393 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2394 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2395 NUM_BANKS(ADDR_SURF_16_BANK) |
2396 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2397 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2398 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2399 break;
2400 case 14: /* Thin 8 bpp. */
2401 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2402 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2403 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2404 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2405 NUM_BANKS(ADDR_SURF_16_BANK) |
2406 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2407 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2408 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2409 break;
2410 case 15: /* Thin 16 bpp. */
2411 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2412 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2413 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2414 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2415 NUM_BANKS(ADDR_SURF_16_BANK) |
2416 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2417 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2418 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2419 break;
2420 case 16: /* Thin 32 bpp. */
2421 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2422 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2423 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2424 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2425 NUM_BANKS(ADDR_SURF_16_BANK) |
2426 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2427 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2428 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2429 break;
2430 case 17: /* Thin 64 bpp. */
2431 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2432 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2433 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2434 TILE_SPLIT(split_equal_to_row_size) |
2435 NUM_BANKS(ADDR_SURF_16_BANK) |
2436 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2437 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2438 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2439 break;
2440 case 21: /* 8 bpp PRT. */
2441 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2442 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2443 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2444 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2445 NUM_BANKS(ADDR_SURF_16_BANK) |
2446 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2447 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2448 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2449 break;
2450 case 22: /* 16 bpp PRT */
2451 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2452 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2453 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2454 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2455 NUM_BANKS(ADDR_SURF_16_BANK) |
2456 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2457 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2458 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2459 break;
2460 case 23: /* 32 bpp PRT */
2461 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2462 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2463 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2464 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2465 NUM_BANKS(ADDR_SURF_16_BANK) |
2466 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2467 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2468 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2469 break;
2470 case 24: /* 64 bpp PRT */
2471 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2472 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2473 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2474 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2475 NUM_BANKS(ADDR_SURF_16_BANK) |
2476 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2478 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2479 break;
2480 case 25: /* 128 bpp PRT */
2481 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2482 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2483 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2484 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2485 NUM_BANKS(ADDR_SURF_8_BANK) |
2486 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2487 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2488 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2489 break;
2490 default:
2491 gb_tile_moden = 0;
2492 break;
2493 }
64d7b8be 2494 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2495 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2496 }
d0ae7fcc 2497 } else if ((rdev->family == CHIP_VERDE) ||
8b02859d
AD		 2498 (rdev->family == CHIP_OLAND) ||
2499 (rdev->family == CHIP_HAINAN)) {
0a96d72b
AD		 2500 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2501 switch (reg_offset) {
2502 case 0: /* non-AA compressed depth or any compressed stencil */
2503 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2504 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2505 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2506 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2507 NUM_BANKS(ADDR_SURF_16_BANK) |
2508 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2509 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2510 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2511 break;
2512 case 1: /* 2xAA/4xAA compressed depth only */
2513 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2515 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2516 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2517 NUM_BANKS(ADDR_SURF_16_BANK) |
2518 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2519 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2520 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2521 break;
2522 case 2: /* 8xAA compressed depth only */
2523 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2524 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2525 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2526 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2527 NUM_BANKS(ADDR_SURF_16_BANK) |
2528 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2529 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2530 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2531 break;
2532 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2533 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2534 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2535 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2536 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2537 NUM_BANKS(ADDR_SURF_16_BANK) |
2538 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2539 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2540 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2541 break;
2542 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2543 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2544 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2545 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2546 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2547 NUM_BANKS(ADDR_SURF_16_BANK) |
2548 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2549 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2550 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2551 break;
2552 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2553 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2554 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2555 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2556 TILE_SPLIT(split_equal_to_row_size) |
2557 NUM_BANKS(ADDR_SURF_16_BANK) |
2558 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2559 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2560 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2561 break;
2562 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2563 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2566 TILE_SPLIT(split_equal_to_row_size) |
2567 NUM_BANKS(ADDR_SURF_16_BANK) |
2568 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2569 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2570 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2571 break;
2572 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2573 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2574 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2575 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2576 TILE_SPLIT(split_equal_to_row_size) |
2577 NUM_BANKS(ADDR_SURF_16_BANK) |
2578 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2579 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2580 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2581 break;
2582 case 8: /* 1D and 1D Array Surfaces */
2583 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2584 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2585 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2586 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2587 NUM_BANKS(ADDR_SURF_16_BANK) |
2588 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2589 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2590 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2591 break;
2592 case 9: /* Displayable maps. */
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2594 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2595 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2596 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2597 NUM_BANKS(ADDR_SURF_16_BANK) |
2598 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2599 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2600 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2601 break;
2602 case 10: /* Display 8bpp. */
2603 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2604 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2605 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2606 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2607 NUM_BANKS(ADDR_SURF_16_BANK) |
2608 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2609 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2610 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2611 break;
2612 case 11: /* Display 16bpp. */
2613 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2614 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2615 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2616 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2617 NUM_BANKS(ADDR_SURF_16_BANK) |
2618 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2619 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2620 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2621 break;
2622 case 12: /* Display 32bpp. */
2623 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2624 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2625 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2626 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2627 NUM_BANKS(ADDR_SURF_16_BANK) |
2628 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2629 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2630 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2631 break;
2632 case 13: /* Thin. */
2633 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2634 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2635 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2636 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2637 NUM_BANKS(ADDR_SURF_16_BANK) |
2638 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2639 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2640 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2641 break;
2642 case 14: /* Thin 8 bpp. */
2643 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2644 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2645 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2646 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2647 NUM_BANKS(ADDR_SURF_16_BANK) |
2648 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2649 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2650 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2651 break;
2652 case 15: /* Thin 16 bpp. */
2653 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2654 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2655 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2656 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2657 NUM_BANKS(ADDR_SURF_16_BANK) |
2658 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2659 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2660 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2661 break;
2662 case 16: /* Thin 32 bpp. */
2663 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2664 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2665 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2666 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2667 NUM_BANKS(ADDR_SURF_16_BANK) |
2668 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2669 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2670 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2671 break;
2672 case 17: /* Thin 64 bpp. */
2673 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2674 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2675 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2676 TILE_SPLIT(split_equal_to_row_size) |
2677 NUM_BANKS(ADDR_SURF_16_BANK) |
2678 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2679 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2680 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2681 break;
2682 case 21: /* 8 bpp PRT. */
2683 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2684 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2685 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2686 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2687 NUM_BANKS(ADDR_SURF_16_BANK) |
2688 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2689 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2690 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2691 break;
2692 case 22: /* 16 bpp PRT */
2693 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2694 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2695 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2696 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2697 NUM_BANKS(ADDR_SURF_16_BANK) |
2698 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2699 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2700 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2701 break;
2702 case 23: /* 32 bpp PRT */
2703 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2704 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2705 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2706 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2707 NUM_BANKS(ADDR_SURF_16_BANK) |
2708 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2711 break;
2712 case 24: /* 64 bpp PRT */
2713 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2714 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2715 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2716 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2717 NUM_BANKS(ADDR_SURF_16_BANK) |
2718 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2719 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2720 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2721 break;
2722 case 25: /* 128 bpp PRT */
2723 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2724 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2725 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2726 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2727 NUM_BANKS(ADDR_SURF_8_BANK) |
2728 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2729 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2730 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2731 break;
2732 default:
2733 gb_tile_moden = 0;
2734 break;
2735 }
64d7b8be 2736 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2737 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2738 }
2739 } else
2740 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2741}
2742
1a8ca750
AD		 2743static void si_select_se_sh(struct radeon_device *rdev,
2744 u32 se_num, u32 sh_num)
2745{
2746 u32 data = INSTANCE_BROADCAST_WRITES;
2747
2748 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2749 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2750 else if (se_num == 0xffffffff)
2751 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2752 else if (sh_num == 0xffffffff)
2753 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2754 else
2755 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2756 WREG32(GRBM_GFX_INDEX, data);
2757}
2758
2759static u32 si_create_bitmask(u32 bit_width)
2760{
2761 u32 i, mask = 0;
2762
2763 for (i = 0; i < bit_width; i++) {
2764 mask <<= 1;
2765 mask |= 1;
2766 }
2767 return mask;
2768}
2769
2770static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2771{
2772 u32 data, mask;
2773
2774 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2775 if (data & 1)
2776 data &= INACTIVE_CUS_MASK;
2777 else
2778 data = 0;
2779 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2780
2781 data >>= INACTIVE_CUS_SHIFT;
2782
2783 mask = si_create_bitmask(cu_per_sh);
2784
2785 return ~data & mask;
2786}
2787
2788static void si_setup_spi(struct radeon_device *rdev,
2789 u32 se_num, u32 sh_per_se,
2790 u32 cu_per_sh)
2791{
2792 int i, j, k;
2793 u32 data, mask, active_cu;
2794
2795 for (i = 0; i < se_num; i++) {
2796 for (j = 0; j < sh_per_se; j++) {
2797 si_select_se_sh(rdev, i, j);
2798 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2799 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2800
2801 mask = 1;
2802 for (k = 0; k < 16; k++) {
2803 mask <<= k;
2804 if (active_cu & mask) {
2805 data &= ~mask;
2806 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2807 break;
2808 }
2809 }
2810 }
2811 }
2812 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2813}
2814
2815static u32 si_get_rb_disabled(struct radeon_device *rdev,
9fadb352 2816 u32 max_rb_num_per_se,
1a8ca750
AD		 2817 u32 sh_per_se)
2818{
2819 u32 data, mask;
2820
2821 data = RREG32(CC_RB_BACKEND_DISABLE);
2822 if (data & 1)
2823 data &= BACKEND_DISABLE_MASK;
2824 else
2825 data = 0;
2826 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2827
2828 data >>= BACKEND_DISABLE_SHIFT;
2829
9fadb352 2830 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
1a8ca750
AD		 2831
2832 return data & mask;
2833}
2834
2835static void si_setup_rb(struct radeon_device *rdev,
2836 u32 se_num, u32 sh_per_se,
9fadb352 2837 u32 max_rb_num_per_se)
1a8ca750
AD		 2838{
2839 int i, j;
2840 u32 data, mask;
2841 u32 disabled_rbs = 0;
2842 u32 enabled_rbs = 0;
2843
2844 for (i = 0; i < se_num; i++) {
2845 for (j = 0; j < sh_per_se; j++) {
2846 si_select_se_sh(rdev, i, j);
9fadb352 2847 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
1a8ca750
AD		 2848 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2849 }
2850 }
2851 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2852
2853 mask = 1;
9fadb352 2854 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
1a8ca750
AD		 2855 if (!(disabled_rbs & mask))
2856 enabled_rbs |= mask;
2857 mask <<= 1;
2858 }
2859
439a1cff
MO		 2860 rdev->config.si.backend_enable_mask = enabled_rbs;
2861
1a8ca750
AD		 2862 for (i = 0; i < se_num; i++) {
2863 si_select_se_sh(rdev, i, 0xffffffff);
2864 data = 0;
2865 for (j = 0; j < sh_per_se; j++) {
2866 switch (enabled_rbs & 3) {
2867 case 1:
2868 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2869 break;
2870 case 2:
2871 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2872 break;
2873 case 3:
2874 default:
2875 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2876 break;
2877 }
2878 enabled_rbs >>= 2;
2879 }
2880 WREG32(PA_SC_RASTER_CONFIG, data);
2881 }
2882 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2883}
2884
0a96d72b
AD		 2885static void si_gpu_init(struct radeon_device *rdev)
2886{
0a96d72b
AD		 2887 u32 gb_addr_config = 0;
2888 u32 mc_shared_chmap, mc_arb_ramcfg;
0a96d72b 2889 u32 sx_debug_1;
0a96d72b
AD		 2890 u32 hdp_host_path_cntl;
2891 u32 tmp;
2892 int i, j;
2893
2894 switch (rdev->family) {
2895 case CHIP_TAHITI:
2896 rdev->config.si.max_shader_engines = 2;
0a96d72b 2897 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 2898 rdev->config.si.max_cu_per_sh = 8;
2899 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2900 rdev->config.si.max_backends_per_se = 4;
2901 rdev->config.si.max_texture_channel_caches = 12;
2902 rdev->config.si.max_gprs = 256;
2903 rdev->config.si.max_gs_threads = 32;
2904 rdev->config.si.max_hw_contexts = 8;
2905
2906 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2907 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2908 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2909 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2910 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2911 break;
2912 case CHIP_PITCAIRN:
2913 rdev->config.si.max_shader_engines = 2;
0a96d72b 2914 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 2915 rdev->config.si.max_cu_per_sh = 5;
2916 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2917 rdev->config.si.max_backends_per_se = 4;
2918 rdev->config.si.max_texture_channel_caches = 8;
2919 rdev->config.si.max_gprs = 256;
2920 rdev->config.si.max_gs_threads = 32;
2921 rdev->config.si.max_hw_contexts = 8;
2922
2923 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2924 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2925 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2926 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2927 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2928 break;
2929 case CHIP_VERDE:
2930 default:
2931 rdev->config.si.max_shader_engines = 1;
0a96d72b 2932 rdev->config.si.max_tile_pipes = 4;
468ef1a5 2933 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 2934 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2935 rdev->config.si.max_backends_per_se = 4;
2936 rdev->config.si.max_texture_channel_caches = 4;
2937 rdev->config.si.max_gprs = 256;
2938 rdev->config.si.max_gs_threads = 32;
2939 rdev->config.si.max_hw_contexts = 8;
2940
d0ae7fcc
AD		 2941 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2942 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2943 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2944 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2945 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2946 break;
2947 case CHIP_OLAND:
2948 rdev->config.si.max_shader_engines = 1;
2949 rdev->config.si.max_tile_pipes = 4;
2950 rdev->config.si.max_cu_per_sh = 6;
2951 rdev->config.si.max_sh_per_se = 1;
2952 rdev->config.si.max_backends_per_se = 2;
2953 rdev->config.si.max_texture_channel_caches = 4;
2954 rdev->config.si.max_gprs = 256;
2955 rdev->config.si.max_gs_threads = 16;
2956 rdev->config.si.max_hw_contexts = 8;
2957
0a96d72b
AD		 2958 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2959 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2960 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2961 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2962 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 2963 break;
8b02859d
AD		 2964 case CHIP_HAINAN:
2965 rdev->config.si.max_shader_engines = 1;
2966 rdev->config.si.max_tile_pipes = 4;
2967 rdev->config.si.max_cu_per_sh = 5;
2968 rdev->config.si.max_sh_per_se = 1;
2969 rdev->config.si.max_backends_per_se = 1;
2970 rdev->config.si.max_texture_channel_caches = 2;
2971 rdev->config.si.max_gprs = 256;
2972 rdev->config.si.max_gs_threads = 16;
2973 rdev->config.si.max_hw_contexts = 8;
2974
2975 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2976 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2977 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2978 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2979 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2980 break;
0a96d72b
AD		 2981 }
2982
2983 /* Initialize HDP */
2984 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2985 WREG32((0x2c14 + j), 0x00000000);
2986 WREG32((0x2c18 + j), 0x00000000);
2987 WREG32((0x2c1c + j), 0x00000000);
2988 WREG32((0x2c20 + j), 0x00000000);
2989 WREG32((0x2c24 + j), 0x00000000);
2990 }
2991
2992 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2993
2994 evergreen_fix_pci_max_read_req_size(rdev);
2995
2996 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2997
2998 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2999 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3000
0a96d72b 3001 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 3002 rdev->config.si.mem_max_burst_length_bytes = 256;
3003 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3004 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3005 if (rdev->config.si.mem_row_size_in_kb > 4)
3006 rdev->config.si.mem_row_size_in_kb = 4;
3007 /* XXX use MC settings? */
3008 rdev->config.si.shader_engine_tile_size = 32;
3009 rdev->config.si.num_gpus = 1;
3010 rdev->config.si.multi_gpu_tile_size = 64;
3011
1a8ca750
AD		 3012 /* fix up row size */
3013 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 3014 switch (rdev->config.si.mem_row_size_in_kb) {
3015 case 1:
3016 default:
3017 gb_addr_config |= ROW_SIZE(0);
3018 break;
3019 case 2:
3020 gb_addr_config |= ROW_SIZE(1);
3021 break;
3022 case 4:
3023 gb_addr_config |= ROW_SIZE(2);
3024 break;
3025 }
3026
0a96d72b
AD		 3027 /* setup tiling info dword. gb_addr_config is not adequate since it does
3028 * not have bank info, so create a custom tiling dword.
3029 * bits 3:0 num_pipes
3030 * bits 7:4 num_banks
3031 * bits 11:8 group_size
3032 * bits 15:12 row_size
3033 */
3034 rdev->config.si.tile_config = 0;
3035 switch (rdev->config.si.num_tile_pipes) {
3036 case 1:
3037 rdev->config.si.tile_config |= (0 << 0);
3038 break;
3039 case 2:
3040 rdev->config.si.tile_config |= (1 << 0);
3041 break;
3042 case 4:
3043 rdev->config.si.tile_config |= (2 << 0);
3044 break;
3045 case 8:
3046 default:
3047 /* XXX what about 12? */
3048 rdev->config.si.tile_config |= (3 << 0);
3049 break;
dca571a6
CK		 3050 }
3051 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3052 case 0: /* four banks */
1a8ca750 3053 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 3054 break;
3055 case 1: /* eight banks */
3056 rdev->config.si.tile_config |= 1 << 4;
3057 break;
3058 case 2: /* sixteen banks */
3059 default:
3060 rdev->config.si.tile_config |= 2 << 4;
3061 break;
3062 }
0a96d72b
AD		 3063 rdev->config.si.tile_config |=
3064 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3065 rdev->config.si.tile_config |=
3066 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3067
0a96d72b
AD		 3068 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3069 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 3070 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 3071 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 3072 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3073 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 3074 if (rdev->has_uvd) {
3075 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3076 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3077 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3078 }
0a96d72b 3079
1a8ca750 3080 si_tiling_mode_table_init(rdev);
0a96d72b 3081
1a8ca750
AD		 3082 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3083 rdev->config.si.max_sh_per_se,
3084 rdev->config.si.max_backends_per_se);
0a96d72b 3085
1a8ca750
AD		 3086 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3087 rdev->config.si.max_sh_per_se,
3088 rdev->config.si.max_cu_per_sh);
0a96d72b 3089
0a96d72b
AD		 3090
3091 /* set HW defaults for 3D engine */
3092 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3093 ROQ_IB2_START(0x2b)));
3094 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3095
3096 sx_debug_1 = RREG32(SX_DEBUG_1);
3097 WREG32(SX_DEBUG_1, sx_debug_1);
3098
3099 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3100
3101 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3102 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3103 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3104 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3105
3106 WREG32(VGT_NUM_INSTANCES, 1);
3107
3108 WREG32(CP_PERFMON_CNTL, 0);
3109
3110 WREG32(SQ_CONFIG, 0);
3111
3112 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3113 FORCE_EOV_MAX_REZ_CNT(255)));
3114
3115 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3116 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3117
3118 WREG32(VGT_GS_VERTEX_REUSE, 16);
3119 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3120
3121 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3122 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3123 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3124 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3125 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3126 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3127 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3128 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3129
3130 tmp = RREG32(HDP_MISC_CNTL);
3131 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3132 WREG32(HDP_MISC_CNTL, tmp);
3133
3134 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3135 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3136
3137 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3138
3139 udelay(50);
3140}
c476dde2 3141
2ece2e8b
AD		 3142/*
3143 * GPU scratch registers helpers function.
3144 */
3145static void si_scratch_init(struct radeon_device *rdev)
3146{
3147 int i;
3148
3149 rdev->scratch.num_reg = 7;
3150 rdev->scratch.reg_base = SCRATCH_REG0;
3151 for (i = 0; i < rdev->scratch.num_reg; i++) {
3152 rdev->scratch.free[i] = true;
3153 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3154 }
3155}
3156
3157void si_fence_ring_emit(struct radeon_device *rdev,
3158 struct radeon_fence *fence)
3159{
3160 struct radeon_ring *ring = &rdev->ring[fence->ring];
3161 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3162
3163 /* flush read cache over gart */
3164 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3165 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3166 radeon_ring_write(ring, 0);
3167 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3168 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3169 PACKET3_TC_ACTION_ENA |
3170 PACKET3_SH_KCACHE_ACTION_ENA |
3171 PACKET3_SH_ICACHE_ACTION_ENA);
3172 radeon_ring_write(ring, 0xFFFFFFFF);
3173 radeon_ring_write(ring, 0);
3174 radeon_ring_write(ring, 10); /* poll interval */
3175 /* EVENT_WRITE_EOP - flush caches, send int */
3176 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3177 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3178 radeon_ring_write(ring, addr & 0xffffffff);
3179 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3180 radeon_ring_write(ring, fence->seq);
3181 radeon_ring_write(ring, 0);
3182}
3183
3184/*
3185 * IB stuff
3186 */
3187void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3188{
876dc9f3 3189 struct radeon_ring *ring = &rdev->ring[ib->ring];
2ece2e8b
AD		 3190 u32 header;
3191
a85a7da4
AD		 3192 if (ib->is_const_ib) {
3193 /* set switch buffer packet before const IB */
3194 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3195 radeon_ring_write(ring, 0);
45df6803 3196
2ece2e8b 3197 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 3198 } else {
89d35807 3199 u32 next_rptr;
a85a7da4 3200 if (ring->rptr_save_reg) {
89d35807 3201 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 3202 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3203 radeon_ring_write(ring, ((ring->rptr_save_reg -
3204 PACKET3_SET_CONFIG_REG_START) >> 2));
3205 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3206 } else if (rdev->wb.enabled) {
3207 next_rptr = ring->wptr + 5 + 4 + 8;
3208 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3209 radeon_ring_write(ring, (1 << 8));
3210 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3211 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3212 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 3213 }
3214
2ece2e8b 3215 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 3216 }
2ece2e8b
AD		 3217
3218 radeon_ring_write(ring, header);
3219 radeon_ring_write(ring,
3220#ifdef __BIG_ENDIAN
3221 (2 << 0) |
3222#endif
3223 (ib->gpu_addr & 0xFFFFFFFC));
3224 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4bf3dd92
CK		 3225 radeon_ring_write(ring, ib->length_dw |
3226 (ib->vm ? (ib->vm->id << 24) : 0));
2ece2e8b 3227
a85a7da4
AD		 3228 if (!ib->is_const_ib) {
3229 /* flush read cache over gart for this vmid */
3230 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3231 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 3232 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
a85a7da4
AD		 3233 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3234 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3235 PACKET3_TC_ACTION_ENA |
3236 PACKET3_SH_KCACHE_ACTION_ENA |
3237 PACKET3_SH_ICACHE_ACTION_ENA);
3238 radeon_ring_write(ring, 0xFFFFFFFF);
3239 radeon_ring_write(ring, 0);
3240 radeon_ring_write(ring, 10); /* poll interval */
3241 }
2ece2e8b
AD		 3242}
3243
48c0c902
AD		 3244/*
3245 * CP.
3246 */
3247static void si_cp_enable(struct radeon_device *rdev, bool enable)
3248{
3249 if (enable)
3250 WREG32(CP_ME_CNTL, 0);
3251 else {
50efa51a
AD		 3252 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3253 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
48c0c902
AD		 3254 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3255 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 3256 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3257 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3258 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 3259 }
3260 udelay(50);
3261}
3262
3263static int si_cp_load_microcode(struct radeon_device *rdev)
3264{
3265 const __be32 *fw_data;
3266 int i;
3267
3268 if (!rdev->me_fw || !rdev->pfp_fw)
3269 return -EINVAL;
3270
3271 si_cp_enable(rdev, false);
3272
3273 /* PFP */
3274 fw_data = (const __be32 *)rdev->pfp_fw->data;
3275 WREG32(CP_PFP_UCODE_ADDR, 0);
3276 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3277 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3278 WREG32(CP_PFP_UCODE_ADDR, 0);
3279
3280 /* CE */
3281 fw_data = (const __be32 *)rdev->ce_fw->data;
3282 WREG32(CP_CE_UCODE_ADDR, 0);
3283 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3284 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3285 WREG32(CP_CE_UCODE_ADDR, 0);
3286
3287 /* ME */
3288 fw_data = (const __be32 *)rdev->me_fw->data;
3289 WREG32(CP_ME_RAM_WADDR, 0);
3290 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3291 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3292 WREG32(CP_ME_RAM_WADDR, 0);
3293
3294 WREG32(CP_PFP_UCODE_ADDR, 0);
3295 WREG32(CP_CE_UCODE_ADDR, 0);
3296 WREG32(CP_ME_RAM_WADDR, 0);
3297 WREG32(CP_ME_RAM_RADDR, 0);
3298 return 0;
3299}
3300
3301static int si_cp_start(struct radeon_device *rdev)
3302{
3303 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3304 int r, i;
3305
3306 r = radeon_ring_lock(rdev, ring, 7 + 4);
3307 if (r) {
3308 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3309 return r;
3310 }
3311 /* init the CP */
3312 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3313 radeon_ring_write(ring, 0x1);
3314 radeon_ring_write(ring, 0x0);
3315 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3316 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3317 radeon_ring_write(ring, 0);
3318 radeon_ring_write(ring, 0);
3319
3320 /* init the CE partitions */
3321 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3322 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3323 radeon_ring_write(ring, 0xc000);
3324 radeon_ring_write(ring, 0xe000);
3325 radeon_ring_unlock_commit(rdev, ring);
3326
3327 si_cp_enable(rdev, true);
3328
3329 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3330 if (r) {
3331 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3332 return r;
3333 }
3334
3335 /* setup clear context state */
3336 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3337 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3338
3339 for (i = 0; i < si_default_size; i++)
3340 radeon_ring_write(ring, si_default_state[i]);
3341
3342 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3343 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3344
3345 /* set clear context state */
3346 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3347 radeon_ring_write(ring, 0);
3348
3349 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3350 radeon_ring_write(ring, 0x00000316);
3351 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3352 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3353
3354 radeon_ring_unlock_commit(rdev, ring);
3355
3356 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3357 ring = &rdev->ring[i];
3358 r = radeon_ring_lock(rdev, ring, 2);
3359
3360 /* clear the compute context state */
3361 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3362 radeon_ring_write(ring, 0);
3363
3364 radeon_ring_unlock_commit(rdev, ring);
3365 }
3366
3367 return 0;
3368}
3369
3370static void si_cp_fini(struct radeon_device *rdev)
3371{
45df6803 3372 struct radeon_ring *ring;
48c0c902 3373 si_cp_enable(rdev, false);
45df6803
CK		 3374
3375 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3376 radeon_ring_fini(rdev, ring);
3377 radeon_scratch_free(rdev, ring->rptr_save_reg);
3378
3379 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3380 radeon_ring_fini(rdev, ring);
3381 radeon_scratch_free(rdev, ring->rptr_save_reg);
3382
3383 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3384 radeon_ring_fini(rdev, ring);
3385 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3386}
3387
3388static int si_cp_resume(struct radeon_device *rdev)
3389{
3390 struct radeon_ring *ring;
3391 u32 tmp;
3392 u32 rb_bufsz;
3393 int r;
3394
811e4d58
AD		 3395 si_enable_gui_idle_interrupt(rdev, false);
3396
48c0c902
AD		 3397 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3398 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3399
3400 /* Set the write pointer delay */
3401 WREG32(CP_RB_WPTR_DELAY, 0);
3402
3403 WREG32(CP_DEBUG, 0);
3404 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3405
3406 /* ring 0 - compute and gfx */
3407 /* Set ring buffer size */
3408 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 3409 rb_bufsz = order_base_2(ring->ring_size / 8);
3410 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3411#ifdef __BIG_ENDIAN
3412 tmp |= BUF_SWAP_32BIT;
3413#endif
3414 WREG32(CP_RB0_CNTL, tmp);
3415
3416 /* Initialize the ring buffer's read and write pointers */
3417 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3418 ring->wptr = 0;
3419 WREG32(CP_RB0_WPTR, ring->wptr);
3420
48fc7f7e 3421 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3422 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3423 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3424
3425 if (rdev->wb.enabled)
3426 WREG32(SCRATCH_UMSK, 0xff);
3427 else {
3428 tmp |= RB_NO_UPDATE;
3429 WREG32(SCRATCH_UMSK, 0);
3430 }
3431
3432 mdelay(1);
3433 WREG32(CP_RB0_CNTL, tmp);
3434
3435 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3436
48c0c902
AD		 3437 /* ring1 - compute only */
3438 /* Set ring buffer size */
3439 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
b72a8925
DV		 3440 rb_bufsz = order_base_2(ring->ring_size / 8);
3441 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3442#ifdef __BIG_ENDIAN
3443 tmp |= BUF_SWAP_32BIT;
3444#endif
3445 WREG32(CP_RB1_CNTL, tmp);
3446
3447 /* Initialize the ring buffer's read and write pointers */
3448 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3449 ring->wptr = 0;
3450 WREG32(CP_RB1_WPTR, ring->wptr);
3451
48fc7f7e 3452 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3453 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3454 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3455
3456 mdelay(1);
3457 WREG32(CP_RB1_CNTL, tmp);
3458
3459 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3460
48c0c902
AD		 3461 /* ring2 - compute only */
3462 /* Set ring buffer size */
3463 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
b72a8925
DV		 3464 rb_bufsz = order_base_2(ring->ring_size / 8);
3465 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3466#ifdef __BIG_ENDIAN
3467 tmp |= BUF_SWAP_32BIT;
3468#endif
3469 WREG32(CP_RB2_CNTL, tmp);
3470
3471 /* Initialize the ring buffer's read and write pointers */
3472 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3473 ring->wptr = 0;
3474 WREG32(CP_RB2_WPTR, ring->wptr);
3475
48fc7f7e 3476 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3477 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3478 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3479
3480 mdelay(1);
3481 WREG32(CP_RB2_CNTL, tmp);
3482
3483 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3484
48c0c902
AD		 3485 /* start the rings */
3486 si_cp_start(rdev);
3487 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3488 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3489 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3490 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3491 if (r) {
3492 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3493 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3494 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3495 return r;
3496 }
3497 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3498 if (r) {
3499 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3500 }
3501 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3502 if (r) {
3503 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3504 }
3505
811e4d58
AD		 3506 si_enable_gui_idle_interrupt(rdev, true);
3507
50efa51a
AD		 3508 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3509 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3510
48c0c902
AD		 3511 return 0;
3512}
3513
2483b4ea 3514u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3515{
014bb209 3516 u32 reset_mask = 0;
1c534671 3517 u32 tmp;
06bc6df0 3518
014bb209
AD		 3519 /* GRBM_STATUS */
3520 tmp = RREG32(GRBM_STATUS);
3521 if (tmp & (PA_BUSY | SC_BUSY |
3522 BCI_BUSY | SX_BUSY |
3523 TA_BUSY | VGT_BUSY |
3524 DB_BUSY | CB_BUSY |
3525 GDS_BUSY | SPI_BUSY |
3526 IA_BUSY | IA_BUSY_NO_DMA))
3527 reset_mask |= RADEON_RESET_GFX;
3528
3529 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3530 CP_BUSY | CP_COHERENCY_BUSY))
3531 reset_mask |= RADEON_RESET_CP;
3532
3533 if (tmp & GRBM_EE_BUSY)
3534 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3535
3536 /* GRBM_STATUS2 */
3537 tmp = RREG32(GRBM_STATUS2);
3538 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3539 reset_mask |= RADEON_RESET_RLC;
3540
3541 /* DMA_STATUS_REG 0 */
3542 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3543 if (!(tmp & DMA_IDLE))
3544 reset_mask |= RADEON_RESET_DMA;
3545
3546 /* DMA_STATUS_REG 1 */
3547 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3548 if (!(tmp & DMA_IDLE))
3549 reset_mask |= RADEON_RESET_DMA1;
3550
3551 /* SRBM_STATUS2 */
3552 tmp = RREG32(SRBM_STATUS2);
3553 if (tmp & DMA_BUSY)
3554 reset_mask |= RADEON_RESET_DMA;
3555
3556 if (tmp & DMA1_BUSY)
3557 reset_mask |= RADEON_RESET_DMA1;
3558
3559 /* SRBM_STATUS */
3560 tmp = RREG32(SRBM_STATUS);
3561
3562 if (tmp & IH_BUSY)
3563 reset_mask |= RADEON_RESET_IH;
3564
3565 if (tmp & SEM_BUSY)
3566 reset_mask |= RADEON_RESET_SEM;
3567
3568 if (tmp & GRBM_RQ_PENDING)
3569 reset_mask |= RADEON_RESET_GRBM;
3570
3571 if (tmp & VMC_BUSY)
3572 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3573
014bb209
AD		 3574 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3575 MCC_BUSY | MCD_BUSY))
3576 reset_mask |= RADEON_RESET_MC;
3577
3578 if (evergreen_is_display_hung(rdev))
3579 reset_mask |= RADEON_RESET_DISPLAY;
3580
3581 /* VM_L2_STATUS */
3582 tmp = RREG32(VM_L2_STATUS);
3583 if (tmp & L2_BUSY)
3584 reset_mask |= RADEON_RESET_VMC;
3585
d808fc88
AD		 3586 /* Skip MC reset as it's mostly likely not hung, just busy */
3587 if (reset_mask & RADEON_RESET_MC) {
3588 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3589 reset_mask &= ~RADEON_RESET_MC;
3590 }
3591
014bb209
AD		 3592 return reset_mask;
3593}
3594
3595static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3596{
3597 struct evergreen_mc_save save;
3598 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3599 u32 tmp;
19fc42ed 3600
06bc6df0 3601 if (reset_mask == 0)
014bb209 3602 return;
06bc6df0
AD		 3603
3604 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3605
1c534671 3606 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3607 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3608 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3609 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3610 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3611
a6f4ae8d
AD		 3612 /* disable PG/CG */
3613 si_fini_pg(rdev);
3614 si_fini_cg(rdev);
3615
3616 /* stop the rlc */
3617 si_rlc_stop(rdev);
3618
1c534671
AD		 3619 /* Disable CP parsing/prefetching */
3620 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3621
3622 if (reset_mask & RADEON_RESET_DMA) {
3623 /* dma0 */
3624 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3625 tmp &= ~DMA_RB_ENABLE;
3626 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3627 }
3628 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3629 /* dma1 */
3630 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3631 tmp &= ~DMA_RB_ENABLE;
3632 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3633 }
3634
f770d78a
AD		 3635 udelay(50);
3636
3637 evergreen_mc_stop(rdev, &save);
3638 if (evergreen_mc_wait_for_idle(rdev)) {
3639 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3640 }
3641
1c534671
AD		 3642 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3643 grbm_soft_reset = SOFT_RESET_CB |
3644 SOFT_RESET_DB |
3645 SOFT_RESET_GDS |
3646 SOFT_RESET_PA |
3647 SOFT_RESET_SC |
3648 SOFT_RESET_BCI |
3649 SOFT_RESET_SPI |
3650 SOFT_RESET_SX |
3651 SOFT_RESET_TC |
3652 SOFT_RESET_TA |
3653 SOFT_RESET_VGT |
3654 SOFT_RESET_IA;
3655 }
3656
3657 if (reset_mask & RADEON_RESET_CP) {
3658 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3659
3660 srbm_soft_reset |= SOFT_RESET_GRBM;
3661 }
06bc6df0
AD		 3662
3663 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3664 srbm_soft_reset |= SOFT_RESET_DMA;
3665
3666 if (reset_mask & RADEON_RESET_DMA1)
3667 srbm_soft_reset |= SOFT_RESET_DMA1;
3668
3669 if (reset_mask & RADEON_RESET_DISPLAY)
3670 srbm_soft_reset |= SOFT_RESET_DC;
3671
3672 if (reset_mask & RADEON_RESET_RLC)
3673 grbm_soft_reset |= SOFT_RESET_RLC;
3674
3675 if (reset_mask & RADEON_RESET_SEM)
3676 srbm_soft_reset |= SOFT_RESET_SEM;
3677
3678 if (reset_mask & RADEON_RESET_IH)
3679 srbm_soft_reset |= SOFT_RESET_IH;
3680
3681 if (reset_mask & RADEON_RESET_GRBM)
3682 srbm_soft_reset |= SOFT_RESET_GRBM;
3683
3684 if (reset_mask & RADEON_RESET_VMC)
3685 srbm_soft_reset |= SOFT_RESET_VMC;
3686
3687 if (reset_mask & RADEON_RESET_MC)
3688 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3689
3690 if (grbm_soft_reset) {
3691 tmp = RREG32(GRBM_SOFT_RESET);
3692 tmp |= grbm_soft_reset;
3693 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3694 WREG32(GRBM_SOFT_RESET, tmp);
3695 tmp = RREG32(GRBM_SOFT_RESET);
3696
3697 udelay(50);
3698
3699 tmp &= ~grbm_soft_reset;
3700 WREG32(GRBM_SOFT_RESET, tmp);
3701 tmp = RREG32(GRBM_SOFT_RESET);
3702 }
3703
3704 if (srbm_soft_reset) {
3705 tmp = RREG32(SRBM_SOFT_RESET);
3706 tmp |= srbm_soft_reset;
3707 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3708 WREG32(SRBM_SOFT_RESET, tmp);
3709 tmp = RREG32(SRBM_SOFT_RESET);
3710
3711 udelay(50);
3712
3713 tmp &= ~srbm_soft_reset;
3714 WREG32(SRBM_SOFT_RESET, tmp);
3715 tmp = RREG32(SRBM_SOFT_RESET);
3716 }
06bc6df0
AD		 3717
3718 /* Wait a little for things to settle down */
3719 udelay(50);
3720
c476dde2 3721 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3722 udelay(50);
3723
1c534671 3724 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3725}
3726
4a5c8ea5
AD		 3727static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3728{
3729 u32 tmp, i;
3730
3731 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3732 tmp |= SPLL_BYPASS_EN;
3733 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3734
3735 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3736 tmp |= SPLL_CTLREQ_CHG;
3737 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3738
3739 for (i = 0; i < rdev->usec_timeout; i++) {
3740 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3741 break;
3742 udelay(1);
3743 }
3744
3745 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3746 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3747 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3748
3749 tmp = RREG32(MPLL_CNTL_MODE);
3750 tmp &= ~MPLL_MCLK_SEL;
3751 WREG32(MPLL_CNTL_MODE, tmp);
3752}
3753
3754static void si_spll_powerdown(struct radeon_device *rdev)
3755{
3756 u32 tmp;
3757
3758 tmp = RREG32(SPLL_CNTL_MODE);
3759 tmp |= SPLL_SW_DIR_CONTROL;
3760 WREG32(SPLL_CNTL_MODE, tmp);
3761
3762 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3763 tmp |= SPLL_RESET;
3764 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3765
3766 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3767 tmp |= SPLL_SLEEP;
3768 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3769
3770 tmp = RREG32(SPLL_CNTL_MODE);
3771 tmp &= ~SPLL_SW_DIR_CONTROL;
3772 WREG32(SPLL_CNTL_MODE, tmp);
3773}
3774
3775static void si_gpu_pci_config_reset(struct radeon_device *rdev)
3776{
3777 struct evergreen_mc_save save;
3778 u32 tmp, i;
3779
3780 dev_info(rdev->dev, "GPU pci config reset\n");
3781
3782 /* disable dpm? */
3783
3784 /* disable cg/pg */
3785 si_fini_pg(rdev);
3786 si_fini_cg(rdev);
3787
3788 /* Disable CP parsing/prefetching */
3789 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3790 /* dma0 */
3791 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3792 tmp &= ~DMA_RB_ENABLE;
3793 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3794 /* dma1 */
3795 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3796 tmp &= ~DMA_RB_ENABLE;
3797 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3798 /* XXX other engines? */
3799
3800 /* halt the rlc, disable cp internal ints */
3801 si_rlc_stop(rdev);
3802
3803 udelay(50);
3804
3805 /* disable mem access */
3806 evergreen_mc_stop(rdev, &save);
3807 if (evergreen_mc_wait_for_idle(rdev)) {
3808 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3809 }
3810
3811 /* set mclk/sclk to bypass */
3812 si_set_clk_bypass_mode(rdev);
3813 /* powerdown spll */
3814 si_spll_powerdown(rdev);
3815 /* disable BM */
3816 pci_clear_master(rdev->pdev);
3817 /* reset */
3818 radeon_pci_config_reset(rdev);
3819 /* wait for asic to come out of reset */
3820 for (i = 0; i < rdev->usec_timeout; i++) {
3821 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3822 break;
3823 udelay(1);
3824 }
3825}
3826
c476dde2
AD		 3827int si_asic_reset(struct radeon_device *rdev)
3828{
014bb209
AD		 3829 u32 reset_mask;
3830
3831 reset_mask = si_gpu_check_soft_reset(rdev);
3832
3833 if (reset_mask)
3834 r600_set_bios_scratch_engine_hung(rdev, true);
3835
4a5c8ea5 3836 /* try soft reset */
014bb209
AD		 3837 si_gpu_soft_reset(rdev, reset_mask);
3838
3839 reset_mask = si_gpu_check_soft_reset(rdev);
3840
4a5c8ea5
AD		 3841 /* try pci config reset */
3842 if (reset_mask && radeon_hard_reset)
3843 si_gpu_pci_config_reset(rdev);
3844
3845 reset_mask = si_gpu_check_soft_reset(rdev);
3846
014bb209
AD		 3847 if (!reset_mask)
3848 r600_set_bios_scratch_engine_hung(rdev, false);
3849
3850 return 0;
c476dde2
AD		 3851}
3852
123bc183
AD		 3853/**
3854 * si_gfx_is_lockup - Check if the GFX engine is locked up
3855 *
3856 * @rdev: radeon_device pointer
3857 * @ring: radeon_ring structure holding ring information
3858 *
3859 * Check if the GFX engine is locked up.
3860 * Returns true if the engine appears to be locked up, false if not.
3861 */
3862bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3863{
3864 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3865
3866 if (!(reset_mask & (RADEON_RESET_GFX |
3867 RADEON_RESET_COMPUTE |
3868 RADEON_RESET_CP))) {
ff212f25 3869 radeon_ring_lockup_update(rdev, ring);
123bc183
AD		 3870 return false;
3871 }
123bc183
AD		 3872 return radeon_ring_test_lockup(rdev, ring);
3873}
3874
d2800ee5
AD		 3875/* MC */
3876static void si_mc_program(struct radeon_device *rdev)
3877{
3878 struct evergreen_mc_save save;
3879 u32 tmp;
3880 int i, j;
3881
3882 /* Initialize HDP */
3883 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3884 WREG32((0x2c14 + j), 0x00000000);
3885 WREG32((0x2c18 + j), 0x00000000);
3886 WREG32((0x2c1c + j), 0x00000000);
3887 WREG32((0x2c20 + j), 0x00000000);
3888 WREG32((0x2c24 + j), 0x00000000);
3889 }
3890 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3891
3892 evergreen_mc_stop(rdev, &save);
3893 if (radeon_mc_wait_for_idle(rdev)) {
3894 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3895 }
5153550a
AD		 3896 if (!ASIC_IS_NODCE(rdev))
3897 /* Lockout access through VGA aperture*/
3898 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 3899 /* Update configuration */
3900 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3901 rdev->mc.vram_start >> 12);
3902 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3903 rdev->mc.vram_end >> 12);
3904 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3905 rdev->vram_scratch.gpu_addr >> 12);
3906 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3907 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3908 WREG32(MC_VM_FB_LOCATION, tmp);
3909 /* XXX double check these! */
3910 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3911 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3912 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3913 WREG32(MC_VM_AGP_BASE, 0);
3914 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3915 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3916 if (radeon_mc_wait_for_idle(rdev)) {
3917 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3918 }
3919 evergreen_mc_resume(rdev, &save);
5153550a
AD		 3920 if (!ASIC_IS_NODCE(rdev)) {
3921 /* we need to own VRAM, so turn off the VGA renderer here
3922 * to stop it overwriting our objects */
3923 rv515_vga_render_disable(rdev);
3924 }
d2800ee5
AD		 3925}
3926
1c49165d
AD		 3927void si_vram_gtt_location(struct radeon_device *rdev,
3928 struct radeon_mc *mc)
d2800ee5
AD		 3929{
3930 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3931 /* leave room for at least 1024M GTT */
3932 dev_warn(rdev->dev, "limiting VRAM\n");
3933 mc->real_vram_size = 0xFFC0000000ULL;
3934 mc->mc_vram_size = 0xFFC0000000ULL;
3935 }
9ed8b1f9 3936 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 3937 rdev->mc.gtt_base_align = 0;
9ed8b1f9 3938 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 3939}
3940
3941static int si_mc_init(struct radeon_device *rdev)
3942{
3943 u32 tmp;
3944 int chansize, numchan;
3945
3946 /* Get VRAM informations */
3947 rdev->mc.vram_is_ddr = true;
3948 tmp = RREG32(MC_ARB_RAMCFG);
3949 if (tmp & CHANSIZE_OVERRIDE) {
3950 chansize = 16;
3951 } else if (tmp & CHANSIZE_MASK) {
3952 chansize = 64;
3953 } else {
3954 chansize = 32;
3955 }
3956 tmp = RREG32(MC_SHARED_CHMAP);
3957 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3958 case 0:
3959 default:
3960 numchan = 1;
3961 break;
3962 case 1:
3963 numchan = 2;
3964 break;
3965 case 2:
3966 numchan = 4;
3967 break;
3968 case 3:
3969 numchan = 8;
3970 break;
3971 case 4:
3972 numchan = 3;
3973 break;
3974 case 5:
3975 numchan = 6;
3976 break;
3977 case 6:
3978 numchan = 10;
3979 break;
3980 case 7:
3981 numchan = 12;
3982 break;
3983 case 8:
3984 numchan = 16;
3985 break;
3986 }
3987 rdev->mc.vram_width = numchan * chansize;
3988 /* Could aper size report 0 ? */
3989 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3990 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3991 /* size in MB on si */
0ca223b0
AD		 3992 tmp = RREG32(CONFIG_MEMSIZE);
3993 /* some boards may have garbage in the upper 16 bits */
3994 if (tmp & 0xffff0000) {
3995 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
3996 if (tmp & 0xffff)
3997 tmp &= 0xffff;
3998 }
3999 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4000 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
d2800ee5
AD		 4001 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4002 si_vram_gtt_location(rdev, &rdev->mc);
4003 radeon_update_bandwidth_info(rdev);
4004
4005 return 0;
4006}
4007
4008/*
4009 * GART
4010 */
4011void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4012{
4013 /* flush hdp cache */
4014 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4015
4016 /* bits 0-15 are the VM contexts0-15 */
4017 WREG32(VM_INVALIDATE_REQUEST, 1);
4018}
4019
1109ca09 4020static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 4021{
4022 int r, i;
4023
4024 if (rdev->gart.robj == NULL) {
4025 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4026 return -EINVAL;
4027 }
4028 r = radeon_gart_table_vram_pin(rdev);
4029 if (r)
4030 return r;
4031 radeon_gart_restore(rdev);
4032 /* Setup TLB control */
4033 WREG32(MC_VM_MX_L1_TLB_CNTL,
4034 (0xA << 7) |
4035 ENABLE_L1_TLB |
4036 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4037 ENABLE_ADVANCED_DRIVER_MODEL |
4038 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4039 /* Setup L2 cache */
4040 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4041 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4042 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4043 EFFECTIVE_L2_QUEUE_SIZE(7) |
4044 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4045 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4046 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4047 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4048 /* setup context0 */
4049 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4050 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4051 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4052 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4053 (u32)(rdev->dummy_page.addr >> 12));
4054 WREG32(VM_CONTEXT0_CNTL2, 0);
4055 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4056 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4057
4058 WREG32(0x15D4, 0);
4059 WREG32(0x15D8, 0);
4060 WREG32(0x15DC, 0);
4061
4062 /* empty context1-15 */
d2800ee5
AD		 4063 /* set vm size, must be a multiple of 4 */
4064 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
c21b328e 4065 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
23d4f1f2
AD		 4066 /* Assign the pt base to something valid for now; the pts used for
4067 * the VMs are determined by the application and setup and assigned
4068 * on the fly in the vm part of radeon_gart.c
4069 */
d2800ee5
AD		 4070 for (i = 1; i < 16; i++) {
4071 if (i < 8)
4072 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4073 rdev->gart.table_addr >> 12);
4074 else
4075 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4076 rdev->gart.table_addr >> 12);
4077 }
4078
4079 /* enable context1-15 */
4080 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4081 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 4082 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 4083 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
ae133a11
CK		 4084 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4085 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4086 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4087 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4088 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4089 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4090 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4091 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4092 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4093 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4094 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4095 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 4096
4097 si_pcie_gart_tlb_flush(rdev);
4098 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4099 (unsigned)(rdev->mc.gtt_size >> 20),
4100 (unsigned long long)rdev->gart.table_addr);
4101 rdev->gart.ready = true;
4102 return 0;
4103}
4104
1109ca09 4105static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5
AD		 4106{
4107 /* Disable all tables */
4108 WREG32(VM_CONTEXT0_CNTL, 0);
4109 WREG32(VM_CONTEXT1_CNTL, 0);
4110 /* Setup TLB control */
4111 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4112 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4113 /* Setup L2 cache */
4114 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4115 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4116 EFFECTIVE_L2_QUEUE_SIZE(7) |
4117 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4118 WREG32(VM_L2_CNTL2, 0);
4119 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4120 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4121 radeon_gart_table_vram_unpin(rdev);
4122}
4123
1109ca09 4124static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 4125{
4126 si_pcie_gart_disable(rdev);
4127 radeon_gart_table_vram_free(rdev);
4128 radeon_gart_fini(rdev);
4129}
4130
498dd8b3
AD		 4131/* vm parser */
4132static bool si_vm_reg_valid(u32 reg)
4133{
4134 /* context regs are fine */
4135 if (reg >= 0x28000)
4136 return true;
4137
4138 /* check config regs */
4139 switch (reg) {
4140 case GRBM_GFX_INDEX:
f418b88a 4141 case CP_STRMOUT_CNTL:
498dd8b3
AD		 4142 case VGT_VTX_VECT_EJECT_REG:
4143 case VGT_CACHE_INVALIDATION:
4144 case VGT_ESGS_RING_SIZE:
4145 case VGT_GSVS_RING_SIZE:
4146 case VGT_GS_VERTEX_REUSE:
4147 case VGT_PRIMITIVE_TYPE:
4148 case VGT_INDEX_TYPE:
4149 case VGT_NUM_INDICES:
4150 case VGT_NUM_INSTANCES:
4151 case VGT_TF_RING_SIZE:
4152 case VGT_HS_OFFCHIP_PARAM:
4153 case VGT_TF_MEMORY_BASE:
4154 case PA_CL_ENHANCE:
4155 case PA_SU_LINE_STIPPLE_VALUE:
4156 case PA_SC_LINE_STIPPLE_STATE:
4157 case PA_SC_ENHANCE:
4158 case SQC_CACHES:
4159 case SPI_STATIC_THREAD_MGMT_1:
4160 case SPI_STATIC_THREAD_MGMT_2:
4161 case SPI_STATIC_THREAD_MGMT_3:
4162 case SPI_PS_MAX_WAVE_ID:
4163 case SPI_CONFIG_CNTL:
4164 case SPI_CONFIG_CNTL_1:
4165 case TA_CNTL_AUX:
4166 return true;
4167 default:
4168 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4169 return false;
4170 }
4171}
4172
4173static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4174 u32 *ib, struct radeon_cs_packet *pkt)
4175{
4176 switch (pkt->opcode) {
4177 case PACKET3_NOP:
4178 case PACKET3_SET_BASE:
4179 case PACKET3_SET_CE_DE_COUNTERS:
4180 case PACKET3_LOAD_CONST_RAM:
4181 case PACKET3_WRITE_CONST_RAM:
4182 case PACKET3_WRITE_CONST_RAM_OFFSET:
4183 case PACKET3_DUMP_CONST_RAM:
4184 case PACKET3_INCREMENT_CE_COUNTER:
4185 case PACKET3_WAIT_ON_DE_COUNTER:
4186 case PACKET3_CE_WRITE:
4187 break;
4188 default:
4189 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4190 return -EINVAL;
4191 }
4192 return 0;
4193}
4194
e5b9e750
TS		 4195static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4196{
4197 u32 start_reg, reg, i;
4198 u32 command = ib[idx + 4];
4199 u32 info = ib[idx + 1];
4200 u32 idx_value = ib[idx];
4201 if (command & PACKET3_CP_DMA_CMD_SAS) {
4202 /* src address space is register */
4203 if (((info & 0x60000000) >> 29) == 0) {
4204 start_reg = idx_value << 2;
4205 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4206 reg = start_reg;
4207 if (!si_vm_reg_valid(reg)) {
4208 DRM_ERROR("CP DMA Bad SRC register\n");
4209 return -EINVAL;
4210 }
4211 } else {
4212 for (i = 0; i < (command & 0x1fffff); i++) {
4213 reg = start_reg + (4 * i);
4214 if (!si_vm_reg_valid(reg)) {
4215 DRM_ERROR("CP DMA Bad SRC register\n");
4216 return -EINVAL;
4217 }
4218 }
4219 }
4220 }
4221 }
4222 if (command & PACKET3_CP_DMA_CMD_DAS) {
4223 /* dst address space is register */
4224 if (((info & 0x00300000) >> 20) == 0) {
4225 start_reg = ib[idx + 2];
4226 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4227 reg = start_reg;
4228 if (!si_vm_reg_valid(reg)) {
4229 DRM_ERROR("CP DMA Bad DST register\n");
4230 return -EINVAL;
4231 }
4232 } else {
4233 for (i = 0; i < (command & 0x1fffff); i++) {
4234 reg = start_reg + (4 * i);
4235 if (!si_vm_reg_valid(reg)) {
4236 DRM_ERROR("CP DMA Bad DST register\n");
4237 return -EINVAL;
4238 }
4239 }
4240 }
4241 }
4242 }
4243 return 0;
4244}
4245
498dd8b3
AD		 4246static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4247 u32 *ib, struct radeon_cs_packet *pkt)
4248{
e5b9e750 4249 int r;
498dd8b3
AD		 4250 u32 idx = pkt->idx + 1;
4251 u32 idx_value = ib[idx];
4252 u32 start_reg, end_reg, reg, i;
4253
4254 switch (pkt->opcode) {
4255 case PACKET3_NOP:
4256 case PACKET3_SET_BASE:
4257 case PACKET3_CLEAR_STATE:
4258 case PACKET3_INDEX_BUFFER_SIZE:
4259 case PACKET3_DISPATCH_DIRECT:
4260 case PACKET3_DISPATCH_INDIRECT:
4261 case PACKET3_ALLOC_GDS:
4262 case PACKET3_WRITE_GDS_RAM:
4263 case PACKET3_ATOMIC_GDS:
4264 case PACKET3_ATOMIC:
4265 case PACKET3_OCCLUSION_QUERY:
4266 case PACKET3_SET_PREDICATION:
4267 case PACKET3_COND_EXEC:
4268 case PACKET3_PRED_EXEC:
4269 case PACKET3_DRAW_INDIRECT:
4270 case PACKET3_DRAW_INDEX_INDIRECT:
4271 case PACKET3_INDEX_BASE:
4272 case PACKET3_DRAW_INDEX_2:
4273 case PACKET3_CONTEXT_CONTROL:
4274 case PACKET3_INDEX_TYPE:
4275 case PACKET3_DRAW_INDIRECT_MULTI:
4276 case PACKET3_DRAW_INDEX_AUTO:
4277 case PACKET3_DRAW_INDEX_IMMD:
4278 case PACKET3_NUM_INSTANCES:
4279 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4280 case PACKET3_STRMOUT_BUFFER_UPDATE:
4281 case PACKET3_DRAW_INDEX_OFFSET_2:
4282 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4283 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4284 case PACKET3_MPEG_INDEX:
4285 case PACKET3_WAIT_REG_MEM:
4286 case PACKET3_MEM_WRITE:
4287 case PACKET3_PFP_SYNC_ME:
4288 case PACKET3_SURFACE_SYNC:
4289 case PACKET3_EVENT_WRITE:
4290 case PACKET3_EVENT_WRITE_EOP:
4291 case PACKET3_EVENT_WRITE_EOS:
4292 case PACKET3_SET_CONTEXT_REG:
4293 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4294 case PACKET3_SET_SH_REG:
4295 case PACKET3_SET_SH_REG_OFFSET:
4296 case PACKET3_INCREMENT_DE_COUNTER:
4297 case PACKET3_WAIT_ON_CE_COUNTER:
4298 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4299 case PACKET3_ME_WRITE:
4300 break;
4301 case PACKET3_COPY_DATA:
4302 if ((idx_value & 0xf00) == 0) {
4303 reg = ib[idx + 3] * 4;
4304 if (!si_vm_reg_valid(reg))
4305 return -EINVAL;
4306 }
4307 break;
4308 case PACKET3_WRITE_DATA:
4309 if ((idx_value & 0xf00) == 0) {
4310 start_reg = ib[idx + 1] * 4;
4311 if (idx_value & 0x10000) {
4312 if (!si_vm_reg_valid(start_reg))
4313 return -EINVAL;
4314 } else {
4315 for (i = 0; i < (pkt->count - 2); i++) {
4316 reg = start_reg + (4 * i);
4317 if (!si_vm_reg_valid(reg))
4318 return -EINVAL;
4319 }
4320 }
4321 }
4322 break;
4323 case PACKET3_COND_WRITE:
4324 if (idx_value & 0x100) {
4325 reg = ib[idx + 5] * 4;
4326 if (!si_vm_reg_valid(reg))
4327 return -EINVAL;
4328 }
4329 break;
4330 case PACKET3_COPY_DW:
4331 if (idx_value & 0x2) {
4332 reg = ib[idx + 3] * 4;
4333 if (!si_vm_reg_valid(reg))
4334 return -EINVAL;
4335 }
4336 break;
4337 case PACKET3_SET_CONFIG_REG:
4338 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4339 end_reg = 4 * pkt->count + start_reg - 4;
4340 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4341 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4342 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4343 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4344 return -EINVAL;
4345 }
4346 for (i = 0; i < pkt->count; i++) {
4347 reg = start_reg + (4 * i);
4348 if (!si_vm_reg_valid(reg))
4349 return -EINVAL;
4350 }
4351 break;
5aa709be 4352 case PACKET3_CP_DMA:
e5b9e750
TS		 4353 r = si_vm_packet3_cp_dma_check(ib, idx);
4354 if (r)
4355 return r;
5aa709be 4356 break;
498dd8b3
AD		 4357 default:
4358 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4359 return -EINVAL;
4360 }
4361 return 0;
4362}
4363
4364static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4365 u32 *ib, struct radeon_cs_packet *pkt)
4366{
e5b9e750 4367 int r;
498dd8b3
AD		 4368 u32 idx = pkt->idx + 1;
4369 u32 idx_value = ib[idx];
4370 u32 start_reg, reg, i;
4371
4372 switch (pkt->opcode) {
4373 case PACKET3_NOP:
4374 case PACKET3_SET_BASE:
4375 case PACKET3_CLEAR_STATE:
4376 case PACKET3_DISPATCH_DIRECT:
4377 case PACKET3_DISPATCH_INDIRECT:
4378 case PACKET3_ALLOC_GDS:
4379 case PACKET3_WRITE_GDS_RAM:
4380 case PACKET3_ATOMIC_GDS:
4381 case PACKET3_ATOMIC:
4382 case PACKET3_OCCLUSION_QUERY:
4383 case PACKET3_SET_PREDICATION:
4384 case PACKET3_COND_EXEC:
4385 case PACKET3_PRED_EXEC:
4386 case PACKET3_CONTEXT_CONTROL:
4387 case PACKET3_STRMOUT_BUFFER_UPDATE:
4388 case PACKET3_WAIT_REG_MEM:
4389 case PACKET3_MEM_WRITE:
4390 case PACKET3_PFP_SYNC_ME:
4391 case PACKET3_SURFACE_SYNC:
4392 case PACKET3_EVENT_WRITE:
4393 case PACKET3_EVENT_WRITE_EOP:
4394 case PACKET3_EVENT_WRITE_EOS:
4395 case PACKET3_SET_CONTEXT_REG:
4396 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4397 case PACKET3_SET_SH_REG:
4398 case PACKET3_SET_SH_REG_OFFSET:
4399 case PACKET3_INCREMENT_DE_COUNTER:
4400 case PACKET3_WAIT_ON_CE_COUNTER:
4401 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4402 case PACKET3_ME_WRITE:
4403 break;
4404 case PACKET3_COPY_DATA:
4405 if ((idx_value & 0xf00) == 0) {
4406 reg = ib[idx + 3] * 4;
4407 if (!si_vm_reg_valid(reg))
4408 return -EINVAL;
4409 }
4410 break;
4411 case PACKET3_WRITE_DATA:
4412 if ((idx_value & 0xf00) == 0) {
4413 start_reg = ib[idx + 1] * 4;
4414 if (idx_value & 0x10000) {
4415 if (!si_vm_reg_valid(start_reg))
4416 return -EINVAL;
4417 } else {
4418 for (i = 0; i < (pkt->count - 2); i++) {
4419 reg = start_reg + (4 * i);
4420 if (!si_vm_reg_valid(reg))
4421 return -EINVAL;
4422 }
4423 }
4424 }
4425 break;
4426 case PACKET3_COND_WRITE:
4427 if (idx_value & 0x100) {
4428 reg = ib[idx + 5] * 4;
4429 if (!si_vm_reg_valid(reg))
4430 return -EINVAL;
4431 }
4432 break;
4433 case PACKET3_COPY_DW:
4434 if (idx_value & 0x2) {
4435 reg = ib[idx + 3] * 4;
4436 if (!si_vm_reg_valid(reg))
4437 return -EINVAL;
4438 }
4439 break;
e5b9e750
TS		 4440 case PACKET3_CP_DMA:
4441 r = si_vm_packet3_cp_dma_check(ib, idx);
4442 if (r)
4443 return r;
4444 break;
498dd8b3
AD		 4445 default:
4446 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4447 return -EINVAL;
4448 }
4449 return 0;
4450}
4451
4452int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4453{
4454 int ret = 0;
4455 u32 idx = 0;
4456 struct radeon_cs_packet pkt;
4457
4458 do {
4459 pkt.idx = idx;
4e872ae2
IH		 4460 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4461 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4462 pkt.one_reg_wr = 0;
4463 switch (pkt.type) {
4e872ae2 4464 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 4465 dev_err(rdev->dev, "Packet0 not allowed!\n");
4466 ret = -EINVAL;
4467 break;
4e872ae2 4468 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4469 idx += 1;
4470 break;
4e872ae2
IH		 4471 case RADEON_PACKET_TYPE3:
4472 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4473 if (ib->is_const_ib)
4474 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4475 else {
876dc9f3 4476 switch (ib->ring) {
498dd8b3
AD		 4477 case RADEON_RING_TYPE_GFX_INDEX:
4478 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4479 break;
4480 case CAYMAN_RING_TYPE_CP1_INDEX:
4481 case CAYMAN_RING_TYPE_CP2_INDEX:
4482 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4483 break;
4484 default:
876dc9f3 4485 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4486 ret = -EINVAL;
4487 break;
4488 }
4489 }
4490 idx += pkt.count + 2;
4491 break;
4492 default:
4493 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4494 ret = -EINVAL;
4495 break;
4496 }
4497 if (ret)
4498 break;
4499 } while (idx < ib->length_dw);
4500
4501 return ret;
4502}
4503
d2800ee5
AD		 4504/*
4505 * vm
4506 */
4507int si_vm_init(struct radeon_device *rdev)
4508{
4509 /* number of VMs */
4510 rdev->vm_manager.nvm = 16;
4511 /* base offset of vram pages */
4512 rdev->vm_manager.vram_base_offset = 0;
4513
4514 return 0;
4515}
4516
4517void si_vm_fini(struct radeon_device *rdev)
4518{
4519}
4520
fbf6dc7a
AD		 4521/**
4522 * si_vm_decode_fault - print human readable fault info
4523 *
4524 * @rdev: radeon_device pointer
4525 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4526 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4527 *
4528 * Print human readable fault information (SI).
4529 */
4530static void si_vm_decode_fault(struct radeon_device *rdev,
4531 u32 status, u32 addr)
4532{
4533 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4534 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4535 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4536 char *block;
4537
4538 if (rdev->family == CHIP_TAHITI) {
4539 switch (mc_id) {
4540 case 160:
4541 case 144:
4542 case 96:
4543 case 80:
4544 case 224:
4545 case 208:
4546 case 32:
4547 case 16:
4548 block = "CB";
4549 break;
4550 case 161:
4551 case 145:
4552 case 97:
4553 case 81:
4554 case 225:
4555 case 209:
4556 case 33:
4557 case 17:
4558 block = "CB_FMASK";
4559 break;
4560 case 162:
4561 case 146:
4562 case 98:
4563 case 82:
4564 case 226:
4565 case 210:
4566 case 34:
4567 case 18:
4568 block = "CB_CMASK";
4569 break;
4570 case 163:
4571 case 147:
4572 case 99:
4573 case 83:
4574 case 227:
4575 case 211:
4576 case 35:
4577 case 19:
4578 block = "CB_IMMED";
4579 break;
4580 case 164:
4581 case 148:
4582 case 100:
4583 case 84:
4584 case 228:
4585 case 212:
4586 case 36:
4587 case 20:
4588 block = "DB";
4589 break;
4590 case 165:
4591 case 149:
4592 case 101:
4593 case 85:
4594 case 229:
4595 case 213:
4596 case 37:
4597 case 21:
4598 block = "DB_HTILE";
4599 break;
4600 case 167:
4601 case 151:
4602 case 103:
4603 case 87:
4604 case 231:
4605 case 215:
4606 case 39:
4607 case 23:
4608 block = "DB_STEN";
4609 break;
4610 case 72:
4611 case 68:
4612 case 64:
4613 case 8:
4614 case 4:
4615 case 0:
4616 case 136:
4617 case 132:
4618 case 128:
4619 case 200:
4620 case 196:
4621 case 192:
4622 block = "TC";
4623 break;
4624 case 112:
4625 case 48:
4626 block = "CP";
4627 break;
4628 case 49:
4629 case 177:
4630 case 50:
4631 case 178:
4632 block = "SH";
4633 break;
4634 case 53:
4635 case 190:
4636 block = "VGT";
4637 break;
4638 case 117:
4639 block = "IH";
4640 break;
4641 case 51:
4642 case 115:
4643 block = "RLC";
4644 break;
4645 case 119:
4646 case 183:
4647 block = "DMA0";
4648 break;
4649 case 61:
4650 block = "DMA1";
4651 break;
4652 case 248:
4653 case 120:
4654 block = "HDP";
4655 break;
4656 default:
4657 block = "unknown";
4658 break;
4659 }
4660 } else {
4661 switch (mc_id) {
4662 case 32:
4663 case 16:
4664 case 96:
4665 case 80:
4666 case 160:
4667 case 144:
4668 case 224:
4669 case 208:
4670 block = "CB";
4671 break;
4672 case 33:
4673 case 17:
4674 case 97:
4675 case 81:
4676 case 161:
4677 case 145:
4678 case 225:
4679 case 209:
4680 block = "CB_FMASK";
4681 break;
4682 case 34:
4683 case 18:
4684 case 98:
4685 case 82:
4686 case 162:
4687 case 146:
4688 case 226:
4689 case 210:
4690 block = "CB_CMASK";
4691 break;
4692 case 35:
4693 case 19:
4694 case 99:
4695 case 83:
4696 case 163:
4697 case 147:
4698 case 227:
4699 case 211:
4700 block = "CB_IMMED";
4701 break;
4702 case 36:
4703 case 20:
4704 case 100:
4705 case 84:
4706 case 164:
4707 case 148:
4708 case 228:
4709 case 212:
4710 block = "DB";
4711 break;
4712 case 37:
4713 case 21:
4714 case 101:
4715 case 85:
4716 case 165:
4717 case 149:
4718 case 229:
4719 case 213:
4720 block = "DB_HTILE";
4721 break;
4722 case 39:
4723 case 23:
4724 case 103:
4725 case 87:
4726 case 167:
4727 case 151:
4728 case 231:
4729 case 215:
4730 block = "DB_STEN";
4731 break;
4732 case 72:
4733 case 68:
4734 case 8:
4735 case 4:
4736 case 136:
4737 case 132:
4738 case 200:
4739 case 196:
4740 block = "TC";
4741 break;
4742 case 112:
4743 case 48:
4744 block = "CP";
4745 break;
4746 case 49:
4747 case 177:
4748 case 50:
4749 case 178:
4750 block = "SH";
4751 break;
4752 case 53:
4753 block = "VGT";
4754 break;
4755 case 117:
4756 block = "IH";
4757 break;
4758 case 51:
4759 case 115:
4760 block = "RLC";
4761 break;
4762 case 119:
4763 case 183:
4764 block = "DMA0";
4765 break;
4766 case 61:
4767 block = "DMA1";
4768 break;
4769 case 248:
4770 case 120:
4771 block = "HDP";
4772 break;
4773 default:
4774 block = "unknown";
4775 break;
4776 }
4777 }
4778
4779 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4780 protections, vmid, addr,
4781 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4782 block, mc_id);
4783}
4784
498522b4 4785void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
d2800ee5 4786{
498522b4 4787 struct radeon_ring *ring = &rdev->ring[ridx];
d2800ee5 4788
ee60e29f 4789 if (vm == NULL)
d2800ee5
AD		 4790 return;
4791
76c44f2c
AD		 4792 /* write new base address */
4793 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4794 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4795 WRITE_DATA_DST_SEL(0)));
4796
ee60e29f 4797 if (vm->id < 8) {
76c44f2c
AD		 4798 radeon_ring_write(ring,
4799 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
ee60e29f 4800 } else {
76c44f2c
AD		 4801 radeon_ring_write(ring,
4802 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
ee60e29f 4803 }
76c44f2c 4804 radeon_ring_write(ring, 0);
fa87e62d 4805 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 4806
d2800ee5 4807 /* flush hdp cache */
76c44f2c
AD		 4808 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4809 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4810 WRITE_DATA_DST_SEL(0)));
4811 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4812 radeon_ring_write(ring, 0);
ee60e29f
CK		 4813 radeon_ring_write(ring, 0x1);
4814
d2800ee5 4815 /* bits 0-15 are the VM contexts0-15 */
76c44f2c
AD		 4816 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4817 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4818 WRITE_DATA_DST_SEL(0)));
4819 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4820 radeon_ring_write(ring, 0);
498522b4 4821 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 4822
4823 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4824 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4825 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 4826}
4827
f8f84ac5
AD		 4828/*
4829 * Power and clock gating
4830 */
4831static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4832{
4833 int i;
4834
4835 for (i = 0; i < rdev->usec_timeout; i++) {
4836 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4837 break;
4838 udelay(1);
4839 }
4840
4841 for (i = 0; i < rdev->usec_timeout; i++) {
4842 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4843 break;
4844 udelay(1);
4845 }
4846}
4847
4848static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4849 bool enable)
4850{
4851 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4852 u32 mask;
4853 int i;
4854
4855 if (enable)
4856 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4857 else
4858 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4859 WREG32(CP_INT_CNTL_RING0, tmp);
4860
4861 if (!enable) {
4862 /* read a gfx register */
4863 tmp = RREG32(DB_DEPTH_INFO);
4864
4865 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4866 for (i = 0; i < rdev->usec_timeout; i++) {
4867 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4868 break;
4869 udelay(1);
4870 }
4871 }
4872}
4873
4874static void si_set_uvd_dcm(struct radeon_device *rdev,
4875 bool sw_mode)
4876{
4877 u32 tmp, tmp2;
4878
4879 tmp = RREG32(UVD_CGC_CTRL);
4880 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4881 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4882
4883 if (sw_mode) {
4884 tmp &= ~0x7ffff800;
4885 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4886 } else {
4887 tmp |= 0x7ffff800;
4888 tmp2 = 0;
4889 }
4890
4891 WREG32(UVD_CGC_CTRL, tmp);
4892 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4893}
4894
22c775ce 4895void si_init_uvd_internal_cg(struct radeon_device *rdev)
f8f84ac5
AD		 4896{
4897 bool hw_mode = true;
4898
4899 if (hw_mode) {
4900 si_set_uvd_dcm(rdev, false);
4901 } else {
4902 u32 tmp = RREG32(UVD_CGC_CTRL);
4903 tmp &= ~DCM;
4904 WREG32(UVD_CGC_CTRL, tmp);
4905 }
4906}
4907
4908static u32 si_halt_rlc(struct radeon_device *rdev)
4909{
4910 u32 data, orig;
4911
4912 orig = data = RREG32(RLC_CNTL);
4913
4914 if (data & RLC_ENABLE) {
4915 data &= ~RLC_ENABLE;
4916 WREG32(RLC_CNTL, data);
4917
4918 si_wait_for_rlc_serdes(rdev);
4919 }
4920
4921 return orig;
4922}
4923
4924static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4925{
4926 u32 tmp;
4927
4928 tmp = RREG32(RLC_CNTL);
4929 if (tmp != rlc)
4930 WREG32(RLC_CNTL, rlc);
4931}
4932
4933static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4934{
4935 u32 data, orig;
4936
4937 orig = data = RREG32(DMA_PG);
e16866ec 4938 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
f8f84ac5
AD		 4939 data |= PG_CNTL_ENABLE;
4940 else
4941 data &= ~PG_CNTL_ENABLE;
4942 if (orig != data)
4943 WREG32(DMA_PG, data);
4944}
4945
4946static void si_init_dma_pg(struct radeon_device *rdev)
4947{
4948 u32 tmp;
4949
4950 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4951 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4952
4953 for (tmp = 0; tmp < 5; tmp++)
4954 WREG32(DMA_PGFSM_WRITE, 0);
4955}
4956
4957static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4958 bool enable)
4959{
4960 u32 tmp;
4961
2b19d17f 4962 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
f8f84ac5
AD		 4963 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4964 WREG32(RLC_TTOP_D, tmp);
4965
4966 tmp = RREG32(RLC_PG_CNTL);
4967 tmp |= GFX_PG_ENABLE;
4968 WREG32(RLC_PG_CNTL, tmp);
4969
4970 tmp = RREG32(RLC_AUTO_PG_CTRL);
4971 tmp |= AUTO_PG_EN;
4972 WREG32(RLC_AUTO_PG_CTRL, tmp);
4973 } else {
4974 tmp = RREG32(RLC_AUTO_PG_CTRL);
4975 tmp &= ~AUTO_PG_EN;
4976 WREG32(RLC_AUTO_PG_CTRL, tmp);
4977
4978 tmp = RREG32(DB_RENDER_CONTROL);
4979 }
4980}
4981
4982static void si_init_gfx_cgpg(struct radeon_device *rdev)
4983{
4984 u32 tmp;
4985
4986 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4987
4988 tmp = RREG32(RLC_PG_CNTL);
4989 tmp |= GFX_PG_SRC;
4990 WREG32(RLC_PG_CNTL, tmp);
4991
4992 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4993
4994 tmp = RREG32(RLC_AUTO_PG_CTRL);
4995
4996 tmp &= ~GRBM_REG_SGIT_MASK;
4997 tmp |= GRBM_REG_SGIT(0x700);
4998 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
4999 WREG32(RLC_AUTO_PG_CTRL, tmp);
5000}
5001
ba19031a 5002static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
f8f84ac5
AD		 5003{
5004 u32 mask = 0, tmp, tmp1;
5005 int i;
5006
5007 si_select_se_sh(rdev, se, sh);
5008 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5009 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5010 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5011
5012 tmp &= 0xffff0000;
5013
5014 tmp |= tmp1;
5015 tmp >>= 16;
5016
5017 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5018 mask <<= 1;
5019 mask |= 1;
5020 }
5021
5022 return (~tmp) & mask;
5023}
5024
5025static void si_init_ao_cu_mask(struct radeon_device *rdev)
5026{
5027 u32 i, j, k, active_cu_number = 0;
5028 u32 mask, counter, cu_bitmap;
5029 u32 tmp = 0;
5030
5031 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5032 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5033 mask = 1;
5034 cu_bitmap = 0;
5035 counter = 0;
5036 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
ba19031a 5037 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
f8f84ac5
AD		 5038 if (counter < 2)
5039 cu_bitmap |= mask;
5040 counter++;
5041 }
5042 mask <<= 1;
5043 }
5044
5045 active_cu_number += counter;
5046 tmp |= (cu_bitmap << (i * 16 + j * 8));
5047 }
5048 }
5049
5050 WREG32(RLC_PG_AO_CU_MASK, tmp);
5051
5052 tmp = RREG32(RLC_MAX_PG_CU);
5053 tmp &= ~MAX_PU_CU_MASK;
5054 tmp |= MAX_PU_CU(active_cu_number);
5055 WREG32(RLC_MAX_PG_CU, tmp);
5056}
5057
5058static void si_enable_cgcg(struct radeon_device *rdev,
5059 bool enable)
5060{
5061 u32 data, orig, tmp;
5062
5063 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5064
e16866ec 5065 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5594a558 5066 si_enable_gui_idle_interrupt(rdev, true);
f8f84ac5 5067
f8f84ac5
AD		 5068 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5069
5070 tmp = si_halt_rlc(rdev);
5071
5072 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5073 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5074 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5075
5076 si_wait_for_rlc_serdes(rdev);
5077
5078 si_update_rlc(rdev, tmp);
5079
5080 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5081
5082 data |= CGCG_EN | CGLS_EN;
5083 } else {
5594a558
AD		 5084 si_enable_gui_idle_interrupt(rdev, false);
5085
f8f84ac5
AD		 5086 RREG32(CB_CGTT_SCLK_CTRL);
5087 RREG32(CB_CGTT_SCLK_CTRL);
5088 RREG32(CB_CGTT_SCLK_CTRL);
5089 RREG32(CB_CGTT_SCLK_CTRL);
5090
5091 data &= ~(CGCG_EN | CGLS_EN);
5092 }
5093
5094 if (orig != data)
5095 WREG32(RLC_CGCG_CGLS_CTRL, data);
5096}
5097
5098static void si_enable_mgcg(struct radeon_device *rdev,
5099 bool enable)
5100{
5101 u32 data, orig, tmp = 0;
5102
e16866ec 5103 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
f8f84ac5
AD		 5104 orig = data = RREG32(CGTS_SM_CTRL_REG);
5105 data = 0x96940200;
5106 if (orig != data)
5107 WREG32(CGTS_SM_CTRL_REG, data);
5108
e16866ec
AD		 5109 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5110 orig = data = RREG32(CP_MEM_SLP_CNTL);
5111 data |= CP_MEM_LS_EN;
5112 if (orig != data)
5113 WREG32(CP_MEM_SLP_CNTL, data);
5114 }
f8f84ac5
AD		 5115
5116 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5117 data &= 0xffffffc0;
5118 if (orig != data)
5119 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5120
5121 tmp = si_halt_rlc(rdev);
5122
5123 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5124 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5125 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5126
5127 si_update_rlc(rdev, tmp);
5128 } else {
5129 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5130 data |= 0x00000003;
5131 if (orig != data)
5132 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5133
5134 data = RREG32(CP_MEM_SLP_CNTL);
5135 if (data & CP_MEM_LS_EN) {
5136 data &= ~CP_MEM_LS_EN;
5137 WREG32(CP_MEM_SLP_CNTL, data);
5138 }
5139 orig = data = RREG32(CGTS_SM_CTRL_REG);
5140 data |= LS_OVERRIDE | OVERRIDE;
5141 if (orig != data)
5142 WREG32(CGTS_SM_CTRL_REG, data);
5143
5144 tmp = si_halt_rlc(rdev);
5145
5146 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5147 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5148 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5149
5150 si_update_rlc(rdev, tmp);
5151 }
5152}
5153
5154static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5155 bool enable)
5156{
5157 u32 orig, data, tmp;
5158
e16866ec 5159 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
f8f84ac5
AD		 5160 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5161 tmp |= 0x3fff;
5162 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5163
5164 orig = data = RREG32(UVD_CGC_CTRL);
5165 data |= DCM;
5166 if (orig != data)
5167 WREG32(UVD_CGC_CTRL, data);
5168
5169 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5170 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5171 } else {
5172 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5173 tmp &= ~0x3fff;
5174 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5175
5176 orig = data = RREG32(UVD_CGC_CTRL);
5177 data &= ~DCM;
5178 if (orig != data)
5179 WREG32(UVD_CGC_CTRL, data);
5180
5181 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5182 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5183 }
5184}
5185
5186static const u32 mc_cg_registers[] =
5187{
5188 MC_HUB_MISC_HUB_CG,
5189 MC_HUB_MISC_SIP_CG,
5190 MC_HUB_MISC_VM_CG,
5191 MC_XPB_CLK_GAT,
5192 ATC_MISC_CG,
5193 MC_CITF_MISC_WR_CG,
5194 MC_CITF_MISC_RD_CG,
5195 MC_CITF_MISC_VM_CG,
5196 VM_L2_CG,
5197};
5198
5199static void si_enable_mc_ls(struct radeon_device *rdev,
5200 bool enable)
5201{
5202 int i;
5203 u32 orig, data;
5204
5205 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5206 orig = data = RREG32(mc_cg_registers[i]);
e16866ec 5207 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
f8f84ac5
AD		 5208 data |= MC_LS_ENABLE;
5209 else
5210 data &= ~MC_LS_ENABLE;
5211 if (data != orig)
5212 WREG32(mc_cg_registers[i], data);
5213 }
5214}
5215
e16866ec
AD		 5216static void si_enable_mc_mgcg(struct radeon_device *rdev,
5217 bool enable)
f8f84ac5 5218{
e16866ec
AD		 5219 int i;
5220 u32 orig, data;
5221
5222 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5223 orig = data = RREG32(mc_cg_registers[i]);
5224 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5225 data |= MC_CG_ENABLE;
5226 else
5227 data &= ~MC_CG_ENABLE;
5228 if (data != orig)
5229 WREG32(mc_cg_registers[i], data);
f8f84ac5
AD		 5230 }
5231}
5232
e16866ec
AD		 5233static void si_enable_dma_mgcg(struct radeon_device *rdev,
5234 bool enable)
f8f84ac5 5235{
e16866ec
AD		 5236 u32 orig, data, offset;
5237 int i;
f8f84ac5 5238
e16866ec
AD		 5239 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5240 for (i = 0; i < 2; i++) {
5241 if (i == 0)
5242 offset = DMA0_REGISTER_OFFSET;
5243 else
5244 offset = DMA1_REGISTER_OFFSET;
5245 orig = data = RREG32(DMA_POWER_CNTL + offset);
5246 data &= ~MEM_POWER_OVERRIDE;
5247 if (data != orig)
5248 WREG32(DMA_POWER_CNTL + offset, data);
5249 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5250 }
5251 } else {
5252 for (i = 0; i < 2; i++) {
5253 if (i == 0)
5254 offset = DMA0_REGISTER_OFFSET;
5255 else
5256 offset = DMA1_REGISTER_OFFSET;
5257 orig = data = RREG32(DMA_POWER_CNTL + offset);
5258 data |= MEM_POWER_OVERRIDE;
5259 if (data != orig)
5260 WREG32(DMA_POWER_CNTL + offset, data);
5261
5262 orig = data = RREG32(DMA_CLK_CTRL + offset);
5263 data = 0xff000000;
5264 if (data != orig)
5265 WREG32(DMA_CLK_CTRL + offset, data);
5266 }
5267 }
f8f84ac5
AD		 5268}
5269
e16866ec
AD		 5270static void si_enable_bif_mgls(struct radeon_device *rdev,
5271 bool enable)
f8f84ac5 5272{
e16866ec 5273 u32 orig, data;
f8f84ac5 5274
e16866ec 5275 orig = data = RREG32_PCIE(PCIE_CNTL2);
f8f84ac5 5276
e16866ec
AD		 5277 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5278 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5279 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5280 else
5281 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5282 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5283
5284 if (orig != data)
5285 WREG32_PCIE(PCIE_CNTL2, data);
f8f84ac5
AD		 5286}
5287
e16866ec
AD		 5288static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5289 bool enable)
f8f84ac5 5290{
e16866ec 5291 u32 orig, data;
f8f84ac5 5292
e16866ec 5293 orig = data = RREG32(HDP_HOST_PATH_CNTL);
f8f84ac5 5294
e16866ec
AD		 5295 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5296 data &= ~CLOCK_GATING_DIS;
5297 else
5298 data |= CLOCK_GATING_DIS;
5299
5300 if (orig != data)
5301 WREG32(HDP_HOST_PATH_CNTL, data);
f8f84ac5
AD		 5302}
5303
e16866ec
AD		 5304static void si_enable_hdp_ls(struct radeon_device *rdev,
5305 bool enable)
347e7592 5306{
e16866ec 5307 u32 orig, data;
347e7592 5308
e16866ec 5309 orig = data = RREG32(HDP_MEM_POWER_LS);
347e7592 5310
e16866ec
AD		 5311 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5312 data |= HDP_LS_ENABLE;
5313 else
5314 data &= ~HDP_LS_ENABLE;
347e7592 5315
e16866ec
AD		 5316 if (orig != data)
5317 WREG32(HDP_MEM_POWER_LS, data);
347e7592
AD		 5318}
5319
68e3a092
AD		 5320static void si_update_cg(struct radeon_device *rdev,
5321 u32 block, bool enable)
347e7592 5322{
e16866ec 5323 if (block & RADEON_CG_BLOCK_GFX) {
811e4d58 5324 si_enable_gui_idle_interrupt(rdev, false);
e16866ec
AD		 5325 /* order matters! */
5326 if (enable) {
5327 si_enable_mgcg(rdev, true);
5328 si_enable_cgcg(rdev, true);
5329 } else {
5330 si_enable_cgcg(rdev, false);
5331 si_enable_mgcg(rdev, false);
347e7592 5332 }
811e4d58 5333 si_enable_gui_idle_interrupt(rdev, true);
347e7592
AD		 5334 }
5335
e16866ec
AD		 5336 if (block & RADEON_CG_BLOCK_MC) {
5337 si_enable_mc_mgcg(rdev, enable);
5338 si_enable_mc_ls(rdev, enable);
347e7592 5339 }
e16866ec
AD		 5340
5341 if (block & RADEON_CG_BLOCK_SDMA) {
5342 si_enable_dma_mgcg(rdev, enable);
347e7592
AD		 5343 }
5344
e16866ec
AD		 5345 if (block & RADEON_CG_BLOCK_BIF) {
5346 si_enable_bif_mgls(rdev, enable);
6d8cf000 5347 }
6d8cf000 5348
e16866ec
AD		 5349 if (block & RADEON_CG_BLOCK_UVD) {
5350 if (rdev->has_uvd) {
5351 si_enable_uvd_mgcg(rdev, enable);
bd8cd539
AD		 5352 }
5353 }
bd8cd539 5354
e16866ec
AD		 5355 if (block & RADEON_CG_BLOCK_HDP) {
5356 si_enable_hdp_mgcg(rdev, enable);
5357 si_enable_hdp_ls(rdev, enable);
347e7592 5358 }
e16866ec 5359}
f8f84ac5
AD		 5360
5361static void si_init_cg(struct radeon_device *rdev)
5362{
e16866ec
AD		 5363 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5364 RADEON_CG_BLOCK_MC |
5365 RADEON_CG_BLOCK_SDMA |
5366 RADEON_CG_BLOCK_BIF |
5367 RADEON_CG_BLOCK_HDP), true);
b2d70917 5368 if (rdev->has_uvd) {
e16866ec 5369 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
f8f84ac5 5370 si_init_uvd_internal_cg(rdev);
347e7592 5371 }
f8f84ac5 5372}
bd8cd539 5373
f8f84ac5
AD		 5374static void si_fini_cg(struct radeon_device *rdev)
5375{
0116e1ef 5376 if (rdev->has_uvd) {
e16866ec 5377 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
347e7592 5378 }
e16866ec
AD		 5379 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5380 RADEON_CG_BLOCK_MC |
5381 RADEON_CG_BLOCK_SDMA |
5382 RADEON_CG_BLOCK_BIF |
5383 RADEON_CG_BLOCK_HDP), false);
5384}
5385
59a82d0e
AD		 5386u32 si_get_csb_size(struct radeon_device *rdev)
5387{
5388 u32 count = 0;
5389 const struct cs_section_def *sect = NULL;
5390 const struct cs_extent_def *ext = NULL;
5391
5392 if (rdev->rlc.cs_data == NULL)
5393 return 0;
5394
5395 /* begin clear state */
5396 count += 2;
5397 /* context control state */
5398 count += 3;
5399
5400 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5401 for (ext = sect->section; ext->extent != NULL; ++ext) {
5402 if (sect->id == SECT_CONTEXT)
5403 count += 2 + ext->reg_count;
5404 else
5405 return 0;
5406 }
bd8cd539 5407 }
59a82d0e
AD		 5408 /* pa_sc_raster_config */
5409 count += 3;
5410 /* end clear state */
5411 count += 2;
5412 /* clear state */
5413 count += 2;
5414
5415 return count;
5416}
5417
5418void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5419{
5420 u32 count = 0, i;
5421 const struct cs_section_def *sect = NULL;
5422 const struct cs_extent_def *ext = NULL;
5423
5424 if (rdev->rlc.cs_data == NULL)
5425 return;
5426 if (buffer == NULL)
5427 return;
5428
6ba81e53
AD		 5429 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5430 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
59a82d0e 5431
6ba81e53
AD		 5432 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5433 buffer[count++] = cpu_to_le32(0x80000000);
5434 buffer[count++] = cpu_to_le32(0x80000000);
59a82d0e
AD		 5435
5436 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5437 for (ext = sect->section; ext->extent != NULL; ++ext) {
5438 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 5439 buffer[count++] =
5440 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5441 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
59a82d0e 5442 for (i = 0; i < ext->reg_count; i++)
6ba81e53 5443 buffer[count++] = cpu_to_le32(ext->extent[i]);
59a82d0e
AD		 5444 } else {
5445 return;
bd8cd539 5446 }
bd8cd539
AD		 5447 }
5448 }
bd8cd539 5449
6ba81e53
AD		 5450 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5451 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
59a82d0e
AD		 5452 switch (rdev->family) {
5453 case CHIP_TAHITI:
5454 case CHIP_PITCAIRN:
6ba81e53 5455 buffer[count++] = cpu_to_le32(0x2a00126a);
59a82d0e
AD		 5456 break;
5457 case CHIP_VERDE:
6ba81e53 5458 buffer[count++] = cpu_to_le32(0x0000124a);
59a82d0e
AD		 5459 break;
5460 case CHIP_OLAND:
6ba81e53 5461 buffer[count++] = cpu_to_le32(0x00000082);
59a82d0e
AD		 5462 break;
5463 case CHIP_HAINAN:
6ba81e53 5464 buffer[count++] = cpu_to_le32(0x00000000);
59a82d0e
AD		 5465 break;
5466 default:
6ba81e53 5467 buffer[count++] = cpu_to_le32(0x00000000);
59a82d0e
AD		 5468 break;
5469 }
5470
6ba81e53
AD		 5471 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5472 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
347e7592 5473
6ba81e53
AD		 5474 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5475 buffer[count++] = cpu_to_le32(0);
59a82d0e
AD		 5476}
5477
f8f84ac5
AD		 5478static void si_init_pg(struct radeon_device *rdev)
5479{
0116e1ef
AD		 5480 if (rdev->pg_flags) {
5481 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5482 si_init_dma_pg(rdev);
0116e1ef 5483 }
f8f84ac5 5484 si_init_ao_cu_mask(rdev);
2b19d17f 5485 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
0116e1ef 5486 si_init_gfx_cgpg(rdev);
aa34dba8
AD		 5487 } else {
5488 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5489 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
0116e1ef 5490 }
ca6ebb39
AD		 5491 si_enable_dma_pg(rdev, true);
5492 si_enable_gfx_cgpg(rdev, true);
f8f84ac5
AD		 5493 } else {
5494 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5495 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5496 }
5497}
5498
5499static void si_fini_pg(struct radeon_device *rdev)
5500{
0116e1ef 5501 if (rdev->pg_flags) {
ca6ebb39
AD		 5502 si_enable_dma_pg(rdev, false);
5503 si_enable_gfx_cgpg(rdev, false);
f8f84ac5 5504 }
347e7592
AD		 5505}
5506
347e7592
AD		 5507/*
5508 * RLC
5509 */
866d83de 5510void si_rlc_reset(struct radeon_device *rdev)
d719cef3 5511{
f8f84ac5 5512 u32 tmp = RREG32(GRBM_SOFT_RESET);
d719cef3 5513
f8f84ac5
AD		 5514 tmp |= SOFT_RESET_RLC;
5515 WREG32(GRBM_SOFT_RESET, tmp);
5516 udelay(50);
5517 tmp &= ~SOFT_RESET_RLC;
5518 WREG32(GRBM_SOFT_RESET, tmp);
5519 udelay(50);
d719cef3
AD		 5520}
5521
347e7592
AD		 5522static void si_rlc_stop(struct radeon_device *rdev)
5523{
5524 WREG32(RLC_CNTL, 0);
d719cef3
AD		 5525
5526 si_enable_gui_idle_interrupt(rdev, false);
5527
5528 si_wait_for_rlc_serdes(rdev);
347e7592
AD		 5529}
5530
5531static void si_rlc_start(struct radeon_device *rdev)
5532{
5533 WREG32(RLC_CNTL, RLC_ENABLE);
d719cef3
AD		 5534
5535 si_enable_gui_idle_interrupt(rdev, true);
5536
5537 udelay(50);
5538}
5539
5540static bool si_lbpw_supported(struct radeon_device *rdev)
5541{
5542 u32 tmp;
5543
5544 /* Enable LBPW only for DDR3 */
5545 tmp = RREG32(MC_SEQ_MISC0);
5546 if ((tmp & 0xF0000000) == 0xB0000000)
5547 return true;
5548 return false;
5549}
5550
5551static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5552{
5553 u32 tmp;
5554
5555 tmp = RREG32(RLC_LB_CNTL);
5556 if (enable)
5557 tmp |= LOAD_BALANCE_ENABLE;
5558 else
5559 tmp &= ~LOAD_BALANCE_ENABLE;
5560 WREG32(RLC_LB_CNTL, tmp);
5561
5562 if (!enable) {
5563 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5564 WREG32(SPI_LB_CU_MASK, 0x00ff);
5565 }
347e7592
AD		 5566}
5567
5568static int si_rlc_resume(struct radeon_device *rdev)
5569{
5570 u32 i;
5571 const __be32 *fw_data;
5572
5573 if (!rdev->rlc_fw)
5574 return -EINVAL;
5575
5576 si_rlc_stop(rdev);
5577
f8f84ac5
AD		 5578 si_rlc_reset(rdev);
5579
5580 si_init_pg(rdev);
5581
5582 si_init_cg(rdev);
5583
347e7592
AD		 5584 WREG32(RLC_RL_BASE, 0);
5585 WREG32(RLC_RL_SIZE, 0);
5586 WREG32(RLC_LB_CNTL, 0);
5587 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5588 WREG32(RLC_LB_CNTR_INIT, 0);
d719cef3 5589 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
347e7592 5590
347e7592
AD		 5591 WREG32(RLC_MC_CNTL, 0);
5592 WREG32(RLC_UCODE_CNTL, 0);
5593
5594 fw_data = (const __be32 *)rdev->rlc_fw->data;
5595 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5596 WREG32(RLC_UCODE_ADDR, i);
5597 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5598 }
5599 WREG32(RLC_UCODE_ADDR, 0);
5600
d719cef3
AD		 5601 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5602
347e7592
AD		 5603 si_rlc_start(rdev);
5604
5605 return 0;
5606}
5607
25a857fb
AD		 5608static void si_enable_interrupts(struct radeon_device *rdev)
5609{
5610 u32 ih_cntl = RREG32(IH_CNTL);
5611 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5612
5613 ih_cntl |= ENABLE_INTR;
5614 ih_rb_cntl |= IH_RB_ENABLE;
5615 WREG32(IH_CNTL, ih_cntl);
5616 WREG32(IH_RB_CNTL, ih_rb_cntl);
5617 rdev->ih.enabled = true;
5618}
5619
5620static void si_disable_interrupts(struct radeon_device *rdev)
5621{
5622 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5623 u32 ih_cntl = RREG32(IH_CNTL);
5624
5625 ih_rb_cntl &= ~IH_RB_ENABLE;
5626 ih_cntl &= ~ENABLE_INTR;
5627 WREG32(IH_RB_CNTL, ih_rb_cntl);
5628 WREG32(IH_CNTL, ih_cntl);
5629 /* set rptr, wptr to 0 */
5630 WREG32(IH_RB_RPTR, 0);
5631 WREG32(IH_RB_WPTR, 0);
5632 rdev->ih.enabled = false;
25a857fb
AD		 5633 rdev->ih.rptr = 0;
5634}
5635
5636static void si_disable_interrupt_state(struct radeon_device *rdev)
5637{
5638 u32 tmp;
5639
811e4d58
AD		 5640 tmp = RREG32(CP_INT_CNTL_RING0) &
5641 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5642 WREG32(CP_INT_CNTL_RING0, tmp);
25a857fb
AD		 5643 WREG32(CP_INT_CNTL_RING1, 0);
5644 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 5645 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5646 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5647 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5648 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 5649 WREG32(GRBM_INT_CNTL, 0);
5153550a
AD		 5650 if (rdev->num_crtc >= 2) {
5651 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5652 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5653 }
25a857fb
AD		 5654 if (rdev->num_crtc >= 4) {
5655 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5656 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5657 }
5658 if (rdev->num_crtc >= 6) {
5659 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5660 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5661 }
5662
5153550a
AD		 5663 if (rdev->num_crtc >= 2) {
5664 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5665 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5666 }
25a857fb
AD		 5667 if (rdev->num_crtc >= 4) {
5668 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5669 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5670 }
5671 if (rdev->num_crtc >= 6) {
5672 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5673 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5674 }
5675
5153550a 5676 if (!ASIC_IS_NODCE(rdev)) {
e9a321c6 5677 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5153550a
AD		 5678
5679 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5680 WREG32(DC_HPD1_INT_CONTROL, tmp);
5681 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5682 WREG32(DC_HPD2_INT_CONTROL, tmp);
5683 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5684 WREG32(DC_HPD3_INT_CONTROL, tmp);
5685 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5686 WREG32(DC_HPD4_INT_CONTROL, tmp);
5687 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5688 WREG32(DC_HPD5_INT_CONTROL, tmp);
5689 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5690 WREG32(DC_HPD6_INT_CONTROL, tmp);
5691 }
25a857fb
AD		 5692}
5693
5694static int si_irq_init(struct radeon_device *rdev)
5695{
5696 int ret = 0;
5697 int rb_bufsz;
5698 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5699
5700 /* allocate ring */
5701 ret = r600_ih_ring_alloc(rdev);
5702 if (ret)
5703 return ret;
5704
5705 /* disable irqs */
5706 si_disable_interrupts(rdev);
5707
5708 /* init rlc */
5709 ret = si_rlc_resume(rdev);
5710 if (ret) {
5711 r600_ih_ring_fini(rdev);
5712 return ret;
5713 }
5714
5715 /* setup interrupt control */
5716 /* set dummy read address to ring address */
5717 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5718 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5719 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5720 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5721 */
5722 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5723 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5724 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5725 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5726
5727 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 5728 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
25a857fb
AD		 5729
5730 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5731 IH_WPTR_OVERFLOW_CLEAR |
5732 (rb_bufsz << 1));
5733
5734 if (rdev->wb.enabled)
5735 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5736
5737 /* set the writeback address whether it's enabled or not */
5738 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5739 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5740
5741 WREG32(IH_RB_CNTL, ih_rb_cntl);
5742
5743 /* set rptr, wptr to 0 */
5744 WREG32(IH_RB_RPTR, 0);
5745 WREG32(IH_RB_WPTR, 0);
5746
5747 /* Default settings for IH_CNTL (disabled at first) */
5748 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5749 /* RPTR_REARM only works if msi's are enabled */
5750 if (rdev->msi_enabled)
5751 ih_cntl |= RPTR_REARM;
5752 WREG32(IH_CNTL, ih_cntl);
5753
5754 /* force the active interrupt state to all disabled */
5755 si_disable_interrupt_state(rdev);
5756
2099810f
DA		 5757 pci_set_master(rdev->pdev);
5758
25a857fb
AD		 5759 /* enable irqs */
5760 si_enable_interrupts(rdev);
5761
5762 return ret;
5763}
5764
5765int si_irq_set(struct radeon_device *rdev)
5766{
811e4d58 5767 u32 cp_int_cntl;
25a857fb
AD		 5768 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5769 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5153550a 5770 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
25a857fb
AD		 5771 u32 grbm_int_cntl = 0;
5772 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
8c5fd7ef 5773 u32 dma_cntl, dma_cntl1;
a9e61410 5774 u32 thermal_int = 0;
25a857fb
AD		 5775
5776 if (!rdev->irq.installed) {
5777 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5778 return -EINVAL;
5779 }
5780 /* don't enable anything if the ih is disabled */
5781 if (!rdev->ih.enabled) {
5782 si_disable_interrupts(rdev);
5783 /* force the active interrupt state to all disabled */
5784 si_disable_interrupt_state(rdev);
5785 return 0;
5786 }
5787
811e4d58
AD		 5788 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
5789 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5790
5153550a
AD		 5791 if (!ASIC_IS_NODCE(rdev)) {
5792 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5793 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5794 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5795 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5796 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5797 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5798 }
25a857fb 5799
8c5fd7ef
AD		 5800 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5801 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5802
a9e61410
AD		 5803 thermal_int = RREG32(CG_THERMAL_INT) &
5804 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5805
25a857fb 5806 /* enable CP interrupts on all rings */
736fc37f 5807 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 5808 DRM_DEBUG("si_irq_set: sw int gfx\n");
5809 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5810 }
736fc37f 5811 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 5812 DRM_DEBUG("si_irq_set: sw int cp1\n");
5813 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5814 }
736fc37f 5815 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 5816 DRM_DEBUG("si_irq_set: sw int cp2\n");
5817 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5818 }
8c5fd7ef
AD		 5819 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5820 DRM_DEBUG("si_irq_set: sw int dma\n");
5821 dma_cntl |= TRAP_ENABLE;
5822 }
5823
5824 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5825 DRM_DEBUG("si_irq_set: sw int dma1\n");
5826 dma_cntl1 |= TRAP_ENABLE;
5827 }
25a857fb 5828 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 5829 atomic_read(&rdev->irq.pflip[0])) {
25a857fb
AD		 5830 DRM_DEBUG("si_irq_set: vblank 0\n");
5831 crtc1 |= VBLANK_INT_MASK;
5832 }
5833 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 5834 atomic_read(&rdev->irq.pflip[1])) {
25a857fb
AD		 5835 DRM_DEBUG("si_irq_set: vblank 1\n");
5836 crtc2 |= VBLANK_INT_MASK;
5837 }
5838 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 5839 atomic_read(&rdev->irq.pflip[2])) {
25a857fb
AD		 5840 DRM_DEBUG("si_irq_set: vblank 2\n");
5841 crtc3 |= VBLANK_INT_MASK;
5842 }
5843 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 5844 atomic_read(&rdev->irq.pflip[3])) {
25a857fb
AD		 5845 DRM_DEBUG("si_irq_set: vblank 3\n");
5846 crtc4 |= VBLANK_INT_MASK;
5847 }
5848 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 5849 atomic_read(&rdev->irq.pflip[4])) {
25a857fb
AD		 5850 DRM_DEBUG("si_irq_set: vblank 4\n");
5851 crtc5 |= VBLANK_INT_MASK;
5852 }
5853 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 5854 atomic_read(&rdev->irq.pflip[5])) {
25a857fb
AD		 5855 DRM_DEBUG("si_irq_set: vblank 5\n");
5856 crtc6 |= VBLANK_INT_MASK;
5857 }
5858 if (rdev->irq.hpd[0]) {
5859 DRM_DEBUG("si_irq_set: hpd 1\n");
5860 hpd1 |= DC_HPDx_INT_EN;
5861 }
5862 if (rdev->irq.hpd[1]) {
5863 DRM_DEBUG("si_irq_set: hpd 2\n");
5864 hpd2 |= DC_HPDx_INT_EN;
5865 }
5866 if (rdev->irq.hpd[2]) {
5867 DRM_DEBUG("si_irq_set: hpd 3\n");
5868 hpd3 |= DC_HPDx_INT_EN;
5869 }
5870 if (rdev->irq.hpd[3]) {
5871 DRM_DEBUG("si_irq_set: hpd 4\n");
5872 hpd4 |= DC_HPDx_INT_EN;
5873 }
5874 if (rdev->irq.hpd[4]) {
5875 DRM_DEBUG("si_irq_set: hpd 5\n");
5876 hpd5 |= DC_HPDx_INT_EN;
5877 }
5878 if (rdev->irq.hpd[5]) {
5879 DRM_DEBUG("si_irq_set: hpd 6\n");
5880 hpd6 |= DC_HPDx_INT_EN;
5881 }
25a857fb
AD		 5882
5883 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5884 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5885 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5886
8c5fd7ef
AD		 5887 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5888 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5889
25a857fb
AD		 5890 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5891
a9e61410
AD		 5892 if (rdev->irq.dpm_thermal) {
5893 DRM_DEBUG("dpm thermal\n");
5894 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5895 }
5896
5153550a
AD		 5897 if (rdev->num_crtc >= 2) {
5898 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5899 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5900 }
25a857fb
AD		 5901 if (rdev->num_crtc >= 4) {
5902 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5903 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5904 }
5905 if (rdev->num_crtc >= 6) {
5906 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5907 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5908 }
5909
5153550a
AD		 5910 if (rdev->num_crtc >= 2) {
5911 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5912 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5913 }
25a857fb
AD		 5914 if (rdev->num_crtc >= 4) {
5915 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5916 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5917 }
5918 if (rdev->num_crtc >= 6) {
5919 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5920 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5921 }
5922
5153550a
AD		 5923 if (!ASIC_IS_NODCE(rdev)) {
5924 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5925 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5926 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5927 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5928 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5929 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5930 }
25a857fb 5931
a9e61410
AD		 5932 WREG32(CG_THERMAL_INT, thermal_int);
5933
25a857fb
AD		 5934 return 0;
5935}
5936
5937static inline void si_irq_ack(struct radeon_device *rdev)
5938{
5939 u32 tmp;
5940
5153550a
AD		 5941 if (ASIC_IS_NODCE(rdev))
5942 return;
5943
25a857fb
AD		 5944 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5945 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5946 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5947 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5948 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5949 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5950 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5951 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5952 if (rdev->num_crtc >= 4) {
5953 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5954 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5955 }
5956 if (rdev->num_crtc >= 6) {
5957 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5958 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5959 }
5960
5961 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5962 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5963 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5964 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5965 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5966 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5967 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5968 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5969 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5970 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5971 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5972 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5973
5974 if (rdev->num_crtc >= 4) {
5975 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5976 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5977 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5978 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5979 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5980 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5981 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5982 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5983 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5984 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5985 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5986 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5987 }
5988
5989 if (rdev->num_crtc >= 6) {
5990 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5991 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5992 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5993 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5994 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5995 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5996 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5997 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5998 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5999 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6000 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6001 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6002 }
6003
6004 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6005 tmp = RREG32(DC_HPD1_INT_CONTROL);
6006 tmp |= DC_HPDx_INT_ACK;
6007 WREG32(DC_HPD1_INT_CONTROL, tmp);
6008 }
6009 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6010 tmp = RREG32(DC_HPD2_INT_CONTROL);
6011 tmp |= DC_HPDx_INT_ACK;
6012 WREG32(DC_HPD2_INT_CONTROL, tmp);
6013 }
6014 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6015 tmp = RREG32(DC_HPD3_INT_CONTROL);
6016 tmp |= DC_HPDx_INT_ACK;
6017 WREG32(DC_HPD3_INT_CONTROL, tmp);
6018 }
6019 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6020 tmp = RREG32(DC_HPD4_INT_CONTROL);
6021 tmp |= DC_HPDx_INT_ACK;
6022 WREG32(DC_HPD4_INT_CONTROL, tmp);
6023 }
6024 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6025 tmp = RREG32(DC_HPD5_INT_CONTROL);
6026 tmp |= DC_HPDx_INT_ACK;
6027 WREG32(DC_HPD5_INT_CONTROL, tmp);
6028 }
6029 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6030 tmp = RREG32(DC_HPD5_INT_CONTROL);
6031 tmp |= DC_HPDx_INT_ACK;
6032 WREG32(DC_HPD6_INT_CONTROL, tmp);
6033 }
6034}
6035
6036static void si_irq_disable(struct radeon_device *rdev)
6037{
6038 si_disable_interrupts(rdev);
6039 /* Wait and acknowledge irq */
6040 mdelay(1);
6041 si_irq_ack(rdev);
6042 si_disable_interrupt_state(rdev);
6043}
6044
6045static void si_irq_suspend(struct radeon_device *rdev)
6046{
6047 si_irq_disable(rdev);
6048 si_rlc_stop(rdev);
6049}
6050
9b136d51
AD		 6051static void si_irq_fini(struct radeon_device *rdev)
6052{
6053 si_irq_suspend(rdev);
6054 r600_ih_ring_fini(rdev);
6055}
6056
25a857fb
AD		 6057static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6058{
6059 u32 wptr, tmp;
6060
6061 if (rdev->wb.enabled)
6062 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6063 else
6064 wptr = RREG32(IH_RB_WPTR);
6065
6066 if (wptr & RB_OVERFLOW) {
6067 /* When a ring buffer overflow happen start parsing interrupt
6068 * from the last not overwritten vector (wptr + 16). Hopefully
6069 * this should allow us to catchup.
6070 */
6071 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6072 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6073 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6074 tmp = RREG32(IH_RB_CNTL);
6075 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6076 WREG32(IH_RB_CNTL, tmp);
6077 }
6078 return (wptr & rdev->ih.ptr_mask);
6079}
6080
6081/* SI IV Ring
6082 * Each IV ring entry is 128 bits:
6083 * [7:0] - interrupt source id
6084 * [31:8] - reserved
6085 * [59:32] - interrupt source data
6086 * [63:60] - reserved
6087 * [71:64] - RINGID
6088 * [79:72] - VMID
6089 * [127:80] - reserved
6090 */
6091int si_irq_process(struct radeon_device *rdev)
6092{
6093 u32 wptr;
6094 u32 rptr;
6095 u32 src_id, src_data, ring_id;
6096 u32 ring_index;
25a857fb 6097 bool queue_hotplug = false;
a9e61410 6098 bool queue_thermal = false;
fbf6dc7a 6099 u32 status, addr;
25a857fb
AD		 6100
6101 if (!rdev->ih.enabled || rdev->shutdown)
6102 return IRQ_NONE;
6103
6104 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 6105
6106restart_ih:
6107 /* is somebody else already processing irqs? */
6108 if (atomic_xchg(&rdev->ih.lock, 1))
6109 return IRQ_NONE;
6110
25a857fb
AD		 6111 rptr = rdev->ih.rptr;
6112 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6113
25a857fb
AD		 6114 /* Order reading of wptr vs. reading of IH ring data */
6115 rmb();
6116
6117 /* display interrupts */
6118 si_irq_ack(rdev);
6119
25a857fb
AD		 6120 while (rptr != wptr) {
6121 /* wptr/rptr are in bytes! */
6122 ring_index = rptr / 4;
6123 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6124 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6125 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6126
6127 switch (src_id) {
6128 case 1: /* D1 vblank/vline */
6129 switch (src_data) {
6130 case 0: /* D1 vblank */
6131 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6132 if (rdev->irq.crtc_vblank_int[0]) {
6133 drm_handle_vblank(rdev->ddev, 0);
6134 rdev->pm.vblank_sync = true;
6135 wake_up(&rdev->irq.vblank_queue);
6136 }
736fc37f 6137 if (atomic_read(&rdev->irq.pflip[0]))
25a857fb
AD		 6138 radeon_crtc_handle_flip(rdev, 0);
6139 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6140 DRM_DEBUG("IH: D1 vblank\n");
6141 }
6142 break;
6143 case 1: /* D1 vline */
6144 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6145 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6146 DRM_DEBUG("IH: D1 vline\n");
6147 }
6148 break;
6149 default:
6150 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6151 break;
6152 }
6153 break;
6154 case 2: /* D2 vblank/vline */
6155 switch (src_data) {
6156 case 0: /* D2 vblank */
6157 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6158 if (rdev->irq.crtc_vblank_int[1]) {
6159 drm_handle_vblank(rdev->ddev, 1);
6160 rdev->pm.vblank_sync = true;
6161 wake_up(&rdev->irq.vblank_queue);
6162 }
736fc37f 6163 if (atomic_read(&rdev->irq.pflip[1]))
25a857fb
AD		 6164 radeon_crtc_handle_flip(rdev, 1);
6165 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6166 DRM_DEBUG("IH: D2 vblank\n");
6167 }
6168 break;
6169 case 1: /* D2 vline */
6170 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6171 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6172 DRM_DEBUG("IH: D2 vline\n");
6173 }
6174 break;
6175 default:
6176 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6177 break;
6178 }
6179 break;
6180 case 3: /* D3 vblank/vline */
6181 switch (src_data) {
6182 case 0: /* D3 vblank */
6183 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6184 if (rdev->irq.crtc_vblank_int[2]) {
6185 drm_handle_vblank(rdev->ddev, 2);
6186 rdev->pm.vblank_sync = true;
6187 wake_up(&rdev->irq.vblank_queue);
6188 }
736fc37f 6189 if (atomic_read(&rdev->irq.pflip[2]))
25a857fb
AD		 6190 radeon_crtc_handle_flip(rdev, 2);
6191 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6192 DRM_DEBUG("IH: D3 vblank\n");
6193 }
6194 break;
6195 case 1: /* D3 vline */
6196 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6197 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6198 DRM_DEBUG("IH: D3 vline\n");
6199 }
6200 break;
6201 default:
6202 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6203 break;
6204 }
6205 break;
6206 case 4: /* D4 vblank/vline */
6207 switch (src_data) {
6208 case 0: /* D4 vblank */
6209 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6210 if (rdev->irq.crtc_vblank_int[3]) {
6211 drm_handle_vblank(rdev->ddev, 3);
6212 rdev->pm.vblank_sync = true;
6213 wake_up(&rdev->irq.vblank_queue);
6214 }
736fc37f 6215 if (atomic_read(&rdev->irq.pflip[3]))
25a857fb
AD		 6216 radeon_crtc_handle_flip(rdev, 3);
6217 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6218 DRM_DEBUG("IH: D4 vblank\n");
6219 }
6220 break;
6221 case 1: /* D4 vline */
6222 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6223 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6224 DRM_DEBUG("IH: D4 vline\n");
6225 }
6226 break;
6227 default:
6228 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6229 break;
6230 }
6231 break;
6232 case 5: /* D5 vblank/vline */
6233 switch (src_data) {
6234 case 0: /* D5 vblank */
6235 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6236 if (rdev->irq.crtc_vblank_int[4]) {
6237 drm_handle_vblank(rdev->ddev, 4);
6238 rdev->pm.vblank_sync = true;
6239 wake_up(&rdev->irq.vblank_queue);
6240 }
736fc37f 6241 if (atomic_read(&rdev->irq.pflip[4]))
25a857fb
AD		 6242 radeon_crtc_handle_flip(rdev, 4);
6243 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6244 DRM_DEBUG("IH: D5 vblank\n");
6245 }
6246 break;
6247 case 1: /* D5 vline */
6248 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6249 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6250 DRM_DEBUG("IH: D5 vline\n");
6251 }
6252 break;
6253 default:
6254 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6255 break;
6256 }
6257 break;
6258 case 6: /* D6 vblank/vline */
6259 switch (src_data) {
6260 case 0: /* D6 vblank */
6261 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6262 if (rdev->irq.crtc_vblank_int[5]) {
6263 drm_handle_vblank(rdev->ddev, 5);
6264 rdev->pm.vblank_sync = true;
6265 wake_up(&rdev->irq.vblank_queue);
6266 }
736fc37f 6267 if (atomic_read(&rdev->irq.pflip[5]))
25a857fb
AD		 6268 radeon_crtc_handle_flip(rdev, 5);
6269 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6270 DRM_DEBUG("IH: D6 vblank\n");
6271 }
6272 break;
6273 case 1: /* D6 vline */
6274 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6275 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6276 DRM_DEBUG("IH: D6 vline\n");
6277 }
6278 break;
6279 default:
6280 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6281 break;
6282 }
6283 break;
6284 case 42: /* HPD hotplug */
6285 switch (src_data) {
6286 case 0:
6287 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6288 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6289 queue_hotplug = true;
6290 DRM_DEBUG("IH: HPD1\n");
6291 }
6292 break;
6293 case 1:
6294 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6295 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6296 queue_hotplug = true;
6297 DRM_DEBUG("IH: HPD2\n");
6298 }
6299 break;
6300 case 2:
6301 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6302 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6303 queue_hotplug = true;
6304 DRM_DEBUG("IH: HPD3\n");
6305 }
6306 break;
6307 case 3:
6308 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6309 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6310 queue_hotplug = true;
6311 DRM_DEBUG("IH: HPD4\n");
6312 }
6313 break;
6314 case 4:
6315 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6316 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6317 queue_hotplug = true;
6318 DRM_DEBUG("IH: HPD5\n");
6319 }
6320 break;
6321 case 5:
6322 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6323 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6324 queue_hotplug = true;
6325 DRM_DEBUG("IH: HPD6\n");
6326 }
6327 break;
6328 default:
6329 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6330 break;
6331 }
6332 break;
b927e1c2
CK		 6333 case 124: /* UVD */
6334 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6335 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6336 break;
ae133a11
CK		 6337 case 146:
6338 case 147:
fbf6dc7a
AD		 6339 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6340 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
ae133a11
CK		 6341 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6342 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
fbf6dc7a 6343 addr);
ae133a11 6344 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
fbf6dc7a
AD		 6345 status);
6346 si_vm_decode_fault(rdev, status, addr);
ae133a11
CK		 6347 /* reset addr and status */
6348 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6349 break;
25a857fb
AD		 6350 case 176: /* RINGID0 CP_INT */
6351 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6352 break;
6353 case 177: /* RINGID1 CP_INT */
6354 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6355 break;
6356 case 178: /* RINGID2 CP_INT */
6357 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6358 break;
6359 case 181: /* CP EOP event */
6360 DRM_DEBUG("IH: CP EOP\n");
6361 switch (ring_id) {
6362 case 0:
6363 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6364 break;
6365 case 1:
6366 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6367 break;
6368 case 2:
6369 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6370 break;
6371 }
6372 break;
8c5fd7ef
AD		 6373 case 224: /* DMA trap event */
6374 DRM_DEBUG("IH: DMA trap\n");
6375 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6376 break;
a9e61410
AD		 6377 case 230: /* thermal low to high */
6378 DRM_DEBUG("IH: thermal low to high\n");
6379 rdev->pm.dpm.thermal.high_to_low = false;
6380 queue_thermal = true;
6381 break;
6382 case 231: /* thermal high to low */
6383 DRM_DEBUG("IH: thermal high to low\n");
6384 rdev->pm.dpm.thermal.high_to_low = true;
6385 queue_thermal = true;
6386 break;
25a857fb
AD		 6387 case 233: /* GUI IDLE */
6388 DRM_DEBUG("IH: GUI idle\n");
25a857fb 6389 break;
8c5fd7ef
AD		 6390 case 244: /* DMA trap event */
6391 DRM_DEBUG("IH: DMA1 trap\n");
6392 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6393 break;
25a857fb
AD		 6394 default:
6395 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6396 break;
6397 }
6398
6399 /* wptr/rptr are in bytes! */
6400 rptr += 16;
6401 rptr &= rdev->ih.ptr_mask;
6402 }
25a857fb
AD		 6403 if (queue_hotplug)
6404 schedule_work(&rdev->hotplug_work);
a9e61410
AD		 6405 if (queue_thermal && rdev->pm.dpm_enabled)
6406 schedule_work(&rdev->pm.dpm.thermal.work);
25a857fb
AD		 6407 rdev->ih.rptr = rptr;
6408 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 6409 atomic_set(&rdev->ih.lock, 0);
6410
6411 /* make sure wptr hasn't changed while processing */
6412 wptr = si_get_ih_wptr(rdev);
6413 if (wptr != rptr)
6414 goto restart_ih;
6415
25a857fb
AD		 6416 return IRQ_HANDLED;
6417}
6418
9b136d51
AD		 6419/*
6420 * startup/shutdown callbacks
6421 */
6422static int si_startup(struct radeon_device *rdev)
6423{
6424 struct radeon_ring *ring;
6425 int r;
6426
b9d305df
AD		 6427 /* enable pcie gen2/3 link */
6428 si_pcie_gen3_enable(rdev);
e0bcf165
AD		 6429 /* enable aspm */
6430 si_program_aspm(rdev);
b9d305df 6431
e5903d39
AD		 6432 /* scratch needs to be initialized before MC */
6433 r = r600_vram_scratch_init(rdev);
6434 if (r)
6435 return r;
6436
6fab3feb
AD		 6437 si_mc_program(rdev);
6438
6c7bccea
AD		 6439 if (!rdev->pm.dpm_enabled) {
6440 r = si_mc_load_microcode(rdev);
6441 if (r) {
6442 DRM_ERROR("Failed to load MC firmware!\n");
6443 return r;
6444 }
9b136d51
AD		 6445 }
6446
9b136d51
AD		 6447 r = si_pcie_gart_enable(rdev);
6448 if (r)
6449 return r;
6450 si_gpu_init(rdev);
6451
9b136d51 6452 /* allocate rlc buffers */
1fd11777
AD		 6453 if (rdev->family == CHIP_VERDE) {
6454 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6455 rdev->rlc.reg_list_size =
6456 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6457 }
6458 rdev->rlc.cs_data = si_cs_data;
6459 r = sumo_rlc_init(rdev);
9b136d51
AD		 6460 if (r) {
6461 DRM_ERROR("Failed to init rlc BOs!\n");
6462 return r;
6463 }
6464
6465 /* allocate wb buffer */
6466 r = radeon_wb_init(rdev);
6467 if (r)
6468 return r;
6469
6470 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6471 if (r) {
6472 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6473 return r;
6474 }
6475
6476 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6477 if (r) {
6478 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6479 return r;
6480 }
6481
6482 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6483 if (r) {
6484 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6485 return r;
6486 }
6487
8c5fd7ef
AD		 6488 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6489 if (r) {
6490 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6491 return r;
6492 }
6493
6494 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6495 if (r) {
6496 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6497 return r;
6498 }
6499
1df0d523 6500 if (rdev->has_uvd) {
e409b128 6501 r = uvd_v2_2_resume(rdev);
1df0d523
AD		 6502 if (!r) {
6503 r = radeon_fence_driver_start_ring(rdev,
6504 R600_RING_TYPE_UVD_INDEX);
6505 if (r)
6506 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6507 }
f2ba57b5 6508 if (r)
1df0d523 6509 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
f2ba57b5 6510 }
f2ba57b5 6511
9b136d51 6512 /* Enable IRQ */
e49f3959
AH		 6513 if (!rdev->irq.installed) {
6514 r = radeon_irq_kms_init(rdev);
6515 if (r)
6516 return r;
6517 }
6518
9b136d51
AD		 6519 r = si_irq_init(rdev);
6520 if (r) {
6521 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6522 radeon_irq_kms_fini(rdev);
6523 return r;
6524 }
6525 si_irq_set(rdev);
6526
6527 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6528 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2e1e6dad 6529 RADEON_CP_PACKET2);
9b136d51
AD		 6530 if (r)
6531 return r;
6532
6533 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6534 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
2e1e6dad 6535 RADEON_CP_PACKET2);
9b136d51
AD		 6536 if (r)
6537 return r;
6538
6539 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6540 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
2e1e6dad 6541 RADEON_CP_PACKET2);
9b136d51
AD		 6542 if (r)
6543 return r;
6544
8c5fd7ef
AD		 6545 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6546 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 6547 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6548 if (r)
6549 return r;
6550
6551 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6552 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 6553 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6554 if (r)
6555 return r;
6556
9b136d51
AD		 6557 r = si_cp_load_microcode(rdev);
6558 if (r)
6559 return r;
6560 r = si_cp_resume(rdev);
6561 if (r)
6562 return r;
6563
8c5fd7ef
AD		 6564 r = cayman_dma_resume(rdev);
6565 if (r)
6566 return r;
6567
1df0d523
AD		 6568 if (rdev->has_uvd) {
6569 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6570 if (ring->ring_size) {
02c9f7fa 6571 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 6572 RADEON_CP_PACKET2);
1df0d523 6573 if (!r)
e409b128 6574 r = uvd_v1_0_init(rdev);
1df0d523
AD		 6575 if (r)
6576 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6577 }
f2ba57b5
CK		 6578 }
6579
2898c348
CK		 6580 r = radeon_ib_pool_init(rdev);
6581 if (r) {
6582 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 6583 return r;
2898c348 6584 }
9b136d51 6585
c6105f24
CK		 6586 r = radeon_vm_manager_init(rdev);
6587 if (r) {
6588 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 6589 return r;
c6105f24 6590 }
9b136d51 6591
b530602f
AD		 6592 r = dce6_audio_init(rdev);
6593 if (r)
6594 return r;
6595
9b136d51
AD		 6596 return 0;
6597}
6598
6599int si_resume(struct radeon_device *rdev)
6600{
6601 int r;
6602
6603 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6604 * posting will perform necessary task to bring back GPU into good
6605 * shape.
6606 */
6607 /* post card */
6608 atom_asic_init(rdev->mode_info.atom_context);
6609
205996c0
AD		 6610 /* init golden registers */
6611 si_init_golden_registers(rdev);
6612
bc6a6295
AD		 6613 if (rdev->pm.pm_method == PM_METHOD_DPM)
6614 radeon_pm_resume(rdev);
6c7bccea 6615
9b136d51
AD		 6616 rdev->accel_working = true;
6617 r = si_startup(rdev);
6618 if (r) {
6619 DRM_ERROR("si startup failed on resume\n");
6620 rdev->accel_working = false;
6621 return r;
6622 }
6623
6624 return r;
6625
6626}
6627
6628int si_suspend(struct radeon_device *rdev)
6629{
6c7bccea 6630 radeon_pm_suspend(rdev);
b530602f 6631 dce6_audio_fini(rdev);
fa3daf9a 6632 radeon_vm_manager_fini(rdev);
9b136d51 6633 si_cp_enable(rdev, false);
8c5fd7ef 6634 cayman_dma_stop(rdev);
1df0d523 6635 if (rdev->has_uvd) {
e409b128 6636 uvd_v1_0_fini(rdev);
1df0d523
AD		 6637 radeon_uvd_suspend(rdev);
6638 }
e16866ec
AD		 6639 si_fini_pg(rdev);
6640 si_fini_cg(rdev);
9b136d51
AD		 6641 si_irq_suspend(rdev);
6642 radeon_wb_disable(rdev);
6643 si_pcie_gart_disable(rdev);
6644 return 0;
6645}
6646
6647/* Plan is to move initialization in that function and use
6648 * helper function so that radeon_device_init pretty much
6649 * do nothing more than calling asic specific function. This
6650 * should also allow to remove a bunch of callback function
6651 * like vram_info.
6652 */
6653int si_init(struct radeon_device *rdev)
6654{
6655 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6656 int r;
6657
9b136d51
AD		 6658 /* Read BIOS */
6659 if (!radeon_get_bios(rdev)) {
6660 if (ASIC_IS_AVIVO(rdev))
6661 return -EINVAL;
6662 }
6663 /* Must be an ATOMBIOS */
6664 if (!rdev->is_atom_bios) {
6665 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6666 return -EINVAL;
6667 }
6668 r = radeon_atombios_init(rdev);
6669 if (r)
6670 return r;
6671
6672 /* Post card if necessary */
6673 if (!radeon_card_posted(rdev)) {
6674 if (!rdev->bios) {
6675 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6676 return -EINVAL;
6677 }
6678 DRM_INFO("GPU not posted. posting now...\n");
6679 atom_asic_init(rdev->mode_info.atom_context);
6680 }
205996c0
AD		 6681 /* init golden registers */
6682 si_init_golden_registers(rdev);
9b136d51
AD		 6683 /* Initialize scratch registers */
6684 si_scratch_init(rdev);
6685 /* Initialize surface registers */
6686 radeon_surface_init(rdev);
6687 /* Initialize clocks */
6688 radeon_get_clock_info(rdev->ddev);
6689
6690 /* Fence driver */
6691 r = radeon_fence_driver_init(rdev);
6692 if (r)
6693 return r;
6694
6695 /* initialize memory controller */
6696 r = si_mc_init(rdev);
6697 if (r)
6698 return r;
6699 /* Memory manager */
6700 r = radeon_bo_init(rdev);
6701 if (r)
6702 return r;
6703
01ac8794
AD		 6704 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6705 !rdev->rlc_fw || !rdev->mc_fw) {
6706 r = si_init_microcode(rdev);
6707 if (r) {
6708 DRM_ERROR("Failed to load firmware!\n");
6709 return r;
6710 }
6711 }
6712
6c7bccea
AD		 6713 /* Initialize power management */
6714 radeon_pm_init(rdev);
6715
9b136d51
AD		 6716 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6717 ring->ring_obj = NULL;
6718 r600_ring_init(rdev, ring, 1024 * 1024);
6719
6720 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6721 ring->ring_obj = NULL;
6722 r600_ring_init(rdev, ring, 1024 * 1024);
6723
6724 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6725 ring->ring_obj = NULL;
6726 r600_ring_init(rdev, ring, 1024 * 1024);
6727
8c5fd7ef
AD		 6728 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6729 ring->ring_obj = NULL;
6730 r600_ring_init(rdev, ring, 64 * 1024);
6731
6732 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6733 ring->ring_obj = NULL;
6734 r600_ring_init(rdev, ring, 64 * 1024);
6735
1df0d523
AD		 6736 if (rdev->has_uvd) {
6737 r = radeon_uvd_init(rdev);
6738 if (!r) {
6739 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6740 ring->ring_obj = NULL;
6741 r600_ring_init(rdev, ring, 4096);
6742 }
f2ba57b5
CK		 6743 }
6744
9b136d51
AD		 6745 rdev->ih.ring_obj = NULL;
6746 r600_ih_ring_init(rdev, 64 * 1024);
6747
6748 r = r600_pcie_gart_init(rdev);
6749 if (r)
6750 return r;
6751
9b136d51 6752 rdev->accel_working = true;
9b136d51
AD		 6753 r = si_startup(rdev);
6754 if (r) {
6755 dev_err(rdev->dev, "disabling GPU acceleration\n");
6756 si_cp_fini(rdev);
8c5fd7ef 6757 cayman_dma_fini(rdev);
9b136d51 6758 si_irq_fini(rdev);
1fd11777 6759 sumo_rlc_fini(rdev);
9b136d51 6760 radeon_wb_fini(rdev);
2898c348 6761 radeon_ib_pool_fini(rdev);
9b136d51
AD		 6762 radeon_vm_manager_fini(rdev);
6763 radeon_irq_kms_fini(rdev);
6764 si_pcie_gart_fini(rdev);
6765 rdev->accel_working = false;
6766 }
6767
6768 /* Don't start up if the MC ucode is missing.
6769 * The default clocks and voltages before the MC ucode
6770 * is loaded are not suffient for advanced operations.
6771 */
6772 if (!rdev->mc_fw) {
6773 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6774 return -EINVAL;
6775 }
6776
6777 return 0;
6778}
6779
6780void si_fini(struct radeon_device *rdev)
6781{
6c7bccea 6782 radeon_pm_fini(rdev);
9b136d51 6783 si_cp_fini(rdev);
8c5fd7ef 6784 cayman_dma_fini(rdev);
f8f84ac5 6785 si_fini_pg(rdev);
e16866ec 6786 si_fini_cg(rdev);
9b136d51 6787 si_irq_fini(rdev);
1fd11777 6788 sumo_rlc_fini(rdev);
9b136d51
AD		 6789 radeon_wb_fini(rdev);
6790 radeon_vm_manager_fini(rdev);
2898c348 6791 radeon_ib_pool_fini(rdev);
9b136d51 6792 radeon_irq_kms_fini(rdev);
2858c00d 6793 if (rdev->has_uvd) {
e409b128 6794 uvd_v1_0_fini(rdev);
1df0d523 6795 radeon_uvd_fini(rdev);
2858c00d 6796 }
9b136d51
AD		 6797 si_pcie_gart_fini(rdev);
6798 r600_vram_scratch_fini(rdev);
6799 radeon_gem_fini(rdev);
9b136d51
AD		 6800 radeon_fence_driver_fini(rdev);
6801 radeon_bo_fini(rdev);
6802 radeon_atombios_fini(rdev);
6803 kfree(rdev->bios);
6804 rdev->bios = NULL;
6805}
6806
6759a0a7 6807/**
d0418894 6808 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 6809 *
6810 * @rdev: radeon_device pointer
6811 *
6812 * Fetches a GPU clock counter snapshot (SI).
6813 * Returns the 64 bit clock counter snapshot.
6814 */
d0418894 6815uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 6816{
6817 uint64_t clock;
6818
6819 mutex_lock(&rdev->gpu_clock_mutex);
6820 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6821 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6822 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6823 mutex_unlock(&rdev->gpu_clock_mutex);
6824 return clock;
6825}
2539eb02 6826
2539eb02
CK		 6827int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6828{
facd112d 6829 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 6830 int r;
6831
4ed10835
CK		 6832 /* bypass vclk and dclk with bclk */
6833 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6834 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6835 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6836
6837 /* put PLL in bypass mode */
6838 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6839
6840 if (!vclk || !dclk) {
6841 /* keep the Bypass mode, put PLL to sleep */
6842 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6843 return 0;
6844 }
6845
facd112d
CK		 6846 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6847 16384, 0x03FFFFFF, 0, 128, 5,
6848 &fb_div, &vclk_div, &dclk_div);
6849 if (r)
6850 return r;
2539eb02
CK		 6851
6852 /* set RESET_ANTI_MUX to 0 */
6853 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6854
6855 /* set VCO_MODE to 1 */
6856 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6857
6858 /* toggle UPLL_SLEEP to 1 then back to 0 */
6859 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6860 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6861
6862 /* deassert UPLL_RESET */
6863 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6864
6865 mdelay(1);
6866
facd112d 6867 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6868 if (r)
6869 return r;
6870
6871 /* assert UPLL_RESET again */
6872 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6873
6874 /* disable spread spectrum. */
6875 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6876
6877 /* set feedback divider */
facd112d 6878 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 6879
6880 /* set ref divider to 0 */
6881 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6882
facd112d 6883 if (fb_div < 307200)
2539eb02
CK		 6884 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6885 else
6886 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6887
6888 /* set PDIV_A and PDIV_B */
6889 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 6890 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 6891 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6892
6893 /* give the PLL some time to settle */
6894 mdelay(15);
6895
6896 /* deassert PLL_RESET */
6897 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6898
6899 mdelay(15);
6900
6901 /* switch from bypass mode to normal mode */
6902 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6903
facd112d 6904 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6905 if (r)
6906 return r;
6907
6908 /* switch VCLK and DCLK selection */
6909 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6910 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6911 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6912
6913 mdelay(100);
6914
6915 return 0;
6916}
b9d305df
AD		 6917
6918static void si_pcie_gen3_enable(struct radeon_device *rdev)
6919{
6920 struct pci_dev *root = rdev->pdev->bus->self;
6921 int bridge_pos, gpu_pos;
6922 u32 speed_cntl, mask, current_data_rate;
6923 int ret, i;
6924 u16 tmp16;
6925
6926 if (radeon_pcie_gen2 == 0)
6927 return;
6928
6929 if (rdev->flags & RADEON_IS_IGP)
6930 return;
6931
6932 if (!(rdev->flags & RADEON_IS_PCIE))
6933 return;
6934
6935 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6936 if (ret != 0)
6937 return;
6938
6939 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6940 return;
6941
6942 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6943 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6944 LC_CURRENT_DATA_RATE_SHIFT;
6945 if (mask & DRM_PCIE_SPEED_80) {
6946 if (current_data_rate == 2) {
6947 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6948 return;
6949 }
6950 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6951 } else if (mask & DRM_PCIE_SPEED_50) {
6952 if (current_data_rate == 1) {
6953 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6954 return;
6955 }
6956 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6957 }
6958
6959 bridge_pos = pci_pcie_cap(root);
6960 if (!bridge_pos)
6961 return;
6962
6963 gpu_pos = pci_pcie_cap(rdev->pdev);
6964 if (!gpu_pos)
6965 return;
6966
6967 if (mask & DRM_PCIE_SPEED_80) {
6968 /* re-try equalization if gen3 is not already enabled */
6969 if (current_data_rate != 2) {
6970 u16 bridge_cfg, gpu_cfg;
6971 u16 bridge_cfg2, gpu_cfg2;
6972 u32 max_lw, current_lw, tmp;
6973
6974 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6975 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6976
6977 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6978 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6979
6980 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6981 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6982
6983 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6984 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6985 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6986
6987 if (current_lw < max_lw) {
6988 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6989 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6990 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6991 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6992 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6993 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6994 }
6995 }
6996
6997 for (i = 0; i < 10; i++) {
6998 /* check status */
6999 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
7000 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7001 break;
7002
7003 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7004 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7005
7006 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7007 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7008
7009 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7010 tmp |= LC_SET_QUIESCE;
7011 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7012
7013 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7014 tmp |= LC_REDO_EQ;
7015 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7016
7017 mdelay(100);
7018
7019 /* linkctl */
7020 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7021 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7022 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7023 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7024
7025 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7026 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7027 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7028 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7029
7030 /* linkctl2 */
7031 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7032 tmp16 &= ~((1 << 4) | (7 << 9));
7033 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7034 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7035
7036 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7037 tmp16 &= ~((1 << 4) | (7 << 9));
7038 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7039 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7040
7041 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7042 tmp &= ~LC_SET_QUIESCE;
7043 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7044 }
7045 }
7046 }
7047
7048 /* set the link speed */
7049 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7050 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7051 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7052
7053 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7054 tmp16 &= ~0xf;
7055 if (mask & DRM_PCIE_SPEED_80)
7056 tmp16 |= 3; /* gen3 */
7057 else if (mask & DRM_PCIE_SPEED_50)
7058 tmp16 |= 2; /* gen2 */
7059 else
7060 tmp16 |= 1; /* gen1 */
7061 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7062
7063 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7064 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7065 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7066
7067 for (i = 0; i < rdev->usec_timeout; i++) {
7068 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7069 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7070 break;
7071 udelay(1);
7072 }
7073}
7074
e0bcf165
AD		 7075static void si_program_aspm(struct radeon_device *rdev)
7076{
7077 u32 data, orig;
7078 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7079 bool disable_clkreq = false;
7080
1294d4a3
AD		 7081 if (radeon_aspm == 0)
7082 return;
7083
e0bcf165
AD		 7084 if (!(rdev->flags & RADEON_IS_PCIE))
7085 return;
7086
7087 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7088 data &= ~LC_XMIT_N_FTS_MASK;
7089 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7090 if (orig != data)
7091 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7092
7093 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7094 data |= LC_GO_TO_RECOVERY;
7095 if (orig != data)
7096 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7097
7098 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7099 data |= P_IGNORE_EDB_ERR;
7100 if (orig != data)
7101 WREG32_PCIE(PCIE_P_CNTL, data);
7102
7103 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7104 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7105 data |= LC_PMI_TO_L1_DIS;
7106 if (!disable_l0s)
7107 data |= LC_L0S_INACTIVITY(7);
7108
7109 if (!disable_l1) {
7110 data |= LC_L1_INACTIVITY(7);
7111 data &= ~LC_PMI_TO_L1_DIS;
7112 if (orig != data)
7113 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7114
7115 if (!disable_plloff_in_l1) {
7116 bool clk_req_support;
7117
7118 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7119 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7120 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7121 if (orig != data)
7122 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7123
7124 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7125 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7126 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7127 if (orig != data)
7128 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7129
7130 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7131 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7132 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7133 if (orig != data)
7134 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7135
7136 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7137 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7138 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7139 if (orig != data)
7140 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7141
7142 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7143 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7144 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7145 if (orig != data)
7146 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7147
7148 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7149 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7150 if (orig != data)
7151 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7152
7153 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7154 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7155 if (orig != data)
7156 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7157
7158 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7159 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7160 if (orig != data)
7161 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7162
7163 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7164 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7165 if (orig != data)
7166 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7167
7168 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7169 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7170 if (orig != data)
7171 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7172
7173 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7174 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7175 if (orig != data)
7176 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7177
7178 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7179 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7180 if (orig != data)
7181 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7182 }
7183 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7184 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7185 data |= LC_DYN_LANES_PWR_STATE(3);
7186 if (orig != data)
7187 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7188
7189 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7190 data &= ~LS2_EXIT_TIME_MASK;
7191 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7192 data |= LS2_EXIT_TIME(5);
7193 if (orig != data)
7194 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7195
7196 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7197 data &= ~LS2_EXIT_TIME_MASK;
7198 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7199 data |= LS2_EXIT_TIME(5);
7200 if (orig != data)
7201 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7202
7203 if (!disable_clkreq) {
7204 struct pci_dev *root = rdev->pdev->bus->self;
7205 u32 lnkcap;
7206
7207 clk_req_support = false;
7208 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7209 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7210 clk_req_support = true;
7211 } else {
7212 clk_req_support = false;
7213 }
7214
7215 if (clk_req_support) {
7216 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7217 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7218 if (orig != data)
7219 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7220
7221 orig = data = RREG32(THM_CLK_CNTL);
7222 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7223 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7224 if (orig != data)
7225 WREG32(THM_CLK_CNTL, data);
7226
7227 orig = data = RREG32(MISC_CLK_CNTL);
7228 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7229 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7230 if (orig != data)
7231 WREG32(MISC_CLK_CNTL, data);
7232
7233 orig = data = RREG32(CG_CLKPIN_CNTL);
7234 data &= ~BCLK_AS_XCLK;
7235 if (orig != data)
7236 WREG32(CG_CLKPIN_CNTL, data);
7237
7238 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7239 data &= ~FORCE_BIF_REFCLK_EN;
7240 if (orig != data)
7241 WREG32(CG_CLKPIN_CNTL_2, data);
7242
7243 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7244 data &= ~MPLL_CLKOUT_SEL_MASK;
7245 data |= MPLL_CLKOUT_SEL(4);
7246 if (orig != data)
7247 WREG32(MPLL_BYPASSCLK_SEL, data);
7248
7249 orig = data = RREG32(SPLL_CNTL_MODE);
7250 data &= ~SPLL_REFCLK_SEL_MASK;
7251 if (orig != data)
7252 WREG32(SPLL_CNTL_MODE, data);
7253 }
7254 }
7255 } else {
7256 if (orig != data)
7257 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7258 }
7259
7260 orig = data = RREG32_PCIE(PCIE_CNTL2);
7261 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7262 if (orig != data)
7263 WREG32_PCIE(PCIE_CNTL2, data);
7264
7265 if (!disable_l0s) {
7266 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7267 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7268 data = RREG32_PCIE(PCIE_LC_STATUS1);
7269 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7270 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7271 data &= ~LC_L0S_INACTIVITY_MASK;
7272 if (orig != data)
7273 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7274 }
7275 }
7276 }
7277}
Linux 6.x block layer and io_uring tree(s)