projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: enable mgcg on CIK
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
0f0de06c 24#include <linux/firmware.h>
0f0de06c
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
760285e7 27#include <drm/drmP.h>
43b3cd99
AD		 28#include "radeon.h"
29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
43b3cd99
AD		 31#include "sid.h"
32#include "atom.h"
48c0c902 33#include "si_blit_shaders.h"
bd8cd539 34#include "clearstate_si.h"
a0ceada6 35#include "radeon_ucode.h"
43b3cd99 36
0f0de06c
AD		 37
38MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
42MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
a9e61410 43MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
0f0de06c
AD		 44MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
45MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
46MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
47MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
48MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
a9e61410 49MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
0f0de06c
AD		 50MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
51MODULE_FIRMWARE("radeon/VERDE_me.bin");
52MODULE_FIRMWARE("radeon/VERDE_ce.bin");
53MODULE_FIRMWARE("radeon/VERDE_mc.bin");
54MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
a9e61410 55MODULE_FIRMWARE("radeon/VERDE_smc.bin");
bcc7f5d2
AD		 56MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
57MODULE_FIRMWARE("radeon/OLAND_me.bin");
58MODULE_FIRMWARE("radeon/OLAND_ce.bin");
59MODULE_FIRMWARE("radeon/OLAND_mc.bin");
60MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
a9e61410 61MODULE_FIRMWARE("radeon/OLAND_smc.bin");
c04c00b4
AD		 62MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
63MODULE_FIRMWARE("radeon/HAINAN_me.bin");
64MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
65MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
66MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
a9e61410 67MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
0f0de06c 68
b9d305df 69static void si_pcie_gen3_enable(struct radeon_device *rdev);
e0bcf165 70static void si_program_aspm(struct radeon_device *rdev);
1fd11777
AD		 71extern void sumo_rlc_fini(struct radeon_device *rdev);
72extern int sumo_rlc_init(struct radeon_device *rdev);
25a857fb
AD		 73extern int r600_ih_ring_alloc(struct radeon_device *rdev);
74extern void r600_ih_ring_fini(struct radeon_device *rdev);
0a96d72b 75extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
c476dde2
AD		 76extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
77extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
ca7db22b 78extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
1c534671 79extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
014bb209 80extern bool evergreen_is_display_hung(struct radeon_device *rdev);
2483b4ea
CK		 81extern void si_dma_vm_set_page(struct radeon_device *rdev,
82 struct radeon_ib *ib,
83 uint64_t pe,
84 uint64_t addr, unsigned count,
85 uint32_t incr, uint32_t flags);
0a96d72b 86
6d8cf000
AD		 87static const u32 verde_rlc_save_restore_register_list[] =
88{
89 (0x8000 << 16) | (0x98f4 >> 2),
90 0x00000000,
91 (0x8040 << 16) | (0x98f4 >> 2),
92 0x00000000,
93 (0x8000 << 16) | (0xe80 >> 2),
94 0x00000000,
95 (0x8040 << 16) | (0xe80 >> 2),
96 0x00000000,
97 (0x8000 << 16) | (0x89bc >> 2),
98 0x00000000,
99 (0x8040 << 16) | (0x89bc >> 2),
100 0x00000000,
101 (0x8000 << 16) | (0x8c1c >> 2),
102 0x00000000,
103 (0x8040 << 16) | (0x8c1c >> 2),
104 0x00000000,
105 (0x9c00 << 16) | (0x98f0 >> 2),
106 0x00000000,
107 (0x9c00 << 16) | (0xe7c >> 2),
108 0x00000000,
109 (0x8000 << 16) | (0x9148 >> 2),
110 0x00000000,
111 (0x8040 << 16) | (0x9148 >> 2),
112 0x00000000,
113 (0x9c00 << 16) | (0x9150 >> 2),
114 0x00000000,
115 (0x9c00 << 16) | (0x897c >> 2),
116 0x00000000,
117 (0x9c00 << 16) | (0x8d8c >> 2),
118 0x00000000,
119 (0x9c00 << 16) | (0xac54 >> 2),
120 0X00000000,
121 0x3,
122 (0x9c00 << 16) | (0x98f8 >> 2),
123 0x00000000,
124 (0x9c00 << 16) | (0x9910 >> 2),
125 0x00000000,
126 (0x9c00 << 16) | (0x9914 >> 2),
127 0x00000000,
128 (0x9c00 << 16) | (0x9918 >> 2),
129 0x00000000,
130 (0x9c00 << 16) | (0x991c >> 2),
131 0x00000000,
132 (0x9c00 << 16) | (0x9920 >> 2),
133 0x00000000,
134 (0x9c00 << 16) | (0x9924 >> 2),
135 0x00000000,
136 (0x9c00 << 16) | (0x9928 >> 2),
137 0x00000000,
138 (0x9c00 << 16) | (0x992c >> 2),
139 0x00000000,
140 (0x9c00 << 16) | (0x9930 >> 2),
141 0x00000000,
142 (0x9c00 << 16) | (0x9934 >> 2),
143 0x00000000,
144 (0x9c00 << 16) | (0x9938 >> 2),
145 0x00000000,
146 (0x9c00 << 16) | (0x993c >> 2),
147 0x00000000,
148 (0x9c00 << 16) | (0x9940 >> 2),
149 0x00000000,
150 (0x9c00 << 16) | (0x9944 >> 2),
151 0x00000000,
152 (0x9c00 << 16) | (0x9948 >> 2),
153 0x00000000,
154 (0x9c00 << 16) | (0x994c >> 2),
155 0x00000000,
156 (0x9c00 << 16) | (0x9950 >> 2),
157 0x00000000,
158 (0x9c00 << 16) | (0x9954 >> 2),
159 0x00000000,
160 (0x9c00 << 16) | (0x9958 >> 2),
161 0x00000000,
162 (0x9c00 << 16) | (0x995c >> 2),
163 0x00000000,
164 (0x9c00 << 16) | (0x9960 >> 2),
165 0x00000000,
166 (0x9c00 << 16) | (0x9964 >> 2),
167 0x00000000,
168 (0x9c00 << 16) | (0x9968 >> 2),
169 0x00000000,
170 (0x9c00 << 16) | (0x996c >> 2),
171 0x00000000,
172 (0x9c00 << 16) | (0x9970 >> 2),
173 0x00000000,
174 (0x9c00 << 16) | (0x9974 >> 2),
175 0x00000000,
176 (0x9c00 << 16) | (0x9978 >> 2),
177 0x00000000,
178 (0x9c00 << 16) | (0x997c >> 2),
179 0x00000000,
180 (0x9c00 << 16) | (0x9980 >> 2),
181 0x00000000,
182 (0x9c00 << 16) | (0x9984 >> 2),
183 0x00000000,
184 (0x9c00 << 16) | (0x9988 >> 2),
185 0x00000000,
186 (0x9c00 << 16) | (0x998c >> 2),
187 0x00000000,
188 (0x9c00 << 16) | (0x8c00 >> 2),
189 0x00000000,
190 (0x9c00 << 16) | (0x8c14 >> 2),
191 0x00000000,
192 (0x9c00 << 16) | (0x8c04 >> 2),
193 0x00000000,
194 (0x9c00 << 16) | (0x8c08 >> 2),
195 0x00000000,
196 (0x8000 << 16) | (0x9b7c >> 2),
197 0x00000000,
198 (0x8040 << 16) | (0x9b7c >> 2),
199 0x00000000,
200 (0x8000 << 16) | (0xe84 >> 2),
201 0x00000000,
202 (0x8040 << 16) | (0xe84 >> 2),
203 0x00000000,
204 (0x8000 << 16) | (0x89c0 >> 2),
205 0x00000000,
206 (0x8040 << 16) | (0x89c0 >> 2),
207 0x00000000,
208 (0x8000 << 16) | (0x914c >> 2),
209 0x00000000,
210 (0x8040 << 16) | (0x914c >> 2),
211 0x00000000,
212 (0x8000 << 16) | (0x8c20 >> 2),
213 0x00000000,
214 (0x8040 << 16) | (0x8c20 >> 2),
215 0x00000000,
216 (0x8000 << 16) | (0x9354 >> 2),
217 0x00000000,
218 (0x8040 << 16) | (0x9354 >> 2),
219 0x00000000,
220 (0x9c00 << 16) | (0x9060 >> 2),
221 0x00000000,
222 (0x9c00 << 16) | (0x9364 >> 2),
223 0x00000000,
224 (0x9c00 << 16) | (0x9100 >> 2),
225 0x00000000,
226 (0x9c00 << 16) | (0x913c >> 2),
227 0x00000000,
228 (0x8000 << 16) | (0x90e0 >> 2),
229 0x00000000,
230 (0x8000 << 16) | (0x90e4 >> 2),
231 0x00000000,
232 (0x8000 << 16) | (0x90e8 >> 2),
233 0x00000000,
234 (0x8040 << 16) | (0x90e0 >> 2),
235 0x00000000,
236 (0x8040 << 16) | (0x90e4 >> 2),
237 0x00000000,
238 (0x8040 << 16) | (0x90e8 >> 2),
239 0x00000000,
240 (0x9c00 << 16) | (0x8bcc >> 2),
241 0x00000000,
242 (0x9c00 << 16) | (0x8b24 >> 2),
243 0x00000000,
244 (0x9c00 << 16) | (0x88c4 >> 2),
245 0x00000000,
246 (0x9c00 << 16) | (0x8e50 >> 2),
247 0x00000000,
248 (0x9c00 << 16) | (0x8c0c >> 2),
249 0x00000000,
250 (0x9c00 << 16) | (0x8e58 >> 2),
251 0x00000000,
252 (0x9c00 << 16) | (0x8e5c >> 2),
253 0x00000000,
254 (0x9c00 << 16) | (0x9508 >> 2),
255 0x00000000,
256 (0x9c00 << 16) | (0x950c >> 2),
257 0x00000000,
258 (0x9c00 << 16) | (0x9494 >> 2),
259 0x00000000,
260 (0x9c00 << 16) | (0xac0c >> 2),
261 0x00000000,
262 (0x9c00 << 16) | (0xac10 >> 2),
263 0x00000000,
264 (0x9c00 << 16) | (0xac14 >> 2),
265 0x00000000,
266 (0x9c00 << 16) | (0xae00 >> 2),
267 0x00000000,
268 (0x9c00 << 16) | (0xac08 >> 2),
269 0x00000000,
270 (0x9c00 << 16) | (0x88d4 >> 2),
271 0x00000000,
272 (0x9c00 << 16) | (0x88c8 >> 2),
273 0x00000000,
274 (0x9c00 << 16) | (0x88cc >> 2),
275 0x00000000,
276 (0x9c00 << 16) | (0x89b0 >> 2),
277 0x00000000,
278 (0x9c00 << 16) | (0x8b10 >> 2),
279 0x00000000,
280 (0x9c00 << 16) | (0x8a14 >> 2),
281 0x00000000,
282 (0x9c00 << 16) | (0x9830 >> 2),
283 0x00000000,
284 (0x9c00 << 16) | (0x9834 >> 2),
285 0x00000000,
286 (0x9c00 << 16) | (0x9838 >> 2),
287 0x00000000,
288 (0x9c00 << 16) | (0x9a10 >> 2),
289 0x00000000,
290 (0x8000 << 16) | (0x9870 >> 2),
291 0x00000000,
292 (0x8000 << 16) | (0x9874 >> 2),
293 0x00000000,
294 (0x8001 << 16) | (0x9870 >> 2),
295 0x00000000,
296 (0x8001 << 16) | (0x9874 >> 2),
297 0x00000000,
298 (0x8040 << 16) | (0x9870 >> 2),
299 0x00000000,
300 (0x8040 << 16) | (0x9874 >> 2),
301 0x00000000,
302 (0x8041 << 16) | (0x9870 >> 2),
303 0x00000000,
304 (0x8041 << 16) | (0x9874 >> 2),
305 0x00000000,
306 0x00000000
307};
308
205996c0
AD		 309static const u32 tahiti_golden_rlc_registers[] =
310{
311 0xc424, 0xffffffff, 0x00601005,
312 0xc47c, 0xffffffff, 0x10104040,
313 0xc488, 0xffffffff, 0x0100000a,
314 0xc314, 0xffffffff, 0x00000800,
315 0xc30c, 0xffffffff, 0x800000f4,
316 0xf4a8, 0xffffffff, 0x00000000
317};
318
319static const u32 tahiti_golden_registers[] =
320{
321 0x9a10, 0x00010000, 0x00018208,
322 0x9830, 0xffffffff, 0x00000000,
323 0x9834, 0xf00fffff, 0x00000400,
324 0x9838, 0x0002021c, 0x00020200,
325 0xc78, 0x00000080, 0x00000000,
326 0xd030, 0x000300c0, 0x00800040,
327 0xd830, 0x000300c0, 0x00800040,
328 0x5bb0, 0x000000f0, 0x00000070,
329 0x5bc0, 0x00200000, 0x50100000,
330 0x7030, 0x31000311, 0x00000011,
331 0x277c, 0x00000003, 0x000007ff,
332 0x240c, 0x000007ff, 0x00000000,
333 0x8a14, 0xf000001f, 0x00000007,
334 0x8b24, 0xffffffff, 0x00ffffff,
335 0x8b10, 0x0000ff0f, 0x00000000,
336 0x28a4c, 0x07ffffff, 0x4e000000,
337 0x28350, 0x3f3f3fff, 0x2a00126a,
338 0x30, 0x000000ff, 0x0040,
339 0x34, 0x00000040, 0x00004040,
340 0x9100, 0x07ffffff, 0x03000000,
341 0x8e88, 0x01ff1f3f, 0x00000000,
342 0x8e84, 0x01ff1f3f, 0x00000000,
343 0x9060, 0x0000007f, 0x00000020,
344 0x9508, 0x00010000, 0x00010000,
345 0xac14, 0x00000200, 0x000002fb,
346 0xac10, 0xffffffff, 0x0000543b,
347 0xac0c, 0xffffffff, 0xa9210876,
348 0x88d0, 0xffffffff, 0x000fff40,
349 0x88d4, 0x0000001f, 0x00000010,
350 0x1410, 0x20000000, 0x20fffed8,
351 0x15c0, 0x000c0fc0, 0x000c0400
352};
353
354static const u32 tahiti_golden_registers2[] =
355{
356 0xc64, 0x00000001, 0x00000001
357};
358
359static const u32 pitcairn_golden_rlc_registers[] =
360{
361 0xc424, 0xffffffff, 0x00601004,
362 0xc47c, 0xffffffff, 0x10102020,
363 0xc488, 0xffffffff, 0x01000020,
364 0xc314, 0xffffffff, 0x00000800,
365 0xc30c, 0xffffffff, 0x800000a4
366};
367
368static const u32 pitcairn_golden_registers[] =
369{
370 0x9a10, 0x00010000, 0x00018208,
371 0x9830, 0xffffffff, 0x00000000,
372 0x9834, 0xf00fffff, 0x00000400,
373 0x9838, 0x0002021c, 0x00020200,
374 0xc78, 0x00000080, 0x00000000,
375 0xd030, 0x000300c0, 0x00800040,
376 0xd830, 0x000300c0, 0x00800040,
377 0x5bb0, 0x000000f0, 0x00000070,
378 0x5bc0, 0x00200000, 0x50100000,
379 0x7030, 0x31000311, 0x00000011,
380 0x2ae4, 0x00073ffe, 0x000022a2,
381 0x240c, 0x000007ff, 0x00000000,
382 0x8a14, 0xf000001f, 0x00000007,
383 0x8b24, 0xffffffff, 0x00ffffff,
384 0x8b10, 0x0000ff0f, 0x00000000,
385 0x28a4c, 0x07ffffff, 0x4e000000,
386 0x28350, 0x3f3f3fff, 0x2a00126a,
387 0x30, 0x000000ff, 0x0040,
388 0x34, 0x00000040, 0x00004040,
389 0x9100, 0x07ffffff, 0x03000000,
390 0x9060, 0x0000007f, 0x00000020,
391 0x9508, 0x00010000, 0x00010000,
392 0xac14, 0x000003ff, 0x000000f7,
393 0xac10, 0xffffffff, 0x00000000,
394 0xac0c, 0xffffffff, 0x32761054,
395 0x88d4, 0x0000001f, 0x00000010,
396 0x15c0, 0x000c0fc0, 0x000c0400
397};
398
399static const u32 verde_golden_rlc_registers[] =
400{
401 0xc424, 0xffffffff, 0x033f1005,
402 0xc47c, 0xffffffff, 0x10808020,
403 0xc488, 0xffffffff, 0x00800008,
404 0xc314, 0xffffffff, 0x00001000,
405 0xc30c, 0xffffffff, 0x80010014
406};
407
408static const u32 verde_golden_registers[] =
409{
410 0x9a10, 0x00010000, 0x00018208,
411 0x9830, 0xffffffff, 0x00000000,
412 0x9834, 0xf00fffff, 0x00000400,
413 0x9838, 0x0002021c, 0x00020200,
414 0xc78, 0x00000080, 0x00000000,
415 0xd030, 0x000300c0, 0x00800040,
416 0xd030, 0x000300c0, 0x00800040,
417 0xd830, 0x000300c0, 0x00800040,
418 0xd830, 0x000300c0, 0x00800040,
419 0x5bb0, 0x000000f0, 0x00000070,
420 0x5bc0, 0x00200000, 0x50100000,
421 0x7030, 0x31000311, 0x00000011,
422 0x2ae4, 0x00073ffe, 0x000022a2,
423 0x2ae4, 0x00073ffe, 0x000022a2,
424 0x2ae4, 0x00073ffe, 0x000022a2,
425 0x240c, 0x000007ff, 0x00000000,
426 0x240c, 0x000007ff, 0x00000000,
427 0x240c, 0x000007ff, 0x00000000,
428 0x8a14, 0xf000001f, 0x00000007,
429 0x8a14, 0xf000001f, 0x00000007,
430 0x8a14, 0xf000001f, 0x00000007,
431 0x8b24, 0xffffffff, 0x00ffffff,
432 0x8b10, 0x0000ff0f, 0x00000000,
433 0x28a4c, 0x07ffffff, 0x4e000000,
434 0x28350, 0x3f3f3fff, 0x0000124a,
435 0x28350, 0x3f3f3fff, 0x0000124a,
436 0x28350, 0x3f3f3fff, 0x0000124a,
437 0x30, 0x000000ff, 0x0040,
438 0x34, 0x00000040, 0x00004040,
439 0x9100, 0x07ffffff, 0x03000000,
440 0x9100, 0x07ffffff, 0x03000000,
441 0x8e88, 0x01ff1f3f, 0x00000000,
442 0x8e88, 0x01ff1f3f, 0x00000000,
443 0x8e88, 0x01ff1f3f, 0x00000000,
444 0x8e84, 0x01ff1f3f, 0x00000000,
445 0x8e84, 0x01ff1f3f, 0x00000000,
446 0x8e84, 0x01ff1f3f, 0x00000000,
447 0x9060, 0x0000007f, 0x00000020,
448 0x9508, 0x00010000, 0x00010000,
449 0xac14, 0x000003ff, 0x00000003,
450 0xac14, 0x000003ff, 0x00000003,
451 0xac14, 0x000003ff, 0x00000003,
452 0xac10, 0xffffffff, 0x00000000,
453 0xac10, 0xffffffff, 0x00000000,
454 0xac10, 0xffffffff, 0x00000000,
455 0xac0c, 0xffffffff, 0x00001032,
456 0xac0c, 0xffffffff, 0x00001032,
457 0xac0c, 0xffffffff, 0x00001032,
458 0x88d4, 0x0000001f, 0x00000010,
459 0x88d4, 0x0000001f, 0x00000010,
460 0x88d4, 0x0000001f, 0x00000010,
461 0x15c0, 0x000c0fc0, 0x000c0400
462};
463
464static const u32 oland_golden_rlc_registers[] =
465{
466 0xc424, 0xffffffff, 0x00601005,
467 0xc47c, 0xffffffff, 0x10104040,
468 0xc488, 0xffffffff, 0x0100000a,
469 0xc314, 0xffffffff, 0x00000800,
470 0xc30c, 0xffffffff, 0x800000f4
471};
472
473static const u32 oland_golden_registers[] =
474{
475 0x9a10, 0x00010000, 0x00018208,
476 0x9830, 0xffffffff, 0x00000000,
477 0x9834, 0xf00fffff, 0x00000400,
478 0x9838, 0x0002021c, 0x00020200,
479 0xc78, 0x00000080, 0x00000000,
480 0xd030, 0x000300c0, 0x00800040,
481 0xd830, 0x000300c0, 0x00800040,
482 0x5bb0, 0x000000f0, 0x00000070,
483 0x5bc0, 0x00200000, 0x50100000,
484 0x7030, 0x31000311, 0x00000011,
485 0x2ae4, 0x00073ffe, 0x000022a2,
486 0x240c, 0x000007ff, 0x00000000,
487 0x8a14, 0xf000001f, 0x00000007,
488 0x8b24, 0xffffffff, 0x00ffffff,
489 0x8b10, 0x0000ff0f, 0x00000000,
490 0x28a4c, 0x07ffffff, 0x4e000000,
491 0x28350, 0x3f3f3fff, 0x00000082,
492 0x30, 0x000000ff, 0x0040,
493 0x34, 0x00000040, 0x00004040,
494 0x9100, 0x07ffffff, 0x03000000,
495 0x9060, 0x0000007f, 0x00000020,
496 0x9508, 0x00010000, 0x00010000,
497 0xac14, 0x000003ff, 0x000000f3,
498 0xac10, 0xffffffff, 0x00000000,
499 0xac0c, 0xffffffff, 0x00003210,
500 0x88d4, 0x0000001f, 0x00000010,
501 0x15c0, 0x000c0fc0, 0x000c0400
502};
503
fffbdda4
AD		 504static const u32 hainan_golden_registers[] =
505{
506 0x9a10, 0x00010000, 0x00018208,
507 0x9830, 0xffffffff, 0x00000000,
508 0x9834, 0xf00fffff, 0x00000400,
509 0x9838, 0x0002021c, 0x00020200,
510 0xd0c0, 0xff000fff, 0x00000100,
511 0xd030, 0x000300c0, 0x00800040,
512 0xd8c0, 0xff000fff, 0x00000100,
513 0xd830, 0x000300c0, 0x00800040,
514 0x2ae4, 0x00073ffe, 0x000022a2,
515 0x240c, 0x000007ff, 0x00000000,
516 0x8a14, 0xf000001f, 0x00000007,
517 0x8b24, 0xffffffff, 0x00ffffff,
518 0x8b10, 0x0000ff0f, 0x00000000,
519 0x28a4c, 0x07ffffff, 0x4e000000,
520 0x28350, 0x3f3f3fff, 0x00000000,
521 0x30, 0x000000ff, 0x0040,
522 0x34, 0x00000040, 0x00004040,
523 0x9100, 0x03e00000, 0x03600000,
524 0x9060, 0x0000007f, 0x00000020,
525 0x9508, 0x00010000, 0x00010000,
526 0xac14, 0x000003ff, 0x000000f1,
527 0xac10, 0xffffffff, 0x00000000,
528 0xac0c, 0xffffffff, 0x00003210,
529 0x88d4, 0x0000001f, 0x00000010,
530 0x15c0, 0x000c0fc0, 0x000c0400
531};
532
533static const u32 hainan_golden_registers2[] =
534{
535 0x98f8, 0xffffffff, 0x02010001
536};
537
205996c0
AD		 538static const u32 tahiti_mgcg_cgcg_init[] =
539{
540 0xc400, 0xffffffff, 0xfffffffc,
541 0x802c, 0xffffffff, 0xe0000000,
542 0x9a60, 0xffffffff, 0x00000100,
543 0x92a4, 0xffffffff, 0x00000100,
544 0xc164, 0xffffffff, 0x00000100,
545 0x9774, 0xffffffff, 0x00000100,
546 0x8984, 0xffffffff, 0x06000100,
547 0x8a18, 0xffffffff, 0x00000100,
548 0x92a0, 0xffffffff, 0x00000100,
549 0xc380, 0xffffffff, 0x00000100,
550 0x8b28, 0xffffffff, 0x00000100,
551 0x9144, 0xffffffff, 0x00000100,
552 0x8d88, 0xffffffff, 0x00000100,
553 0x8d8c, 0xffffffff, 0x00000100,
554 0x9030, 0xffffffff, 0x00000100,
555 0x9034, 0xffffffff, 0x00000100,
556 0x9038, 0xffffffff, 0x00000100,
557 0x903c, 0xffffffff, 0x00000100,
558 0xad80, 0xffffffff, 0x00000100,
559 0xac54, 0xffffffff, 0x00000100,
560 0x897c, 0xffffffff, 0x06000100,
561 0x9868, 0xffffffff, 0x00000100,
562 0x9510, 0xffffffff, 0x00000100,
563 0xaf04, 0xffffffff, 0x00000100,
564 0xae04, 0xffffffff, 0x00000100,
565 0x949c, 0xffffffff, 0x00000100,
566 0x802c, 0xffffffff, 0xe0000000,
567 0x9160, 0xffffffff, 0x00010000,
568 0x9164, 0xffffffff, 0x00030002,
569 0x9168, 0xffffffff, 0x00040007,
570 0x916c, 0xffffffff, 0x00060005,
571 0x9170, 0xffffffff, 0x00090008,
572 0x9174, 0xffffffff, 0x00020001,
573 0x9178, 0xffffffff, 0x00040003,
574 0x917c, 0xffffffff, 0x00000007,
575 0x9180, 0xffffffff, 0x00060005,
576 0x9184, 0xffffffff, 0x00090008,
577 0x9188, 0xffffffff, 0x00030002,
578 0x918c, 0xffffffff, 0x00050004,
579 0x9190, 0xffffffff, 0x00000008,
580 0x9194, 0xffffffff, 0x00070006,
581 0x9198, 0xffffffff, 0x000a0009,
582 0x919c, 0xffffffff, 0x00040003,
583 0x91a0, 0xffffffff, 0x00060005,
584 0x91a4, 0xffffffff, 0x00000009,
585 0x91a8, 0xffffffff, 0x00080007,
586 0x91ac, 0xffffffff, 0x000b000a,
587 0x91b0, 0xffffffff, 0x00050004,
588 0x91b4, 0xffffffff, 0x00070006,
589 0x91b8, 0xffffffff, 0x0008000b,
590 0x91bc, 0xffffffff, 0x000a0009,
591 0x91c0, 0xffffffff, 0x000d000c,
592 0x91c4, 0xffffffff, 0x00060005,
593 0x91c8, 0xffffffff, 0x00080007,
594 0x91cc, 0xffffffff, 0x0000000b,
595 0x91d0, 0xffffffff, 0x000a0009,
596 0x91d4, 0xffffffff, 0x000d000c,
597 0x91d8, 0xffffffff, 0x00070006,
598 0x91dc, 0xffffffff, 0x00090008,
599 0x91e0, 0xffffffff, 0x0000000c,
600 0x91e4, 0xffffffff, 0x000b000a,
601 0x91e8, 0xffffffff, 0x000e000d,
602 0x91ec, 0xffffffff, 0x00080007,
603 0x91f0, 0xffffffff, 0x000a0009,
604 0x91f4, 0xffffffff, 0x0000000d,
605 0x91f8, 0xffffffff, 0x000c000b,
606 0x91fc, 0xffffffff, 0x000f000e,
607 0x9200, 0xffffffff, 0x00090008,
608 0x9204, 0xffffffff, 0x000b000a,
609 0x9208, 0xffffffff, 0x000c000f,
610 0x920c, 0xffffffff, 0x000e000d,
611 0x9210, 0xffffffff, 0x00110010,
612 0x9214, 0xffffffff, 0x000a0009,
613 0x9218, 0xffffffff, 0x000c000b,
614 0x921c, 0xffffffff, 0x0000000f,
615 0x9220, 0xffffffff, 0x000e000d,
616 0x9224, 0xffffffff, 0x00110010,
617 0x9228, 0xffffffff, 0x000b000a,
618 0x922c, 0xffffffff, 0x000d000c,
619 0x9230, 0xffffffff, 0x00000010,
620 0x9234, 0xffffffff, 0x000f000e,
621 0x9238, 0xffffffff, 0x00120011,
622 0x923c, 0xffffffff, 0x000c000b,
623 0x9240, 0xffffffff, 0x000e000d,
624 0x9244, 0xffffffff, 0x00000011,
625 0x9248, 0xffffffff, 0x0010000f,
626 0x924c, 0xffffffff, 0x00130012,
627 0x9250, 0xffffffff, 0x000d000c,
628 0x9254, 0xffffffff, 0x000f000e,
629 0x9258, 0xffffffff, 0x00100013,
630 0x925c, 0xffffffff, 0x00120011,
631 0x9260, 0xffffffff, 0x00150014,
632 0x9264, 0xffffffff, 0x000e000d,
633 0x9268, 0xffffffff, 0x0010000f,
634 0x926c, 0xffffffff, 0x00000013,
635 0x9270, 0xffffffff, 0x00120011,
636 0x9274, 0xffffffff, 0x00150014,
637 0x9278, 0xffffffff, 0x000f000e,
638 0x927c, 0xffffffff, 0x00110010,
639 0x9280, 0xffffffff, 0x00000014,
640 0x9284, 0xffffffff, 0x00130012,
641 0x9288, 0xffffffff, 0x00160015,
642 0x928c, 0xffffffff, 0x0010000f,
643 0x9290, 0xffffffff, 0x00120011,
644 0x9294, 0xffffffff, 0x00000015,
645 0x9298, 0xffffffff, 0x00140013,
646 0x929c, 0xffffffff, 0x00170016,
647 0x9150, 0xffffffff, 0x96940200,
648 0x8708, 0xffffffff, 0x00900100,
649 0xc478, 0xffffffff, 0x00000080,
650 0xc404, 0xffffffff, 0x0020003f,
651 0x30, 0xffffffff, 0x0000001c,
652 0x34, 0x000f0000, 0x000f0000,
653 0x160c, 0xffffffff, 0x00000100,
654 0x1024, 0xffffffff, 0x00000100,
655 0x102c, 0x00000101, 0x00000000,
656 0x20a8, 0xffffffff, 0x00000104,
657 0x264c, 0x000c0000, 0x000c0000,
658 0x2648, 0x000c0000, 0x000c0000,
659 0x55e4, 0xff000fff, 0x00000100,
660 0x55e8, 0x00000001, 0x00000001,
661 0x2f50, 0x00000001, 0x00000001,
662 0x30cc, 0xc0000fff, 0x00000104,
663 0xc1e4, 0x00000001, 0x00000001,
664 0xd0c0, 0xfffffff0, 0x00000100,
665 0xd8c0, 0xfffffff0, 0x00000100
666};
667
668static const u32 pitcairn_mgcg_cgcg_init[] =
669{
670 0xc400, 0xffffffff, 0xfffffffc,
671 0x802c, 0xffffffff, 0xe0000000,
672 0x9a60, 0xffffffff, 0x00000100,
673 0x92a4, 0xffffffff, 0x00000100,
674 0xc164, 0xffffffff, 0x00000100,
675 0x9774, 0xffffffff, 0x00000100,
676 0x8984, 0xffffffff, 0x06000100,
677 0x8a18, 0xffffffff, 0x00000100,
678 0x92a0, 0xffffffff, 0x00000100,
679 0xc380, 0xffffffff, 0x00000100,
680 0x8b28, 0xffffffff, 0x00000100,
681 0x9144, 0xffffffff, 0x00000100,
682 0x8d88, 0xffffffff, 0x00000100,
683 0x8d8c, 0xffffffff, 0x00000100,
684 0x9030, 0xffffffff, 0x00000100,
685 0x9034, 0xffffffff, 0x00000100,
686 0x9038, 0xffffffff, 0x00000100,
687 0x903c, 0xffffffff, 0x00000100,
688 0xad80, 0xffffffff, 0x00000100,
689 0xac54, 0xffffffff, 0x00000100,
690 0x897c, 0xffffffff, 0x06000100,
691 0x9868, 0xffffffff, 0x00000100,
692 0x9510, 0xffffffff, 0x00000100,
693 0xaf04, 0xffffffff, 0x00000100,
694 0xae04, 0xffffffff, 0x00000100,
695 0x949c, 0xffffffff, 0x00000100,
696 0x802c, 0xffffffff, 0xe0000000,
697 0x9160, 0xffffffff, 0x00010000,
698 0x9164, 0xffffffff, 0x00030002,
699 0x9168, 0xffffffff, 0x00040007,
700 0x916c, 0xffffffff, 0x00060005,
701 0x9170, 0xffffffff, 0x00090008,
702 0x9174, 0xffffffff, 0x00020001,
703 0x9178, 0xffffffff, 0x00040003,
704 0x917c, 0xffffffff, 0x00000007,
705 0x9180, 0xffffffff, 0x00060005,
706 0x9184, 0xffffffff, 0x00090008,
707 0x9188, 0xffffffff, 0x00030002,
708 0x918c, 0xffffffff, 0x00050004,
709 0x9190, 0xffffffff, 0x00000008,
710 0x9194, 0xffffffff, 0x00070006,
711 0x9198, 0xffffffff, 0x000a0009,
712 0x919c, 0xffffffff, 0x00040003,
713 0x91a0, 0xffffffff, 0x00060005,
714 0x91a4, 0xffffffff, 0x00000009,
715 0x91a8, 0xffffffff, 0x00080007,
716 0x91ac, 0xffffffff, 0x000b000a,
717 0x91b0, 0xffffffff, 0x00050004,
718 0x91b4, 0xffffffff, 0x00070006,
719 0x91b8, 0xffffffff, 0x0008000b,
720 0x91bc, 0xffffffff, 0x000a0009,
721 0x91c0, 0xffffffff, 0x000d000c,
722 0x9200, 0xffffffff, 0x00090008,
723 0x9204, 0xffffffff, 0x000b000a,
724 0x9208, 0xffffffff, 0x000c000f,
725 0x920c, 0xffffffff, 0x000e000d,
726 0x9210, 0xffffffff, 0x00110010,
727 0x9214, 0xffffffff, 0x000a0009,
728 0x9218, 0xffffffff, 0x000c000b,
729 0x921c, 0xffffffff, 0x0000000f,
730 0x9220, 0xffffffff, 0x000e000d,
731 0x9224, 0xffffffff, 0x00110010,
732 0x9228, 0xffffffff, 0x000b000a,
733 0x922c, 0xffffffff, 0x000d000c,
734 0x9230, 0xffffffff, 0x00000010,
735 0x9234, 0xffffffff, 0x000f000e,
736 0x9238, 0xffffffff, 0x00120011,
737 0x923c, 0xffffffff, 0x000c000b,
738 0x9240, 0xffffffff, 0x000e000d,
739 0x9244, 0xffffffff, 0x00000011,
740 0x9248, 0xffffffff, 0x0010000f,
741 0x924c, 0xffffffff, 0x00130012,
742 0x9250, 0xffffffff, 0x000d000c,
743 0x9254, 0xffffffff, 0x000f000e,
744 0x9258, 0xffffffff, 0x00100013,
745 0x925c, 0xffffffff, 0x00120011,
746 0x9260, 0xffffffff, 0x00150014,
747 0x9150, 0xffffffff, 0x96940200,
748 0x8708, 0xffffffff, 0x00900100,
749 0xc478, 0xffffffff, 0x00000080,
750 0xc404, 0xffffffff, 0x0020003f,
751 0x30, 0xffffffff, 0x0000001c,
752 0x34, 0x000f0000, 0x000f0000,
753 0x160c, 0xffffffff, 0x00000100,
754 0x1024, 0xffffffff, 0x00000100,
755 0x102c, 0x00000101, 0x00000000,
756 0x20a8, 0xffffffff, 0x00000104,
757 0x55e4, 0xff000fff, 0x00000100,
758 0x55e8, 0x00000001, 0x00000001,
759 0x2f50, 0x00000001, 0x00000001,
760 0x30cc, 0xc0000fff, 0x00000104,
761 0xc1e4, 0x00000001, 0x00000001,
762 0xd0c0, 0xfffffff0, 0x00000100,
763 0xd8c0, 0xfffffff0, 0x00000100
764};
765
766static const u32 verde_mgcg_cgcg_init[] =
767{
768 0xc400, 0xffffffff, 0xfffffffc,
769 0x802c, 0xffffffff, 0xe0000000,
770 0x9a60, 0xffffffff, 0x00000100,
771 0x92a4, 0xffffffff, 0x00000100,
772 0xc164, 0xffffffff, 0x00000100,
773 0x9774, 0xffffffff, 0x00000100,
774 0x8984, 0xffffffff, 0x06000100,
775 0x8a18, 0xffffffff, 0x00000100,
776 0x92a0, 0xffffffff, 0x00000100,
777 0xc380, 0xffffffff, 0x00000100,
778 0x8b28, 0xffffffff, 0x00000100,
779 0x9144, 0xffffffff, 0x00000100,
780 0x8d88, 0xffffffff, 0x00000100,
781 0x8d8c, 0xffffffff, 0x00000100,
782 0x9030, 0xffffffff, 0x00000100,
783 0x9034, 0xffffffff, 0x00000100,
784 0x9038, 0xffffffff, 0x00000100,
785 0x903c, 0xffffffff, 0x00000100,
786 0xad80, 0xffffffff, 0x00000100,
787 0xac54, 0xffffffff, 0x00000100,
788 0x897c, 0xffffffff, 0x06000100,
789 0x9868, 0xffffffff, 0x00000100,
790 0x9510, 0xffffffff, 0x00000100,
791 0xaf04, 0xffffffff, 0x00000100,
792 0xae04, 0xffffffff, 0x00000100,
793 0x949c, 0xffffffff, 0x00000100,
794 0x802c, 0xffffffff, 0xe0000000,
795 0x9160, 0xffffffff, 0x00010000,
796 0x9164, 0xffffffff, 0x00030002,
797 0x9168, 0xffffffff, 0x00040007,
798 0x916c, 0xffffffff, 0x00060005,
799 0x9170, 0xffffffff, 0x00090008,
800 0x9174, 0xffffffff, 0x00020001,
801 0x9178, 0xffffffff, 0x00040003,
802 0x917c, 0xffffffff, 0x00000007,
803 0x9180, 0xffffffff, 0x00060005,
804 0x9184, 0xffffffff, 0x00090008,
805 0x9188, 0xffffffff, 0x00030002,
806 0x918c, 0xffffffff, 0x00050004,
807 0x9190, 0xffffffff, 0x00000008,
808 0x9194, 0xffffffff, 0x00070006,
809 0x9198, 0xffffffff, 0x000a0009,
810 0x919c, 0xffffffff, 0x00040003,
811 0x91a0, 0xffffffff, 0x00060005,
812 0x91a4, 0xffffffff, 0x00000009,
813 0x91a8, 0xffffffff, 0x00080007,
814 0x91ac, 0xffffffff, 0x000b000a,
815 0x91b0, 0xffffffff, 0x00050004,
816 0x91b4, 0xffffffff, 0x00070006,
817 0x91b8, 0xffffffff, 0x0008000b,
818 0x91bc, 0xffffffff, 0x000a0009,
819 0x91c0, 0xffffffff, 0x000d000c,
820 0x9200, 0xffffffff, 0x00090008,
821 0x9204, 0xffffffff, 0x000b000a,
822 0x9208, 0xffffffff, 0x000c000f,
823 0x920c, 0xffffffff, 0x000e000d,
824 0x9210, 0xffffffff, 0x00110010,
825 0x9214, 0xffffffff, 0x000a0009,
826 0x9218, 0xffffffff, 0x000c000b,
827 0x921c, 0xffffffff, 0x0000000f,
828 0x9220, 0xffffffff, 0x000e000d,
829 0x9224, 0xffffffff, 0x00110010,
830 0x9228, 0xffffffff, 0x000b000a,
831 0x922c, 0xffffffff, 0x000d000c,
832 0x9230, 0xffffffff, 0x00000010,
833 0x9234, 0xffffffff, 0x000f000e,
834 0x9238, 0xffffffff, 0x00120011,
835 0x923c, 0xffffffff, 0x000c000b,
836 0x9240, 0xffffffff, 0x000e000d,
837 0x9244, 0xffffffff, 0x00000011,
838 0x9248, 0xffffffff, 0x0010000f,
839 0x924c, 0xffffffff, 0x00130012,
840 0x9250, 0xffffffff, 0x000d000c,
841 0x9254, 0xffffffff, 0x000f000e,
842 0x9258, 0xffffffff, 0x00100013,
843 0x925c, 0xffffffff, 0x00120011,
844 0x9260, 0xffffffff, 0x00150014,
845 0x9150, 0xffffffff, 0x96940200,
846 0x8708, 0xffffffff, 0x00900100,
847 0xc478, 0xffffffff, 0x00000080,
848 0xc404, 0xffffffff, 0x0020003f,
849 0x30, 0xffffffff, 0x0000001c,
850 0x34, 0x000f0000, 0x000f0000,
851 0x160c, 0xffffffff, 0x00000100,
852 0x1024, 0xffffffff, 0x00000100,
853 0x102c, 0x00000101, 0x00000000,
854 0x20a8, 0xffffffff, 0x00000104,
855 0x264c, 0x000c0000, 0x000c0000,
856 0x2648, 0x000c0000, 0x000c0000,
857 0x55e4, 0xff000fff, 0x00000100,
858 0x55e8, 0x00000001, 0x00000001,
859 0x2f50, 0x00000001, 0x00000001,
860 0x30cc, 0xc0000fff, 0x00000104,
861 0xc1e4, 0x00000001, 0x00000001,
862 0xd0c0, 0xfffffff0, 0x00000100,
863 0xd8c0, 0xfffffff0, 0x00000100
864};
865
866static const u32 oland_mgcg_cgcg_init[] =
867{
868 0xc400, 0xffffffff, 0xfffffffc,
869 0x802c, 0xffffffff, 0xe0000000,
870 0x9a60, 0xffffffff, 0x00000100,
871 0x92a4, 0xffffffff, 0x00000100,
872 0xc164, 0xffffffff, 0x00000100,
873 0x9774, 0xffffffff, 0x00000100,
874 0x8984, 0xffffffff, 0x06000100,
875 0x8a18, 0xffffffff, 0x00000100,
876 0x92a0, 0xffffffff, 0x00000100,
877 0xc380, 0xffffffff, 0x00000100,
878 0x8b28, 0xffffffff, 0x00000100,
879 0x9144, 0xffffffff, 0x00000100,
880 0x8d88, 0xffffffff, 0x00000100,
881 0x8d8c, 0xffffffff, 0x00000100,
882 0x9030, 0xffffffff, 0x00000100,
883 0x9034, 0xffffffff, 0x00000100,
884 0x9038, 0xffffffff, 0x00000100,
885 0x903c, 0xffffffff, 0x00000100,
886 0xad80, 0xffffffff, 0x00000100,
887 0xac54, 0xffffffff, 0x00000100,
888 0x897c, 0xffffffff, 0x06000100,
889 0x9868, 0xffffffff, 0x00000100,
890 0x9510, 0xffffffff, 0x00000100,
891 0xaf04, 0xffffffff, 0x00000100,
892 0xae04, 0xffffffff, 0x00000100,
893 0x949c, 0xffffffff, 0x00000100,
894 0x802c, 0xffffffff, 0xe0000000,
895 0x9160, 0xffffffff, 0x00010000,
896 0x9164, 0xffffffff, 0x00030002,
897 0x9168, 0xffffffff, 0x00040007,
898 0x916c, 0xffffffff, 0x00060005,
899 0x9170, 0xffffffff, 0x00090008,
900 0x9174, 0xffffffff, 0x00020001,
901 0x9178, 0xffffffff, 0x00040003,
902 0x917c, 0xffffffff, 0x00000007,
903 0x9180, 0xffffffff, 0x00060005,
904 0x9184, 0xffffffff, 0x00090008,
905 0x9188, 0xffffffff, 0x00030002,
906 0x918c, 0xffffffff, 0x00050004,
907 0x9190, 0xffffffff, 0x00000008,
908 0x9194, 0xffffffff, 0x00070006,
909 0x9198, 0xffffffff, 0x000a0009,
910 0x919c, 0xffffffff, 0x00040003,
911 0x91a0, 0xffffffff, 0x00060005,
912 0x91a4, 0xffffffff, 0x00000009,
913 0x91a8, 0xffffffff, 0x00080007,
914 0x91ac, 0xffffffff, 0x000b000a,
915 0x91b0, 0xffffffff, 0x00050004,
916 0x91b4, 0xffffffff, 0x00070006,
917 0x91b8, 0xffffffff, 0x0008000b,
918 0x91bc, 0xffffffff, 0x000a0009,
919 0x91c0, 0xffffffff, 0x000d000c,
920 0x91c4, 0xffffffff, 0x00060005,
921 0x91c8, 0xffffffff, 0x00080007,
922 0x91cc, 0xffffffff, 0x0000000b,
923 0x91d0, 0xffffffff, 0x000a0009,
924 0x91d4, 0xffffffff, 0x000d000c,
925 0x9150, 0xffffffff, 0x96940200,
926 0x8708, 0xffffffff, 0x00900100,
927 0xc478, 0xffffffff, 0x00000080,
928 0xc404, 0xffffffff, 0x0020003f,
929 0x30, 0xffffffff, 0x0000001c,
930 0x34, 0x000f0000, 0x000f0000,
931 0x160c, 0xffffffff, 0x00000100,
932 0x1024, 0xffffffff, 0x00000100,
933 0x102c, 0x00000101, 0x00000000,
934 0x20a8, 0xffffffff, 0x00000104,
935 0x264c, 0x000c0000, 0x000c0000,
936 0x2648, 0x000c0000, 0x000c0000,
937 0x55e4, 0xff000fff, 0x00000100,
938 0x55e8, 0x00000001, 0x00000001,
939 0x2f50, 0x00000001, 0x00000001,
940 0x30cc, 0xc0000fff, 0x00000104,
941 0xc1e4, 0x00000001, 0x00000001,
942 0xd0c0, 0xfffffff0, 0x00000100,
943 0xd8c0, 0xfffffff0, 0x00000100
944};
945
fffbdda4
AD		 946static const u32 hainan_mgcg_cgcg_init[] =
947{
948 0xc400, 0xffffffff, 0xfffffffc,
949 0x802c, 0xffffffff, 0xe0000000,
950 0x9a60, 0xffffffff, 0x00000100,
951 0x92a4, 0xffffffff, 0x00000100,
952 0xc164, 0xffffffff, 0x00000100,
953 0x9774, 0xffffffff, 0x00000100,
954 0x8984, 0xffffffff, 0x06000100,
955 0x8a18, 0xffffffff, 0x00000100,
956 0x92a0, 0xffffffff, 0x00000100,
957 0xc380, 0xffffffff, 0x00000100,
958 0x8b28, 0xffffffff, 0x00000100,
959 0x9144, 0xffffffff, 0x00000100,
960 0x8d88, 0xffffffff, 0x00000100,
961 0x8d8c, 0xffffffff, 0x00000100,
962 0x9030, 0xffffffff, 0x00000100,
963 0x9034, 0xffffffff, 0x00000100,
964 0x9038, 0xffffffff, 0x00000100,
965 0x903c, 0xffffffff, 0x00000100,
966 0xad80, 0xffffffff, 0x00000100,
967 0xac54, 0xffffffff, 0x00000100,
968 0x897c, 0xffffffff, 0x06000100,
969 0x9868, 0xffffffff, 0x00000100,
970 0x9510, 0xffffffff, 0x00000100,
971 0xaf04, 0xffffffff, 0x00000100,
972 0xae04, 0xffffffff, 0x00000100,
973 0x949c, 0xffffffff, 0x00000100,
974 0x802c, 0xffffffff, 0xe0000000,
975 0x9160, 0xffffffff, 0x00010000,
976 0x9164, 0xffffffff, 0x00030002,
977 0x9168, 0xffffffff, 0x00040007,
978 0x916c, 0xffffffff, 0x00060005,
979 0x9170, 0xffffffff, 0x00090008,
980 0x9174, 0xffffffff, 0x00020001,
981 0x9178, 0xffffffff, 0x00040003,
982 0x917c, 0xffffffff, 0x00000007,
983 0x9180, 0xffffffff, 0x00060005,
984 0x9184, 0xffffffff, 0x00090008,
985 0x9188, 0xffffffff, 0x00030002,
986 0x918c, 0xffffffff, 0x00050004,
987 0x9190, 0xffffffff, 0x00000008,
988 0x9194, 0xffffffff, 0x00070006,
989 0x9198, 0xffffffff, 0x000a0009,
990 0x919c, 0xffffffff, 0x00040003,
991 0x91a0, 0xffffffff, 0x00060005,
992 0x91a4, 0xffffffff, 0x00000009,
993 0x91a8, 0xffffffff, 0x00080007,
994 0x91ac, 0xffffffff, 0x000b000a,
995 0x91b0, 0xffffffff, 0x00050004,
996 0x91b4, 0xffffffff, 0x00070006,
997 0x91b8, 0xffffffff, 0x0008000b,
998 0x91bc, 0xffffffff, 0x000a0009,
999 0x91c0, 0xffffffff, 0x000d000c,
1000 0x91c4, 0xffffffff, 0x00060005,
1001 0x91c8, 0xffffffff, 0x00080007,
1002 0x91cc, 0xffffffff, 0x0000000b,
1003 0x91d0, 0xffffffff, 0x000a0009,
1004 0x91d4, 0xffffffff, 0x000d000c,
1005 0x9150, 0xffffffff, 0x96940200,
1006 0x8708, 0xffffffff, 0x00900100,
1007 0xc478, 0xffffffff, 0x00000080,
1008 0xc404, 0xffffffff, 0x0020003f,
1009 0x30, 0xffffffff, 0x0000001c,
1010 0x34, 0x000f0000, 0x000f0000,
1011 0x160c, 0xffffffff, 0x00000100,
1012 0x1024, 0xffffffff, 0x00000100,
1013 0x20a8, 0xffffffff, 0x00000104,
1014 0x264c, 0x000c0000, 0x000c0000,
1015 0x2648, 0x000c0000, 0x000c0000,
1016 0x2f50, 0x00000001, 0x00000001,
1017 0x30cc, 0xc0000fff, 0x00000104,
1018 0xc1e4, 0x00000001, 0x00000001,
1019 0xd0c0, 0xfffffff0, 0x00000100,
1020 0xd8c0, 0xfffffff0, 0x00000100
1021};
1022
205996c0
AD		 1023static u32 verde_pg_init[] =
1024{
1025 0x353c, 0xffffffff, 0x40000,
1026 0x3538, 0xffffffff, 0x200010ff,
1027 0x353c, 0xffffffff, 0x0,
1028 0x353c, 0xffffffff, 0x0,
1029 0x353c, 0xffffffff, 0x0,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x7007,
1033 0x3538, 0xffffffff, 0x300010ff,
1034 0x353c, 0xffffffff, 0x0,
1035 0x353c, 0xffffffff, 0x0,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x400000,
1040 0x3538, 0xffffffff, 0x100010ff,
1041 0x353c, 0xffffffff, 0x0,
1042 0x353c, 0xffffffff, 0x0,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x120200,
1047 0x3538, 0xffffffff, 0x500010ff,
1048 0x353c, 0xffffffff, 0x0,
1049 0x353c, 0xffffffff, 0x0,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x1e1e16,
1054 0x3538, 0xffffffff, 0x600010ff,
1055 0x353c, 0xffffffff, 0x0,
1056 0x353c, 0xffffffff, 0x0,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x0,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x171f1e,
1061 0x3538, 0xffffffff, 0x700010ff,
1062 0x353c, 0xffffffff, 0x0,
1063 0x353c, 0xffffffff, 0x0,
1064 0x353c, 0xffffffff, 0x0,
1065 0x353c, 0xffffffff, 0x0,
1066 0x353c, 0xffffffff, 0x0,
1067 0x353c, 0xffffffff, 0x0,
1068 0x3538, 0xffffffff, 0x9ff,
1069 0x3500, 0xffffffff, 0x0,
1070 0x3504, 0xffffffff, 0x10000800,
1071 0x3504, 0xffffffff, 0xf,
1072 0x3504, 0xffffffff, 0xf,
1073 0x3500, 0xffffffff, 0x4,
1074 0x3504, 0xffffffff, 0x1000051e,
1075 0x3504, 0xffffffff, 0xffff,
1076 0x3504, 0xffffffff, 0xffff,
1077 0x3500, 0xffffffff, 0x8,
1078 0x3504, 0xffffffff, 0x80500,
1079 0x3500, 0xffffffff, 0x12,
1080 0x3504, 0xffffffff, 0x9050c,
1081 0x3500, 0xffffffff, 0x1d,
1082 0x3504, 0xffffffff, 0xb052c,
1083 0x3500, 0xffffffff, 0x2a,
1084 0x3504, 0xffffffff, 0x1053e,
1085 0x3500, 0xffffffff, 0x2d,
1086 0x3504, 0xffffffff, 0x10546,
1087 0x3500, 0xffffffff, 0x30,
1088 0x3504, 0xffffffff, 0xa054e,
1089 0x3500, 0xffffffff, 0x3c,
1090 0x3504, 0xffffffff, 0x1055f,
1091 0x3500, 0xffffffff, 0x3f,
1092 0x3504, 0xffffffff, 0x10567,
1093 0x3500, 0xffffffff, 0x42,
1094 0x3504, 0xffffffff, 0x1056f,
1095 0x3500, 0xffffffff, 0x45,
1096 0x3504, 0xffffffff, 0x10572,
1097 0x3500, 0xffffffff, 0x48,
1098 0x3504, 0xffffffff, 0x20575,
1099 0x3500, 0xffffffff, 0x4c,
1100 0x3504, 0xffffffff, 0x190801,
1101 0x3500, 0xffffffff, 0x67,
1102 0x3504, 0xffffffff, 0x1082a,
1103 0x3500, 0xffffffff, 0x6a,
1104 0x3504, 0xffffffff, 0x1b082d,
1105 0x3500, 0xffffffff, 0x87,
1106 0x3504, 0xffffffff, 0x310851,
1107 0x3500, 0xffffffff, 0xba,
1108 0x3504, 0xffffffff, 0x891,
1109 0x3500, 0xffffffff, 0xbc,
1110 0x3504, 0xffffffff, 0x893,
1111 0x3500, 0xffffffff, 0xbe,
1112 0x3504, 0xffffffff, 0x20895,
1113 0x3500, 0xffffffff, 0xc2,
1114 0x3504, 0xffffffff, 0x20899,
1115 0x3500, 0xffffffff, 0xc6,
1116 0x3504, 0xffffffff, 0x2089d,
1117 0x3500, 0xffffffff, 0xca,
1118 0x3504, 0xffffffff, 0x8a1,
1119 0x3500, 0xffffffff, 0xcc,
1120 0x3504, 0xffffffff, 0x8a3,
1121 0x3500, 0xffffffff, 0xce,
1122 0x3504, 0xffffffff, 0x308a5,
1123 0x3500, 0xffffffff, 0xd3,
1124 0x3504, 0xffffffff, 0x6d08cd,
1125 0x3500, 0xffffffff, 0x142,
1126 0x3504, 0xffffffff, 0x2000095a,
1127 0x3504, 0xffffffff, 0x1,
1128 0x3500, 0xffffffff, 0x144,
1129 0x3504, 0xffffffff, 0x301f095b,
1130 0x3500, 0xffffffff, 0x165,
1131 0x3504, 0xffffffff, 0xc094d,
1132 0x3500, 0xffffffff, 0x173,
1133 0x3504, 0xffffffff, 0xf096d,
1134 0x3500, 0xffffffff, 0x184,
1135 0x3504, 0xffffffff, 0x15097f,
1136 0x3500, 0xffffffff, 0x19b,
1137 0x3504, 0xffffffff, 0xc0998,
1138 0x3500, 0xffffffff, 0x1a9,
1139 0x3504, 0xffffffff, 0x409a7,
1140 0x3500, 0xffffffff, 0x1af,
1141 0x3504, 0xffffffff, 0xcdc,
1142 0x3500, 0xffffffff, 0x1b1,
1143 0x3504, 0xffffffff, 0x800,
1144 0x3508, 0xffffffff, 0x6c9b2000,
1145 0x3510, 0xfc00, 0x2000,
1146 0x3544, 0xffffffff, 0xfc0,
1147 0x28d4, 0x00000100, 0x100
1148};
1149
1150static void si_init_golden_registers(struct radeon_device *rdev)
1151{
1152 switch (rdev->family) {
1153 case CHIP_TAHITI:
1154 radeon_program_register_sequence(rdev,
1155 tahiti_golden_registers,
1156 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1157 radeon_program_register_sequence(rdev,
1158 tahiti_golden_rlc_registers,
1159 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1160 radeon_program_register_sequence(rdev,
1161 tahiti_mgcg_cgcg_init,
1162 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1163 radeon_program_register_sequence(rdev,
1164 tahiti_golden_registers2,
1165 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1166 break;
1167 case CHIP_PITCAIRN:
1168 radeon_program_register_sequence(rdev,
1169 pitcairn_golden_registers,
1170 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1171 radeon_program_register_sequence(rdev,
1172 pitcairn_golden_rlc_registers,
1173 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1174 radeon_program_register_sequence(rdev,
1175 pitcairn_mgcg_cgcg_init,
1176 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1177 break;
1178 case CHIP_VERDE:
1179 radeon_program_register_sequence(rdev,
1180 verde_golden_registers,
1181 (const u32)ARRAY_SIZE(verde_golden_registers));
1182 radeon_program_register_sequence(rdev,
1183 verde_golden_rlc_registers,
1184 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1185 radeon_program_register_sequence(rdev,
1186 verde_mgcg_cgcg_init,
1187 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1188 radeon_program_register_sequence(rdev,
1189 verde_pg_init,
1190 (const u32)ARRAY_SIZE(verde_pg_init));
1191 break;
1192 case CHIP_OLAND:
1193 radeon_program_register_sequence(rdev,
1194 oland_golden_registers,
1195 (const u32)ARRAY_SIZE(oland_golden_registers));
1196 radeon_program_register_sequence(rdev,
1197 oland_golden_rlc_registers,
1198 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1199 radeon_program_register_sequence(rdev,
1200 oland_mgcg_cgcg_init,
1201 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1202 break;
fffbdda4
AD		 1203 case CHIP_HAINAN:
1204 radeon_program_register_sequence(rdev,
1205 hainan_golden_registers,
1206 (const u32)ARRAY_SIZE(hainan_golden_registers));
1207 radeon_program_register_sequence(rdev,
1208 hainan_golden_registers2,
1209 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1210 radeon_program_register_sequence(rdev,
1211 hainan_mgcg_cgcg_init,
1212 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1213 break;
205996c0
AD		 1214 default:
1215 break;
1216 }
1217}
1218
454d2e2a
AD		 1219#define PCIE_BUS_CLK 10000
1220#define TCLK (PCIE_BUS_CLK / 10)
1221
1222/**
1223 * si_get_xclk - get the xclk
1224 *
1225 * @rdev: radeon_device pointer
1226 *
1227 * Returns the reference clock used by the gfx engine
1228 * (SI).
1229 */
1230u32 si_get_xclk(struct radeon_device *rdev)
1231{
1232 u32 reference_clock = rdev->clock.spll.reference_freq;
1233 u32 tmp;
1234
1235 tmp = RREG32(CG_CLKPIN_CNTL_2);
1236 if (tmp & MUX_TCLK_TO_XCLK)
1237 return TCLK;
1238
1239 tmp = RREG32(CG_CLKPIN_CNTL);
1240 if (tmp & XTALIN_DIVIDE)
1241 return reference_clock / 4;
1242
1243 return reference_clock;
1244}
1245
1bd47d2e
AD		 1246/* get temperature in millidegrees */
1247int si_get_temp(struct radeon_device *rdev)
1248{
1249 u32 temp;
1250 int actual_temp = 0;
1251
1252 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1253 CTF_TEMP_SHIFT;
1254
1255 if (temp & 0x200)
1256 actual_temp = 255;
1257 else
1258 actual_temp = temp & 0x1ff;
1259
1260 actual_temp = (actual_temp * 1000);
1261
1262 return actual_temp;
1263}
1264
8b074dd6
AD		 1265#define TAHITI_IO_MC_REGS_SIZE 36
1266
1267static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1268 {0x0000006f, 0x03044000},
1269 {0x00000070, 0x0480c018},
1270 {0x00000071, 0x00000040},
1271 {0x00000072, 0x01000000},
1272 {0x00000074, 0x000000ff},
1273 {0x00000075, 0x00143400},
1274 {0x00000076, 0x08ec0800},
1275 {0x00000077, 0x040000cc},
1276 {0x00000079, 0x00000000},
1277 {0x0000007a, 0x21000409},
1278 {0x0000007c, 0x00000000},
1279 {0x0000007d, 0xe8000000},
1280 {0x0000007e, 0x044408a8},
1281 {0x0000007f, 0x00000003},
1282 {0x00000080, 0x00000000},
1283 {0x00000081, 0x01000000},
1284 {0x00000082, 0x02000000},
1285 {0x00000083, 0x00000000},
1286 {0x00000084, 0xe3f3e4f4},
1287 {0x00000085, 0x00052024},
1288 {0x00000087, 0x00000000},
1289 {0x00000088, 0x66036603},
1290 {0x00000089, 0x01000000},
1291 {0x0000008b, 0x1c0a0000},
1292 {0x0000008c, 0xff010000},
1293 {0x0000008e, 0xffffefff},
1294 {0x0000008f, 0xfff3efff},
1295 {0x00000090, 0xfff3efbf},
1296 {0x00000094, 0x00101101},
1297 {0x00000095, 0x00000fff},
1298 {0x00000096, 0x00116fff},
1299 {0x00000097, 0x60010000},
1300 {0x00000098, 0x10010000},
1301 {0x00000099, 0x00006000},
1302 {0x0000009a, 0x00001000},
1303 {0x0000009f, 0x00a77400}
1304};
1305
1306static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1307 {0x0000006f, 0x03044000},
1308 {0x00000070, 0x0480c018},
1309 {0x00000071, 0x00000040},
1310 {0x00000072, 0x01000000},
1311 {0x00000074, 0x000000ff},
1312 {0x00000075, 0x00143400},
1313 {0x00000076, 0x08ec0800},
1314 {0x00000077, 0x040000cc},
1315 {0x00000079, 0x00000000},
1316 {0x0000007a, 0x21000409},
1317 {0x0000007c, 0x00000000},
1318 {0x0000007d, 0xe8000000},
1319 {0x0000007e, 0x044408a8},
1320 {0x0000007f, 0x00000003},
1321 {0x00000080, 0x00000000},
1322 {0x00000081, 0x01000000},
1323 {0x00000082, 0x02000000},
1324 {0x00000083, 0x00000000},
1325 {0x00000084, 0xe3f3e4f4},
1326 {0x00000085, 0x00052024},
1327 {0x00000087, 0x00000000},
1328 {0x00000088, 0x66036603},
1329 {0x00000089, 0x01000000},
1330 {0x0000008b, 0x1c0a0000},
1331 {0x0000008c, 0xff010000},
1332 {0x0000008e, 0xffffefff},
1333 {0x0000008f, 0xfff3efff},
1334 {0x00000090, 0xfff3efbf},
1335 {0x00000094, 0x00101101},
1336 {0x00000095, 0x00000fff},
1337 {0x00000096, 0x00116fff},
1338 {0x00000097, 0x60010000},
1339 {0x00000098, 0x10010000},
1340 {0x00000099, 0x00006000},
1341 {0x0000009a, 0x00001000},
1342 {0x0000009f, 0x00a47400}
1343};
1344
1345static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1346 {0x0000006f, 0x03044000},
1347 {0x00000070, 0x0480c018},
1348 {0x00000071, 0x00000040},
1349 {0x00000072, 0x01000000},
1350 {0x00000074, 0x000000ff},
1351 {0x00000075, 0x00143400},
1352 {0x00000076, 0x08ec0800},
1353 {0x00000077, 0x040000cc},
1354 {0x00000079, 0x00000000},
1355 {0x0000007a, 0x21000409},
1356 {0x0000007c, 0x00000000},
1357 {0x0000007d, 0xe8000000},
1358 {0x0000007e, 0x044408a8},
1359 {0x0000007f, 0x00000003},
1360 {0x00000080, 0x00000000},
1361 {0x00000081, 0x01000000},
1362 {0x00000082, 0x02000000},
1363 {0x00000083, 0x00000000},
1364 {0x00000084, 0xe3f3e4f4},
1365 {0x00000085, 0x00052024},
1366 {0x00000087, 0x00000000},
1367 {0x00000088, 0x66036603},
1368 {0x00000089, 0x01000000},
1369 {0x0000008b, 0x1c0a0000},
1370 {0x0000008c, 0xff010000},
1371 {0x0000008e, 0xffffefff},
1372 {0x0000008f, 0xfff3efff},
1373 {0x00000090, 0xfff3efbf},
1374 {0x00000094, 0x00101101},
1375 {0x00000095, 0x00000fff},
1376 {0x00000096, 0x00116fff},
1377 {0x00000097, 0x60010000},
1378 {0x00000098, 0x10010000},
1379 {0x00000099, 0x00006000},
1380 {0x0000009a, 0x00001000},
1381 {0x0000009f, 0x00a37400}
1382};
1383
bcc7f5d2
AD		 1384static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1385 {0x0000006f, 0x03044000},
1386 {0x00000070, 0x0480c018},
1387 {0x00000071, 0x00000040},
1388 {0x00000072, 0x01000000},
1389 {0x00000074, 0x000000ff},
1390 {0x00000075, 0x00143400},
1391 {0x00000076, 0x08ec0800},
1392 {0x00000077, 0x040000cc},
1393 {0x00000079, 0x00000000},
1394 {0x0000007a, 0x21000409},
1395 {0x0000007c, 0x00000000},
1396 {0x0000007d, 0xe8000000},
1397 {0x0000007e, 0x044408a8},
1398 {0x0000007f, 0x00000003},
1399 {0x00000080, 0x00000000},
1400 {0x00000081, 0x01000000},
1401 {0x00000082, 0x02000000},
1402 {0x00000083, 0x00000000},
1403 {0x00000084, 0xe3f3e4f4},
1404 {0x00000085, 0x00052024},
1405 {0x00000087, 0x00000000},
1406 {0x00000088, 0x66036603},
1407 {0x00000089, 0x01000000},
1408 {0x0000008b, 0x1c0a0000},
1409 {0x0000008c, 0xff010000},
1410 {0x0000008e, 0xffffefff},
1411 {0x0000008f, 0xfff3efff},
1412 {0x00000090, 0xfff3efbf},
1413 {0x00000094, 0x00101101},
1414 {0x00000095, 0x00000fff},
1415 {0x00000096, 0x00116fff},
1416 {0x00000097, 0x60010000},
1417 {0x00000098, 0x10010000},
1418 {0x00000099, 0x00006000},
1419 {0x0000009a, 0x00001000},
1420 {0x0000009f, 0x00a17730}
1421};
1422
c04c00b4
AD		 1423static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1424 {0x0000006f, 0x03044000},
1425 {0x00000070, 0x0480c018},
1426 {0x00000071, 0x00000040},
1427 {0x00000072, 0x01000000},
1428 {0x00000074, 0x000000ff},
1429 {0x00000075, 0x00143400},
1430 {0x00000076, 0x08ec0800},
1431 {0x00000077, 0x040000cc},
1432 {0x00000079, 0x00000000},
1433 {0x0000007a, 0x21000409},
1434 {0x0000007c, 0x00000000},
1435 {0x0000007d, 0xe8000000},
1436 {0x0000007e, 0x044408a8},
1437 {0x0000007f, 0x00000003},
1438 {0x00000080, 0x00000000},
1439 {0x00000081, 0x01000000},
1440 {0x00000082, 0x02000000},
1441 {0x00000083, 0x00000000},
1442 {0x00000084, 0xe3f3e4f4},
1443 {0x00000085, 0x00052024},
1444 {0x00000087, 0x00000000},
1445 {0x00000088, 0x66036603},
1446 {0x00000089, 0x01000000},
1447 {0x0000008b, 0x1c0a0000},
1448 {0x0000008c, 0xff010000},
1449 {0x0000008e, 0xffffefff},
1450 {0x0000008f, 0xfff3efff},
1451 {0x00000090, 0xfff3efbf},
1452 {0x00000094, 0x00101101},
1453 {0x00000095, 0x00000fff},
1454 {0x00000096, 0x00116fff},
1455 {0x00000097, 0x60010000},
1456 {0x00000098, 0x10010000},
1457 {0x00000099, 0x00006000},
1458 {0x0000009a, 0x00001000},
1459 {0x0000009f, 0x00a07730}
1460};
1461
8b074dd6
AD		 1462/* ucode loading */
1463static int si_mc_load_microcode(struct radeon_device *rdev)
1464{
1465 const __be32 *fw_data;
1466 u32 running, blackout = 0;
1467 u32 *io_mc_regs;
1468 int i, ucode_size, regs_size;
1469
1470 if (!rdev->mc_fw)
1471 return -EINVAL;
1472
1473 switch (rdev->family) {
1474 case CHIP_TAHITI:
1475 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1476 ucode_size = SI_MC_UCODE_SIZE;
1477 regs_size = TAHITI_IO_MC_REGS_SIZE;
1478 break;
1479 case CHIP_PITCAIRN:
1480 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1481 ucode_size = SI_MC_UCODE_SIZE;
1482 regs_size = TAHITI_IO_MC_REGS_SIZE;
1483 break;
1484 case CHIP_VERDE:
1485 default:
1486 io_mc_regs = (u32 *)&verde_io_mc_regs;
1487 ucode_size = SI_MC_UCODE_SIZE;
1488 regs_size = TAHITI_IO_MC_REGS_SIZE;
1489 break;
bcc7f5d2
AD		 1490 case CHIP_OLAND:
1491 io_mc_regs = (u32 *)&oland_io_mc_regs;
1492 ucode_size = OLAND_MC_UCODE_SIZE;
1493 regs_size = TAHITI_IO_MC_REGS_SIZE;
1494 break;
c04c00b4
AD		 1495 case CHIP_HAINAN:
1496 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1497 ucode_size = OLAND_MC_UCODE_SIZE;
1498 regs_size = TAHITI_IO_MC_REGS_SIZE;
1499 break;
8b074dd6
AD		 1500 }
1501
1502 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1503
1504 if (running == 0) {
1505 if (running) {
1506 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1507 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1508 }
1509
1510 /* reset the engine and set to writable */
1511 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1512 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1513
1514 /* load mc io regs */
1515 for (i = 0; i < regs_size; i++) {
1516 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1517 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1518 }
1519 /* load the MC ucode */
1520 fw_data = (const __be32 *)rdev->mc_fw->data;
1521 for (i = 0; i < ucode_size; i++)
1522 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1523
1524 /* put the engine back into the active state */
1525 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1526 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1527 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1528
1529 /* wait for training to complete */
1530 for (i = 0; i < rdev->usec_timeout; i++) {
1531 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1532 break;
1533 udelay(1);
1534 }
1535 for (i = 0; i < rdev->usec_timeout; i++) {
1536 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1537 break;
1538 udelay(1);
1539 }
1540
1541 if (running)
1542 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1543 }
1544
1545 return 0;
1546}
1547
0f0de06c
AD		 1548static int si_init_microcode(struct radeon_device *rdev)
1549{
0f0de06c
AD		 1550 const char *chip_name;
1551 const char *rlc_chip_name;
1552 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
a9e61410 1553 size_t smc_req_size;
0f0de06c
AD		 1554 char fw_name[30];
1555 int err;
1556
1557 DRM_DEBUG("\n");
1558
0f0de06c
AD		 1559 switch (rdev->family) {
1560 case CHIP_TAHITI:
1561 chip_name = "TAHITI";
1562 rlc_chip_name = "TAHITI";
1563 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1564 me_req_size = SI_PM4_UCODE_SIZE * 4;
1565 ce_req_size = SI_CE_UCODE_SIZE * 4;
1566 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1567 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1568 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1569 break;
1570 case CHIP_PITCAIRN:
1571 chip_name = "PITCAIRN";
1572 rlc_chip_name = "PITCAIRN";
1573 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1574 me_req_size = SI_PM4_UCODE_SIZE * 4;
1575 ce_req_size = SI_CE_UCODE_SIZE * 4;
1576 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1577 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1578 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1579 break;
1580 case CHIP_VERDE:
1581 chip_name = "VERDE";
1582 rlc_chip_name = "VERDE";
1583 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1584 me_req_size = SI_PM4_UCODE_SIZE * 4;
1585 ce_req_size = SI_CE_UCODE_SIZE * 4;
1586 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1587 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1588 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
0f0de06c 1589 break;
bcc7f5d2
AD		 1590 case CHIP_OLAND:
1591 chip_name = "OLAND";
1592 rlc_chip_name = "OLAND";
1593 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1594 me_req_size = SI_PM4_UCODE_SIZE * 4;
1595 ce_req_size = SI_CE_UCODE_SIZE * 4;
1596 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1597 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1598 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
bcc7f5d2 1599 break;
c04c00b4
AD		 1600 case CHIP_HAINAN:
1601 chip_name = "HAINAN";
1602 rlc_chip_name = "HAINAN";
1603 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1604 me_req_size = SI_PM4_UCODE_SIZE * 4;
1605 ce_req_size = SI_CE_UCODE_SIZE * 4;
1606 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1607 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1608 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
c04c00b4 1609 break;
0f0de06c
AD		 1610 default: BUG();
1611 }
1612
1613 DRM_INFO("Loading %s Microcode\n", chip_name);
1614
1615 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
0a168933 1616 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
0f0de06c
AD		 1617 if (err)
1618 goto out;
1619 if (rdev->pfp_fw->size != pfp_req_size) {
1620 printk(KERN_ERR
1621 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1622 rdev->pfp_fw->size, fw_name);
1623 err = -EINVAL;
1624 goto out;
1625 }
1626
1627 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
0a168933 1628 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
0f0de06c
AD		 1629 if (err)
1630 goto out;
1631 if (rdev->me_fw->size != me_req_size) {
1632 printk(KERN_ERR
1633 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1634 rdev->me_fw->size, fw_name);
1635 err = -EINVAL;
1636 }
1637
1638 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
0a168933 1639 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
0f0de06c
AD		 1640 if (err)
1641 goto out;
1642 if (rdev->ce_fw->size != ce_req_size) {
1643 printk(KERN_ERR
1644 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1645 rdev->ce_fw->size, fw_name);
1646 err = -EINVAL;
1647 }
1648
1649 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
0a168933 1650 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
0f0de06c
AD		 1651 if (err)
1652 goto out;
1653 if (rdev->rlc_fw->size != rlc_req_size) {
1654 printk(KERN_ERR
1655 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1656 rdev->rlc_fw->size, fw_name);
1657 err = -EINVAL;
1658 }
1659
1660 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
0a168933 1661 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
0f0de06c
AD		 1662 if (err)
1663 goto out;
1664 if (rdev->mc_fw->size != mc_req_size) {
1665 printk(KERN_ERR
1666 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1667 rdev->mc_fw->size, fw_name);
1668 err = -EINVAL;
1669 }
1670
a9e61410 1671 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
0a168933 1672 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
8a53fa23
AD		 1673 if (err) {
1674 printk(KERN_ERR
1675 "smc: error loading firmware \"%s\"\n",
1676 fw_name);
1677 release_firmware(rdev->smc_fw);
1678 rdev->smc_fw = NULL;
1679 } else if (rdev->smc_fw->size != smc_req_size) {
a9e61410
AD		 1680 printk(KERN_ERR
1681 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1682 rdev->smc_fw->size, fw_name);
1683 err = -EINVAL;
1684 }
1685
0f0de06c 1686out:
0f0de06c
AD		 1687 if (err) {
1688 if (err != -EINVAL)
1689 printk(KERN_ERR
1690 "si_cp: Failed to load firmware \"%s\"\n",
1691 fw_name);
1692 release_firmware(rdev->pfp_fw);
1693 rdev->pfp_fw = NULL;
1694 release_firmware(rdev->me_fw);
1695 rdev->me_fw = NULL;
1696 release_firmware(rdev->ce_fw);
1697 rdev->ce_fw = NULL;
1698 release_firmware(rdev->rlc_fw);
1699 rdev->rlc_fw = NULL;
1700 release_firmware(rdev->mc_fw);
1701 rdev->mc_fw = NULL;
a9e61410
AD		 1702 release_firmware(rdev->smc_fw);
1703 rdev->smc_fw = NULL;
0f0de06c
AD		 1704 }
1705 return err;
1706}
1707
43b3cd99
AD		 1708/* watermark setup */
1709static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1710 struct radeon_crtc *radeon_crtc,
1711 struct drm_display_mode *mode,
1712 struct drm_display_mode *other_mode)
1713{
1714 u32 tmp;
1715 /*
1716 * Line Buffer Setup
1717 * There are 3 line buffers, each one shared by 2 display controllers.
1718 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1719 * the display controllers. The paritioning is done via one of four
1720 * preset allocations specified in bits 21:20:
1721 * 0 - half lb
1722 * 2 - whole lb, other crtc must be disabled
1723 */
1724 /* this can get tricky if we have two large displays on a paired group
1725 * of crtcs. Ideally for multiple large displays we'd assign them to
1726 * non-linked crtcs for maximum line buffer allocation.
1727 */
1728 if (radeon_crtc->base.enabled && mode) {
1729 if (other_mode)
1730 tmp = 0; /* 1/2 */
1731 else
1732 tmp = 2; /* whole */
1733 } else
1734 tmp = 0;
1735
1736 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1737 DC_LB_MEMORY_CONFIG(tmp));
1738
1739 if (radeon_crtc->base.enabled && mode) {
1740 switch (tmp) {
1741 case 0:
1742 default:
1743 return 4096 * 2;
1744 case 2:
1745 return 8192 * 2;
1746 }
1747 }
1748
1749 /* controller not enabled, so no lb used */
1750 return 0;
1751}
1752
ca7db22b 1753static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 1754{
1755 u32 tmp = RREG32(MC_SHARED_CHMAP);
1756
1757 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1758 case 0:
1759 default:
1760 return 1;
1761 case 1:
1762 return 2;
1763 case 2:
1764 return 4;
1765 case 3:
1766 return 8;
1767 case 4:
1768 return 3;
1769 case 5:
1770 return 6;
1771 case 6:
1772 return 10;
1773 case 7:
1774 return 12;
1775 case 8:
1776 return 16;
1777 }
1778}
1779
1780struct dce6_wm_params {
1781 u32 dram_channels; /* number of dram channels */
1782 u32 yclk; /* bandwidth per dram data pin in kHz */
1783 u32 sclk; /* engine clock in kHz */
1784 u32 disp_clk; /* display clock in kHz */
1785 u32 src_width; /* viewport width */
1786 u32 active_time; /* active display time in ns */
1787 u32 blank_time; /* blank time in ns */
1788 bool interlaced; /* mode is interlaced */
1789 fixed20_12 vsc; /* vertical scale ratio */
1790 u32 num_heads; /* number of active crtcs */
1791 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1792 u32 lb_size; /* line buffer allocated to pipe */
1793 u32 vtaps; /* vertical scaler taps */
1794};
1795
1796static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1797{
1798 /* Calculate raw DRAM Bandwidth */
1799 fixed20_12 dram_efficiency; /* 0.7 */
1800 fixed20_12 yclk, dram_channels, bandwidth;
1801 fixed20_12 a;
1802
1803 a.full = dfixed_const(1000);
1804 yclk.full = dfixed_const(wm->yclk);
1805 yclk.full = dfixed_div(yclk, a);
1806 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1807 a.full = dfixed_const(10);
1808 dram_efficiency.full = dfixed_const(7);
1809 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1810 bandwidth.full = dfixed_mul(dram_channels, yclk);
1811 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1812
1813 return dfixed_trunc(bandwidth);
1814}
1815
1816static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1817{
1818 /* Calculate DRAM Bandwidth and the part allocated to display. */
1819 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1820 fixed20_12 yclk, dram_channels, bandwidth;
1821 fixed20_12 a;
1822
1823 a.full = dfixed_const(1000);
1824 yclk.full = dfixed_const(wm->yclk);
1825 yclk.full = dfixed_div(yclk, a);
1826 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1827 a.full = dfixed_const(10);
1828 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1829 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1830 bandwidth.full = dfixed_mul(dram_channels, yclk);
1831 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1832
1833 return dfixed_trunc(bandwidth);
1834}
1835
1836static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1837{
1838 /* Calculate the display Data return Bandwidth */
1839 fixed20_12 return_efficiency; /* 0.8 */
1840 fixed20_12 sclk, bandwidth;
1841 fixed20_12 a;
1842
1843 a.full = dfixed_const(1000);
1844 sclk.full = dfixed_const(wm->sclk);
1845 sclk.full = dfixed_div(sclk, a);
1846 a.full = dfixed_const(10);
1847 return_efficiency.full = dfixed_const(8);
1848 return_efficiency.full = dfixed_div(return_efficiency, a);
1849 a.full = dfixed_const(32);
1850 bandwidth.full = dfixed_mul(a, sclk);
1851 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1852
1853 return dfixed_trunc(bandwidth);
1854}
1855
1856static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1857{
1858 return 32;
1859}
1860
1861static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1862{
1863 /* Calculate the DMIF Request Bandwidth */
1864 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1865 fixed20_12 disp_clk, sclk, bandwidth;
1866 fixed20_12 a, b1, b2;
1867 u32 min_bandwidth;
1868
1869 a.full = dfixed_const(1000);
1870 disp_clk.full = dfixed_const(wm->disp_clk);
1871 disp_clk.full = dfixed_div(disp_clk, a);
1872 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1873 b1.full = dfixed_mul(a, disp_clk);
1874
1875 a.full = dfixed_const(1000);
1876 sclk.full = dfixed_const(wm->sclk);
1877 sclk.full = dfixed_div(sclk, a);
1878 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1879 b2.full = dfixed_mul(a, sclk);
1880
1881 a.full = dfixed_const(10);
1882 disp_clk_request_efficiency.full = dfixed_const(8);
1883 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1884
1885 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1886
1887 a.full = dfixed_const(min_bandwidth);
1888 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1889
1890 return dfixed_trunc(bandwidth);
1891}
1892
1893static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1894{
1895 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1896 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1897 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1898 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1899
1900 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1901}
1902
1903static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1904{
1905 /* Calculate the display mode Average Bandwidth
1906 * DisplayMode should contain the source and destination dimensions,
1907 * timing, etc.
1908 */
1909 fixed20_12 bpp;
1910 fixed20_12 line_time;
1911 fixed20_12 src_width;
1912 fixed20_12 bandwidth;
1913 fixed20_12 a;
1914
1915 a.full = dfixed_const(1000);
1916 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1917 line_time.full = dfixed_div(line_time, a);
1918 bpp.full = dfixed_const(wm->bytes_per_pixel);
1919 src_width.full = dfixed_const(wm->src_width);
1920 bandwidth.full = dfixed_mul(src_width, bpp);
1921 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1922 bandwidth.full = dfixed_div(bandwidth, line_time);
1923
1924 return dfixed_trunc(bandwidth);
1925}
1926
1927static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1928{
1929 /* First calcualte the latency in ns */
1930 u32 mc_latency = 2000; /* 2000 ns. */
1931 u32 available_bandwidth = dce6_available_bandwidth(wm);
1932 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1933 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1934 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1935 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1936 (wm->num_heads * cursor_line_pair_return_time);
1937 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1938 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1939 u32 tmp, dmif_size = 12288;
1940 fixed20_12 a, b, c;
1941
1942 if (wm->num_heads == 0)
1943 return 0;
1944
1945 a.full = dfixed_const(2);
1946 b.full = dfixed_const(1);
1947 if ((wm->vsc.full > a.full) ||
1948 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1949 (wm->vtaps >= 5) ||
1950 ((wm->vsc.full >= a.full) && wm->interlaced))
1951 max_src_lines_per_dst_line = 4;
1952 else
1953 max_src_lines_per_dst_line = 2;
1954
1955 a.full = dfixed_const(available_bandwidth);
1956 b.full = dfixed_const(wm->num_heads);
1957 a.full = dfixed_div(a, b);
1958
1959 b.full = dfixed_const(mc_latency + 512);
1960 c.full = dfixed_const(wm->disp_clk);
1961 b.full = dfixed_div(b, c);
1962
1963 c.full = dfixed_const(dmif_size);
1964 b.full = dfixed_div(c, b);
1965
1966 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1967
1968 b.full = dfixed_const(1000);
1969 c.full = dfixed_const(wm->disp_clk);
1970 b.full = dfixed_div(c, b);
1971 c.full = dfixed_const(wm->bytes_per_pixel);
1972 b.full = dfixed_mul(b, c);
1973
1974 lb_fill_bw = min(tmp, dfixed_trunc(b));
1975
1976 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1977 b.full = dfixed_const(1000);
1978 c.full = dfixed_const(lb_fill_bw);
1979 b.full = dfixed_div(c, b);
1980 a.full = dfixed_div(a, b);
1981 line_fill_time = dfixed_trunc(a);
1982
1983 if (line_fill_time < wm->active_time)
1984 return latency;
1985 else
1986 return latency + (line_fill_time - wm->active_time);
1987
1988}
1989
1990static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1991{
1992 if (dce6_average_bandwidth(wm) <=
1993 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
1994 return true;
1995 else
1996 return false;
1997};
1998
1999static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2000{
2001 if (dce6_average_bandwidth(wm) <=
2002 (dce6_available_bandwidth(wm) / wm->num_heads))
2003 return true;
2004 else
2005 return false;
2006};
2007
2008static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2009{
2010 u32 lb_partitions = wm->lb_size / wm->src_width;
2011 u32 line_time = wm->active_time + wm->blank_time;
2012 u32 latency_tolerant_lines;
2013 u32 latency_hiding;
2014 fixed20_12 a;
2015
2016 a.full = dfixed_const(1);
2017 if (wm->vsc.full > a.full)
2018 latency_tolerant_lines = 1;
2019 else {
2020 if (lb_partitions <= (wm->vtaps + 1))
2021 latency_tolerant_lines = 1;
2022 else
2023 latency_tolerant_lines = 2;
2024 }
2025
2026 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2027
2028 if (dce6_latency_watermark(wm) <= latency_hiding)
2029 return true;
2030 else
2031 return false;
2032}
2033
2034static void dce6_program_watermarks(struct radeon_device *rdev,
2035 struct radeon_crtc *radeon_crtc,
2036 u32 lb_size, u32 num_heads)
2037{
2038 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 2039 struct dce6_wm_params wm_low, wm_high;
2040 u32 dram_channels;
43b3cd99
AD		 2041 u32 pixel_period;
2042 u32 line_time = 0;
2043 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2044 u32 priority_a_mark = 0, priority_b_mark = 0;
2045 u32 priority_a_cnt = PRIORITY_OFF;
2046 u32 priority_b_cnt = PRIORITY_OFF;
2047 u32 tmp, arb_control3;
2048 fixed20_12 a, b, c;
2049
2050 if (radeon_crtc->base.enabled && num_heads && mode) {
2051 pixel_period = 1000000 / (u32)mode->clock;
2052 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2053 priority_a_cnt = 0;
2054 priority_b_cnt = 0;
2055
ca7db22b 2056 if (rdev->family == CHIP_ARUBA)
c696e53f 2057 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 2058 else
c696e53f
AD		 2059 dram_channels = si_get_number_of_dram_channels(rdev);
2060
2061 /* watermark for high clocks */
2062 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2063 wm_high.yclk =
2064 radeon_dpm_get_mclk(rdev, false) * 10;
2065 wm_high.sclk =
2066 radeon_dpm_get_sclk(rdev, false) * 10;
2067 } else {
2068 wm_high.yclk = rdev->pm.current_mclk * 10;
2069 wm_high.sclk = rdev->pm.current_sclk * 10;
2070 }
2071
2072 wm_high.disp_clk = mode->clock;
2073 wm_high.src_width = mode->crtc_hdisplay;
2074 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2075 wm_high.blank_time = line_time - wm_high.active_time;
2076 wm_high.interlaced = false;
2077 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2078 wm_high.interlaced = true;
2079 wm_high.vsc = radeon_crtc->vsc;
2080 wm_high.vtaps = 1;
2081 if (radeon_crtc->rmx_type != RMX_OFF)
2082 wm_high.vtaps = 2;
2083 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2084 wm_high.lb_size = lb_size;
2085 wm_high.dram_channels = dram_channels;
2086 wm_high.num_heads = num_heads;
2087
2088 /* watermark for low clocks */
2089 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2090 wm_low.yclk =
2091 radeon_dpm_get_mclk(rdev, true) * 10;
2092 wm_low.sclk =
2093 radeon_dpm_get_sclk(rdev, true) * 10;
2094 } else {
2095 wm_low.yclk = rdev->pm.current_mclk * 10;
2096 wm_low.sclk = rdev->pm.current_sclk * 10;
2097 }
2098
2099 wm_low.disp_clk = mode->clock;
2100 wm_low.src_width = mode->crtc_hdisplay;
2101 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2102 wm_low.blank_time = line_time - wm_low.active_time;
2103 wm_low.interlaced = false;
2104 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2105 wm_low.interlaced = true;
2106 wm_low.vsc = radeon_crtc->vsc;
2107 wm_low.vtaps = 1;
2108 if (radeon_crtc->rmx_type != RMX_OFF)
2109 wm_low.vtaps = 2;
2110 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2111 wm_low.lb_size = lb_size;
2112 wm_low.dram_channels = dram_channels;
2113 wm_low.num_heads = num_heads;
43b3cd99
AD		 2114
2115 /* set for high clocks */
c696e53f 2116 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 2117 /* set for low clocks */
c696e53f 2118 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 2119
2120 /* possibly force display priority to high */
2121 /* should really do this at mode validation time... */
c696e53f
AD		 2122 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2123 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2124 !dce6_check_latency_hiding(&wm_high) ||
2125 (rdev->disp_priority == 2)) {
2126 DRM_DEBUG_KMS("force priority to high\n");
2127 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2128 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2129 }
2130 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2131 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2132 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 2133 (rdev->disp_priority == 2)) {
2134 DRM_DEBUG_KMS("force priority to high\n");
2135 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2136 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2137 }
2138
2139 a.full = dfixed_const(1000);
2140 b.full = dfixed_const(mode->clock);
2141 b.full = dfixed_div(b, a);
2142 c.full = dfixed_const(latency_watermark_a);
2143 c.full = dfixed_mul(c, b);
2144 c.full = dfixed_mul(c, radeon_crtc->hsc);
2145 c.full = dfixed_div(c, a);
2146 a.full = dfixed_const(16);
2147 c.full = dfixed_div(c, a);
2148 priority_a_mark = dfixed_trunc(c);
2149 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2150
2151 a.full = dfixed_const(1000);
2152 b.full = dfixed_const(mode->clock);
2153 b.full = dfixed_div(b, a);
2154 c.full = dfixed_const(latency_watermark_b);
2155 c.full = dfixed_mul(c, b);
2156 c.full = dfixed_mul(c, radeon_crtc->hsc);
2157 c.full = dfixed_div(c, a);
2158 a.full = dfixed_const(16);
2159 c.full = dfixed_div(c, a);
2160 priority_b_mark = dfixed_trunc(c);
2161 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2162 }
2163
2164 /* select wm A */
2165 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2166 tmp = arb_control3;
2167 tmp &= ~LATENCY_WATERMARK_MASK(3);
2168 tmp |= LATENCY_WATERMARK_MASK(1);
2169 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2170 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2171 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2172 LATENCY_HIGH_WATERMARK(line_time)));
2173 /* select wm B */
2174 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2175 tmp &= ~LATENCY_WATERMARK_MASK(3);
2176 tmp |= LATENCY_WATERMARK_MASK(2);
2177 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2178 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2179 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2180 LATENCY_HIGH_WATERMARK(line_time)));
2181 /* restore original selection */
2182 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2183
2184 /* write the priority marks */
2185 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2186 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2187
7178d2a6
AD		 2188 /* save values for DPM */
2189 radeon_crtc->line_time = line_time;
2190 radeon_crtc->wm_high = latency_watermark_a;
2191 radeon_crtc->wm_low = latency_watermark_b;
43b3cd99
AD		 2192}
2193
2194void dce6_bandwidth_update(struct radeon_device *rdev)
2195{
2196 struct drm_display_mode *mode0 = NULL;
2197 struct drm_display_mode *mode1 = NULL;
2198 u32 num_heads = 0, lb_size;
2199 int i;
2200
2201 radeon_update_display_priority(rdev);
2202
2203 for (i = 0; i < rdev->num_crtc; i++) {
2204 if (rdev->mode_info.crtcs[i]->base.enabled)
2205 num_heads++;
2206 }
2207 for (i = 0; i < rdev->num_crtc; i += 2) {
2208 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2209 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2210 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2211 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2212 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2213 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2214 }
2215}
2216
0a96d72b
AD		 2217/*
2218 * Core functions
2219 */
0a96d72b
AD		 2220static void si_tiling_mode_table_init(struct radeon_device *rdev)
2221{
2222 const u32 num_tile_mode_states = 32;
2223 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2224
2225 switch (rdev->config.si.mem_row_size_in_kb) {
2226 case 1:
2227 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2228 break;
2229 case 2:
2230 default:
2231 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2232 break;
2233 case 4:
2234 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2235 break;
2236 }
2237
2238 if ((rdev->family == CHIP_TAHITI) ||
2239 (rdev->family == CHIP_PITCAIRN)) {
2240 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2241 switch (reg_offset) {
2242 case 0: /* non-AA compressed depth or any compressed stencil */
2243 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2244 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2245 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2246 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2247 NUM_BANKS(ADDR_SURF_16_BANK) |
2248 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2249 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2250 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2251 break;
2252 case 1: /* 2xAA/4xAA compressed depth only */
2253 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2254 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2255 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2256 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2257 NUM_BANKS(ADDR_SURF_16_BANK) |
2258 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2259 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2260 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2261 break;
2262 case 2: /* 8xAA compressed depth only */
2263 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2264 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2265 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2266 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2267 NUM_BANKS(ADDR_SURF_16_BANK) |
2268 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2269 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2270 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2271 break;
2272 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2273 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2274 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2275 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2276 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2277 NUM_BANKS(ADDR_SURF_16_BANK) |
2278 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2279 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2280 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2281 break;
2282 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2283 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2284 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2285 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2286 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2287 NUM_BANKS(ADDR_SURF_16_BANK) |
2288 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2289 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2290 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2291 break;
2292 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2293 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2294 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2295 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2296 TILE_SPLIT(split_equal_to_row_size) |
2297 NUM_BANKS(ADDR_SURF_16_BANK) |
2298 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2299 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2300 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2301 break;
2302 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2303 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2304 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2305 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2306 TILE_SPLIT(split_equal_to_row_size) |
2307 NUM_BANKS(ADDR_SURF_16_BANK) |
2308 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2309 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2310 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2311 break;
2312 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2313 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2314 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2315 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2316 TILE_SPLIT(split_equal_to_row_size) |
2317 NUM_BANKS(ADDR_SURF_16_BANK) |
2318 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2319 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2320 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2321 break;
2322 case 8: /* 1D and 1D Array Surfaces */
2323 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2324 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2325 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2326 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2327 NUM_BANKS(ADDR_SURF_16_BANK) |
2328 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2329 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2330 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2331 break;
2332 case 9: /* Displayable maps. */
2333 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2334 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2335 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2336 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2337 NUM_BANKS(ADDR_SURF_16_BANK) |
2338 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2339 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2340 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2341 break;
2342 case 10: /* Display 8bpp. */
2343 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2344 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2345 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2346 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2347 NUM_BANKS(ADDR_SURF_16_BANK) |
2348 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2349 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2350 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2351 break;
2352 case 11: /* Display 16bpp. */
2353 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2354 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2355 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2356 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2357 NUM_BANKS(ADDR_SURF_16_BANK) |
2358 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2359 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2360 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2361 break;
2362 case 12: /* Display 32bpp. */
2363 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2364 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2365 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2366 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2367 NUM_BANKS(ADDR_SURF_16_BANK) |
2368 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2369 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2370 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2371 break;
2372 case 13: /* Thin. */
2373 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2374 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2375 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2376 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2377 NUM_BANKS(ADDR_SURF_16_BANK) |
2378 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2379 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2380 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2381 break;
2382 case 14: /* Thin 8 bpp. */
2383 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2384 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2385 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2386 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2387 NUM_BANKS(ADDR_SURF_16_BANK) |
2388 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2389 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2390 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2391 break;
2392 case 15: /* Thin 16 bpp. */
2393 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2394 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2395 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2396 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2397 NUM_BANKS(ADDR_SURF_16_BANK) |
2398 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2399 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2400 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2401 break;
2402 case 16: /* Thin 32 bpp. */
2403 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2404 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2405 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2406 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2407 NUM_BANKS(ADDR_SURF_16_BANK) |
2408 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2409 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2410 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2411 break;
2412 case 17: /* Thin 64 bpp. */
2413 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2414 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2415 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2416 TILE_SPLIT(split_equal_to_row_size) |
2417 NUM_BANKS(ADDR_SURF_16_BANK) |
2418 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2419 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2420 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2421 break;
2422 case 21: /* 8 bpp PRT. */
2423 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2424 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2425 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2426 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2427 NUM_BANKS(ADDR_SURF_16_BANK) |
2428 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2429 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2430 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2431 break;
2432 case 22: /* 16 bpp PRT */
2433 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2434 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2435 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2436 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2437 NUM_BANKS(ADDR_SURF_16_BANK) |
2438 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2439 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2440 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2441 break;
2442 case 23: /* 32 bpp PRT */
2443 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2444 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2445 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2446 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2447 NUM_BANKS(ADDR_SURF_16_BANK) |
2448 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2449 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2450 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2451 break;
2452 case 24: /* 64 bpp PRT */
2453 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2454 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2455 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2456 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2457 NUM_BANKS(ADDR_SURF_16_BANK) |
2458 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2459 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2460 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2461 break;
2462 case 25: /* 128 bpp PRT */
2463 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2464 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2465 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2466 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2467 NUM_BANKS(ADDR_SURF_8_BANK) |
2468 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2469 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2470 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2471 break;
2472 default:
2473 gb_tile_moden = 0;
2474 break;
2475 }
64d7b8be 2476 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2477 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2478 }
d0ae7fcc 2479 } else if ((rdev->family == CHIP_VERDE) ||
8b02859d
AD		 2480 (rdev->family == CHIP_OLAND) ||
2481 (rdev->family == CHIP_HAINAN)) {
0a96d72b
AD		 2482 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2483 switch (reg_offset) {
2484 case 0: /* non-AA compressed depth or any compressed stencil */
2485 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2486 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2487 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2488 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2489 NUM_BANKS(ADDR_SURF_16_BANK) |
2490 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2491 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2492 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2493 break;
2494 case 1: /* 2xAA/4xAA compressed depth only */
2495 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2496 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2497 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2498 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2499 NUM_BANKS(ADDR_SURF_16_BANK) |
2500 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2501 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2502 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2503 break;
2504 case 2: /* 8xAA compressed depth only */
2505 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2506 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2507 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2508 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2509 NUM_BANKS(ADDR_SURF_16_BANK) |
2510 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2511 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2512 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2513 break;
2514 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2515 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2516 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2517 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2518 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2519 NUM_BANKS(ADDR_SURF_16_BANK) |
2520 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2521 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2522 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2523 break;
2524 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2526 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2528 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2529 NUM_BANKS(ADDR_SURF_16_BANK) |
2530 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2531 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2532 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2533 break;
2534 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2535 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2536 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2537 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2538 TILE_SPLIT(split_equal_to_row_size) |
2539 NUM_BANKS(ADDR_SURF_16_BANK) |
2540 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2541 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2542 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2543 break;
2544 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2545 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2546 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2547 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2548 TILE_SPLIT(split_equal_to_row_size) |
2549 NUM_BANKS(ADDR_SURF_16_BANK) |
2550 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2551 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2552 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2553 break;
2554 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2555 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2556 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2557 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2558 TILE_SPLIT(split_equal_to_row_size) |
2559 NUM_BANKS(ADDR_SURF_16_BANK) |
2560 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2561 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2562 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2563 break;
2564 case 8: /* 1D and 1D Array Surfaces */
2565 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2566 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2567 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2568 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2569 NUM_BANKS(ADDR_SURF_16_BANK) |
2570 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2571 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2572 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2573 break;
2574 case 9: /* Displayable maps. */
2575 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2576 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2577 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2578 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2579 NUM_BANKS(ADDR_SURF_16_BANK) |
2580 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2581 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2582 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2583 break;
2584 case 10: /* Display 8bpp. */
2585 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2586 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2587 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2588 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2589 NUM_BANKS(ADDR_SURF_16_BANK) |
2590 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2591 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2592 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2593 break;
2594 case 11: /* Display 16bpp. */
2595 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2596 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2597 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2598 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2599 NUM_BANKS(ADDR_SURF_16_BANK) |
2600 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2601 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2602 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2603 break;
2604 case 12: /* Display 32bpp. */
2605 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2606 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2607 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2608 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2609 NUM_BANKS(ADDR_SURF_16_BANK) |
2610 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2611 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2612 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2613 break;
2614 case 13: /* Thin. */
2615 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2616 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2617 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2618 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2619 NUM_BANKS(ADDR_SURF_16_BANK) |
2620 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2621 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2622 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2623 break;
2624 case 14: /* Thin 8 bpp. */
2625 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2626 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2627 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2628 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2629 NUM_BANKS(ADDR_SURF_16_BANK) |
2630 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2631 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2632 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2633 break;
2634 case 15: /* Thin 16 bpp. */
2635 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2636 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2637 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2638 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2639 NUM_BANKS(ADDR_SURF_16_BANK) |
2640 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2641 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2642 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2643 break;
2644 case 16: /* Thin 32 bpp. */
2645 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2646 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2647 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2648 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2649 NUM_BANKS(ADDR_SURF_16_BANK) |
2650 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2651 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2652 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2653 break;
2654 case 17: /* Thin 64 bpp. */
2655 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2656 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2657 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2658 TILE_SPLIT(split_equal_to_row_size) |
2659 NUM_BANKS(ADDR_SURF_16_BANK) |
2660 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2661 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2662 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2663 break;
2664 case 21: /* 8 bpp PRT. */
2665 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2666 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2667 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2668 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2669 NUM_BANKS(ADDR_SURF_16_BANK) |
2670 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2671 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2672 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2673 break;
2674 case 22: /* 16 bpp PRT */
2675 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2676 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2677 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2678 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2679 NUM_BANKS(ADDR_SURF_16_BANK) |
2680 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2681 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2682 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2683 break;
2684 case 23: /* 32 bpp PRT */
2685 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2686 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2687 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2688 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2689 NUM_BANKS(ADDR_SURF_16_BANK) |
2690 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2693 break;
2694 case 24: /* 64 bpp PRT */
2695 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2696 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2697 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2698 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2699 NUM_BANKS(ADDR_SURF_16_BANK) |
2700 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2701 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2702 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2703 break;
2704 case 25: /* 128 bpp PRT */
2705 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2706 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2707 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2708 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2709 NUM_BANKS(ADDR_SURF_8_BANK) |
2710 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2711 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2712 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2713 break;
2714 default:
2715 gb_tile_moden = 0;
2716 break;
2717 }
64d7b8be 2718 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2719 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2720 }
2721 } else
2722 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2723}
2724
1a8ca750
AD		 2725static void si_select_se_sh(struct radeon_device *rdev,
2726 u32 se_num, u32 sh_num)
2727{
2728 u32 data = INSTANCE_BROADCAST_WRITES;
2729
2730 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2731 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2732 else if (se_num == 0xffffffff)
2733 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2734 else if (sh_num == 0xffffffff)
2735 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2736 else
2737 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2738 WREG32(GRBM_GFX_INDEX, data);
2739}
2740
2741static u32 si_create_bitmask(u32 bit_width)
2742{
2743 u32 i, mask = 0;
2744
2745 for (i = 0; i < bit_width; i++) {
2746 mask <<= 1;
2747 mask |= 1;
2748 }
2749 return mask;
2750}
2751
2752static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2753{
2754 u32 data, mask;
2755
2756 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2757 if (data & 1)
2758 data &= INACTIVE_CUS_MASK;
2759 else
2760 data = 0;
2761 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2762
2763 data >>= INACTIVE_CUS_SHIFT;
2764
2765 mask = si_create_bitmask(cu_per_sh);
2766
2767 return ~data & mask;
2768}
2769
2770static void si_setup_spi(struct radeon_device *rdev,
2771 u32 se_num, u32 sh_per_se,
2772 u32 cu_per_sh)
2773{
2774 int i, j, k;
2775 u32 data, mask, active_cu;
2776
2777 for (i = 0; i < se_num; i++) {
2778 for (j = 0; j < sh_per_se; j++) {
2779 si_select_se_sh(rdev, i, j);
2780 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2781 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2782
2783 mask = 1;
2784 for (k = 0; k < 16; k++) {
2785 mask <<= k;
2786 if (active_cu & mask) {
2787 data &= ~mask;
2788 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2789 break;
2790 }
2791 }
2792 }
2793 }
2794 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2795}
2796
2797static u32 si_get_rb_disabled(struct radeon_device *rdev,
2798 u32 max_rb_num, u32 se_num,
2799 u32 sh_per_se)
2800{
2801 u32 data, mask;
2802
2803 data = RREG32(CC_RB_BACKEND_DISABLE);
2804 if (data & 1)
2805 data &= BACKEND_DISABLE_MASK;
2806 else
2807 data = 0;
2808 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2809
2810 data >>= BACKEND_DISABLE_SHIFT;
2811
2812 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2813
2814 return data & mask;
2815}
2816
2817static void si_setup_rb(struct radeon_device *rdev,
2818 u32 se_num, u32 sh_per_se,
2819 u32 max_rb_num)
2820{
2821 int i, j;
2822 u32 data, mask;
2823 u32 disabled_rbs = 0;
2824 u32 enabled_rbs = 0;
2825
2826 for (i = 0; i < se_num; i++) {
2827 for (j = 0; j < sh_per_se; j++) {
2828 si_select_se_sh(rdev, i, j);
2829 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2830 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2831 }
2832 }
2833 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2834
2835 mask = 1;
2836 for (i = 0; i < max_rb_num; i++) {
2837 if (!(disabled_rbs & mask))
2838 enabled_rbs |= mask;
2839 mask <<= 1;
2840 }
2841
2842 for (i = 0; i < se_num; i++) {
2843 si_select_se_sh(rdev, i, 0xffffffff);
2844 data = 0;
2845 for (j = 0; j < sh_per_se; j++) {
2846 switch (enabled_rbs & 3) {
2847 case 1:
2848 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2849 break;
2850 case 2:
2851 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2852 break;
2853 case 3:
2854 default:
2855 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2856 break;
2857 }
2858 enabled_rbs >>= 2;
2859 }
2860 WREG32(PA_SC_RASTER_CONFIG, data);
2861 }
2862 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2863}
2864
0a96d72b
AD		 2865static void si_gpu_init(struct radeon_device *rdev)
2866{
0a96d72b
AD		 2867 u32 gb_addr_config = 0;
2868 u32 mc_shared_chmap, mc_arb_ramcfg;
0a96d72b 2869 u32 sx_debug_1;
0a96d72b
AD		 2870 u32 hdp_host_path_cntl;
2871 u32 tmp;
2872 int i, j;
2873
2874 switch (rdev->family) {
2875 case CHIP_TAHITI:
2876 rdev->config.si.max_shader_engines = 2;
0a96d72b 2877 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 2878 rdev->config.si.max_cu_per_sh = 8;
2879 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2880 rdev->config.si.max_backends_per_se = 4;
2881 rdev->config.si.max_texture_channel_caches = 12;
2882 rdev->config.si.max_gprs = 256;
2883 rdev->config.si.max_gs_threads = 32;
2884 rdev->config.si.max_hw_contexts = 8;
2885
2886 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2887 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2888 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2889 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2890 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2891 break;
2892 case CHIP_PITCAIRN:
2893 rdev->config.si.max_shader_engines = 2;
0a96d72b 2894 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 2895 rdev->config.si.max_cu_per_sh = 5;
2896 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2897 rdev->config.si.max_backends_per_se = 4;
2898 rdev->config.si.max_texture_channel_caches = 8;
2899 rdev->config.si.max_gprs = 256;
2900 rdev->config.si.max_gs_threads = 32;
2901 rdev->config.si.max_hw_contexts = 8;
2902
2903 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2904 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2905 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2906 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2907 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2908 break;
2909 case CHIP_VERDE:
2910 default:
2911 rdev->config.si.max_shader_engines = 1;
0a96d72b 2912 rdev->config.si.max_tile_pipes = 4;
468ef1a5 2913 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 2914 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2915 rdev->config.si.max_backends_per_se = 4;
2916 rdev->config.si.max_texture_channel_caches = 4;
2917 rdev->config.si.max_gprs = 256;
2918 rdev->config.si.max_gs_threads = 32;
2919 rdev->config.si.max_hw_contexts = 8;
2920
d0ae7fcc
AD		 2921 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2922 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2923 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2924 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2925 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2926 break;
2927 case CHIP_OLAND:
2928 rdev->config.si.max_shader_engines = 1;
2929 rdev->config.si.max_tile_pipes = 4;
2930 rdev->config.si.max_cu_per_sh = 6;
2931 rdev->config.si.max_sh_per_se = 1;
2932 rdev->config.si.max_backends_per_se = 2;
2933 rdev->config.si.max_texture_channel_caches = 4;
2934 rdev->config.si.max_gprs = 256;
2935 rdev->config.si.max_gs_threads = 16;
2936 rdev->config.si.max_hw_contexts = 8;
2937
0a96d72b
AD		 2938 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2939 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2940 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2941 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2942 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 2943 break;
8b02859d
AD		 2944 case CHIP_HAINAN:
2945 rdev->config.si.max_shader_engines = 1;
2946 rdev->config.si.max_tile_pipes = 4;
2947 rdev->config.si.max_cu_per_sh = 5;
2948 rdev->config.si.max_sh_per_se = 1;
2949 rdev->config.si.max_backends_per_se = 1;
2950 rdev->config.si.max_texture_channel_caches = 2;
2951 rdev->config.si.max_gprs = 256;
2952 rdev->config.si.max_gs_threads = 16;
2953 rdev->config.si.max_hw_contexts = 8;
2954
2955 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2956 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2957 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2958 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2959 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2960 break;
0a96d72b
AD		 2961 }
2962
2963 /* Initialize HDP */
2964 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2965 WREG32((0x2c14 + j), 0x00000000);
2966 WREG32((0x2c18 + j), 0x00000000);
2967 WREG32((0x2c1c + j), 0x00000000);
2968 WREG32((0x2c20 + j), 0x00000000);
2969 WREG32((0x2c24 + j), 0x00000000);
2970 }
2971
2972 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2973
2974 evergreen_fix_pci_max_read_req_size(rdev);
2975
2976 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2977
2978 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2979 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2980
0a96d72b 2981 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 2982 rdev->config.si.mem_max_burst_length_bytes = 256;
2983 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2984 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2985 if (rdev->config.si.mem_row_size_in_kb > 4)
2986 rdev->config.si.mem_row_size_in_kb = 4;
2987 /* XXX use MC settings? */
2988 rdev->config.si.shader_engine_tile_size = 32;
2989 rdev->config.si.num_gpus = 1;
2990 rdev->config.si.multi_gpu_tile_size = 64;
2991
1a8ca750
AD		 2992 /* fix up row size */
2993 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 2994 switch (rdev->config.si.mem_row_size_in_kb) {
2995 case 1:
2996 default:
2997 gb_addr_config |= ROW_SIZE(0);
2998 break;
2999 case 2:
3000 gb_addr_config |= ROW_SIZE(1);
3001 break;
3002 case 4:
3003 gb_addr_config |= ROW_SIZE(2);
3004 break;
3005 }
3006
0a96d72b
AD		 3007 /* setup tiling info dword. gb_addr_config is not adequate since it does
3008 * not have bank info, so create a custom tiling dword.
3009 * bits 3:0 num_pipes
3010 * bits 7:4 num_banks
3011 * bits 11:8 group_size
3012 * bits 15:12 row_size
3013 */
3014 rdev->config.si.tile_config = 0;
3015 switch (rdev->config.si.num_tile_pipes) {
3016 case 1:
3017 rdev->config.si.tile_config |= (0 << 0);
3018 break;
3019 case 2:
3020 rdev->config.si.tile_config |= (1 << 0);
3021 break;
3022 case 4:
3023 rdev->config.si.tile_config |= (2 << 0);
3024 break;
3025 case 8:
3026 default:
3027 /* XXX what about 12? */
3028 rdev->config.si.tile_config |= (3 << 0);
3029 break;
dca571a6
CK		 3030 }
3031 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3032 case 0: /* four banks */
1a8ca750 3033 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 3034 break;
3035 case 1: /* eight banks */
3036 rdev->config.si.tile_config |= 1 << 4;
3037 break;
3038 case 2: /* sixteen banks */
3039 default:
3040 rdev->config.si.tile_config |= 2 << 4;
3041 break;
3042 }
0a96d72b
AD		 3043 rdev->config.si.tile_config |=
3044 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3045 rdev->config.si.tile_config |=
3046 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3047
0a96d72b
AD		 3048 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3049 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 3050 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 3051 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 3052 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3053 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 3054 if (rdev->has_uvd) {
3055 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3056 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3057 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3058 }
0a96d72b 3059
1a8ca750 3060 si_tiling_mode_table_init(rdev);
0a96d72b 3061
1a8ca750
AD		 3062 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3063 rdev->config.si.max_sh_per_se,
3064 rdev->config.si.max_backends_per_se);
0a96d72b 3065
1a8ca750
AD		 3066 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3067 rdev->config.si.max_sh_per_se,
3068 rdev->config.si.max_cu_per_sh);
0a96d72b 3069
0a96d72b
AD		 3070
3071 /* set HW defaults for 3D engine */
3072 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3073 ROQ_IB2_START(0x2b)));
3074 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3075
3076 sx_debug_1 = RREG32(SX_DEBUG_1);
3077 WREG32(SX_DEBUG_1, sx_debug_1);
3078
3079 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3080
3081 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3082 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3083 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3084 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3085
3086 WREG32(VGT_NUM_INSTANCES, 1);
3087
3088 WREG32(CP_PERFMON_CNTL, 0);
3089
3090 WREG32(SQ_CONFIG, 0);
3091
3092 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3093 FORCE_EOV_MAX_REZ_CNT(255)));
3094
3095 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3096 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3097
3098 WREG32(VGT_GS_VERTEX_REUSE, 16);
3099 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3100
3101 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3102 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3103 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3104 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3105 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3106 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3107 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3108 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3109
3110 tmp = RREG32(HDP_MISC_CNTL);
3111 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3112 WREG32(HDP_MISC_CNTL, tmp);
3113
3114 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3115 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3116
3117 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3118
3119 udelay(50);
3120}
c476dde2 3121
2ece2e8b
AD		 3122/*
3123 * GPU scratch registers helpers function.
3124 */
3125static void si_scratch_init(struct radeon_device *rdev)
3126{
3127 int i;
3128
3129 rdev->scratch.num_reg = 7;
3130 rdev->scratch.reg_base = SCRATCH_REG0;
3131 for (i = 0; i < rdev->scratch.num_reg; i++) {
3132 rdev->scratch.free[i] = true;
3133 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3134 }
3135}
3136
3137void si_fence_ring_emit(struct radeon_device *rdev,
3138 struct radeon_fence *fence)
3139{
3140 struct radeon_ring *ring = &rdev->ring[fence->ring];
3141 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3142
3143 /* flush read cache over gart */
3144 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3145 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3146 radeon_ring_write(ring, 0);
3147 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3148 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3149 PACKET3_TC_ACTION_ENA |
3150 PACKET3_SH_KCACHE_ACTION_ENA |
3151 PACKET3_SH_ICACHE_ACTION_ENA);
3152 radeon_ring_write(ring, 0xFFFFFFFF);
3153 radeon_ring_write(ring, 0);
3154 radeon_ring_write(ring, 10); /* poll interval */
3155 /* EVENT_WRITE_EOP - flush caches, send int */
3156 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3157 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3158 radeon_ring_write(ring, addr & 0xffffffff);
3159 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3160 radeon_ring_write(ring, fence->seq);
3161 radeon_ring_write(ring, 0);
3162}
3163
3164/*
3165 * IB stuff
3166 */
3167void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3168{
876dc9f3 3169 struct radeon_ring *ring = &rdev->ring[ib->ring];
2ece2e8b
AD		 3170 u32 header;
3171
a85a7da4
AD		 3172 if (ib->is_const_ib) {
3173 /* set switch buffer packet before const IB */
3174 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3175 radeon_ring_write(ring, 0);
45df6803 3176
2ece2e8b 3177 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 3178 } else {
89d35807 3179 u32 next_rptr;
a85a7da4 3180 if (ring->rptr_save_reg) {
89d35807 3181 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 3182 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3183 radeon_ring_write(ring, ((ring->rptr_save_reg -
3184 PACKET3_SET_CONFIG_REG_START) >> 2));
3185 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3186 } else if (rdev->wb.enabled) {
3187 next_rptr = ring->wptr + 5 + 4 + 8;
3188 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3189 radeon_ring_write(ring, (1 << 8));
3190 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3191 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3192 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 3193 }
3194
2ece2e8b 3195 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 3196 }
2ece2e8b
AD		 3197
3198 radeon_ring_write(ring, header);
3199 radeon_ring_write(ring,
3200#ifdef __BIG_ENDIAN
3201 (2 << 0) |
3202#endif
3203 (ib->gpu_addr & 0xFFFFFFFC));
3204 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4bf3dd92
CK		 3205 radeon_ring_write(ring, ib->length_dw |
3206 (ib->vm ? (ib->vm->id << 24) : 0));
2ece2e8b 3207
a85a7da4
AD		 3208 if (!ib->is_const_ib) {
3209 /* flush read cache over gart for this vmid */
3210 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3211 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 3212 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
a85a7da4
AD		 3213 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3214 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3215 PACKET3_TC_ACTION_ENA |
3216 PACKET3_SH_KCACHE_ACTION_ENA |
3217 PACKET3_SH_ICACHE_ACTION_ENA);
3218 radeon_ring_write(ring, 0xFFFFFFFF);
3219 radeon_ring_write(ring, 0);
3220 radeon_ring_write(ring, 10); /* poll interval */
3221 }
2ece2e8b
AD		 3222}
3223
48c0c902
AD		 3224/*
3225 * CP.
3226 */
3227static void si_cp_enable(struct radeon_device *rdev, bool enable)
3228{
3229 if (enable)
3230 WREG32(CP_ME_CNTL, 0);
3231 else {
3232 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3233 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3234 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 3235 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3236 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3237 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 3238 }
3239 udelay(50);
3240}
3241
3242static int si_cp_load_microcode(struct radeon_device *rdev)
3243{
3244 const __be32 *fw_data;
3245 int i;
3246
3247 if (!rdev->me_fw || !rdev->pfp_fw)
3248 return -EINVAL;
3249
3250 si_cp_enable(rdev, false);
3251
3252 /* PFP */
3253 fw_data = (const __be32 *)rdev->pfp_fw->data;
3254 WREG32(CP_PFP_UCODE_ADDR, 0);
3255 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3256 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3257 WREG32(CP_PFP_UCODE_ADDR, 0);
3258
3259 /* CE */
3260 fw_data = (const __be32 *)rdev->ce_fw->data;
3261 WREG32(CP_CE_UCODE_ADDR, 0);
3262 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3263 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3264 WREG32(CP_CE_UCODE_ADDR, 0);
3265
3266 /* ME */
3267 fw_data = (const __be32 *)rdev->me_fw->data;
3268 WREG32(CP_ME_RAM_WADDR, 0);
3269 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3270 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3271 WREG32(CP_ME_RAM_WADDR, 0);
3272
3273 WREG32(CP_PFP_UCODE_ADDR, 0);
3274 WREG32(CP_CE_UCODE_ADDR, 0);
3275 WREG32(CP_ME_RAM_WADDR, 0);
3276 WREG32(CP_ME_RAM_RADDR, 0);
3277 return 0;
3278}
3279
3280static int si_cp_start(struct radeon_device *rdev)
3281{
3282 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3283 int r, i;
3284
3285 r = radeon_ring_lock(rdev, ring, 7 + 4);
3286 if (r) {
3287 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3288 return r;
3289 }
3290 /* init the CP */
3291 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3292 radeon_ring_write(ring, 0x1);
3293 radeon_ring_write(ring, 0x0);
3294 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3295 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3296 radeon_ring_write(ring, 0);
3297 radeon_ring_write(ring, 0);
3298
3299 /* init the CE partitions */
3300 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3301 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3302 radeon_ring_write(ring, 0xc000);
3303 radeon_ring_write(ring, 0xe000);
3304 radeon_ring_unlock_commit(rdev, ring);
3305
3306 si_cp_enable(rdev, true);
3307
3308 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3309 if (r) {
3310 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3311 return r;
3312 }
3313
3314 /* setup clear context state */
3315 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3316 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3317
3318 for (i = 0; i < si_default_size; i++)
3319 radeon_ring_write(ring, si_default_state[i]);
3320
3321 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3322 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3323
3324 /* set clear context state */
3325 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3326 radeon_ring_write(ring, 0);
3327
3328 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3329 radeon_ring_write(ring, 0x00000316);
3330 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3331 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3332
3333 radeon_ring_unlock_commit(rdev, ring);
3334
3335 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3336 ring = &rdev->ring[i];
3337 r = radeon_ring_lock(rdev, ring, 2);
3338
3339 /* clear the compute context state */
3340 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3341 radeon_ring_write(ring, 0);
3342
3343 radeon_ring_unlock_commit(rdev, ring);
3344 }
3345
3346 return 0;
3347}
3348
3349static void si_cp_fini(struct radeon_device *rdev)
3350{
45df6803 3351 struct radeon_ring *ring;
48c0c902 3352 si_cp_enable(rdev, false);
45df6803
CK		 3353
3354 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3355 radeon_ring_fini(rdev, ring);
3356 radeon_scratch_free(rdev, ring->rptr_save_reg);
3357
3358 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3359 radeon_ring_fini(rdev, ring);
3360 radeon_scratch_free(rdev, ring->rptr_save_reg);
3361
3362 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3363 radeon_ring_fini(rdev, ring);
3364 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3365}
3366
3367static int si_cp_resume(struct radeon_device *rdev)
3368{
3369 struct radeon_ring *ring;
3370 u32 tmp;
3371 u32 rb_bufsz;
3372 int r;
3373
48c0c902
AD		 3374 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3375 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3376
3377 /* Set the write pointer delay */
3378 WREG32(CP_RB_WPTR_DELAY, 0);
3379
3380 WREG32(CP_DEBUG, 0);
3381 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3382
3383 /* ring 0 - compute and gfx */
3384 /* Set ring buffer size */
3385 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3386 rb_bufsz = drm_order(ring->ring_size / 8);
3387 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3388#ifdef __BIG_ENDIAN
3389 tmp |= BUF_SWAP_32BIT;
3390#endif
3391 WREG32(CP_RB0_CNTL, tmp);
3392
3393 /* Initialize the ring buffer's read and write pointers */
3394 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3395 ring->wptr = 0;
3396 WREG32(CP_RB0_WPTR, ring->wptr);
3397
48fc7f7e 3398 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3399 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3400 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3401
3402 if (rdev->wb.enabled)
3403 WREG32(SCRATCH_UMSK, 0xff);
3404 else {
3405 tmp |= RB_NO_UPDATE;
3406 WREG32(SCRATCH_UMSK, 0);
3407 }
3408
3409 mdelay(1);
3410 WREG32(CP_RB0_CNTL, tmp);
3411
3412 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3413
3414 ring->rptr = RREG32(CP_RB0_RPTR);
3415
3416 /* ring1 - compute only */
3417 /* Set ring buffer size */
3418 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3419 rb_bufsz = drm_order(ring->ring_size / 8);
3420 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3421#ifdef __BIG_ENDIAN
3422 tmp |= BUF_SWAP_32BIT;
3423#endif
3424 WREG32(CP_RB1_CNTL, tmp);
3425
3426 /* Initialize the ring buffer's read and write pointers */
3427 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3428 ring->wptr = 0;
3429 WREG32(CP_RB1_WPTR, ring->wptr);
3430
48fc7f7e 3431 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3432 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3433 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3434
3435 mdelay(1);
3436 WREG32(CP_RB1_CNTL, tmp);
3437
3438 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3439
3440 ring->rptr = RREG32(CP_RB1_RPTR);
3441
3442 /* ring2 - compute only */
3443 /* Set ring buffer size */
3444 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3445 rb_bufsz = drm_order(ring->ring_size / 8);
3446 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3447#ifdef __BIG_ENDIAN
3448 tmp |= BUF_SWAP_32BIT;
3449#endif
3450 WREG32(CP_RB2_CNTL, tmp);
3451
3452 /* Initialize the ring buffer's read and write pointers */
3453 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3454 ring->wptr = 0;
3455 WREG32(CP_RB2_WPTR, ring->wptr);
3456
48fc7f7e 3457 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3458 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3459 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3460
3461 mdelay(1);
3462 WREG32(CP_RB2_CNTL, tmp);
3463
3464 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3465
3466 ring->rptr = RREG32(CP_RB2_RPTR);
3467
3468 /* start the rings */
3469 si_cp_start(rdev);
3470 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3471 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3472 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3473 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3474 if (r) {
3475 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3476 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3477 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3478 return r;
3479 }
3480 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3481 if (r) {
3482 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3483 }
3484 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3485 if (r) {
3486 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3487 }
3488
3489 return 0;
3490}
3491
2483b4ea 3492u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3493{
014bb209 3494 u32 reset_mask = 0;
1c534671 3495 u32 tmp;
06bc6df0 3496
014bb209
AD		 3497 /* GRBM_STATUS */
3498 tmp = RREG32(GRBM_STATUS);
3499 if (tmp & (PA_BUSY | SC_BUSY |
3500 BCI_BUSY | SX_BUSY |
3501 TA_BUSY | VGT_BUSY |
3502 DB_BUSY | CB_BUSY |
3503 GDS_BUSY | SPI_BUSY |
3504 IA_BUSY | IA_BUSY_NO_DMA))
3505 reset_mask |= RADEON_RESET_GFX;
3506
3507 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3508 CP_BUSY | CP_COHERENCY_BUSY))
3509 reset_mask |= RADEON_RESET_CP;
3510
3511 if (tmp & GRBM_EE_BUSY)
3512 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3513
3514 /* GRBM_STATUS2 */
3515 tmp = RREG32(GRBM_STATUS2);
3516 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3517 reset_mask |= RADEON_RESET_RLC;
3518
3519 /* DMA_STATUS_REG 0 */
3520 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3521 if (!(tmp & DMA_IDLE))
3522 reset_mask |= RADEON_RESET_DMA;
3523
3524 /* DMA_STATUS_REG 1 */
3525 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3526 if (!(tmp & DMA_IDLE))
3527 reset_mask |= RADEON_RESET_DMA1;
3528
3529 /* SRBM_STATUS2 */
3530 tmp = RREG32(SRBM_STATUS2);
3531 if (tmp & DMA_BUSY)
3532 reset_mask |= RADEON_RESET_DMA;
3533
3534 if (tmp & DMA1_BUSY)
3535 reset_mask |= RADEON_RESET_DMA1;
3536
3537 /* SRBM_STATUS */
3538 tmp = RREG32(SRBM_STATUS);
3539
3540 if (tmp & IH_BUSY)
3541 reset_mask |= RADEON_RESET_IH;
3542
3543 if (tmp & SEM_BUSY)
3544 reset_mask |= RADEON_RESET_SEM;
3545
3546 if (tmp & GRBM_RQ_PENDING)
3547 reset_mask |= RADEON_RESET_GRBM;
3548
3549 if (tmp & VMC_BUSY)
3550 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3551
014bb209
AD		 3552 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3553 MCC_BUSY | MCD_BUSY))
3554 reset_mask |= RADEON_RESET_MC;
3555
3556 if (evergreen_is_display_hung(rdev))
3557 reset_mask |= RADEON_RESET_DISPLAY;
3558
3559 /* VM_L2_STATUS */
3560 tmp = RREG32(VM_L2_STATUS);
3561 if (tmp & L2_BUSY)
3562 reset_mask |= RADEON_RESET_VMC;
3563
d808fc88
AD		 3564 /* Skip MC reset as it's mostly likely not hung, just busy */
3565 if (reset_mask & RADEON_RESET_MC) {
3566 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3567 reset_mask &= ~RADEON_RESET_MC;
3568 }
3569
014bb209
AD		 3570 return reset_mask;
3571}
3572
3573static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3574{
3575 struct evergreen_mc_save save;
3576 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3577 u32 tmp;
19fc42ed 3578
06bc6df0 3579 if (reset_mask == 0)
014bb209 3580 return;
06bc6df0
AD		 3581
3582 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3583
1c534671 3584 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3585 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3586 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3587 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3588 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3589
1c534671
AD		 3590 /* Disable CP parsing/prefetching */
3591 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3592
3593 if (reset_mask & RADEON_RESET_DMA) {
3594 /* dma0 */
3595 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3596 tmp &= ~DMA_RB_ENABLE;
3597 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3598 }
3599 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3600 /* dma1 */
3601 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3602 tmp &= ~DMA_RB_ENABLE;
3603 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3604 }
3605
f770d78a
AD		 3606 udelay(50);
3607
3608 evergreen_mc_stop(rdev, &save);
3609 if (evergreen_mc_wait_for_idle(rdev)) {
3610 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3611 }
3612
1c534671
AD		 3613 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3614 grbm_soft_reset = SOFT_RESET_CB |
3615 SOFT_RESET_DB |
3616 SOFT_RESET_GDS |
3617 SOFT_RESET_PA |
3618 SOFT_RESET_SC |
3619 SOFT_RESET_BCI |
3620 SOFT_RESET_SPI |
3621 SOFT_RESET_SX |
3622 SOFT_RESET_TC |
3623 SOFT_RESET_TA |
3624 SOFT_RESET_VGT |
3625 SOFT_RESET_IA;
3626 }
3627
3628 if (reset_mask & RADEON_RESET_CP) {
3629 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3630
3631 srbm_soft_reset |= SOFT_RESET_GRBM;
3632 }
06bc6df0
AD		 3633
3634 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3635 srbm_soft_reset |= SOFT_RESET_DMA;
3636
3637 if (reset_mask & RADEON_RESET_DMA1)
3638 srbm_soft_reset |= SOFT_RESET_DMA1;
3639
3640 if (reset_mask & RADEON_RESET_DISPLAY)
3641 srbm_soft_reset |= SOFT_RESET_DC;
3642
3643 if (reset_mask & RADEON_RESET_RLC)
3644 grbm_soft_reset |= SOFT_RESET_RLC;
3645
3646 if (reset_mask & RADEON_RESET_SEM)
3647 srbm_soft_reset |= SOFT_RESET_SEM;
3648
3649 if (reset_mask & RADEON_RESET_IH)
3650 srbm_soft_reset |= SOFT_RESET_IH;
3651
3652 if (reset_mask & RADEON_RESET_GRBM)
3653 srbm_soft_reset |= SOFT_RESET_GRBM;
3654
3655 if (reset_mask & RADEON_RESET_VMC)
3656 srbm_soft_reset |= SOFT_RESET_VMC;
3657
3658 if (reset_mask & RADEON_RESET_MC)
3659 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3660
3661 if (grbm_soft_reset) {
3662 tmp = RREG32(GRBM_SOFT_RESET);
3663 tmp |= grbm_soft_reset;
3664 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3665 WREG32(GRBM_SOFT_RESET, tmp);
3666 tmp = RREG32(GRBM_SOFT_RESET);
3667
3668 udelay(50);
3669
3670 tmp &= ~grbm_soft_reset;
3671 WREG32(GRBM_SOFT_RESET, tmp);
3672 tmp = RREG32(GRBM_SOFT_RESET);
3673 }
3674
3675 if (srbm_soft_reset) {
3676 tmp = RREG32(SRBM_SOFT_RESET);
3677 tmp |= srbm_soft_reset;
3678 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3679 WREG32(SRBM_SOFT_RESET, tmp);
3680 tmp = RREG32(SRBM_SOFT_RESET);
3681
3682 udelay(50);
3683
3684 tmp &= ~srbm_soft_reset;
3685 WREG32(SRBM_SOFT_RESET, tmp);
3686 tmp = RREG32(SRBM_SOFT_RESET);
3687 }
06bc6df0
AD		 3688
3689 /* Wait a little for things to settle down */
3690 udelay(50);
3691
c476dde2 3692 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3693 udelay(50);
3694
1c534671 3695 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3696}
3697
3698int si_asic_reset(struct radeon_device *rdev)
3699{
014bb209
AD		 3700 u32 reset_mask;
3701
3702 reset_mask = si_gpu_check_soft_reset(rdev);
3703
3704 if (reset_mask)
3705 r600_set_bios_scratch_engine_hung(rdev, true);
3706
3707 si_gpu_soft_reset(rdev, reset_mask);
3708
3709 reset_mask = si_gpu_check_soft_reset(rdev);
3710
3711 if (!reset_mask)
3712 r600_set_bios_scratch_engine_hung(rdev, false);
3713
3714 return 0;
c476dde2
AD		 3715}
3716
123bc183
AD		 3717/**
3718 * si_gfx_is_lockup - Check if the GFX engine is locked up
3719 *
3720 * @rdev: radeon_device pointer
3721 * @ring: radeon_ring structure holding ring information
3722 *
3723 * Check if the GFX engine is locked up.
3724 * Returns true if the engine appears to be locked up, false if not.
3725 */
3726bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3727{
3728 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3729
3730 if (!(reset_mask & (RADEON_RESET_GFX |
3731 RADEON_RESET_COMPUTE |
3732 RADEON_RESET_CP))) {
3733 radeon_ring_lockup_update(ring);
3734 return false;
3735 }
3736 /* force CP activities */
3737 radeon_ring_force_activity(rdev, ring);
3738 return radeon_ring_test_lockup(rdev, ring);
3739}
3740
d2800ee5
AD		 3741/* MC */
3742static void si_mc_program(struct radeon_device *rdev)
3743{
3744 struct evergreen_mc_save save;
3745 u32 tmp;
3746 int i, j;
3747
3748 /* Initialize HDP */
3749 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3750 WREG32((0x2c14 + j), 0x00000000);
3751 WREG32((0x2c18 + j), 0x00000000);
3752 WREG32((0x2c1c + j), 0x00000000);
3753 WREG32((0x2c20 + j), 0x00000000);
3754 WREG32((0x2c24 + j), 0x00000000);
3755 }
3756 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3757
3758 evergreen_mc_stop(rdev, &save);
3759 if (radeon_mc_wait_for_idle(rdev)) {
3760 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3761 }
5153550a
AD		 3762 if (!ASIC_IS_NODCE(rdev))
3763 /* Lockout access through VGA aperture*/
3764 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 3765 /* Update configuration */
3766 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3767 rdev->mc.vram_start >> 12);
3768 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3769 rdev->mc.vram_end >> 12);
3770 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3771 rdev->vram_scratch.gpu_addr >> 12);
3772 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3773 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3774 WREG32(MC_VM_FB_LOCATION, tmp);
3775 /* XXX double check these! */
3776 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3777 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3778 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3779 WREG32(MC_VM_AGP_BASE, 0);
3780 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3781 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3782 if (radeon_mc_wait_for_idle(rdev)) {
3783 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3784 }
3785 evergreen_mc_resume(rdev, &save);
5153550a
AD		 3786 if (!ASIC_IS_NODCE(rdev)) {
3787 /* we need to own VRAM, so turn off the VGA renderer here
3788 * to stop it overwriting our objects */
3789 rv515_vga_render_disable(rdev);
3790 }
d2800ee5
AD		 3791}
3792
1c49165d
AD		 3793void si_vram_gtt_location(struct radeon_device *rdev,
3794 struct radeon_mc *mc)
d2800ee5
AD		 3795{
3796 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3797 /* leave room for at least 1024M GTT */
3798 dev_warn(rdev->dev, "limiting VRAM\n");
3799 mc->real_vram_size = 0xFFC0000000ULL;
3800 mc->mc_vram_size = 0xFFC0000000ULL;
3801 }
9ed8b1f9 3802 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 3803 rdev->mc.gtt_base_align = 0;
9ed8b1f9 3804 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 3805}
3806
3807static int si_mc_init(struct radeon_device *rdev)
3808{
3809 u32 tmp;
3810 int chansize, numchan;
3811
3812 /* Get VRAM informations */
3813 rdev->mc.vram_is_ddr = true;
3814 tmp = RREG32(MC_ARB_RAMCFG);
3815 if (tmp & CHANSIZE_OVERRIDE) {
3816 chansize = 16;
3817 } else if (tmp & CHANSIZE_MASK) {
3818 chansize = 64;
3819 } else {
3820 chansize = 32;
3821 }
3822 tmp = RREG32(MC_SHARED_CHMAP);
3823 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3824 case 0:
3825 default:
3826 numchan = 1;
3827 break;
3828 case 1:
3829 numchan = 2;
3830 break;
3831 case 2:
3832 numchan = 4;
3833 break;
3834 case 3:
3835 numchan = 8;
3836 break;
3837 case 4:
3838 numchan = 3;
3839 break;
3840 case 5:
3841 numchan = 6;
3842 break;
3843 case 6:
3844 numchan = 10;
3845 break;
3846 case 7:
3847 numchan = 12;
3848 break;
3849 case 8:
3850 numchan = 16;
3851 break;
3852 }
3853 rdev->mc.vram_width = numchan * chansize;
3854 /* Could aper size report 0 ? */
3855 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3856 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3857 /* size in MB on si */
fc986034
NOS		 3858 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3859 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
d2800ee5
AD		 3860 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3861 si_vram_gtt_location(rdev, &rdev->mc);
3862 radeon_update_bandwidth_info(rdev);
3863
3864 return 0;
3865}
3866
3867/*
3868 * GART
3869 */
3870void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3871{
3872 /* flush hdp cache */
3873 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3874
3875 /* bits 0-15 are the VM contexts0-15 */
3876 WREG32(VM_INVALIDATE_REQUEST, 1);
3877}
3878
1109ca09 3879static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 3880{
3881 int r, i;
3882
3883 if (rdev->gart.robj == NULL) {
3884 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3885 return -EINVAL;
3886 }
3887 r = radeon_gart_table_vram_pin(rdev);
3888 if (r)
3889 return r;
3890 radeon_gart_restore(rdev);
3891 /* Setup TLB control */
3892 WREG32(MC_VM_MX_L1_TLB_CNTL,
3893 (0xA << 7) |
3894 ENABLE_L1_TLB |
3895 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3896 ENABLE_ADVANCED_DRIVER_MODEL |
3897 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3898 /* Setup L2 cache */
3899 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3900 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3901 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3902 EFFECTIVE_L2_QUEUE_SIZE(7) |
3903 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3904 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3905 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3906 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3907 /* setup context0 */
3908 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3909 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3910 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3911 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3912 (u32)(rdev->dummy_page.addr >> 12));
3913 WREG32(VM_CONTEXT0_CNTL2, 0);
3914 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3915 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3916
3917 WREG32(0x15D4, 0);
3918 WREG32(0x15D8, 0);
3919 WREG32(0x15DC, 0);
3920
3921 /* empty context1-15 */
d2800ee5
AD		 3922 /* set vm size, must be a multiple of 4 */
3923 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
c21b328e 3924 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
23d4f1f2
AD		 3925 /* Assign the pt base to something valid for now; the pts used for
3926 * the VMs are determined by the application and setup and assigned
3927 * on the fly in the vm part of radeon_gart.c
3928 */
d2800ee5
AD		 3929 for (i = 1; i < 16; i++) {
3930 if (i < 8)
3931 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3932 rdev->gart.table_addr >> 12);
3933 else
3934 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3935 rdev->gart.table_addr >> 12);
3936 }
3937
3938 /* enable context1-15 */
3939 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3940 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 3941 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 3942 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
ae133a11
CK		 3943 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3944 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3945 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3946 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3947 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3948 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3949 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3950 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3951 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3952 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3953 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3954 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 3955
3956 si_pcie_gart_tlb_flush(rdev);
3957 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3958 (unsigned)(rdev->mc.gtt_size >> 20),
3959 (unsigned long long)rdev->gart.table_addr);
3960 rdev->gart.ready = true;
3961 return 0;
3962}
3963
1109ca09 3964static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5
AD		 3965{
3966 /* Disable all tables */
3967 WREG32(VM_CONTEXT0_CNTL, 0);
3968 WREG32(VM_CONTEXT1_CNTL, 0);
3969 /* Setup TLB control */
3970 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3971 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3972 /* Setup L2 cache */
3973 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3974 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3975 EFFECTIVE_L2_QUEUE_SIZE(7) |
3976 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3977 WREG32(VM_L2_CNTL2, 0);
3978 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3979 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3980 radeon_gart_table_vram_unpin(rdev);
3981}
3982
1109ca09 3983static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 3984{
3985 si_pcie_gart_disable(rdev);
3986 radeon_gart_table_vram_free(rdev);
3987 radeon_gart_fini(rdev);
3988}
3989
498dd8b3
AD		 3990/* vm parser */
3991static bool si_vm_reg_valid(u32 reg)
3992{
3993 /* context regs are fine */
3994 if (reg >= 0x28000)
3995 return true;
3996
3997 /* check config regs */
3998 switch (reg) {
3999 case GRBM_GFX_INDEX:
f418b88a 4000 case CP_STRMOUT_CNTL:
498dd8b3
AD		 4001 case VGT_VTX_VECT_EJECT_REG:
4002 case VGT_CACHE_INVALIDATION:
4003 case VGT_ESGS_RING_SIZE:
4004 case VGT_GSVS_RING_SIZE:
4005 case VGT_GS_VERTEX_REUSE:
4006 case VGT_PRIMITIVE_TYPE:
4007 case VGT_INDEX_TYPE:
4008 case VGT_NUM_INDICES:
4009 case VGT_NUM_INSTANCES:
4010 case VGT_TF_RING_SIZE:
4011 case VGT_HS_OFFCHIP_PARAM:
4012 case VGT_TF_MEMORY_BASE:
4013 case PA_CL_ENHANCE:
4014 case PA_SU_LINE_STIPPLE_VALUE:
4015 case PA_SC_LINE_STIPPLE_STATE:
4016 case PA_SC_ENHANCE:
4017 case SQC_CACHES:
4018 case SPI_STATIC_THREAD_MGMT_1:
4019 case SPI_STATIC_THREAD_MGMT_2:
4020 case SPI_STATIC_THREAD_MGMT_3:
4021 case SPI_PS_MAX_WAVE_ID:
4022 case SPI_CONFIG_CNTL:
4023 case SPI_CONFIG_CNTL_1:
4024 case TA_CNTL_AUX:
4025 return true;
4026 default:
4027 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4028 return false;
4029 }
4030}
4031
4032static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4033 u32 *ib, struct radeon_cs_packet *pkt)
4034{
4035 switch (pkt->opcode) {
4036 case PACKET3_NOP:
4037 case PACKET3_SET_BASE:
4038 case PACKET3_SET_CE_DE_COUNTERS:
4039 case PACKET3_LOAD_CONST_RAM:
4040 case PACKET3_WRITE_CONST_RAM:
4041 case PACKET3_WRITE_CONST_RAM_OFFSET:
4042 case PACKET3_DUMP_CONST_RAM:
4043 case PACKET3_INCREMENT_CE_COUNTER:
4044 case PACKET3_WAIT_ON_DE_COUNTER:
4045 case PACKET3_CE_WRITE:
4046 break;
4047 default:
4048 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4049 return -EINVAL;
4050 }
4051 return 0;
4052}
4053
4054static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4055 u32 *ib, struct radeon_cs_packet *pkt)
4056{
4057 u32 idx = pkt->idx + 1;
4058 u32 idx_value = ib[idx];
4059 u32 start_reg, end_reg, reg, i;
5aa709be 4060 u32 command, info;
498dd8b3
AD		 4061
4062 switch (pkt->opcode) {
4063 case PACKET3_NOP:
4064 case PACKET3_SET_BASE:
4065 case PACKET3_CLEAR_STATE:
4066 case PACKET3_INDEX_BUFFER_SIZE:
4067 case PACKET3_DISPATCH_DIRECT:
4068 case PACKET3_DISPATCH_INDIRECT:
4069 case PACKET3_ALLOC_GDS:
4070 case PACKET3_WRITE_GDS_RAM:
4071 case PACKET3_ATOMIC_GDS:
4072 case PACKET3_ATOMIC:
4073 case PACKET3_OCCLUSION_QUERY:
4074 case PACKET3_SET_PREDICATION:
4075 case PACKET3_COND_EXEC:
4076 case PACKET3_PRED_EXEC:
4077 case PACKET3_DRAW_INDIRECT:
4078 case PACKET3_DRAW_INDEX_INDIRECT:
4079 case PACKET3_INDEX_BASE:
4080 case PACKET3_DRAW_INDEX_2:
4081 case PACKET3_CONTEXT_CONTROL:
4082 case PACKET3_INDEX_TYPE:
4083 case PACKET3_DRAW_INDIRECT_MULTI:
4084 case PACKET3_DRAW_INDEX_AUTO:
4085 case PACKET3_DRAW_INDEX_IMMD:
4086 case PACKET3_NUM_INSTANCES:
4087 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4088 case PACKET3_STRMOUT_BUFFER_UPDATE:
4089 case PACKET3_DRAW_INDEX_OFFSET_2:
4090 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4091 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4092 case PACKET3_MPEG_INDEX:
4093 case PACKET3_WAIT_REG_MEM:
4094 case PACKET3_MEM_WRITE:
4095 case PACKET3_PFP_SYNC_ME:
4096 case PACKET3_SURFACE_SYNC:
4097 case PACKET3_EVENT_WRITE:
4098 case PACKET3_EVENT_WRITE_EOP:
4099 case PACKET3_EVENT_WRITE_EOS:
4100 case PACKET3_SET_CONTEXT_REG:
4101 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4102 case PACKET3_SET_SH_REG:
4103 case PACKET3_SET_SH_REG_OFFSET:
4104 case PACKET3_INCREMENT_DE_COUNTER:
4105 case PACKET3_WAIT_ON_CE_COUNTER:
4106 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4107 case PACKET3_ME_WRITE:
4108 break;
4109 case PACKET3_COPY_DATA:
4110 if ((idx_value & 0xf00) == 0) {
4111 reg = ib[idx + 3] * 4;
4112 if (!si_vm_reg_valid(reg))
4113 return -EINVAL;
4114 }
4115 break;
4116 case PACKET3_WRITE_DATA:
4117 if ((idx_value & 0xf00) == 0) {
4118 start_reg = ib[idx + 1] * 4;
4119 if (idx_value & 0x10000) {
4120 if (!si_vm_reg_valid(start_reg))
4121 return -EINVAL;
4122 } else {
4123 for (i = 0; i < (pkt->count - 2); i++) {
4124 reg = start_reg + (4 * i);
4125 if (!si_vm_reg_valid(reg))
4126 return -EINVAL;
4127 }
4128 }
4129 }
4130 break;
4131 case PACKET3_COND_WRITE:
4132 if (idx_value & 0x100) {
4133 reg = ib[idx + 5] * 4;
4134 if (!si_vm_reg_valid(reg))
4135 return -EINVAL;
4136 }
4137 break;
4138 case PACKET3_COPY_DW:
4139 if (idx_value & 0x2) {
4140 reg = ib[idx + 3] * 4;
4141 if (!si_vm_reg_valid(reg))
4142 return -EINVAL;
4143 }
4144 break;
4145 case PACKET3_SET_CONFIG_REG:
4146 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4147 end_reg = 4 * pkt->count + start_reg - 4;
4148 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4149 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4150 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4151 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4152 return -EINVAL;
4153 }
4154 for (i = 0; i < pkt->count; i++) {
4155 reg = start_reg + (4 * i);
4156 if (!si_vm_reg_valid(reg))
4157 return -EINVAL;
4158 }
4159 break;
5aa709be
AD		 4160 case PACKET3_CP_DMA:
4161 command = ib[idx + 4];
4162 info = ib[idx + 1];
4163 if (command & PACKET3_CP_DMA_CMD_SAS) {
4164 /* src address space is register */
4165 if (((info & 0x60000000) >> 29) == 0) {
4166 start_reg = idx_value << 2;
4167 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4168 reg = start_reg;
4169 if (!si_vm_reg_valid(reg)) {
4170 DRM_ERROR("CP DMA Bad SRC register\n");
4171 return -EINVAL;
4172 }
4173 } else {
4174 for (i = 0; i < (command & 0x1fffff); i++) {
4175 reg = start_reg + (4 * i);
4176 if (!si_vm_reg_valid(reg)) {
4177 DRM_ERROR("CP DMA Bad SRC register\n");
4178 return -EINVAL;
4179 }
4180 }
4181 }
4182 }
4183 }
4184 if (command & PACKET3_CP_DMA_CMD_DAS) {
4185 /* dst address space is register */
4186 if (((info & 0x00300000) >> 20) == 0) {
4187 start_reg = ib[idx + 2];
4188 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4189 reg = start_reg;
4190 if (!si_vm_reg_valid(reg)) {
4191 DRM_ERROR("CP DMA Bad DST register\n");
4192 return -EINVAL;
4193 }
4194 } else {
4195 for (i = 0; i < (command & 0x1fffff); i++) {
4196 reg = start_reg + (4 * i);
4197 if (!si_vm_reg_valid(reg)) {
4198 DRM_ERROR("CP DMA Bad DST register\n");
4199 return -EINVAL;
4200 }
4201 }
4202 }
4203 }
4204 }
4205 break;
498dd8b3
AD		 4206 default:
4207 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4208 return -EINVAL;
4209 }
4210 return 0;
4211}
4212
4213static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4214 u32 *ib, struct radeon_cs_packet *pkt)
4215{
4216 u32 idx = pkt->idx + 1;
4217 u32 idx_value = ib[idx];
4218 u32 start_reg, reg, i;
4219
4220 switch (pkt->opcode) {
4221 case PACKET3_NOP:
4222 case PACKET3_SET_BASE:
4223 case PACKET3_CLEAR_STATE:
4224 case PACKET3_DISPATCH_DIRECT:
4225 case PACKET3_DISPATCH_INDIRECT:
4226 case PACKET3_ALLOC_GDS:
4227 case PACKET3_WRITE_GDS_RAM:
4228 case PACKET3_ATOMIC_GDS:
4229 case PACKET3_ATOMIC:
4230 case PACKET3_OCCLUSION_QUERY:
4231 case PACKET3_SET_PREDICATION:
4232 case PACKET3_COND_EXEC:
4233 case PACKET3_PRED_EXEC:
4234 case PACKET3_CONTEXT_CONTROL:
4235 case PACKET3_STRMOUT_BUFFER_UPDATE:
4236 case PACKET3_WAIT_REG_MEM:
4237 case PACKET3_MEM_WRITE:
4238 case PACKET3_PFP_SYNC_ME:
4239 case PACKET3_SURFACE_SYNC:
4240 case PACKET3_EVENT_WRITE:
4241 case PACKET3_EVENT_WRITE_EOP:
4242 case PACKET3_EVENT_WRITE_EOS:
4243 case PACKET3_SET_CONTEXT_REG:
4244 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4245 case PACKET3_SET_SH_REG:
4246 case PACKET3_SET_SH_REG_OFFSET:
4247 case PACKET3_INCREMENT_DE_COUNTER:
4248 case PACKET3_WAIT_ON_CE_COUNTER:
4249 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4250 case PACKET3_ME_WRITE:
4251 break;
4252 case PACKET3_COPY_DATA:
4253 if ((idx_value & 0xf00) == 0) {
4254 reg = ib[idx + 3] * 4;
4255 if (!si_vm_reg_valid(reg))
4256 return -EINVAL;
4257 }
4258 break;
4259 case PACKET3_WRITE_DATA:
4260 if ((idx_value & 0xf00) == 0) {
4261 start_reg = ib[idx + 1] * 4;
4262 if (idx_value & 0x10000) {
4263 if (!si_vm_reg_valid(start_reg))
4264 return -EINVAL;
4265 } else {
4266 for (i = 0; i < (pkt->count - 2); i++) {
4267 reg = start_reg + (4 * i);
4268 if (!si_vm_reg_valid(reg))
4269 return -EINVAL;
4270 }
4271 }
4272 }
4273 break;
4274 case PACKET3_COND_WRITE:
4275 if (idx_value & 0x100) {
4276 reg = ib[idx + 5] * 4;
4277 if (!si_vm_reg_valid(reg))
4278 return -EINVAL;
4279 }
4280 break;
4281 case PACKET3_COPY_DW:
4282 if (idx_value & 0x2) {
4283 reg = ib[idx + 3] * 4;
4284 if (!si_vm_reg_valid(reg))
4285 return -EINVAL;
4286 }
4287 break;
4288 default:
4289 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4290 return -EINVAL;
4291 }
4292 return 0;
4293}
4294
4295int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4296{
4297 int ret = 0;
4298 u32 idx = 0;
4299 struct radeon_cs_packet pkt;
4300
4301 do {
4302 pkt.idx = idx;
4e872ae2
IH		 4303 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4304 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4305 pkt.one_reg_wr = 0;
4306 switch (pkt.type) {
4e872ae2 4307 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 4308 dev_err(rdev->dev, "Packet0 not allowed!\n");
4309 ret = -EINVAL;
4310 break;
4e872ae2 4311 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4312 idx += 1;
4313 break;
4e872ae2
IH		 4314 case RADEON_PACKET_TYPE3:
4315 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4316 if (ib->is_const_ib)
4317 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4318 else {
876dc9f3 4319 switch (ib->ring) {
498dd8b3
AD		 4320 case RADEON_RING_TYPE_GFX_INDEX:
4321 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4322 break;
4323 case CAYMAN_RING_TYPE_CP1_INDEX:
4324 case CAYMAN_RING_TYPE_CP2_INDEX:
4325 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4326 break;
4327 default:
876dc9f3 4328 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4329 ret = -EINVAL;
4330 break;
4331 }
4332 }
4333 idx += pkt.count + 2;
4334 break;
4335 default:
4336 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4337 ret = -EINVAL;
4338 break;
4339 }
4340 if (ret)
4341 break;
4342 } while (idx < ib->length_dw);
4343
4344 return ret;
4345}
4346
d2800ee5
AD		 4347/*
4348 * vm
4349 */
4350int si_vm_init(struct radeon_device *rdev)
4351{
4352 /* number of VMs */
4353 rdev->vm_manager.nvm = 16;
4354 /* base offset of vram pages */
4355 rdev->vm_manager.vram_base_offset = 0;
4356
4357 return 0;
4358}
4359
4360void si_vm_fini(struct radeon_device *rdev)
4361{
4362}
4363
fbf6dc7a
AD		 4364/**
4365 * si_vm_decode_fault - print human readable fault info
4366 *
4367 * @rdev: radeon_device pointer
4368 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4369 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4370 *
4371 * Print human readable fault information (SI).
4372 */
4373static void si_vm_decode_fault(struct radeon_device *rdev,
4374 u32 status, u32 addr)
4375{
4376 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4377 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4378 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4379 char *block;
4380
4381 if (rdev->family == CHIP_TAHITI) {
4382 switch (mc_id) {
4383 case 160:
4384 case 144:
4385 case 96:
4386 case 80:
4387 case 224:
4388 case 208:
4389 case 32:
4390 case 16:
4391 block = "CB";
4392 break;
4393 case 161:
4394 case 145:
4395 case 97:
4396 case 81:
4397 case 225:
4398 case 209:
4399 case 33:
4400 case 17:
4401 block = "CB_FMASK";
4402 break;
4403 case 162:
4404 case 146:
4405 case 98:
4406 case 82:
4407 case 226:
4408 case 210:
4409 case 34:
4410 case 18:
4411 block = "CB_CMASK";
4412 break;
4413 case 163:
4414 case 147:
4415 case 99:
4416 case 83:
4417 case 227:
4418 case 211:
4419 case 35:
4420 case 19:
4421 block = "CB_IMMED";
4422 break;
4423 case 164:
4424 case 148:
4425 case 100:
4426 case 84:
4427 case 228:
4428 case 212:
4429 case 36:
4430 case 20:
4431 block = "DB";
4432 break;
4433 case 165:
4434 case 149:
4435 case 101:
4436 case 85:
4437 case 229:
4438 case 213:
4439 case 37:
4440 case 21:
4441 block = "DB_HTILE";
4442 break;
4443 case 167:
4444 case 151:
4445 case 103:
4446 case 87:
4447 case 231:
4448 case 215:
4449 case 39:
4450 case 23:
4451 block = "DB_STEN";
4452 break;
4453 case 72:
4454 case 68:
4455 case 64:
4456 case 8:
4457 case 4:
4458 case 0:
4459 case 136:
4460 case 132:
4461 case 128:
4462 case 200:
4463 case 196:
4464 case 192:
4465 block = "TC";
4466 break;
4467 case 112:
4468 case 48:
4469 block = "CP";
4470 break;
4471 case 49:
4472 case 177:
4473 case 50:
4474 case 178:
4475 block = "SH";
4476 break;
4477 case 53:
4478 case 190:
4479 block = "VGT";
4480 break;
4481 case 117:
4482 block = "IH";
4483 break;
4484 case 51:
4485 case 115:
4486 block = "RLC";
4487 break;
4488 case 119:
4489 case 183:
4490 block = "DMA0";
4491 break;
4492 case 61:
4493 block = "DMA1";
4494 break;
4495 case 248:
4496 case 120:
4497 block = "HDP";
4498 break;
4499 default:
4500 block = "unknown";
4501 break;
4502 }
4503 } else {
4504 switch (mc_id) {
4505 case 32:
4506 case 16:
4507 case 96:
4508 case 80:
4509 case 160:
4510 case 144:
4511 case 224:
4512 case 208:
4513 block = "CB";
4514 break;
4515 case 33:
4516 case 17:
4517 case 97:
4518 case 81:
4519 case 161:
4520 case 145:
4521 case 225:
4522 case 209:
4523 block = "CB_FMASK";
4524 break;
4525 case 34:
4526 case 18:
4527 case 98:
4528 case 82:
4529 case 162:
4530 case 146:
4531 case 226:
4532 case 210:
4533 block = "CB_CMASK";
4534 break;
4535 case 35:
4536 case 19:
4537 case 99:
4538 case 83:
4539 case 163:
4540 case 147:
4541 case 227:
4542 case 211:
4543 block = "CB_IMMED";
4544 break;
4545 case 36:
4546 case 20:
4547 case 100:
4548 case 84:
4549 case 164:
4550 case 148:
4551 case 228:
4552 case 212:
4553 block = "DB";
4554 break;
4555 case 37:
4556 case 21:
4557 case 101:
4558 case 85:
4559 case 165:
4560 case 149:
4561 case 229:
4562 case 213:
4563 block = "DB_HTILE";
4564 break;
4565 case 39:
4566 case 23:
4567 case 103:
4568 case 87:
4569 case 167:
4570 case 151:
4571 case 231:
4572 case 215:
4573 block = "DB_STEN";
4574 break;
4575 case 72:
4576 case 68:
4577 case 8:
4578 case 4:
4579 case 136:
4580 case 132:
4581 case 200:
4582 case 196:
4583 block = "TC";
4584 break;
4585 case 112:
4586 case 48:
4587 block = "CP";
4588 break;
4589 case 49:
4590 case 177:
4591 case 50:
4592 case 178:
4593 block = "SH";
4594 break;
4595 case 53:
4596 block = "VGT";
4597 break;
4598 case 117:
4599 block = "IH";
4600 break;
4601 case 51:
4602 case 115:
4603 block = "RLC";
4604 break;
4605 case 119:
4606 case 183:
4607 block = "DMA0";
4608 break;
4609 case 61:
4610 block = "DMA1";
4611 break;
4612 case 248:
4613 case 120:
4614 block = "HDP";
4615 break;
4616 default:
4617 block = "unknown";
4618 break;
4619 }
4620 }
4621
4622 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4623 protections, vmid, addr,
4624 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4625 block, mc_id);
4626}
4627
82ffd92b
AD		 4628/**
4629 * si_vm_set_page - update the page tables using the CP
4630 *
4631 * @rdev: radeon_device pointer
43f1214a 4632 * @ib: indirect buffer to fill with commands
82ffd92b
AD		 4633 * @pe: addr of the page entry
4634 * @addr: dst addr to write into pe
4635 * @count: number of page entries to update
4636 * @incr: increase next addr by incr bytes
4637 * @flags: access flags
4638 *
43f1214a 4639 * Update the page tables using the CP (SI).
82ffd92b 4640 */
43f1214a
AD		 4641void si_vm_set_page(struct radeon_device *rdev,
4642 struct radeon_ib *ib,
4643 uint64_t pe,
82ffd92b
AD		 4644 uint64_t addr, unsigned count,
4645 uint32_t incr, uint32_t flags)
d2800ee5 4646{
82ffd92b 4647 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
deab48f1
AD		 4648 uint64_t value;
4649 unsigned ndw;
4650
4651 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4652 while (count) {
4653 ndw = 2 + count * 2;
4654 if (ndw > 0x3FFE)
4655 ndw = 0x3FFE;
4656
43f1214a
AD		 4657 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4658 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4659 WRITE_DATA_DST_SEL(1));
4660 ib->ptr[ib->length_dw++] = pe;
4661 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
deab48f1
AD		 4662 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4663 if (flags & RADEON_VM_PAGE_SYSTEM) {
4664 value = radeon_vm_map_gart(rdev, addr);
4665 value &= 0xFFFFFFFFFFFFF000ULL;
4666 } else if (flags & RADEON_VM_PAGE_VALID) {
4667 value = addr;
4668 } else {
4669 value = 0;
4670 }
4671 addr += incr;
4672 value |= r600_flags;
43f1214a
AD		 4673 ib->ptr[ib->length_dw++] = value;
4674 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4675 }
4676 }
4677 } else {
4678 /* DMA */
2483b4ea 4679 si_dma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
82ffd92b 4680 }
d2800ee5
AD		 4681}
4682
498522b4 4683void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
d2800ee5 4684{
498522b4 4685 struct radeon_ring *ring = &rdev->ring[ridx];
d2800ee5 4686
ee60e29f 4687 if (vm == NULL)
d2800ee5
AD		 4688 return;
4689
76c44f2c
AD		 4690 /* write new base address */
4691 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4692 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4693 WRITE_DATA_DST_SEL(0)));
4694
ee60e29f 4695 if (vm->id < 8) {
76c44f2c
AD		 4696 radeon_ring_write(ring,
4697 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
ee60e29f 4698 } else {
76c44f2c
AD		 4699 radeon_ring_write(ring,
4700 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
ee60e29f 4701 }
76c44f2c 4702 radeon_ring_write(ring, 0);
fa87e62d 4703 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 4704
d2800ee5 4705 /* flush hdp cache */
76c44f2c
AD		 4706 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4707 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4708 WRITE_DATA_DST_SEL(0)));
4709 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4710 radeon_ring_write(ring, 0);
ee60e29f
CK		 4711 radeon_ring_write(ring, 0x1);
4712
d2800ee5 4713 /* bits 0-15 are the VM contexts0-15 */
76c44f2c
AD		 4714 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4715 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4716 WRITE_DATA_DST_SEL(0)));
4717 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4718 radeon_ring_write(ring, 0);
498522b4 4719 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 4720
4721 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4722 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4723 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 4724}
4725
f8f84ac5
AD		 4726/*
4727 * Power and clock gating
4728 */
4729static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4730{
4731 int i;
4732
4733 for (i = 0; i < rdev->usec_timeout; i++) {
4734 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4735 break;
4736 udelay(1);
4737 }
4738
4739 for (i = 0; i < rdev->usec_timeout; i++) {
4740 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4741 break;
4742 udelay(1);
4743 }
4744}
4745
4746static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4747 bool enable)
4748{
4749 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4750 u32 mask;
4751 int i;
4752
4753 if (enable)
4754 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4755 else
4756 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4757 WREG32(CP_INT_CNTL_RING0, tmp);
4758
4759 if (!enable) {
4760 /* read a gfx register */
4761 tmp = RREG32(DB_DEPTH_INFO);
4762
4763 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4764 for (i = 0; i < rdev->usec_timeout; i++) {
4765 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4766 break;
4767 udelay(1);
4768 }
4769 }
4770}
4771
4772static void si_set_uvd_dcm(struct radeon_device *rdev,
4773 bool sw_mode)
4774{
4775 u32 tmp, tmp2;
4776
4777 tmp = RREG32(UVD_CGC_CTRL);
4778 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4779 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4780
4781 if (sw_mode) {
4782 tmp &= ~0x7ffff800;
4783 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4784 } else {
4785 tmp |= 0x7ffff800;
4786 tmp2 = 0;
4787 }
4788
4789 WREG32(UVD_CGC_CTRL, tmp);
4790 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4791}
4792
22c775ce 4793void si_init_uvd_internal_cg(struct radeon_device *rdev)
f8f84ac5
AD		 4794{
4795 bool hw_mode = true;
4796
4797 if (hw_mode) {
4798 si_set_uvd_dcm(rdev, false);
4799 } else {
4800 u32 tmp = RREG32(UVD_CGC_CTRL);
4801 tmp &= ~DCM;
4802 WREG32(UVD_CGC_CTRL, tmp);
4803 }
4804}
4805
4806static u32 si_halt_rlc(struct radeon_device *rdev)
4807{
4808 u32 data, orig;
4809
4810 orig = data = RREG32(RLC_CNTL);
4811
4812 if (data & RLC_ENABLE) {
4813 data &= ~RLC_ENABLE;
4814 WREG32(RLC_CNTL, data);
4815
4816 si_wait_for_rlc_serdes(rdev);
4817 }
4818
4819 return orig;
4820}
4821
4822static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4823{
4824 u32 tmp;
4825
4826 tmp = RREG32(RLC_CNTL);
4827 if (tmp != rlc)
4828 WREG32(RLC_CNTL, rlc);
4829}
4830
4831static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4832{
4833 u32 data, orig;
4834
4835 orig = data = RREG32(DMA_PG);
e16866ec 4836 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
f8f84ac5
AD		 4837 data |= PG_CNTL_ENABLE;
4838 else
4839 data &= ~PG_CNTL_ENABLE;
4840 if (orig != data)
4841 WREG32(DMA_PG, data);
4842}
4843
4844static void si_init_dma_pg(struct radeon_device *rdev)
4845{
4846 u32 tmp;
4847
4848 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4849 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4850
4851 for (tmp = 0; tmp < 5; tmp++)
4852 WREG32(DMA_PGFSM_WRITE, 0);
4853}
4854
4855static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4856 bool enable)
4857{
4858 u32 tmp;
4859
e16866ec 4860 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_CG)) {
f8f84ac5
AD		 4861 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4862 WREG32(RLC_TTOP_D, tmp);
4863
4864 tmp = RREG32(RLC_PG_CNTL);
4865 tmp |= GFX_PG_ENABLE;
4866 WREG32(RLC_PG_CNTL, tmp);
4867
4868 tmp = RREG32(RLC_AUTO_PG_CTRL);
4869 tmp |= AUTO_PG_EN;
4870 WREG32(RLC_AUTO_PG_CTRL, tmp);
4871 } else {
4872 tmp = RREG32(RLC_AUTO_PG_CTRL);
4873 tmp &= ~AUTO_PG_EN;
4874 WREG32(RLC_AUTO_PG_CTRL, tmp);
4875
4876 tmp = RREG32(DB_RENDER_CONTROL);
4877 }
4878}
4879
4880static void si_init_gfx_cgpg(struct radeon_device *rdev)
4881{
4882 u32 tmp;
4883
4884 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4885
4886 tmp = RREG32(RLC_PG_CNTL);
4887 tmp |= GFX_PG_SRC;
4888 WREG32(RLC_PG_CNTL, tmp);
4889
4890 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4891
4892 tmp = RREG32(RLC_AUTO_PG_CTRL);
4893
4894 tmp &= ~GRBM_REG_SGIT_MASK;
4895 tmp |= GRBM_REG_SGIT(0x700);
4896 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
4897 WREG32(RLC_AUTO_PG_CTRL, tmp);
4898}
4899
ba19031a 4900static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
f8f84ac5
AD		 4901{
4902 u32 mask = 0, tmp, tmp1;
4903 int i;
4904
4905 si_select_se_sh(rdev, se, sh);
4906 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
4907 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
4908 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4909
4910 tmp &= 0xffff0000;
4911
4912 tmp |= tmp1;
4913 tmp >>= 16;
4914
4915 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
4916 mask <<= 1;
4917 mask |= 1;
4918 }
4919
4920 return (~tmp) & mask;
4921}
4922
4923static void si_init_ao_cu_mask(struct radeon_device *rdev)
4924{
4925 u32 i, j, k, active_cu_number = 0;
4926 u32 mask, counter, cu_bitmap;
4927 u32 tmp = 0;
4928
4929 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
4930 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
4931 mask = 1;
4932 cu_bitmap = 0;
4933 counter = 0;
4934 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
ba19031a 4935 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
f8f84ac5
AD		 4936 if (counter < 2)
4937 cu_bitmap |= mask;
4938 counter++;
4939 }
4940 mask <<= 1;
4941 }
4942
4943 active_cu_number += counter;
4944 tmp |= (cu_bitmap << (i * 16 + j * 8));
4945 }
4946 }
4947
4948 WREG32(RLC_PG_AO_CU_MASK, tmp);
4949
4950 tmp = RREG32(RLC_MAX_PG_CU);
4951 tmp &= ~MAX_PU_CU_MASK;
4952 tmp |= MAX_PU_CU(active_cu_number);
4953 WREG32(RLC_MAX_PG_CU, tmp);
4954}
4955
4956static void si_enable_cgcg(struct radeon_device *rdev,
4957 bool enable)
4958{
4959 u32 data, orig, tmp;
4960
4961 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
4962
e16866ec 4963 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5594a558
AD		 4964 si_enable_gui_idle_interrupt(rdev, true);
4965
f8f84ac5
AD		 4966 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
4967
4968 tmp = si_halt_rlc(rdev);
4969
4970 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4971 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4972 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
4973
4974 si_wait_for_rlc_serdes(rdev);
4975
4976 si_update_rlc(rdev, tmp);
4977
4978 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
4979
4980 data |= CGCG_EN | CGLS_EN;
4981 } else {
5594a558
AD		 4982 si_enable_gui_idle_interrupt(rdev, false);
4983
f8f84ac5
AD		 4984 RREG32(CB_CGTT_SCLK_CTRL);
4985 RREG32(CB_CGTT_SCLK_CTRL);
4986 RREG32(CB_CGTT_SCLK_CTRL);
4987 RREG32(CB_CGTT_SCLK_CTRL);
4988
4989 data &= ~(CGCG_EN | CGLS_EN);
4990 }
4991
4992 if (orig != data)
4993 WREG32(RLC_CGCG_CGLS_CTRL, data);
4994}
4995
4996static void si_enable_mgcg(struct radeon_device *rdev,
4997 bool enable)
4998{
4999 u32 data, orig, tmp = 0;
5000
e16866ec 5001 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
f8f84ac5
AD		 5002 orig = data = RREG32(CGTS_SM_CTRL_REG);
5003 data = 0x96940200;
5004 if (orig != data)
5005 WREG32(CGTS_SM_CTRL_REG, data);
5006
e16866ec
AD		 5007 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5008 orig = data = RREG32(CP_MEM_SLP_CNTL);
5009 data |= CP_MEM_LS_EN;
5010 if (orig != data)
5011 WREG32(CP_MEM_SLP_CNTL, data);
5012 }
f8f84ac5
AD		 5013
5014 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5015 data &= 0xffffffc0;
5016 if (orig != data)
5017 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5018
5019 tmp = si_halt_rlc(rdev);
5020
5021 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5022 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5023 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5024
5025 si_update_rlc(rdev, tmp);
5026 } else {
5027 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5028 data |= 0x00000003;
5029 if (orig != data)
5030 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5031
5032 data = RREG32(CP_MEM_SLP_CNTL);
5033 if (data & CP_MEM_LS_EN) {
5034 data &= ~CP_MEM_LS_EN;
5035 WREG32(CP_MEM_SLP_CNTL, data);
5036 }
5037 orig = data = RREG32(CGTS_SM_CTRL_REG);
5038 data |= LS_OVERRIDE | OVERRIDE;
5039 if (orig != data)
5040 WREG32(CGTS_SM_CTRL_REG, data);
5041
5042 tmp = si_halt_rlc(rdev);
5043
5044 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5045 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5046 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5047
5048 si_update_rlc(rdev, tmp);
5049 }
5050}
5051
5052static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5053 bool enable)
5054{
5055 u32 orig, data, tmp;
5056
e16866ec 5057 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
f8f84ac5
AD		 5058 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5059 tmp |= 0x3fff;
5060 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5061
5062 orig = data = RREG32(UVD_CGC_CTRL);
5063 data |= DCM;
5064 if (orig != data)
5065 WREG32(UVD_CGC_CTRL, data);
5066
5067 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5068 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5069 } else {
5070 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5071 tmp &= ~0x3fff;
5072 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5073
5074 orig = data = RREG32(UVD_CGC_CTRL);
5075 data &= ~DCM;
5076 if (orig != data)
5077 WREG32(UVD_CGC_CTRL, data);
5078
5079 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5080 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5081 }
5082}
5083
5084static const u32 mc_cg_registers[] =
5085{
5086 MC_HUB_MISC_HUB_CG,
5087 MC_HUB_MISC_SIP_CG,
5088 MC_HUB_MISC_VM_CG,
5089 MC_XPB_CLK_GAT,
5090 ATC_MISC_CG,
5091 MC_CITF_MISC_WR_CG,
5092 MC_CITF_MISC_RD_CG,
5093 MC_CITF_MISC_VM_CG,
5094 VM_L2_CG,
5095};
5096
5097static void si_enable_mc_ls(struct radeon_device *rdev,
5098 bool enable)
5099{
5100 int i;
5101 u32 orig, data;
5102
5103 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5104 orig = data = RREG32(mc_cg_registers[i]);
e16866ec 5105 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
f8f84ac5
AD		 5106 data |= MC_LS_ENABLE;
5107 else
5108 data &= ~MC_LS_ENABLE;
5109 if (data != orig)
5110 WREG32(mc_cg_registers[i], data);
5111 }
5112}
5113
e16866ec
AD		 5114static void si_enable_mc_mgcg(struct radeon_device *rdev,
5115 bool enable)
5116{
5117 int i;
5118 u32 orig, data;
5119
5120 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5121 orig = data = RREG32(mc_cg_registers[i]);
5122 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5123 data |= MC_CG_ENABLE;
5124 else
5125 data &= ~MC_CG_ENABLE;
5126 if (data != orig)
5127 WREG32(mc_cg_registers[i], data);
5128 }
5129}
5130
5131static void si_enable_dma_mgcg(struct radeon_device *rdev,
5132 bool enable)
5133{
5134 u32 orig, data, offset;
5135 int i;
5136
5137 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5138 for (i = 0; i < 2; i++) {
5139 if (i == 0)
5140 offset = DMA0_REGISTER_OFFSET;
5141 else
5142 offset = DMA1_REGISTER_OFFSET;
5143 orig = data = RREG32(DMA_POWER_CNTL + offset);
5144 data &= ~MEM_POWER_OVERRIDE;
5145 if (data != orig)
5146 WREG32(DMA_POWER_CNTL + offset, data);
5147 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5148 }
5149 } else {
5150 for (i = 0; i < 2; i++) {
5151 if (i == 0)
5152 offset = DMA0_REGISTER_OFFSET;
5153 else
5154 offset = DMA1_REGISTER_OFFSET;
5155 orig = data = RREG32(DMA_POWER_CNTL + offset);
5156 data |= MEM_POWER_OVERRIDE;
5157 if (data != orig)
5158 WREG32(DMA_POWER_CNTL + offset, data);
5159
5160 orig = data = RREG32(DMA_CLK_CTRL + offset);
5161 data = 0xff000000;
5162 if (data != orig)
5163 WREG32(DMA_CLK_CTRL + offset, data);
5164 }
5165 }
5166}
5167
5168static void si_enable_bif_mgls(struct radeon_device *rdev,
5169 bool enable)
5170{
5171 u32 orig, data;
5172
5173 orig = data = RREG32_PCIE(PCIE_CNTL2);
5174
5175 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5176 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5177 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5178 else
5179 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5180 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5181
5182 if (orig != data)
5183 WREG32_PCIE(PCIE_CNTL2, data);
5184}
5185
5186static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5187 bool enable)
5188{
5189 u32 orig, data;
5190
5191 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5192
5193 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5194 data &= ~CLOCK_GATING_DIS;
5195 else
5196 data |= CLOCK_GATING_DIS;
5197
5198 if (orig != data)
5199 WREG32(HDP_HOST_PATH_CNTL, data);
5200}
5201
5202static void si_enable_hdp_ls(struct radeon_device *rdev,
5203 bool enable)
5204{
5205 u32 orig, data;
5206
5207 orig = data = RREG32(HDP_MEM_POWER_LS);
5208
5209 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5210 data |= HDP_LS_ENABLE;
5211 else
5212 data &= ~HDP_LS_ENABLE;
5213
5214 if (orig != data)
5215 WREG32(HDP_MEM_POWER_LS, data);
5216}
5217
5218void si_update_cg(struct radeon_device *rdev,
5219 u32 block, bool enable)
5220{
5221 if (block & RADEON_CG_BLOCK_GFX) {
5222 /* order matters! */
5223 if (enable) {
5224 si_enable_mgcg(rdev, true);
5225 si_enable_cgcg(rdev, true);
5226 } else {
5227 si_enable_cgcg(rdev, false);
5228 si_enable_mgcg(rdev, false);
5229 }
5230 }
5231
5232 if (block & RADEON_CG_BLOCK_MC) {
5233 si_enable_mc_mgcg(rdev, enable);
5234 si_enable_mc_ls(rdev, enable);
5235 }
5236
5237 if (block & RADEON_CG_BLOCK_SDMA) {
5238 si_enable_dma_mgcg(rdev, enable);
5239 }
5240
5241 if (block & RADEON_CG_BLOCK_BIF) {
5242 si_enable_bif_mgls(rdev, enable);
5243 }
5244
5245 if (block & RADEON_CG_BLOCK_UVD) {
5246 if (rdev->has_uvd) {
5247 si_enable_uvd_mgcg(rdev, enable);
5248 }
5249 }
5250
5251 if (block & RADEON_CG_BLOCK_HDP) {
5252 si_enable_hdp_mgcg(rdev, enable);
5253 si_enable_hdp_ls(rdev, enable);
5254 }
5255}
f8f84ac5
AD		 5256
5257static void si_init_cg(struct radeon_device *rdev)
5258{
e16866ec
AD		 5259 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5260 RADEON_CG_BLOCK_MC |
5261 RADEON_CG_BLOCK_SDMA |
5262 RADEON_CG_BLOCK_BIF |
5263 RADEON_CG_BLOCK_HDP), true);
b2d70917 5264 if (rdev->has_uvd) {
e16866ec 5265 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
f8f84ac5
AD		 5266 si_init_uvd_internal_cg(rdev);
5267 }
5268}
5269
5270static void si_fini_cg(struct radeon_device *rdev)
5271{
0116e1ef 5272 if (rdev->has_uvd) {
e16866ec 5273 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
0116e1ef 5274 }
e16866ec
AD		 5275 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5276 RADEON_CG_BLOCK_MC |
5277 RADEON_CG_BLOCK_SDMA |
5278 RADEON_CG_BLOCK_BIF |
5279 RADEON_CG_BLOCK_HDP), false);
5280}
5281
59a82d0e
AD		 5282u32 si_get_csb_size(struct radeon_device *rdev)
5283{
5284 u32 count = 0;
5285 const struct cs_section_def *sect = NULL;
5286 const struct cs_extent_def *ext = NULL;
5287
5288 if (rdev->rlc.cs_data == NULL)
5289 return 0;
5290
5291 /* begin clear state */
5292 count += 2;
5293 /* context control state */
5294 count += 3;
5295
5296 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5297 for (ext = sect->section; ext->extent != NULL; ++ext) {
5298 if (sect->id == SECT_CONTEXT)
5299 count += 2 + ext->reg_count;
5300 else
5301 return 0;
5302 }
5303 }
5304 /* pa_sc_raster_config */
5305 count += 3;
5306 /* end clear state */
5307 count += 2;
5308 /* clear state */
5309 count += 2;
5310
5311 return count;
5312}
5313
5314void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5315{
5316 u32 count = 0, i;
5317 const struct cs_section_def *sect = NULL;
5318 const struct cs_extent_def *ext = NULL;
5319
5320 if (rdev->rlc.cs_data == NULL)
5321 return;
5322 if (buffer == NULL)
5323 return;
5324
5325 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5326 buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE;
5327
5328 buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1);
5329 buffer[count++] = 0x80000000;
5330 buffer[count++] = 0x80000000;
5331
5332 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5333 for (ext = sect->section; ext->extent != NULL; ++ext) {
5334 if (sect->id == SECT_CONTEXT) {
5335 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count);
5336 buffer[count++] = ext->reg_index - 0xa000;
5337 for (i = 0; i < ext->reg_count; i++)
5338 buffer[count++] = ext->extent[i];
5339 } else {
5340 return;
5341 }
5342 }
5343 }
5344
5345 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 1);
5346 buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START;
5347 switch (rdev->family) {
5348 case CHIP_TAHITI:
5349 case CHIP_PITCAIRN:
5350 buffer[count++] = 0x2a00126a;
5351 break;
5352 case CHIP_VERDE:
5353 buffer[count++] = 0x0000124a;
5354 break;
5355 case CHIP_OLAND:
5356 buffer[count++] = 0x00000082;
5357 break;
5358 case CHIP_HAINAN:
5359 buffer[count++] = 0x00000000;
5360 break;
5361 default:
5362 buffer[count++] = 0x00000000;
5363 break;
5364 }
5365
5366 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5367 buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE;
5368
5369 buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0);
5370 buffer[count++] = 0;
5371}
5372
f8f84ac5
AD		 5373static void si_init_pg(struct radeon_device *rdev)
5374{
0116e1ef
AD		 5375 if (rdev->pg_flags) {
5376 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5377 si_init_dma_pg(rdev);
0116e1ef 5378 }
f8f84ac5 5379 si_init_ao_cu_mask(rdev);
0116e1ef
AD		 5380 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_CG) {
5381 si_init_gfx_cgpg(rdev);
0116e1ef 5382 }
ca6ebb39
AD		 5383 si_enable_dma_pg(rdev, true);
5384 si_enable_gfx_cgpg(rdev, true);
f8f84ac5
AD		 5385 } else {
5386 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5387 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5388 }
5389}
5390
5391static void si_fini_pg(struct radeon_device *rdev)
5392{
0116e1ef 5393 if (rdev->pg_flags) {
ca6ebb39
AD		 5394 si_enable_dma_pg(rdev, false);
5395 si_enable_gfx_cgpg(rdev, false);
f8f84ac5
AD		 5396 }
5397}
5398
347e7592
AD		 5399/*
5400 * RLC
5401 */
866d83de 5402void si_rlc_reset(struct radeon_device *rdev)
d719cef3 5403{
f8f84ac5 5404 u32 tmp = RREG32(GRBM_SOFT_RESET);
d719cef3 5405
f8f84ac5
AD		 5406 tmp |= SOFT_RESET_RLC;
5407 WREG32(GRBM_SOFT_RESET, tmp);
5408 udelay(50);
5409 tmp &= ~SOFT_RESET_RLC;
5410 WREG32(GRBM_SOFT_RESET, tmp);
5411 udelay(50);
d719cef3
AD		 5412}
5413
347e7592
AD		 5414static void si_rlc_stop(struct radeon_device *rdev)
5415{
5416 WREG32(RLC_CNTL, 0);
d719cef3
AD		 5417
5418 si_enable_gui_idle_interrupt(rdev, false);
5419
5420 si_wait_for_rlc_serdes(rdev);
347e7592
AD		 5421}
5422
5423static void si_rlc_start(struct radeon_device *rdev)
5424{
5425 WREG32(RLC_CNTL, RLC_ENABLE);
d719cef3
AD		 5426
5427 si_enable_gui_idle_interrupt(rdev, true);
5428
5429 udelay(50);
5430}
5431
5432static bool si_lbpw_supported(struct radeon_device *rdev)
5433{
5434 u32 tmp;
5435
5436 /* Enable LBPW only for DDR3 */
5437 tmp = RREG32(MC_SEQ_MISC0);
5438 if ((tmp & 0xF0000000) == 0xB0000000)
5439 return true;
5440 return false;
5441}
5442
5443static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5444{
5445 u32 tmp;
5446
5447 tmp = RREG32(RLC_LB_CNTL);
5448 if (enable)
5449 tmp |= LOAD_BALANCE_ENABLE;
5450 else
5451 tmp &= ~LOAD_BALANCE_ENABLE;
5452 WREG32(RLC_LB_CNTL, tmp);
5453
5454 if (!enable) {
5455 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5456 WREG32(SPI_LB_CU_MASK, 0x00ff);
5457 }
347e7592
AD		 5458}
5459
5460static int si_rlc_resume(struct radeon_device *rdev)
5461{
5462 u32 i;
5463 const __be32 *fw_data;
5464
5465 if (!rdev->rlc_fw)
5466 return -EINVAL;
5467
5468 si_rlc_stop(rdev);
5469
f8f84ac5
AD		 5470 si_rlc_reset(rdev);
5471
5472 si_init_pg(rdev);
5473
5474 si_init_cg(rdev);
5475
347e7592
AD		 5476 WREG32(RLC_RL_BASE, 0);
5477 WREG32(RLC_RL_SIZE, 0);
5478 WREG32(RLC_LB_CNTL, 0);
5479 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5480 WREG32(RLC_LB_CNTR_INIT, 0);
d719cef3 5481 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
347e7592 5482
347e7592
AD		 5483 WREG32(RLC_MC_CNTL, 0);
5484 WREG32(RLC_UCODE_CNTL, 0);
5485
5486 fw_data = (const __be32 *)rdev->rlc_fw->data;
5487 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5488 WREG32(RLC_UCODE_ADDR, i);
5489 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5490 }
5491 WREG32(RLC_UCODE_ADDR, 0);
5492
d719cef3
AD		 5493 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5494
347e7592
AD		 5495 si_rlc_start(rdev);
5496
5497 return 0;
5498}
5499
25a857fb
AD		 5500static void si_enable_interrupts(struct radeon_device *rdev)
5501{
5502 u32 ih_cntl = RREG32(IH_CNTL);
5503 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5504
5505 ih_cntl |= ENABLE_INTR;
5506 ih_rb_cntl |= IH_RB_ENABLE;
5507 WREG32(IH_CNTL, ih_cntl);
5508 WREG32(IH_RB_CNTL, ih_rb_cntl);
5509 rdev->ih.enabled = true;
5510}
5511
5512static void si_disable_interrupts(struct radeon_device *rdev)
5513{
5514 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5515 u32 ih_cntl = RREG32(IH_CNTL);
5516
5517 ih_rb_cntl &= ~IH_RB_ENABLE;
5518 ih_cntl &= ~ENABLE_INTR;
5519 WREG32(IH_RB_CNTL, ih_rb_cntl);
5520 WREG32(IH_CNTL, ih_cntl);
5521 /* set rptr, wptr to 0 */
5522 WREG32(IH_RB_RPTR, 0);
5523 WREG32(IH_RB_WPTR, 0);
5524 rdev->ih.enabled = false;
25a857fb
AD		 5525 rdev->ih.rptr = 0;
5526}
5527
5528static void si_disable_interrupt_state(struct radeon_device *rdev)
5529{
5530 u32 tmp;
5531
5532 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5533 WREG32(CP_INT_CNTL_RING1, 0);
5534 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 5535 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5536 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5537 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5538 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 5539 WREG32(GRBM_INT_CNTL, 0);
5153550a
AD		 5540 if (rdev->num_crtc >= 2) {
5541 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5542 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5543 }
25a857fb
AD		 5544 if (rdev->num_crtc >= 4) {
5545 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5546 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5547 }
5548 if (rdev->num_crtc >= 6) {
5549 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5550 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5551 }
5552
5153550a
AD		 5553 if (rdev->num_crtc >= 2) {
5554 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5555 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5556 }
25a857fb
AD		 5557 if (rdev->num_crtc >= 4) {
5558 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5559 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5560 }
5561 if (rdev->num_crtc >= 6) {
5562 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5563 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5564 }
5565
5153550a
AD		 5566 if (!ASIC_IS_NODCE(rdev)) {
5567 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
5568
5569 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5570 WREG32(DC_HPD1_INT_CONTROL, tmp);
5571 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5572 WREG32(DC_HPD2_INT_CONTROL, tmp);
5573 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5574 WREG32(DC_HPD3_INT_CONTROL, tmp);
5575 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5576 WREG32(DC_HPD4_INT_CONTROL, tmp);
5577 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5578 WREG32(DC_HPD5_INT_CONTROL, tmp);
5579 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5580 WREG32(DC_HPD6_INT_CONTROL, tmp);
5581 }
25a857fb
AD		 5582}
5583
5584static int si_irq_init(struct radeon_device *rdev)
5585{
5586 int ret = 0;
5587 int rb_bufsz;
5588 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5589
5590 /* allocate ring */
5591 ret = r600_ih_ring_alloc(rdev);
5592 if (ret)
5593 return ret;
5594
5595 /* disable irqs */
5596 si_disable_interrupts(rdev);
5597
5598 /* init rlc */
5599 ret = si_rlc_resume(rdev);
5600 if (ret) {
5601 r600_ih_ring_fini(rdev);
5602 return ret;
5603 }
5604
5605 /* setup interrupt control */
5606 /* set dummy read address to ring address */
5607 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5608 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5609 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5610 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5611 */
5612 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5613 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5614 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5615 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5616
5617 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5618 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
5619
5620 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5621 IH_WPTR_OVERFLOW_CLEAR |
5622 (rb_bufsz << 1));
5623
5624 if (rdev->wb.enabled)
5625 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5626
5627 /* set the writeback address whether it's enabled or not */
5628 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5629 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5630
5631 WREG32(IH_RB_CNTL, ih_rb_cntl);
5632
5633 /* set rptr, wptr to 0 */
5634 WREG32(IH_RB_RPTR, 0);
5635 WREG32(IH_RB_WPTR, 0);
5636
5637 /* Default settings for IH_CNTL (disabled at first) */
5638 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5639 /* RPTR_REARM only works if msi's are enabled */
5640 if (rdev->msi_enabled)
5641 ih_cntl |= RPTR_REARM;
5642 WREG32(IH_CNTL, ih_cntl);
5643
5644 /* force the active interrupt state to all disabled */
5645 si_disable_interrupt_state(rdev);
5646
2099810f
DA		 5647 pci_set_master(rdev->pdev);
5648
25a857fb
AD		 5649 /* enable irqs */
5650 si_enable_interrupts(rdev);
5651
5652 return ret;
5653}
5654
5655int si_irq_set(struct radeon_device *rdev)
5656{
5657 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
5658 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5659 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5153550a 5660 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
25a857fb
AD		 5661 u32 grbm_int_cntl = 0;
5662 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
8c5fd7ef 5663 u32 dma_cntl, dma_cntl1;
a9e61410 5664 u32 thermal_int = 0;
25a857fb
AD		 5665
5666 if (!rdev->irq.installed) {
5667 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5668 return -EINVAL;
5669 }
5670 /* don't enable anything if the ih is disabled */
5671 if (!rdev->ih.enabled) {
5672 si_disable_interrupts(rdev);
5673 /* force the active interrupt state to all disabled */
5674 si_disable_interrupt_state(rdev);
5675 return 0;
5676 }
5677
5153550a
AD		 5678 if (!ASIC_IS_NODCE(rdev)) {
5679 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5680 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5681 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5682 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5683 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5684 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5685 }
25a857fb 5686
8c5fd7ef
AD		 5687 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5688 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5689
a9e61410
AD		 5690 thermal_int = RREG32(CG_THERMAL_INT) &
5691 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5692
25a857fb 5693 /* enable CP interrupts on all rings */
736fc37f 5694 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 5695 DRM_DEBUG("si_irq_set: sw int gfx\n");
5696 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5697 }
736fc37f 5698 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 5699 DRM_DEBUG("si_irq_set: sw int cp1\n");
5700 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5701 }
736fc37f 5702 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 5703 DRM_DEBUG("si_irq_set: sw int cp2\n");
5704 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5705 }
8c5fd7ef
AD		 5706 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5707 DRM_DEBUG("si_irq_set: sw int dma\n");
5708 dma_cntl |= TRAP_ENABLE;
5709 }
5710
5711 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5712 DRM_DEBUG("si_irq_set: sw int dma1\n");
5713 dma_cntl1 |= TRAP_ENABLE;
5714 }
25a857fb 5715 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 5716 atomic_read(&rdev->irq.pflip[0])) {
25a857fb
AD		 5717 DRM_DEBUG("si_irq_set: vblank 0\n");
5718 crtc1 |= VBLANK_INT_MASK;
5719 }
5720 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 5721 atomic_read(&rdev->irq.pflip[1])) {
25a857fb
AD		 5722 DRM_DEBUG("si_irq_set: vblank 1\n");
5723 crtc2 |= VBLANK_INT_MASK;
5724 }
5725 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 5726 atomic_read(&rdev->irq.pflip[2])) {
25a857fb
AD		 5727 DRM_DEBUG("si_irq_set: vblank 2\n");
5728 crtc3 |= VBLANK_INT_MASK;
5729 }
5730 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 5731 atomic_read(&rdev->irq.pflip[3])) {
25a857fb
AD		 5732 DRM_DEBUG("si_irq_set: vblank 3\n");
5733 crtc4 |= VBLANK_INT_MASK;
5734 }
5735 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 5736 atomic_read(&rdev->irq.pflip[4])) {
25a857fb
AD		 5737 DRM_DEBUG("si_irq_set: vblank 4\n");
5738 crtc5 |= VBLANK_INT_MASK;
5739 }
5740 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 5741 atomic_read(&rdev->irq.pflip[5])) {
25a857fb
AD		 5742 DRM_DEBUG("si_irq_set: vblank 5\n");
5743 crtc6 |= VBLANK_INT_MASK;
5744 }
5745 if (rdev->irq.hpd[0]) {
5746 DRM_DEBUG("si_irq_set: hpd 1\n");
5747 hpd1 |= DC_HPDx_INT_EN;
5748 }
5749 if (rdev->irq.hpd[1]) {
5750 DRM_DEBUG("si_irq_set: hpd 2\n");
5751 hpd2 |= DC_HPDx_INT_EN;
5752 }
5753 if (rdev->irq.hpd[2]) {
5754 DRM_DEBUG("si_irq_set: hpd 3\n");
5755 hpd3 |= DC_HPDx_INT_EN;
5756 }
5757 if (rdev->irq.hpd[3]) {
5758 DRM_DEBUG("si_irq_set: hpd 4\n");
5759 hpd4 |= DC_HPDx_INT_EN;
5760 }
5761 if (rdev->irq.hpd[4]) {
5762 DRM_DEBUG("si_irq_set: hpd 5\n");
5763 hpd5 |= DC_HPDx_INT_EN;
5764 }
5765 if (rdev->irq.hpd[5]) {
5766 DRM_DEBUG("si_irq_set: hpd 6\n");
5767 hpd6 |= DC_HPDx_INT_EN;
5768 }
25a857fb
AD		 5769
5770 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5771 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5772 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5773
8c5fd7ef
AD		 5774 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5775 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5776
25a857fb
AD		 5777 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5778
a9e61410
AD		 5779 if (rdev->irq.dpm_thermal) {
5780 DRM_DEBUG("dpm thermal\n");
5781 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5782 }
5783
5153550a
AD		 5784 if (rdev->num_crtc >= 2) {
5785 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5786 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5787 }
25a857fb
AD		 5788 if (rdev->num_crtc >= 4) {
5789 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5790 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5791 }
5792 if (rdev->num_crtc >= 6) {
5793 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5794 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5795 }
5796
5153550a
AD		 5797 if (rdev->num_crtc >= 2) {
5798 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5799 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5800 }
25a857fb
AD		 5801 if (rdev->num_crtc >= 4) {
5802 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5803 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5804 }
5805 if (rdev->num_crtc >= 6) {
5806 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5807 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5808 }
5809
5153550a
AD		 5810 if (!ASIC_IS_NODCE(rdev)) {
5811 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5812 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5813 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5814 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5815 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5816 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5817 }
25a857fb 5818
a9e61410
AD		 5819 WREG32(CG_THERMAL_INT, thermal_int);
5820
25a857fb
AD		 5821 return 0;
5822}
5823
5824static inline void si_irq_ack(struct radeon_device *rdev)
5825{
5826 u32 tmp;
5827
5153550a
AD		 5828 if (ASIC_IS_NODCE(rdev))
5829 return;
5830
25a857fb
AD		 5831 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5832 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5833 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5834 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5835 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5836 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5837 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5838 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5839 if (rdev->num_crtc >= 4) {
5840 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5841 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5842 }
5843 if (rdev->num_crtc >= 6) {
5844 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5845 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5846 }
5847
5848 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5849 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5850 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5851 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5852 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5853 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5854 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5855 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5856 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5857 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5858 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5859 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5860
5861 if (rdev->num_crtc >= 4) {
5862 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5863 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5864 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5865 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5866 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5867 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5868 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5869 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5870 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5871 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5872 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5873 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5874 }
5875
5876 if (rdev->num_crtc >= 6) {
5877 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5878 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5879 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5880 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5881 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5882 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5883 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5884 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5885 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5886 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5887 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5888 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5889 }
5890
5891 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5892 tmp = RREG32(DC_HPD1_INT_CONTROL);
5893 tmp |= DC_HPDx_INT_ACK;
5894 WREG32(DC_HPD1_INT_CONTROL, tmp);
5895 }
5896 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5897 tmp = RREG32(DC_HPD2_INT_CONTROL);
5898 tmp |= DC_HPDx_INT_ACK;
5899 WREG32(DC_HPD2_INT_CONTROL, tmp);
5900 }
5901 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5902 tmp = RREG32(DC_HPD3_INT_CONTROL);
5903 tmp |= DC_HPDx_INT_ACK;
5904 WREG32(DC_HPD3_INT_CONTROL, tmp);
5905 }
5906 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5907 tmp = RREG32(DC_HPD4_INT_CONTROL);
5908 tmp |= DC_HPDx_INT_ACK;
5909 WREG32(DC_HPD4_INT_CONTROL, tmp);
5910 }
5911 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5912 tmp = RREG32(DC_HPD5_INT_CONTROL);
5913 tmp |= DC_HPDx_INT_ACK;
5914 WREG32(DC_HPD5_INT_CONTROL, tmp);
5915 }
5916 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5917 tmp = RREG32(DC_HPD5_INT_CONTROL);
5918 tmp |= DC_HPDx_INT_ACK;
5919 WREG32(DC_HPD6_INT_CONTROL, tmp);
5920 }
5921}
5922
5923static void si_irq_disable(struct radeon_device *rdev)
5924{
5925 si_disable_interrupts(rdev);
5926 /* Wait and acknowledge irq */
5927 mdelay(1);
5928 si_irq_ack(rdev);
5929 si_disable_interrupt_state(rdev);
5930}
5931
5932static void si_irq_suspend(struct radeon_device *rdev)
5933{
5934 si_irq_disable(rdev);
5935 si_rlc_stop(rdev);
5936}
5937
9b136d51
AD		 5938static void si_irq_fini(struct radeon_device *rdev)
5939{
5940 si_irq_suspend(rdev);
5941 r600_ih_ring_fini(rdev);
5942}
5943
25a857fb
AD		 5944static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
5945{
5946 u32 wptr, tmp;
5947
5948 if (rdev->wb.enabled)
5949 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
5950 else
5951 wptr = RREG32(IH_RB_WPTR);
5952
5953 if (wptr & RB_OVERFLOW) {
5954 /* When a ring buffer overflow happen start parsing interrupt
5955 * from the last not overwritten vector (wptr + 16). Hopefully
5956 * this should allow us to catchup.
5957 */
5958 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
5959 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
5960 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
5961 tmp = RREG32(IH_RB_CNTL);
5962 tmp |= IH_WPTR_OVERFLOW_CLEAR;
5963 WREG32(IH_RB_CNTL, tmp);
5964 }
5965 return (wptr & rdev->ih.ptr_mask);
5966}
5967
5968/* SI IV Ring
5969 * Each IV ring entry is 128 bits:
5970 * [7:0] - interrupt source id
5971 * [31:8] - reserved
5972 * [59:32] - interrupt source data
5973 * [63:60] - reserved
5974 * [71:64] - RINGID
5975 * [79:72] - VMID
5976 * [127:80] - reserved
5977 */
5978int si_irq_process(struct radeon_device *rdev)
5979{
5980 u32 wptr;
5981 u32 rptr;
5982 u32 src_id, src_data, ring_id;
5983 u32 ring_index;
25a857fb 5984 bool queue_hotplug = false;
a9e61410 5985 bool queue_thermal = false;
fbf6dc7a 5986 u32 status, addr;
25a857fb
AD		 5987
5988 if (!rdev->ih.enabled || rdev->shutdown)
5989 return IRQ_NONE;
5990
5991 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 5992
5993restart_ih:
5994 /* is somebody else already processing irqs? */
5995 if (atomic_xchg(&rdev->ih.lock, 1))
5996 return IRQ_NONE;
5997
25a857fb
AD		 5998 rptr = rdev->ih.rptr;
5999 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6000
25a857fb
AD		 6001 /* Order reading of wptr vs. reading of IH ring data */
6002 rmb();
6003
6004 /* display interrupts */
6005 si_irq_ack(rdev);
6006
25a857fb
AD		 6007 while (rptr != wptr) {
6008 /* wptr/rptr are in bytes! */
6009 ring_index = rptr / 4;
6010 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6011 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6012 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6013
6014 switch (src_id) {
6015 case 1: /* D1 vblank/vline */
6016 switch (src_data) {
6017 case 0: /* D1 vblank */
6018 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6019 if (rdev->irq.crtc_vblank_int[0]) {
6020 drm_handle_vblank(rdev->ddev, 0);
6021 rdev->pm.vblank_sync = true;
6022 wake_up(&rdev->irq.vblank_queue);
6023 }
736fc37f 6024 if (atomic_read(&rdev->irq.pflip[0]))
25a857fb
AD		 6025 radeon_crtc_handle_flip(rdev, 0);
6026 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6027 DRM_DEBUG("IH: D1 vblank\n");
6028 }
6029 break;
6030 case 1: /* D1 vline */
6031 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6032 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6033 DRM_DEBUG("IH: D1 vline\n");
6034 }
6035 break;
6036 default:
6037 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6038 break;
6039 }
6040 break;
6041 case 2: /* D2 vblank/vline */
6042 switch (src_data) {
6043 case 0: /* D2 vblank */
6044 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6045 if (rdev->irq.crtc_vblank_int[1]) {
6046 drm_handle_vblank(rdev->ddev, 1);
6047 rdev->pm.vblank_sync = true;
6048 wake_up(&rdev->irq.vblank_queue);
6049 }
736fc37f 6050 if (atomic_read(&rdev->irq.pflip[1]))
25a857fb
AD		 6051 radeon_crtc_handle_flip(rdev, 1);
6052 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6053 DRM_DEBUG("IH: D2 vblank\n");
6054 }
6055 break;
6056 case 1: /* D2 vline */
6057 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6058 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6059 DRM_DEBUG("IH: D2 vline\n");
6060 }
6061 break;
6062 default:
6063 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6064 break;
6065 }
6066 break;
6067 case 3: /* D3 vblank/vline */
6068 switch (src_data) {
6069 case 0: /* D3 vblank */
6070 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6071 if (rdev->irq.crtc_vblank_int[2]) {
6072 drm_handle_vblank(rdev->ddev, 2);
6073 rdev->pm.vblank_sync = true;
6074 wake_up(&rdev->irq.vblank_queue);
6075 }
736fc37f 6076 if (atomic_read(&rdev->irq.pflip[2]))
25a857fb
AD		 6077 radeon_crtc_handle_flip(rdev, 2);
6078 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6079 DRM_DEBUG("IH: D3 vblank\n");
6080 }
6081 break;
6082 case 1: /* D3 vline */
6083 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6084 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6085 DRM_DEBUG("IH: D3 vline\n");
6086 }
6087 break;
6088 default:
6089 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6090 break;
6091 }
6092 break;
6093 case 4: /* D4 vblank/vline */
6094 switch (src_data) {
6095 case 0: /* D4 vblank */
6096 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6097 if (rdev->irq.crtc_vblank_int[3]) {
6098 drm_handle_vblank(rdev->ddev, 3);
6099 rdev->pm.vblank_sync = true;
6100 wake_up(&rdev->irq.vblank_queue);
6101 }
736fc37f 6102 if (atomic_read(&rdev->irq.pflip[3]))
25a857fb
AD		 6103 radeon_crtc_handle_flip(rdev, 3);
6104 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6105 DRM_DEBUG("IH: D4 vblank\n");
6106 }
6107 break;
6108 case 1: /* D4 vline */
6109 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6110 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6111 DRM_DEBUG("IH: D4 vline\n");
6112 }
6113 break;
6114 default:
6115 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6116 break;
6117 }
6118 break;
6119 case 5: /* D5 vblank/vline */
6120 switch (src_data) {
6121 case 0: /* D5 vblank */
6122 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6123 if (rdev->irq.crtc_vblank_int[4]) {
6124 drm_handle_vblank(rdev->ddev, 4);
6125 rdev->pm.vblank_sync = true;
6126 wake_up(&rdev->irq.vblank_queue);
6127 }
736fc37f 6128 if (atomic_read(&rdev->irq.pflip[4]))
25a857fb
AD		 6129 radeon_crtc_handle_flip(rdev, 4);
6130 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6131 DRM_DEBUG("IH: D5 vblank\n");
6132 }
6133 break;
6134 case 1: /* D5 vline */
6135 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6136 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6137 DRM_DEBUG("IH: D5 vline\n");
6138 }
6139 break;
6140 default:
6141 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6142 break;
6143 }
6144 break;
6145 case 6: /* D6 vblank/vline */
6146 switch (src_data) {
6147 case 0: /* D6 vblank */
6148 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6149 if (rdev->irq.crtc_vblank_int[5]) {
6150 drm_handle_vblank(rdev->ddev, 5);
6151 rdev->pm.vblank_sync = true;
6152 wake_up(&rdev->irq.vblank_queue);
6153 }
736fc37f 6154 if (atomic_read(&rdev->irq.pflip[5]))
25a857fb
AD		 6155 radeon_crtc_handle_flip(rdev, 5);
6156 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6157 DRM_DEBUG("IH: D6 vblank\n");
6158 }
6159 break;
6160 case 1: /* D6 vline */
6161 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6162 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6163 DRM_DEBUG("IH: D6 vline\n");
6164 }
6165 break;
6166 default:
6167 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6168 break;
6169 }
6170 break;
6171 case 42: /* HPD hotplug */
6172 switch (src_data) {
6173 case 0:
6174 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6175 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6176 queue_hotplug = true;
6177 DRM_DEBUG("IH: HPD1\n");
6178 }
6179 break;
6180 case 1:
6181 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6182 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6183 queue_hotplug = true;
6184 DRM_DEBUG("IH: HPD2\n");
6185 }
6186 break;
6187 case 2:
6188 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6189 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6190 queue_hotplug = true;
6191 DRM_DEBUG("IH: HPD3\n");
6192 }
6193 break;
6194 case 3:
6195 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6196 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6197 queue_hotplug = true;
6198 DRM_DEBUG("IH: HPD4\n");
6199 }
6200 break;
6201 case 4:
6202 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6203 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6204 queue_hotplug = true;
6205 DRM_DEBUG("IH: HPD5\n");
6206 }
6207 break;
6208 case 5:
6209 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6210 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6211 queue_hotplug = true;
6212 DRM_DEBUG("IH: HPD6\n");
6213 }
6214 break;
6215 default:
6216 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6217 break;
6218 }
6219 break;
ae133a11
CK		 6220 case 146:
6221 case 147:
fbf6dc7a
AD		 6222 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6223 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
ae133a11
CK		 6224 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6225 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
fbf6dc7a 6226 addr);
ae133a11 6227 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
fbf6dc7a
AD		 6228 status);
6229 si_vm_decode_fault(rdev, status, addr);
ae133a11
CK		 6230 /* reset addr and status */
6231 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6232 break;
25a857fb
AD		 6233 case 176: /* RINGID0 CP_INT */
6234 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6235 break;
6236 case 177: /* RINGID1 CP_INT */
6237 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6238 break;
6239 case 178: /* RINGID2 CP_INT */
6240 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6241 break;
6242 case 181: /* CP EOP event */
6243 DRM_DEBUG("IH: CP EOP\n");
6244 switch (ring_id) {
6245 case 0:
6246 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6247 break;
6248 case 1:
6249 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6250 break;
6251 case 2:
6252 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6253 break;
6254 }
6255 break;
8c5fd7ef
AD		 6256 case 224: /* DMA trap event */
6257 DRM_DEBUG("IH: DMA trap\n");
6258 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6259 break;
a9e61410
AD		 6260 case 230: /* thermal low to high */
6261 DRM_DEBUG("IH: thermal low to high\n");
6262 rdev->pm.dpm.thermal.high_to_low = false;
6263 queue_thermal = true;
6264 break;
6265 case 231: /* thermal high to low */
6266 DRM_DEBUG("IH: thermal high to low\n");
6267 rdev->pm.dpm.thermal.high_to_low = true;
6268 queue_thermal = true;
6269 break;
25a857fb
AD		 6270 case 233: /* GUI IDLE */
6271 DRM_DEBUG("IH: GUI idle\n");
25a857fb 6272 break;
8c5fd7ef
AD		 6273 case 244: /* DMA trap event */
6274 DRM_DEBUG("IH: DMA1 trap\n");
6275 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6276 break;
25a857fb
AD		 6277 default:
6278 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6279 break;
6280 }
6281
6282 /* wptr/rptr are in bytes! */
6283 rptr += 16;
6284 rptr &= rdev->ih.ptr_mask;
6285 }
25a857fb
AD		 6286 if (queue_hotplug)
6287 schedule_work(&rdev->hotplug_work);
a9e61410
AD		 6288 if (queue_thermal && rdev->pm.dpm_enabled)
6289 schedule_work(&rdev->pm.dpm.thermal.work);
25a857fb
AD		 6290 rdev->ih.rptr = rptr;
6291 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 6292 atomic_set(&rdev->ih.lock, 0);
6293
6294 /* make sure wptr hasn't changed while processing */
6295 wptr = si_get_ih_wptr(rdev);
6296 if (wptr != rptr)
6297 goto restart_ih;
6298
25a857fb
AD		 6299 return IRQ_HANDLED;
6300}
6301
9b136d51
AD		 6302/*
6303 * startup/shutdown callbacks
6304 */
6305static int si_startup(struct radeon_device *rdev)
6306{
6307 struct radeon_ring *ring;
6308 int r;
6309
b9d305df
AD		 6310 /* enable pcie gen2/3 link */
6311 si_pcie_gen3_enable(rdev);
e0bcf165
AD		 6312 /* enable aspm */
6313 si_program_aspm(rdev);
b9d305df 6314
6fab3feb
AD		 6315 si_mc_program(rdev);
6316
9b136d51
AD		 6317 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6318 !rdev->rlc_fw || !rdev->mc_fw) {
6319 r = si_init_microcode(rdev);
6320 if (r) {
6321 DRM_ERROR("Failed to load firmware!\n");
6322 return r;
6323 }
6324 }
6325
6326 r = si_mc_load_microcode(rdev);
6327 if (r) {
6328 DRM_ERROR("Failed to load MC firmware!\n");
6329 return r;
6330 }
6331
6332 r = r600_vram_scratch_init(rdev);
6333 if (r)
6334 return r;
6335
9b136d51
AD		 6336 r = si_pcie_gart_enable(rdev);
6337 if (r)
6338 return r;
6339 si_gpu_init(rdev);
6340
9b136d51 6341 /* allocate rlc buffers */
1fd11777
AD		 6342 if (rdev->family == CHIP_VERDE) {
6343 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6344 rdev->rlc.reg_list_size =
6345 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6346 }
6347 rdev->rlc.cs_data = si_cs_data;
6348 r = sumo_rlc_init(rdev);
9b136d51
AD		 6349 if (r) {
6350 DRM_ERROR("Failed to init rlc BOs!\n");
6351 return r;
6352 }
6353
6354 /* allocate wb buffer */
6355 r = radeon_wb_init(rdev);
6356 if (r)
6357 return r;
6358
6359 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6360 if (r) {
6361 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6362 return r;
6363 }
6364
6365 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6366 if (r) {
6367 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6368 return r;
6369 }
6370
6371 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6372 if (r) {
6373 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6374 return r;
6375 }
6376
8c5fd7ef
AD		 6377 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6378 if (r) {
6379 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6380 return r;
6381 }
6382
6383 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6384 if (r) {
6385 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6386 return r;
6387 }
6388
1df0d523 6389 if (rdev->has_uvd) {
e409b128 6390 r = uvd_v2_2_resume(rdev);
1df0d523
AD		 6391 if (!r) {
6392 r = radeon_fence_driver_start_ring(rdev,
6393 R600_RING_TYPE_UVD_INDEX);
6394 if (r)
6395 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6396 }
f2ba57b5 6397 if (r)
1df0d523 6398 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
f2ba57b5 6399 }
f2ba57b5 6400
9b136d51 6401 /* Enable IRQ */
e49f3959
AH		 6402 if (!rdev->irq.installed) {
6403 r = radeon_irq_kms_init(rdev);
6404 if (r)
6405 return r;
6406 }
6407
9b136d51
AD		 6408 r = si_irq_init(rdev);
6409 if (r) {
6410 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6411 radeon_irq_kms_fini(rdev);
6412 return r;
6413 }
6414 si_irq_set(rdev);
6415
6416 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6417 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6418 CP_RB0_RPTR, CP_RB0_WPTR,
2e1e6dad 6419 RADEON_CP_PACKET2);
9b136d51
AD		 6420 if (r)
6421 return r;
6422
6423 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6424 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6425 CP_RB1_RPTR, CP_RB1_WPTR,
2e1e6dad 6426 RADEON_CP_PACKET2);
9b136d51
AD		 6427 if (r)
6428 return r;
6429
6430 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6431 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6432 CP_RB2_RPTR, CP_RB2_WPTR,
2e1e6dad 6433 RADEON_CP_PACKET2);
9b136d51
AD		 6434 if (r)
6435 return r;
6436
8c5fd7ef
AD		 6437 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6438 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6439 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
6440 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
2e1e6dad 6441 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6442 if (r)
6443 return r;
6444
6445 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6446 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6447 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
6448 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
2e1e6dad 6449 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6450 if (r)
6451 return r;
6452
9b136d51
AD		 6453 r = si_cp_load_microcode(rdev);
6454 if (r)
6455 return r;
6456 r = si_cp_resume(rdev);
6457 if (r)
6458 return r;
6459
8c5fd7ef
AD		 6460 r = cayman_dma_resume(rdev);
6461 if (r)
6462 return r;
6463
1df0d523
AD		 6464 if (rdev->has_uvd) {
6465 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6466 if (ring->ring_size) {
02c9f7fa 6467 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
1df0d523 6468 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2e1e6dad 6469 RADEON_CP_PACKET2);
1df0d523 6470 if (!r)
e409b128 6471 r = uvd_v1_0_init(rdev);
1df0d523
AD		 6472 if (r)
6473 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6474 }
f2ba57b5
CK		 6475 }
6476
2898c348
CK		 6477 r = radeon_ib_pool_init(rdev);
6478 if (r) {
6479 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 6480 return r;
2898c348 6481 }
9b136d51 6482
c6105f24
CK		 6483 r = radeon_vm_manager_init(rdev);
6484 if (r) {
6485 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 6486 return r;
c6105f24 6487 }
9b136d51 6488
b530602f
AD		 6489 r = dce6_audio_init(rdev);
6490 if (r)
6491 return r;
6492
9b136d51
AD		 6493 return 0;
6494}
6495
6496int si_resume(struct radeon_device *rdev)
6497{
6498 int r;
6499
6500 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6501 * posting will perform necessary task to bring back GPU into good
6502 * shape.
6503 */
6504 /* post card */
6505 atom_asic_init(rdev->mode_info.atom_context);
6506
205996c0
AD		 6507 /* init golden registers */
6508 si_init_golden_registers(rdev);
6509
9b136d51
AD		 6510 rdev->accel_working = true;
6511 r = si_startup(rdev);
6512 if (r) {
6513 DRM_ERROR("si startup failed on resume\n");
6514 rdev->accel_working = false;
6515 return r;
6516 }
6517
6518 return r;
6519
6520}
6521
6522int si_suspend(struct radeon_device *rdev)
6523{
b530602f 6524 dce6_audio_fini(rdev);
fa3daf9a 6525 radeon_vm_manager_fini(rdev);
9b136d51 6526 si_cp_enable(rdev, false);
8c5fd7ef 6527 cayman_dma_stop(rdev);
1df0d523 6528 if (rdev->has_uvd) {
e409b128 6529 uvd_v1_0_fini(rdev);
1df0d523
AD		 6530 radeon_uvd_suspend(rdev);
6531 }
e16866ec
AD		 6532 si_fini_pg(rdev);
6533 si_fini_cg(rdev);
9b136d51
AD		 6534 si_irq_suspend(rdev);
6535 radeon_wb_disable(rdev);
6536 si_pcie_gart_disable(rdev);
6537 return 0;
6538}
6539
6540/* Plan is to move initialization in that function and use
6541 * helper function so that radeon_device_init pretty much
6542 * do nothing more than calling asic specific function. This
6543 * should also allow to remove a bunch of callback function
6544 * like vram_info.
6545 */
6546int si_init(struct radeon_device *rdev)
6547{
6548 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6549 int r;
6550
9b136d51
AD		 6551 /* Read BIOS */
6552 if (!radeon_get_bios(rdev)) {
6553 if (ASIC_IS_AVIVO(rdev))
6554 return -EINVAL;
6555 }
6556 /* Must be an ATOMBIOS */
6557 if (!rdev->is_atom_bios) {
6558 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6559 return -EINVAL;
6560 }
6561 r = radeon_atombios_init(rdev);
6562 if (r)
6563 return r;
6564
6565 /* Post card if necessary */
6566 if (!radeon_card_posted(rdev)) {
6567 if (!rdev->bios) {
6568 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6569 return -EINVAL;
6570 }
6571 DRM_INFO("GPU not posted. posting now...\n");
6572 atom_asic_init(rdev->mode_info.atom_context);
6573 }
205996c0
AD		 6574 /* init golden registers */
6575 si_init_golden_registers(rdev);
9b136d51
AD		 6576 /* Initialize scratch registers */
6577 si_scratch_init(rdev);
6578 /* Initialize surface registers */
6579 radeon_surface_init(rdev);
6580 /* Initialize clocks */
6581 radeon_get_clock_info(rdev->ddev);
6582
6583 /* Fence driver */
6584 r = radeon_fence_driver_init(rdev);
6585 if (r)
6586 return r;
6587
6588 /* initialize memory controller */
6589 r = si_mc_init(rdev);
6590 if (r)
6591 return r;
6592 /* Memory manager */
6593 r = radeon_bo_init(rdev);
6594 if (r)
6595 return r;
6596
9b136d51
AD		 6597 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6598 ring->ring_obj = NULL;
6599 r600_ring_init(rdev, ring, 1024 * 1024);
6600
6601 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6602 ring->ring_obj = NULL;
6603 r600_ring_init(rdev, ring, 1024 * 1024);
6604
6605 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6606 ring->ring_obj = NULL;
6607 r600_ring_init(rdev, ring, 1024 * 1024);
6608
8c5fd7ef
AD		 6609 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6610 ring->ring_obj = NULL;
6611 r600_ring_init(rdev, ring, 64 * 1024);
6612
6613 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6614 ring->ring_obj = NULL;
6615 r600_ring_init(rdev, ring, 64 * 1024);
6616
1df0d523
AD		 6617 if (rdev->has_uvd) {
6618 r = radeon_uvd_init(rdev);
6619 if (!r) {
6620 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6621 ring->ring_obj = NULL;
6622 r600_ring_init(rdev, ring, 4096);
6623 }
f2ba57b5
CK		 6624 }
6625
9b136d51
AD		 6626 rdev->ih.ring_obj = NULL;
6627 r600_ih_ring_init(rdev, 64 * 1024);
6628
6629 r = r600_pcie_gart_init(rdev);
6630 if (r)
6631 return r;
6632
9b136d51 6633 rdev->accel_working = true;
9b136d51
AD		 6634 r = si_startup(rdev);
6635 if (r) {
6636 dev_err(rdev->dev, "disabling GPU acceleration\n");
6637 si_cp_fini(rdev);
8c5fd7ef 6638 cayman_dma_fini(rdev);
9b136d51 6639 si_irq_fini(rdev);
1fd11777 6640 sumo_rlc_fini(rdev);
9b136d51 6641 radeon_wb_fini(rdev);
2898c348 6642 radeon_ib_pool_fini(rdev);
9b136d51
AD		 6643 radeon_vm_manager_fini(rdev);
6644 radeon_irq_kms_fini(rdev);
6645 si_pcie_gart_fini(rdev);
6646 rdev->accel_working = false;
6647 }
6648
6649 /* Don't start up if the MC ucode is missing.
6650 * The default clocks and voltages before the MC ucode
6651 * is loaded are not suffient for advanced operations.
6652 */
6653 if (!rdev->mc_fw) {
6654 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6655 return -EINVAL;
6656 }
6657
6658 return 0;
6659}
6660
6661void si_fini(struct radeon_device *rdev)
6662{
9b136d51 6663 si_cp_fini(rdev);
8c5fd7ef 6664 cayman_dma_fini(rdev);
e16866ec
AD		 6665 si_fini_pg(rdev);
6666 si_fini_cg(rdev);
9b136d51 6667 si_irq_fini(rdev);
1fd11777 6668 sumo_rlc_fini(rdev);
9b136d51
AD		 6669 radeon_wb_fini(rdev);
6670 radeon_vm_manager_fini(rdev);
2898c348 6671 radeon_ib_pool_fini(rdev);
9b136d51 6672 radeon_irq_kms_fini(rdev);
2858c00d 6673 if (rdev->has_uvd) {
e409b128 6674 uvd_v1_0_fini(rdev);
1df0d523 6675 radeon_uvd_fini(rdev);
2858c00d 6676 }
9b136d51
AD		 6677 si_pcie_gart_fini(rdev);
6678 r600_vram_scratch_fini(rdev);
6679 radeon_gem_fini(rdev);
9b136d51
AD		 6680 radeon_fence_driver_fini(rdev);
6681 radeon_bo_fini(rdev);
6682 radeon_atombios_fini(rdev);
6683 kfree(rdev->bios);
6684 rdev->bios = NULL;
6685}
6686
6759a0a7 6687/**
d0418894 6688 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 6689 *
6690 * @rdev: radeon_device pointer
6691 *
6692 * Fetches a GPU clock counter snapshot (SI).
6693 * Returns the 64 bit clock counter snapshot.
6694 */
d0418894 6695uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 6696{
6697 uint64_t clock;
6698
6699 mutex_lock(&rdev->gpu_clock_mutex);
6700 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6701 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6702 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6703 mutex_unlock(&rdev->gpu_clock_mutex);
6704 return clock;
6705}
2539eb02 6706
2539eb02
CK		 6707int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6708{
facd112d 6709 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 6710 int r;
6711
4ed10835
CK		 6712 /* bypass vclk and dclk with bclk */
6713 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6714 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6715 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6716
6717 /* put PLL in bypass mode */
6718 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6719
6720 if (!vclk || !dclk) {
6721 /* keep the Bypass mode, put PLL to sleep */
6722 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6723 return 0;
6724 }
6725
facd112d
CK		 6726 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6727 16384, 0x03FFFFFF, 0, 128, 5,
6728 &fb_div, &vclk_div, &dclk_div);
6729 if (r)
6730 return r;
2539eb02
CK		 6731
6732 /* set RESET_ANTI_MUX to 0 */
6733 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6734
6735 /* set VCO_MODE to 1 */
6736 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6737
6738 /* toggle UPLL_SLEEP to 1 then back to 0 */
6739 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6740 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6741
6742 /* deassert UPLL_RESET */
6743 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6744
6745 mdelay(1);
6746
facd112d 6747 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6748 if (r)
6749 return r;
6750
6751 /* assert UPLL_RESET again */
6752 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6753
6754 /* disable spread spectrum. */
6755 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6756
6757 /* set feedback divider */
facd112d 6758 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 6759
6760 /* set ref divider to 0 */
6761 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6762
facd112d 6763 if (fb_div < 307200)
2539eb02
CK		 6764 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6765 else
6766 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6767
6768 /* set PDIV_A and PDIV_B */
6769 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 6770 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 6771 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6772
6773 /* give the PLL some time to settle */
6774 mdelay(15);
6775
6776 /* deassert PLL_RESET */
6777 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6778
6779 mdelay(15);
6780
6781 /* switch from bypass mode to normal mode */
6782 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6783
facd112d 6784 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6785 if (r)
6786 return r;
6787
6788 /* switch VCLK and DCLK selection */
6789 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6790 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6791 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6792
6793 mdelay(100);
6794
6795 return 0;
6796}
b9d305df
AD		 6797
6798static void si_pcie_gen3_enable(struct radeon_device *rdev)
6799{
6800 struct pci_dev *root = rdev->pdev->bus->self;
6801 int bridge_pos, gpu_pos;
6802 u32 speed_cntl, mask, current_data_rate;
6803 int ret, i;
6804 u16 tmp16;
6805
6806 if (radeon_pcie_gen2 == 0)
6807 return;
6808
6809 if (rdev->flags & RADEON_IS_IGP)
6810 return;
6811
6812 if (!(rdev->flags & RADEON_IS_PCIE))
6813 return;
6814
6815 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6816 if (ret != 0)
6817 return;
6818
6819 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6820 return;
6821
6822 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6823 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6824 LC_CURRENT_DATA_RATE_SHIFT;
6825 if (mask & DRM_PCIE_SPEED_80) {
6826 if (current_data_rate == 2) {
6827 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6828 return;
6829 }
6830 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6831 } else if (mask & DRM_PCIE_SPEED_50) {
6832 if (current_data_rate == 1) {
6833 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6834 return;
6835 }
6836 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6837 }
6838
6839 bridge_pos = pci_pcie_cap(root);
6840 if (!bridge_pos)
6841 return;
6842
6843 gpu_pos = pci_pcie_cap(rdev->pdev);
6844 if (!gpu_pos)
6845 return;
6846
6847 if (mask & DRM_PCIE_SPEED_80) {
6848 /* re-try equalization if gen3 is not already enabled */
6849 if (current_data_rate != 2) {
6850 u16 bridge_cfg, gpu_cfg;
6851 u16 bridge_cfg2, gpu_cfg2;
6852 u32 max_lw, current_lw, tmp;
6853
6854 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6855 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6856
6857 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6858 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6859
6860 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6861 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6862
6863 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6864 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6865 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6866
6867 if (current_lw < max_lw) {
6868 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6869 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6870 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6871 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6872 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6873 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6874 }
6875 }
6876
6877 for (i = 0; i < 10; i++) {
6878 /* check status */
6879 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6880 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6881 break;
6882
6883 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6884 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6885
6886 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6887 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6888
6889 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6890 tmp |= LC_SET_QUIESCE;
6891 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6892
6893 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6894 tmp |= LC_REDO_EQ;
6895 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6896
6897 mdelay(100);
6898
6899 /* linkctl */
6900 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6901 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6902 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6903 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6904
6905 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
6906 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6907 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
6908 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6909
6910 /* linkctl2 */
6911 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
6912 tmp16 &= ~((1 << 4) | (7 << 9));
6913 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
6914 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
6915
6916 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6917 tmp16 &= ~((1 << 4) | (7 << 9));
6918 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
6919 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6920
6921 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6922 tmp &= ~LC_SET_QUIESCE;
6923 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6924 }
6925 }
6926 }
6927
6928 /* set the link speed */
6929 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
6930 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
6931 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6932
6933 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6934 tmp16 &= ~0xf;
6935 if (mask & DRM_PCIE_SPEED_80)
6936 tmp16 |= 3; /* gen3 */
6937 else if (mask & DRM_PCIE_SPEED_50)
6938 tmp16 |= 2; /* gen2 */
6939 else
6940 tmp16 |= 1; /* gen1 */
6941 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6942
6943 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6944 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
6945 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6946
6947 for (i = 0; i < rdev->usec_timeout; i++) {
6948 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6949 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
6950 break;
6951 udelay(1);
6952 }
6953}
6954
e0bcf165
AD		 6955static void si_program_aspm(struct radeon_device *rdev)
6956{
6957 u32 data, orig;
6958 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
6959 bool disable_clkreq = false;
6960
1294d4a3
AD		 6961 if (radeon_aspm == 0)
6962 return;
6963
e0bcf165
AD		 6964 if (!(rdev->flags & RADEON_IS_PCIE))
6965 return;
6966
6967 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6968 data &= ~LC_XMIT_N_FTS_MASK;
6969 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
6970 if (orig != data)
6971 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
6972
6973 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
6974 data |= LC_GO_TO_RECOVERY;
6975 if (orig != data)
6976 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
6977
6978 orig = data = RREG32_PCIE(PCIE_P_CNTL);
6979 data |= P_IGNORE_EDB_ERR;
6980 if (orig != data)
6981 WREG32_PCIE(PCIE_P_CNTL, data);
6982
6983 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6984 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
6985 data |= LC_PMI_TO_L1_DIS;
6986 if (!disable_l0s)
6987 data |= LC_L0S_INACTIVITY(7);
6988
6989 if (!disable_l1) {
6990 data |= LC_L1_INACTIVITY(7);
6991 data &= ~LC_PMI_TO_L1_DIS;
6992 if (orig != data)
6993 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6994
6995 if (!disable_plloff_in_l1) {
6996 bool clk_req_support;
6997
6998 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6999 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7000 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7001 if (orig != data)
7002 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7003
7004 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7005 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7006 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7007 if (orig != data)
7008 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7009
7010 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7011 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7012 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7013 if (orig != data)
7014 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7015
7016 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7017 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7018 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7019 if (orig != data)
7020 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7021
7022 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7023 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7024 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7025 if (orig != data)
7026 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7027
7028 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7029 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7030 if (orig != data)
7031 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7032
7033 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7034 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7035 if (orig != data)
7036 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7037
7038 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7039 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7040 if (orig != data)
7041 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7042
7043 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7044 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7045 if (orig != data)
7046 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7047
7048 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7049 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7050 if (orig != data)
7051 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7052
7053 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7054 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7055 if (orig != data)
7056 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7057
7058 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7059 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7060 if (orig != data)
7061 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7062 }
7063 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7064 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7065 data |= LC_DYN_LANES_PWR_STATE(3);
7066 if (orig != data)
7067 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7068
7069 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7070 data &= ~LS2_EXIT_TIME_MASK;
7071 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7072 data |= LS2_EXIT_TIME(5);
7073 if (orig != data)
7074 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7075
7076 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7077 data &= ~LS2_EXIT_TIME_MASK;
7078 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7079 data |= LS2_EXIT_TIME(5);
7080 if (orig != data)
7081 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7082
7083 if (!disable_clkreq) {
7084 struct pci_dev *root = rdev->pdev->bus->self;
7085 u32 lnkcap;
7086
7087 clk_req_support = false;
7088 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7089 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7090 clk_req_support = true;
7091 } else {
7092 clk_req_support = false;
7093 }
7094
7095 if (clk_req_support) {
7096 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7097 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7098 if (orig != data)
7099 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7100
7101 orig = data = RREG32(THM_CLK_CNTL);
7102 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7103 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7104 if (orig != data)
7105 WREG32(THM_CLK_CNTL, data);
7106
7107 orig = data = RREG32(MISC_CLK_CNTL);
7108 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7109 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7110 if (orig != data)
7111 WREG32(MISC_CLK_CNTL, data);
7112
7113 orig = data = RREG32(CG_CLKPIN_CNTL);
7114 data &= ~BCLK_AS_XCLK;
7115 if (orig != data)
7116 WREG32(CG_CLKPIN_CNTL, data);
7117
7118 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7119 data &= ~FORCE_BIF_REFCLK_EN;
7120 if (orig != data)
7121 WREG32(CG_CLKPIN_CNTL_2, data);
7122
7123 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7124 data &= ~MPLL_CLKOUT_SEL_MASK;
7125 data |= MPLL_CLKOUT_SEL(4);
7126 if (orig != data)
7127 WREG32(MPLL_BYPASSCLK_SEL, data);
7128
7129 orig = data = RREG32(SPLL_CNTL_MODE);
7130 data &= ~SPLL_REFCLK_SEL_MASK;
7131 if (orig != data)
7132 WREG32(SPLL_CNTL_MODE, data);
7133 }
7134 }
7135 } else {
7136 if (orig != data)
7137 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7138 }
7139
7140 orig = data = RREG32_PCIE(PCIE_CNTL2);
7141 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7142 if (orig != data)
7143 WREG32_PCIE(PCIE_CNTL2, data);
7144
7145 if (!disable_l0s) {
7146 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7147 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7148 data = RREG32_PCIE(PCIE_LC_STATUS1);
7149 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7150 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7151 data &= ~LC_L0S_INACTIVITY_MASK;
7152 if (orig != data)
7153 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7154 }
7155 }
7156 }
7157}
Linux 6.x block layer and io_uring tree(s)