projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon/dpm: only need to reprogram uvd if uvd pg is enabled
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
0f0de06c 24#include <linux/firmware.h>
0f0de06c
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
760285e7 27#include <drm/drmP.h>
43b3cd99
AD		 28#include "radeon.h"
29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
43b3cd99
AD		 31#include "sid.h"
32#include "atom.h"
48c0c902 33#include "si_blit_shaders.h"
bd8cd539 34#include "clearstate_si.h"
a0ceada6 35#include "radeon_ucode.h"
43b3cd99 36
0f0de06c
AD		 37
38MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
42MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
a9e61410 43MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
0f0de06c
AD		 44MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
45MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
46MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
47MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
48MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
a9e61410 49MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
0f0de06c
AD		 50MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
51MODULE_FIRMWARE("radeon/VERDE_me.bin");
52MODULE_FIRMWARE("radeon/VERDE_ce.bin");
53MODULE_FIRMWARE("radeon/VERDE_mc.bin");
54MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
a9e61410 55MODULE_FIRMWARE("radeon/VERDE_smc.bin");
bcc7f5d2
AD		 56MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
57MODULE_FIRMWARE("radeon/OLAND_me.bin");
58MODULE_FIRMWARE("radeon/OLAND_ce.bin");
59MODULE_FIRMWARE("radeon/OLAND_mc.bin");
60MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
a9e61410 61MODULE_FIRMWARE("radeon/OLAND_smc.bin");
c04c00b4
AD		 62MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
63MODULE_FIRMWARE("radeon/HAINAN_me.bin");
64MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
65MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
66MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
a9e61410 67MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
0f0de06c 68
b9d305df 69static void si_pcie_gen3_enable(struct radeon_device *rdev);
e0bcf165 70static void si_program_aspm(struct radeon_device *rdev);
1fd11777
AD		 71extern void sumo_rlc_fini(struct radeon_device *rdev);
72extern int sumo_rlc_init(struct radeon_device *rdev);
25a857fb
AD		 73extern int r600_ih_ring_alloc(struct radeon_device *rdev);
74extern void r600_ih_ring_fini(struct radeon_device *rdev);
0a96d72b 75extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
c476dde2
AD		 76extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
77extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
ca7db22b 78extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
1c534671 79extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
014bb209 80extern bool evergreen_is_display_hung(struct radeon_device *rdev);
2483b4ea
CK		 81extern void si_dma_vm_set_page(struct radeon_device *rdev,
82 struct radeon_ib *ib,
83 uint64_t pe,
84 uint64_t addr, unsigned count,
85 uint32_t incr, uint32_t flags);
0a96d72b 86
6d8cf000
AD		 87static const u32 verde_rlc_save_restore_register_list[] =
88{
89 (0x8000 << 16) | (0x98f4 >> 2),
90 0x00000000,
91 (0x8040 << 16) | (0x98f4 >> 2),
92 0x00000000,
93 (0x8000 << 16) | (0xe80 >> 2),
94 0x00000000,
95 (0x8040 << 16) | (0xe80 >> 2),
96 0x00000000,
97 (0x8000 << 16) | (0x89bc >> 2),
98 0x00000000,
99 (0x8040 << 16) | (0x89bc >> 2),
100 0x00000000,
101 (0x8000 << 16) | (0x8c1c >> 2),
102 0x00000000,
103 (0x8040 << 16) | (0x8c1c >> 2),
104 0x00000000,
105 (0x9c00 << 16) | (0x98f0 >> 2),
106 0x00000000,
107 (0x9c00 << 16) | (0xe7c >> 2),
108 0x00000000,
109 (0x8000 << 16) | (0x9148 >> 2),
110 0x00000000,
111 (0x8040 << 16) | (0x9148 >> 2),
112 0x00000000,
113 (0x9c00 << 16) | (0x9150 >> 2),
114 0x00000000,
115 (0x9c00 << 16) | (0x897c >> 2),
116 0x00000000,
117 (0x9c00 << 16) | (0x8d8c >> 2),
118 0x00000000,
119 (0x9c00 << 16) | (0xac54 >> 2),
120 0X00000000,
121 0x3,
122 (0x9c00 << 16) | (0x98f8 >> 2),
123 0x00000000,
124 (0x9c00 << 16) | (0x9910 >> 2),
125 0x00000000,
126 (0x9c00 << 16) | (0x9914 >> 2),
127 0x00000000,
128 (0x9c00 << 16) | (0x9918 >> 2),
129 0x00000000,
130 (0x9c00 << 16) | (0x991c >> 2),
131 0x00000000,
132 (0x9c00 << 16) | (0x9920 >> 2),
133 0x00000000,
134 (0x9c00 << 16) | (0x9924 >> 2),
135 0x00000000,
136 (0x9c00 << 16) | (0x9928 >> 2),
137 0x00000000,
138 (0x9c00 << 16) | (0x992c >> 2),
139 0x00000000,
140 (0x9c00 << 16) | (0x9930 >> 2),
141 0x00000000,
142 (0x9c00 << 16) | (0x9934 >> 2),
143 0x00000000,
144 (0x9c00 << 16) | (0x9938 >> 2),
145 0x00000000,
146 (0x9c00 << 16) | (0x993c >> 2),
147 0x00000000,
148 (0x9c00 << 16) | (0x9940 >> 2),
149 0x00000000,
150 (0x9c00 << 16) | (0x9944 >> 2),
151 0x00000000,
152 (0x9c00 << 16) | (0x9948 >> 2),
153 0x00000000,
154 (0x9c00 << 16) | (0x994c >> 2),
155 0x00000000,
156 (0x9c00 << 16) | (0x9950 >> 2),
157 0x00000000,
158 (0x9c00 << 16) | (0x9954 >> 2),
159 0x00000000,
160 (0x9c00 << 16) | (0x9958 >> 2),
161 0x00000000,
162 (0x9c00 << 16) | (0x995c >> 2),
163 0x00000000,
164 (0x9c00 << 16) | (0x9960 >> 2),
165 0x00000000,
166 (0x9c00 << 16) | (0x9964 >> 2),
167 0x00000000,
168 (0x9c00 << 16) | (0x9968 >> 2),
169 0x00000000,
170 (0x9c00 << 16) | (0x996c >> 2),
171 0x00000000,
172 (0x9c00 << 16) | (0x9970 >> 2),
173 0x00000000,
174 (0x9c00 << 16) | (0x9974 >> 2),
175 0x00000000,
176 (0x9c00 << 16) | (0x9978 >> 2),
177 0x00000000,
178 (0x9c00 << 16) | (0x997c >> 2),
179 0x00000000,
180 (0x9c00 << 16) | (0x9980 >> 2),
181 0x00000000,
182 (0x9c00 << 16) | (0x9984 >> 2),
183 0x00000000,
184 (0x9c00 << 16) | (0x9988 >> 2),
185 0x00000000,
186 (0x9c00 << 16) | (0x998c >> 2),
187 0x00000000,
188 (0x9c00 << 16) | (0x8c00 >> 2),
189 0x00000000,
190 (0x9c00 << 16) | (0x8c14 >> 2),
191 0x00000000,
192 (0x9c00 << 16) | (0x8c04 >> 2),
193 0x00000000,
194 (0x9c00 << 16) | (0x8c08 >> 2),
195 0x00000000,
196 (0x8000 << 16) | (0x9b7c >> 2),
197 0x00000000,
198 (0x8040 << 16) | (0x9b7c >> 2),
199 0x00000000,
200 (0x8000 << 16) | (0xe84 >> 2),
201 0x00000000,
202 (0x8040 << 16) | (0xe84 >> 2),
203 0x00000000,
204 (0x8000 << 16) | (0x89c0 >> 2),
205 0x00000000,
206 (0x8040 << 16) | (0x89c0 >> 2),
207 0x00000000,
208 (0x8000 << 16) | (0x914c >> 2),
209 0x00000000,
210 (0x8040 << 16) | (0x914c >> 2),
211 0x00000000,
212 (0x8000 << 16) | (0x8c20 >> 2),
213 0x00000000,
214 (0x8040 << 16) | (0x8c20 >> 2),
215 0x00000000,
216 (0x8000 << 16) | (0x9354 >> 2),
217 0x00000000,
218 (0x8040 << 16) | (0x9354 >> 2),
219 0x00000000,
220 (0x9c00 << 16) | (0x9060 >> 2),
221 0x00000000,
222 (0x9c00 << 16) | (0x9364 >> 2),
223 0x00000000,
224 (0x9c00 << 16) | (0x9100 >> 2),
225 0x00000000,
226 (0x9c00 << 16) | (0x913c >> 2),
227 0x00000000,
228 (0x8000 << 16) | (0x90e0 >> 2),
229 0x00000000,
230 (0x8000 << 16) | (0x90e4 >> 2),
231 0x00000000,
232 (0x8000 << 16) | (0x90e8 >> 2),
233 0x00000000,
234 (0x8040 << 16) | (0x90e0 >> 2),
235 0x00000000,
236 (0x8040 << 16) | (0x90e4 >> 2),
237 0x00000000,
238 (0x8040 << 16) | (0x90e8 >> 2),
239 0x00000000,
240 (0x9c00 << 16) | (0x8bcc >> 2),
241 0x00000000,
242 (0x9c00 << 16) | (0x8b24 >> 2),
243 0x00000000,
244 (0x9c00 << 16) | (0x88c4 >> 2),
245 0x00000000,
246 (0x9c00 << 16) | (0x8e50 >> 2),
247 0x00000000,
248 (0x9c00 << 16) | (0x8c0c >> 2),
249 0x00000000,
250 (0x9c00 << 16) | (0x8e58 >> 2),
251 0x00000000,
252 (0x9c00 << 16) | (0x8e5c >> 2),
253 0x00000000,
254 (0x9c00 << 16) | (0x9508 >> 2),
255 0x00000000,
256 (0x9c00 << 16) | (0x950c >> 2),
257 0x00000000,
258 (0x9c00 << 16) | (0x9494 >> 2),
259 0x00000000,
260 (0x9c00 << 16) | (0xac0c >> 2),
261 0x00000000,
262 (0x9c00 << 16) | (0xac10 >> 2),
263 0x00000000,
264 (0x9c00 << 16) | (0xac14 >> 2),
265 0x00000000,
266 (0x9c00 << 16) | (0xae00 >> 2),
267 0x00000000,
268 (0x9c00 << 16) | (0xac08 >> 2),
269 0x00000000,
270 (0x9c00 << 16) | (0x88d4 >> 2),
271 0x00000000,
272 (0x9c00 << 16) | (0x88c8 >> 2),
273 0x00000000,
274 (0x9c00 << 16) | (0x88cc >> 2),
275 0x00000000,
276 (0x9c00 << 16) | (0x89b0 >> 2),
277 0x00000000,
278 (0x9c00 << 16) | (0x8b10 >> 2),
279 0x00000000,
280 (0x9c00 << 16) | (0x8a14 >> 2),
281 0x00000000,
282 (0x9c00 << 16) | (0x9830 >> 2),
283 0x00000000,
284 (0x9c00 << 16) | (0x9834 >> 2),
285 0x00000000,
286 (0x9c00 << 16) | (0x9838 >> 2),
287 0x00000000,
288 (0x9c00 << 16) | (0x9a10 >> 2),
289 0x00000000,
290 (0x8000 << 16) | (0x9870 >> 2),
291 0x00000000,
292 (0x8000 << 16) | (0x9874 >> 2),
293 0x00000000,
294 (0x8001 << 16) | (0x9870 >> 2),
295 0x00000000,
296 (0x8001 << 16) | (0x9874 >> 2),
297 0x00000000,
298 (0x8040 << 16) | (0x9870 >> 2),
299 0x00000000,
300 (0x8040 << 16) | (0x9874 >> 2),
301 0x00000000,
302 (0x8041 << 16) | (0x9870 >> 2),
303 0x00000000,
304 (0x8041 << 16) | (0x9874 >> 2),
305 0x00000000,
306 0x00000000
307};
308
205996c0
AD		 309static const u32 tahiti_golden_rlc_registers[] =
310{
311 0xc424, 0xffffffff, 0x00601005,
312 0xc47c, 0xffffffff, 0x10104040,
313 0xc488, 0xffffffff, 0x0100000a,
314 0xc314, 0xffffffff, 0x00000800,
315 0xc30c, 0xffffffff, 0x800000f4,
316 0xf4a8, 0xffffffff, 0x00000000
317};
318
319static const u32 tahiti_golden_registers[] =
320{
321 0x9a10, 0x00010000, 0x00018208,
322 0x9830, 0xffffffff, 0x00000000,
323 0x9834, 0xf00fffff, 0x00000400,
324 0x9838, 0x0002021c, 0x00020200,
325 0xc78, 0x00000080, 0x00000000,
326 0xd030, 0x000300c0, 0x00800040,
327 0xd830, 0x000300c0, 0x00800040,
328 0x5bb0, 0x000000f0, 0x00000070,
329 0x5bc0, 0x00200000, 0x50100000,
330 0x7030, 0x31000311, 0x00000011,
331 0x277c, 0x00000003, 0x000007ff,
332 0x240c, 0x000007ff, 0x00000000,
333 0x8a14, 0xf000001f, 0x00000007,
334 0x8b24, 0xffffffff, 0x00ffffff,
335 0x8b10, 0x0000ff0f, 0x00000000,
336 0x28a4c, 0x07ffffff, 0x4e000000,
337 0x28350, 0x3f3f3fff, 0x2a00126a,
338 0x30, 0x000000ff, 0x0040,
339 0x34, 0x00000040, 0x00004040,
340 0x9100, 0x07ffffff, 0x03000000,
341 0x8e88, 0x01ff1f3f, 0x00000000,
342 0x8e84, 0x01ff1f3f, 0x00000000,
343 0x9060, 0x0000007f, 0x00000020,
344 0x9508, 0x00010000, 0x00010000,
345 0xac14, 0x00000200, 0x000002fb,
346 0xac10, 0xffffffff, 0x0000543b,
347 0xac0c, 0xffffffff, 0xa9210876,
348 0x88d0, 0xffffffff, 0x000fff40,
349 0x88d4, 0x0000001f, 0x00000010,
350 0x1410, 0x20000000, 0x20fffed8,
351 0x15c0, 0x000c0fc0, 0x000c0400
352};
353
354static const u32 tahiti_golden_registers2[] =
355{
356 0xc64, 0x00000001, 0x00000001
357};
358
359static const u32 pitcairn_golden_rlc_registers[] =
360{
361 0xc424, 0xffffffff, 0x00601004,
362 0xc47c, 0xffffffff, 0x10102020,
363 0xc488, 0xffffffff, 0x01000020,
364 0xc314, 0xffffffff, 0x00000800,
365 0xc30c, 0xffffffff, 0x800000a4
366};
367
368static const u32 pitcairn_golden_registers[] =
369{
370 0x9a10, 0x00010000, 0x00018208,
371 0x9830, 0xffffffff, 0x00000000,
372 0x9834, 0xf00fffff, 0x00000400,
373 0x9838, 0x0002021c, 0x00020200,
374 0xc78, 0x00000080, 0x00000000,
375 0xd030, 0x000300c0, 0x00800040,
376 0xd830, 0x000300c0, 0x00800040,
377 0x5bb0, 0x000000f0, 0x00000070,
378 0x5bc0, 0x00200000, 0x50100000,
379 0x7030, 0x31000311, 0x00000011,
380 0x2ae4, 0x00073ffe, 0x000022a2,
381 0x240c, 0x000007ff, 0x00000000,
382 0x8a14, 0xf000001f, 0x00000007,
383 0x8b24, 0xffffffff, 0x00ffffff,
384 0x8b10, 0x0000ff0f, 0x00000000,
385 0x28a4c, 0x07ffffff, 0x4e000000,
386 0x28350, 0x3f3f3fff, 0x2a00126a,
387 0x30, 0x000000ff, 0x0040,
388 0x34, 0x00000040, 0x00004040,
389 0x9100, 0x07ffffff, 0x03000000,
390 0x9060, 0x0000007f, 0x00000020,
391 0x9508, 0x00010000, 0x00010000,
392 0xac14, 0x000003ff, 0x000000f7,
393 0xac10, 0xffffffff, 0x00000000,
394 0xac0c, 0xffffffff, 0x32761054,
395 0x88d4, 0x0000001f, 0x00000010,
396 0x15c0, 0x000c0fc0, 0x000c0400
397};
398
399static const u32 verde_golden_rlc_registers[] =
400{
401 0xc424, 0xffffffff, 0x033f1005,
402 0xc47c, 0xffffffff, 0x10808020,
403 0xc488, 0xffffffff, 0x00800008,
404 0xc314, 0xffffffff, 0x00001000,
405 0xc30c, 0xffffffff, 0x80010014
406};
407
408static const u32 verde_golden_registers[] =
409{
410 0x9a10, 0x00010000, 0x00018208,
411 0x9830, 0xffffffff, 0x00000000,
412 0x9834, 0xf00fffff, 0x00000400,
413 0x9838, 0x0002021c, 0x00020200,
414 0xc78, 0x00000080, 0x00000000,
415 0xd030, 0x000300c0, 0x00800040,
416 0xd030, 0x000300c0, 0x00800040,
417 0xd830, 0x000300c0, 0x00800040,
418 0xd830, 0x000300c0, 0x00800040,
419 0x5bb0, 0x000000f0, 0x00000070,
420 0x5bc0, 0x00200000, 0x50100000,
421 0x7030, 0x31000311, 0x00000011,
422 0x2ae4, 0x00073ffe, 0x000022a2,
423 0x2ae4, 0x00073ffe, 0x000022a2,
424 0x2ae4, 0x00073ffe, 0x000022a2,
425 0x240c, 0x000007ff, 0x00000000,
426 0x240c, 0x000007ff, 0x00000000,
427 0x240c, 0x000007ff, 0x00000000,
428 0x8a14, 0xf000001f, 0x00000007,
429 0x8a14, 0xf000001f, 0x00000007,
430 0x8a14, 0xf000001f, 0x00000007,
431 0x8b24, 0xffffffff, 0x00ffffff,
432 0x8b10, 0x0000ff0f, 0x00000000,
433 0x28a4c, 0x07ffffff, 0x4e000000,
434 0x28350, 0x3f3f3fff, 0x0000124a,
435 0x28350, 0x3f3f3fff, 0x0000124a,
436 0x28350, 0x3f3f3fff, 0x0000124a,
437 0x30, 0x000000ff, 0x0040,
438 0x34, 0x00000040, 0x00004040,
439 0x9100, 0x07ffffff, 0x03000000,
440 0x9100, 0x07ffffff, 0x03000000,
441 0x8e88, 0x01ff1f3f, 0x00000000,
442 0x8e88, 0x01ff1f3f, 0x00000000,
443 0x8e88, 0x01ff1f3f, 0x00000000,
444 0x8e84, 0x01ff1f3f, 0x00000000,
445 0x8e84, 0x01ff1f3f, 0x00000000,
446 0x8e84, 0x01ff1f3f, 0x00000000,
447 0x9060, 0x0000007f, 0x00000020,
448 0x9508, 0x00010000, 0x00010000,
449 0xac14, 0x000003ff, 0x00000003,
450 0xac14, 0x000003ff, 0x00000003,
451 0xac14, 0x000003ff, 0x00000003,
452 0xac10, 0xffffffff, 0x00000000,
453 0xac10, 0xffffffff, 0x00000000,
454 0xac10, 0xffffffff, 0x00000000,
455 0xac0c, 0xffffffff, 0x00001032,
456 0xac0c, 0xffffffff, 0x00001032,
457 0xac0c, 0xffffffff, 0x00001032,
458 0x88d4, 0x0000001f, 0x00000010,
459 0x88d4, 0x0000001f, 0x00000010,
460 0x88d4, 0x0000001f, 0x00000010,
461 0x15c0, 0x000c0fc0, 0x000c0400
462};
463
464static const u32 oland_golden_rlc_registers[] =
465{
466 0xc424, 0xffffffff, 0x00601005,
467 0xc47c, 0xffffffff, 0x10104040,
468 0xc488, 0xffffffff, 0x0100000a,
469 0xc314, 0xffffffff, 0x00000800,
470 0xc30c, 0xffffffff, 0x800000f4
471};
472
473static const u32 oland_golden_registers[] =
474{
475 0x9a10, 0x00010000, 0x00018208,
476 0x9830, 0xffffffff, 0x00000000,
477 0x9834, 0xf00fffff, 0x00000400,
478 0x9838, 0x0002021c, 0x00020200,
479 0xc78, 0x00000080, 0x00000000,
480 0xd030, 0x000300c0, 0x00800040,
481 0xd830, 0x000300c0, 0x00800040,
482 0x5bb0, 0x000000f0, 0x00000070,
483 0x5bc0, 0x00200000, 0x50100000,
484 0x7030, 0x31000311, 0x00000011,
485 0x2ae4, 0x00073ffe, 0x000022a2,
486 0x240c, 0x000007ff, 0x00000000,
487 0x8a14, 0xf000001f, 0x00000007,
488 0x8b24, 0xffffffff, 0x00ffffff,
489 0x8b10, 0x0000ff0f, 0x00000000,
490 0x28a4c, 0x07ffffff, 0x4e000000,
491 0x28350, 0x3f3f3fff, 0x00000082,
492 0x30, 0x000000ff, 0x0040,
493 0x34, 0x00000040, 0x00004040,
494 0x9100, 0x07ffffff, 0x03000000,
495 0x9060, 0x0000007f, 0x00000020,
496 0x9508, 0x00010000, 0x00010000,
497 0xac14, 0x000003ff, 0x000000f3,
498 0xac10, 0xffffffff, 0x00000000,
499 0xac0c, 0xffffffff, 0x00003210,
500 0x88d4, 0x0000001f, 0x00000010,
501 0x15c0, 0x000c0fc0, 0x000c0400
502};
503
fffbdda4
AD		 504static const u32 hainan_golden_registers[] =
505{
506 0x9a10, 0x00010000, 0x00018208,
507 0x9830, 0xffffffff, 0x00000000,
508 0x9834, 0xf00fffff, 0x00000400,
509 0x9838, 0x0002021c, 0x00020200,
510 0xd0c0, 0xff000fff, 0x00000100,
511 0xd030, 0x000300c0, 0x00800040,
512 0xd8c0, 0xff000fff, 0x00000100,
513 0xd830, 0x000300c0, 0x00800040,
514 0x2ae4, 0x00073ffe, 0x000022a2,
515 0x240c, 0x000007ff, 0x00000000,
516 0x8a14, 0xf000001f, 0x00000007,
517 0x8b24, 0xffffffff, 0x00ffffff,
518 0x8b10, 0x0000ff0f, 0x00000000,
519 0x28a4c, 0x07ffffff, 0x4e000000,
520 0x28350, 0x3f3f3fff, 0x00000000,
521 0x30, 0x000000ff, 0x0040,
522 0x34, 0x00000040, 0x00004040,
523 0x9100, 0x03e00000, 0x03600000,
524 0x9060, 0x0000007f, 0x00000020,
525 0x9508, 0x00010000, 0x00010000,
526 0xac14, 0x000003ff, 0x000000f1,
527 0xac10, 0xffffffff, 0x00000000,
528 0xac0c, 0xffffffff, 0x00003210,
529 0x88d4, 0x0000001f, 0x00000010,
530 0x15c0, 0x000c0fc0, 0x000c0400
531};
532
533static const u32 hainan_golden_registers2[] =
534{
535 0x98f8, 0xffffffff, 0x02010001
536};
537
205996c0
AD		 538static const u32 tahiti_mgcg_cgcg_init[] =
539{
540 0xc400, 0xffffffff, 0xfffffffc,
541 0x802c, 0xffffffff, 0xe0000000,
542 0x9a60, 0xffffffff, 0x00000100,
543 0x92a4, 0xffffffff, 0x00000100,
544 0xc164, 0xffffffff, 0x00000100,
545 0x9774, 0xffffffff, 0x00000100,
546 0x8984, 0xffffffff, 0x06000100,
547 0x8a18, 0xffffffff, 0x00000100,
548 0x92a0, 0xffffffff, 0x00000100,
549 0xc380, 0xffffffff, 0x00000100,
550 0x8b28, 0xffffffff, 0x00000100,
551 0x9144, 0xffffffff, 0x00000100,
552 0x8d88, 0xffffffff, 0x00000100,
553 0x8d8c, 0xffffffff, 0x00000100,
554 0x9030, 0xffffffff, 0x00000100,
555 0x9034, 0xffffffff, 0x00000100,
556 0x9038, 0xffffffff, 0x00000100,
557 0x903c, 0xffffffff, 0x00000100,
558 0xad80, 0xffffffff, 0x00000100,
559 0xac54, 0xffffffff, 0x00000100,
560 0x897c, 0xffffffff, 0x06000100,
561 0x9868, 0xffffffff, 0x00000100,
562 0x9510, 0xffffffff, 0x00000100,
563 0xaf04, 0xffffffff, 0x00000100,
564 0xae04, 0xffffffff, 0x00000100,
565 0x949c, 0xffffffff, 0x00000100,
566 0x802c, 0xffffffff, 0xe0000000,
567 0x9160, 0xffffffff, 0x00010000,
568 0x9164, 0xffffffff, 0x00030002,
569 0x9168, 0xffffffff, 0x00040007,
570 0x916c, 0xffffffff, 0x00060005,
571 0x9170, 0xffffffff, 0x00090008,
572 0x9174, 0xffffffff, 0x00020001,
573 0x9178, 0xffffffff, 0x00040003,
574 0x917c, 0xffffffff, 0x00000007,
575 0x9180, 0xffffffff, 0x00060005,
576 0x9184, 0xffffffff, 0x00090008,
577 0x9188, 0xffffffff, 0x00030002,
578 0x918c, 0xffffffff, 0x00050004,
579 0x9190, 0xffffffff, 0x00000008,
580 0x9194, 0xffffffff, 0x00070006,
581 0x9198, 0xffffffff, 0x000a0009,
582 0x919c, 0xffffffff, 0x00040003,
583 0x91a0, 0xffffffff, 0x00060005,
584 0x91a4, 0xffffffff, 0x00000009,
585 0x91a8, 0xffffffff, 0x00080007,
586 0x91ac, 0xffffffff, 0x000b000a,
587 0x91b0, 0xffffffff, 0x00050004,
588 0x91b4, 0xffffffff, 0x00070006,
589 0x91b8, 0xffffffff, 0x0008000b,
590 0x91bc, 0xffffffff, 0x000a0009,
591 0x91c0, 0xffffffff, 0x000d000c,
592 0x91c4, 0xffffffff, 0x00060005,
593 0x91c8, 0xffffffff, 0x00080007,
594 0x91cc, 0xffffffff, 0x0000000b,
595 0x91d0, 0xffffffff, 0x000a0009,
596 0x91d4, 0xffffffff, 0x000d000c,
597 0x91d8, 0xffffffff, 0x00070006,
598 0x91dc, 0xffffffff, 0x00090008,
599 0x91e0, 0xffffffff, 0x0000000c,
600 0x91e4, 0xffffffff, 0x000b000a,
601 0x91e8, 0xffffffff, 0x000e000d,
602 0x91ec, 0xffffffff, 0x00080007,
603 0x91f0, 0xffffffff, 0x000a0009,
604 0x91f4, 0xffffffff, 0x0000000d,
605 0x91f8, 0xffffffff, 0x000c000b,
606 0x91fc, 0xffffffff, 0x000f000e,
607 0x9200, 0xffffffff, 0x00090008,
608 0x9204, 0xffffffff, 0x000b000a,
609 0x9208, 0xffffffff, 0x000c000f,
610 0x920c, 0xffffffff, 0x000e000d,
611 0x9210, 0xffffffff, 0x00110010,
612 0x9214, 0xffffffff, 0x000a0009,
613 0x9218, 0xffffffff, 0x000c000b,
614 0x921c, 0xffffffff, 0x0000000f,
615 0x9220, 0xffffffff, 0x000e000d,
616 0x9224, 0xffffffff, 0x00110010,
617 0x9228, 0xffffffff, 0x000b000a,
618 0x922c, 0xffffffff, 0x000d000c,
619 0x9230, 0xffffffff, 0x00000010,
620 0x9234, 0xffffffff, 0x000f000e,
621 0x9238, 0xffffffff, 0x00120011,
622 0x923c, 0xffffffff, 0x000c000b,
623 0x9240, 0xffffffff, 0x000e000d,
624 0x9244, 0xffffffff, 0x00000011,
625 0x9248, 0xffffffff, 0x0010000f,
626 0x924c, 0xffffffff, 0x00130012,
627 0x9250, 0xffffffff, 0x000d000c,
628 0x9254, 0xffffffff, 0x000f000e,
629 0x9258, 0xffffffff, 0x00100013,
630 0x925c, 0xffffffff, 0x00120011,
631 0x9260, 0xffffffff, 0x00150014,
632 0x9264, 0xffffffff, 0x000e000d,
633 0x9268, 0xffffffff, 0x0010000f,
634 0x926c, 0xffffffff, 0x00000013,
635 0x9270, 0xffffffff, 0x00120011,
636 0x9274, 0xffffffff, 0x00150014,
637 0x9278, 0xffffffff, 0x000f000e,
638 0x927c, 0xffffffff, 0x00110010,
639 0x9280, 0xffffffff, 0x00000014,
640 0x9284, 0xffffffff, 0x00130012,
641 0x9288, 0xffffffff, 0x00160015,
642 0x928c, 0xffffffff, 0x0010000f,
643 0x9290, 0xffffffff, 0x00120011,
644 0x9294, 0xffffffff, 0x00000015,
645 0x9298, 0xffffffff, 0x00140013,
646 0x929c, 0xffffffff, 0x00170016,
647 0x9150, 0xffffffff, 0x96940200,
648 0x8708, 0xffffffff, 0x00900100,
649 0xc478, 0xffffffff, 0x00000080,
650 0xc404, 0xffffffff, 0x0020003f,
651 0x30, 0xffffffff, 0x0000001c,
652 0x34, 0x000f0000, 0x000f0000,
653 0x160c, 0xffffffff, 0x00000100,
654 0x1024, 0xffffffff, 0x00000100,
655 0x102c, 0x00000101, 0x00000000,
656 0x20a8, 0xffffffff, 0x00000104,
657 0x264c, 0x000c0000, 0x000c0000,
658 0x2648, 0x000c0000, 0x000c0000,
659 0x55e4, 0xff000fff, 0x00000100,
660 0x55e8, 0x00000001, 0x00000001,
661 0x2f50, 0x00000001, 0x00000001,
662 0x30cc, 0xc0000fff, 0x00000104,
663 0xc1e4, 0x00000001, 0x00000001,
664 0xd0c0, 0xfffffff0, 0x00000100,
665 0xd8c0, 0xfffffff0, 0x00000100
666};
667
668static const u32 pitcairn_mgcg_cgcg_init[] =
669{
670 0xc400, 0xffffffff, 0xfffffffc,
671 0x802c, 0xffffffff, 0xe0000000,
672 0x9a60, 0xffffffff, 0x00000100,
673 0x92a4, 0xffffffff, 0x00000100,
674 0xc164, 0xffffffff, 0x00000100,
675 0x9774, 0xffffffff, 0x00000100,
676 0x8984, 0xffffffff, 0x06000100,
677 0x8a18, 0xffffffff, 0x00000100,
678 0x92a0, 0xffffffff, 0x00000100,
679 0xc380, 0xffffffff, 0x00000100,
680 0x8b28, 0xffffffff, 0x00000100,
681 0x9144, 0xffffffff, 0x00000100,
682 0x8d88, 0xffffffff, 0x00000100,
683 0x8d8c, 0xffffffff, 0x00000100,
684 0x9030, 0xffffffff, 0x00000100,
685 0x9034, 0xffffffff, 0x00000100,
686 0x9038, 0xffffffff, 0x00000100,
687 0x903c, 0xffffffff, 0x00000100,
688 0xad80, 0xffffffff, 0x00000100,
689 0xac54, 0xffffffff, 0x00000100,
690 0x897c, 0xffffffff, 0x06000100,
691 0x9868, 0xffffffff, 0x00000100,
692 0x9510, 0xffffffff, 0x00000100,
693 0xaf04, 0xffffffff, 0x00000100,
694 0xae04, 0xffffffff, 0x00000100,
695 0x949c, 0xffffffff, 0x00000100,
696 0x802c, 0xffffffff, 0xe0000000,
697 0x9160, 0xffffffff, 0x00010000,
698 0x9164, 0xffffffff, 0x00030002,
699 0x9168, 0xffffffff, 0x00040007,
700 0x916c, 0xffffffff, 0x00060005,
701 0x9170, 0xffffffff, 0x00090008,
702 0x9174, 0xffffffff, 0x00020001,
703 0x9178, 0xffffffff, 0x00040003,
704 0x917c, 0xffffffff, 0x00000007,
705 0x9180, 0xffffffff, 0x00060005,
706 0x9184, 0xffffffff, 0x00090008,
707 0x9188, 0xffffffff, 0x00030002,
708 0x918c, 0xffffffff, 0x00050004,
709 0x9190, 0xffffffff, 0x00000008,
710 0x9194, 0xffffffff, 0x00070006,
711 0x9198, 0xffffffff, 0x000a0009,
712 0x919c, 0xffffffff, 0x00040003,
713 0x91a0, 0xffffffff, 0x00060005,
714 0x91a4, 0xffffffff, 0x00000009,
715 0x91a8, 0xffffffff, 0x00080007,
716 0x91ac, 0xffffffff, 0x000b000a,
717 0x91b0, 0xffffffff, 0x00050004,
718 0x91b4, 0xffffffff, 0x00070006,
719 0x91b8, 0xffffffff, 0x0008000b,
720 0x91bc, 0xffffffff, 0x000a0009,
721 0x91c0, 0xffffffff, 0x000d000c,
722 0x9200, 0xffffffff, 0x00090008,
723 0x9204, 0xffffffff, 0x000b000a,
724 0x9208, 0xffffffff, 0x000c000f,
725 0x920c, 0xffffffff, 0x000e000d,
726 0x9210, 0xffffffff, 0x00110010,
727 0x9214, 0xffffffff, 0x000a0009,
728 0x9218, 0xffffffff, 0x000c000b,
729 0x921c, 0xffffffff, 0x0000000f,
730 0x9220, 0xffffffff, 0x000e000d,
731 0x9224, 0xffffffff, 0x00110010,
732 0x9228, 0xffffffff, 0x000b000a,
733 0x922c, 0xffffffff, 0x000d000c,
734 0x9230, 0xffffffff, 0x00000010,
735 0x9234, 0xffffffff, 0x000f000e,
736 0x9238, 0xffffffff, 0x00120011,
737 0x923c, 0xffffffff, 0x000c000b,
738 0x9240, 0xffffffff, 0x000e000d,
739 0x9244, 0xffffffff, 0x00000011,
740 0x9248, 0xffffffff, 0x0010000f,
741 0x924c, 0xffffffff, 0x00130012,
742 0x9250, 0xffffffff, 0x000d000c,
743 0x9254, 0xffffffff, 0x000f000e,
744 0x9258, 0xffffffff, 0x00100013,
745 0x925c, 0xffffffff, 0x00120011,
746 0x9260, 0xffffffff, 0x00150014,
747 0x9150, 0xffffffff, 0x96940200,
748 0x8708, 0xffffffff, 0x00900100,
749 0xc478, 0xffffffff, 0x00000080,
750 0xc404, 0xffffffff, 0x0020003f,
751 0x30, 0xffffffff, 0x0000001c,
752 0x34, 0x000f0000, 0x000f0000,
753 0x160c, 0xffffffff, 0x00000100,
754 0x1024, 0xffffffff, 0x00000100,
755 0x102c, 0x00000101, 0x00000000,
756 0x20a8, 0xffffffff, 0x00000104,
757 0x55e4, 0xff000fff, 0x00000100,
758 0x55e8, 0x00000001, 0x00000001,
759 0x2f50, 0x00000001, 0x00000001,
760 0x30cc, 0xc0000fff, 0x00000104,
761 0xc1e4, 0x00000001, 0x00000001,
762 0xd0c0, 0xfffffff0, 0x00000100,
763 0xd8c0, 0xfffffff0, 0x00000100
764};
765
766static const u32 verde_mgcg_cgcg_init[] =
767{
768 0xc400, 0xffffffff, 0xfffffffc,
769 0x802c, 0xffffffff, 0xe0000000,
770 0x9a60, 0xffffffff, 0x00000100,
771 0x92a4, 0xffffffff, 0x00000100,
772 0xc164, 0xffffffff, 0x00000100,
773 0x9774, 0xffffffff, 0x00000100,
774 0x8984, 0xffffffff, 0x06000100,
775 0x8a18, 0xffffffff, 0x00000100,
776 0x92a0, 0xffffffff, 0x00000100,
777 0xc380, 0xffffffff, 0x00000100,
778 0x8b28, 0xffffffff, 0x00000100,
779 0x9144, 0xffffffff, 0x00000100,
780 0x8d88, 0xffffffff, 0x00000100,
781 0x8d8c, 0xffffffff, 0x00000100,
782 0x9030, 0xffffffff, 0x00000100,
783 0x9034, 0xffffffff, 0x00000100,
784 0x9038, 0xffffffff, 0x00000100,
785 0x903c, 0xffffffff, 0x00000100,
786 0xad80, 0xffffffff, 0x00000100,
787 0xac54, 0xffffffff, 0x00000100,
788 0x897c, 0xffffffff, 0x06000100,
789 0x9868, 0xffffffff, 0x00000100,
790 0x9510, 0xffffffff, 0x00000100,
791 0xaf04, 0xffffffff, 0x00000100,
792 0xae04, 0xffffffff, 0x00000100,
793 0x949c, 0xffffffff, 0x00000100,
794 0x802c, 0xffffffff, 0xe0000000,
795 0x9160, 0xffffffff, 0x00010000,
796 0x9164, 0xffffffff, 0x00030002,
797 0x9168, 0xffffffff, 0x00040007,
798 0x916c, 0xffffffff, 0x00060005,
799 0x9170, 0xffffffff, 0x00090008,
800 0x9174, 0xffffffff, 0x00020001,
801 0x9178, 0xffffffff, 0x00040003,
802 0x917c, 0xffffffff, 0x00000007,
803 0x9180, 0xffffffff, 0x00060005,
804 0x9184, 0xffffffff, 0x00090008,
805 0x9188, 0xffffffff, 0x00030002,
806 0x918c, 0xffffffff, 0x00050004,
807 0x9190, 0xffffffff, 0x00000008,
808 0x9194, 0xffffffff, 0x00070006,
809 0x9198, 0xffffffff, 0x000a0009,
810 0x919c, 0xffffffff, 0x00040003,
811 0x91a0, 0xffffffff, 0x00060005,
812 0x91a4, 0xffffffff, 0x00000009,
813 0x91a8, 0xffffffff, 0x00080007,
814 0x91ac, 0xffffffff, 0x000b000a,
815 0x91b0, 0xffffffff, 0x00050004,
816 0x91b4, 0xffffffff, 0x00070006,
817 0x91b8, 0xffffffff, 0x0008000b,
818 0x91bc, 0xffffffff, 0x000a0009,
819 0x91c0, 0xffffffff, 0x000d000c,
820 0x9200, 0xffffffff, 0x00090008,
821 0x9204, 0xffffffff, 0x000b000a,
822 0x9208, 0xffffffff, 0x000c000f,
823 0x920c, 0xffffffff, 0x000e000d,
824 0x9210, 0xffffffff, 0x00110010,
825 0x9214, 0xffffffff, 0x000a0009,
826 0x9218, 0xffffffff, 0x000c000b,
827 0x921c, 0xffffffff, 0x0000000f,
828 0x9220, 0xffffffff, 0x000e000d,
829 0x9224, 0xffffffff, 0x00110010,
830 0x9228, 0xffffffff, 0x000b000a,
831 0x922c, 0xffffffff, 0x000d000c,
832 0x9230, 0xffffffff, 0x00000010,
833 0x9234, 0xffffffff, 0x000f000e,
834 0x9238, 0xffffffff, 0x00120011,
835 0x923c, 0xffffffff, 0x000c000b,
836 0x9240, 0xffffffff, 0x000e000d,
837 0x9244, 0xffffffff, 0x00000011,
838 0x9248, 0xffffffff, 0x0010000f,
839 0x924c, 0xffffffff, 0x00130012,
840 0x9250, 0xffffffff, 0x000d000c,
841 0x9254, 0xffffffff, 0x000f000e,
842 0x9258, 0xffffffff, 0x00100013,
843 0x925c, 0xffffffff, 0x00120011,
844 0x9260, 0xffffffff, 0x00150014,
845 0x9150, 0xffffffff, 0x96940200,
846 0x8708, 0xffffffff, 0x00900100,
847 0xc478, 0xffffffff, 0x00000080,
848 0xc404, 0xffffffff, 0x0020003f,
849 0x30, 0xffffffff, 0x0000001c,
850 0x34, 0x000f0000, 0x000f0000,
851 0x160c, 0xffffffff, 0x00000100,
852 0x1024, 0xffffffff, 0x00000100,
853 0x102c, 0x00000101, 0x00000000,
854 0x20a8, 0xffffffff, 0x00000104,
855 0x264c, 0x000c0000, 0x000c0000,
856 0x2648, 0x000c0000, 0x000c0000,
857 0x55e4, 0xff000fff, 0x00000100,
858 0x55e8, 0x00000001, 0x00000001,
859 0x2f50, 0x00000001, 0x00000001,
860 0x30cc, 0xc0000fff, 0x00000104,
861 0xc1e4, 0x00000001, 0x00000001,
862 0xd0c0, 0xfffffff0, 0x00000100,
863 0xd8c0, 0xfffffff0, 0x00000100
864};
865
866static const u32 oland_mgcg_cgcg_init[] =
867{
868 0xc400, 0xffffffff, 0xfffffffc,
869 0x802c, 0xffffffff, 0xe0000000,
870 0x9a60, 0xffffffff, 0x00000100,
871 0x92a4, 0xffffffff, 0x00000100,
872 0xc164, 0xffffffff, 0x00000100,
873 0x9774, 0xffffffff, 0x00000100,
874 0x8984, 0xffffffff, 0x06000100,
875 0x8a18, 0xffffffff, 0x00000100,
876 0x92a0, 0xffffffff, 0x00000100,
877 0xc380, 0xffffffff, 0x00000100,
878 0x8b28, 0xffffffff, 0x00000100,
879 0x9144, 0xffffffff, 0x00000100,
880 0x8d88, 0xffffffff, 0x00000100,
881 0x8d8c, 0xffffffff, 0x00000100,
882 0x9030, 0xffffffff, 0x00000100,
883 0x9034, 0xffffffff, 0x00000100,
884 0x9038, 0xffffffff, 0x00000100,
885 0x903c, 0xffffffff, 0x00000100,
886 0xad80, 0xffffffff, 0x00000100,
887 0xac54, 0xffffffff, 0x00000100,
888 0x897c, 0xffffffff, 0x06000100,
889 0x9868, 0xffffffff, 0x00000100,
890 0x9510, 0xffffffff, 0x00000100,
891 0xaf04, 0xffffffff, 0x00000100,
892 0xae04, 0xffffffff, 0x00000100,
893 0x949c, 0xffffffff, 0x00000100,
894 0x802c, 0xffffffff, 0xe0000000,
895 0x9160, 0xffffffff, 0x00010000,
896 0x9164, 0xffffffff, 0x00030002,
897 0x9168, 0xffffffff, 0x00040007,
898 0x916c, 0xffffffff, 0x00060005,
899 0x9170, 0xffffffff, 0x00090008,
900 0x9174, 0xffffffff, 0x00020001,
901 0x9178, 0xffffffff, 0x00040003,
902 0x917c, 0xffffffff, 0x00000007,
903 0x9180, 0xffffffff, 0x00060005,
904 0x9184, 0xffffffff, 0x00090008,
905 0x9188, 0xffffffff, 0x00030002,
906 0x918c, 0xffffffff, 0x00050004,
907 0x9190, 0xffffffff, 0x00000008,
908 0x9194, 0xffffffff, 0x00070006,
909 0x9198, 0xffffffff, 0x000a0009,
910 0x919c, 0xffffffff, 0x00040003,
911 0x91a0, 0xffffffff, 0x00060005,
912 0x91a4, 0xffffffff, 0x00000009,
913 0x91a8, 0xffffffff, 0x00080007,
914 0x91ac, 0xffffffff, 0x000b000a,
915 0x91b0, 0xffffffff, 0x00050004,
916 0x91b4, 0xffffffff, 0x00070006,
917 0x91b8, 0xffffffff, 0x0008000b,
918 0x91bc, 0xffffffff, 0x000a0009,
919 0x91c0, 0xffffffff, 0x000d000c,
920 0x91c4, 0xffffffff, 0x00060005,
921 0x91c8, 0xffffffff, 0x00080007,
922 0x91cc, 0xffffffff, 0x0000000b,
923 0x91d0, 0xffffffff, 0x000a0009,
924 0x91d4, 0xffffffff, 0x000d000c,
925 0x9150, 0xffffffff, 0x96940200,
926 0x8708, 0xffffffff, 0x00900100,
927 0xc478, 0xffffffff, 0x00000080,
928 0xc404, 0xffffffff, 0x0020003f,
929 0x30, 0xffffffff, 0x0000001c,
930 0x34, 0x000f0000, 0x000f0000,
931 0x160c, 0xffffffff, 0x00000100,
932 0x1024, 0xffffffff, 0x00000100,
933 0x102c, 0x00000101, 0x00000000,
934 0x20a8, 0xffffffff, 0x00000104,
935 0x264c, 0x000c0000, 0x000c0000,
936 0x2648, 0x000c0000, 0x000c0000,
937 0x55e4, 0xff000fff, 0x00000100,
938 0x55e8, 0x00000001, 0x00000001,
939 0x2f50, 0x00000001, 0x00000001,
940 0x30cc, 0xc0000fff, 0x00000104,
941 0xc1e4, 0x00000001, 0x00000001,
942 0xd0c0, 0xfffffff0, 0x00000100,
943 0xd8c0, 0xfffffff0, 0x00000100
944};
945
fffbdda4
AD		 946static const u32 hainan_mgcg_cgcg_init[] =
947{
948 0xc400, 0xffffffff, 0xfffffffc,
949 0x802c, 0xffffffff, 0xe0000000,
950 0x9a60, 0xffffffff, 0x00000100,
951 0x92a4, 0xffffffff, 0x00000100,
952 0xc164, 0xffffffff, 0x00000100,
953 0x9774, 0xffffffff, 0x00000100,
954 0x8984, 0xffffffff, 0x06000100,
955 0x8a18, 0xffffffff, 0x00000100,
956 0x92a0, 0xffffffff, 0x00000100,
957 0xc380, 0xffffffff, 0x00000100,
958 0x8b28, 0xffffffff, 0x00000100,
959 0x9144, 0xffffffff, 0x00000100,
960 0x8d88, 0xffffffff, 0x00000100,
961 0x8d8c, 0xffffffff, 0x00000100,
962 0x9030, 0xffffffff, 0x00000100,
963 0x9034, 0xffffffff, 0x00000100,
964 0x9038, 0xffffffff, 0x00000100,
965 0x903c, 0xffffffff, 0x00000100,
966 0xad80, 0xffffffff, 0x00000100,
967 0xac54, 0xffffffff, 0x00000100,
968 0x897c, 0xffffffff, 0x06000100,
969 0x9868, 0xffffffff, 0x00000100,
970 0x9510, 0xffffffff, 0x00000100,
971 0xaf04, 0xffffffff, 0x00000100,
972 0xae04, 0xffffffff, 0x00000100,
973 0x949c, 0xffffffff, 0x00000100,
974 0x802c, 0xffffffff, 0xe0000000,
975 0x9160, 0xffffffff, 0x00010000,
976 0x9164, 0xffffffff, 0x00030002,
977 0x9168, 0xffffffff, 0x00040007,
978 0x916c, 0xffffffff, 0x00060005,
979 0x9170, 0xffffffff, 0x00090008,
980 0x9174, 0xffffffff, 0x00020001,
981 0x9178, 0xffffffff, 0x00040003,
982 0x917c, 0xffffffff, 0x00000007,
983 0x9180, 0xffffffff, 0x00060005,
984 0x9184, 0xffffffff, 0x00090008,
985 0x9188, 0xffffffff, 0x00030002,
986 0x918c, 0xffffffff, 0x00050004,
987 0x9190, 0xffffffff, 0x00000008,
988 0x9194, 0xffffffff, 0x00070006,
989 0x9198, 0xffffffff, 0x000a0009,
990 0x919c, 0xffffffff, 0x00040003,
991 0x91a0, 0xffffffff, 0x00060005,
992 0x91a4, 0xffffffff, 0x00000009,
993 0x91a8, 0xffffffff, 0x00080007,
994 0x91ac, 0xffffffff, 0x000b000a,
995 0x91b0, 0xffffffff, 0x00050004,
996 0x91b4, 0xffffffff, 0x00070006,
997 0x91b8, 0xffffffff, 0x0008000b,
998 0x91bc, 0xffffffff, 0x000a0009,
999 0x91c0, 0xffffffff, 0x000d000c,
1000 0x91c4, 0xffffffff, 0x00060005,
1001 0x91c8, 0xffffffff, 0x00080007,
1002 0x91cc, 0xffffffff, 0x0000000b,
1003 0x91d0, 0xffffffff, 0x000a0009,
1004 0x91d4, 0xffffffff, 0x000d000c,
1005 0x9150, 0xffffffff, 0x96940200,
1006 0x8708, 0xffffffff, 0x00900100,
1007 0xc478, 0xffffffff, 0x00000080,
1008 0xc404, 0xffffffff, 0x0020003f,
1009 0x30, 0xffffffff, 0x0000001c,
1010 0x34, 0x000f0000, 0x000f0000,
1011 0x160c, 0xffffffff, 0x00000100,
1012 0x1024, 0xffffffff, 0x00000100,
1013 0x20a8, 0xffffffff, 0x00000104,
1014 0x264c, 0x000c0000, 0x000c0000,
1015 0x2648, 0x000c0000, 0x000c0000,
1016 0x2f50, 0x00000001, 0x00000001,
1017 0x30cc, 0xc0000fff, 0x00000104,
1018 0xc1e4, 0x00000001, 0x00000001,
1019 0xd0c0, 0xfffffff0, 0x00000100,
1020 0xd8c0, 0xfffffff0, 0x00000100
1021};
1022
205996c0
AD		 1023static u32 verde_pg_init[] =
1024{
1025 0x353c, 0xffffffff, 0x40000,
1026 0x3538, 0xffffffff, 0x200010ff,
1027 0x353c, 0xffffffff, 0x0,
1028 0x353c, 0xffffffff, 0x0,
1029 0x353c, 0xffffffff, 0x0,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x7007,
1033 0x3538, 0xffffffff, 0x300010ff,
1034 0x353c, 0xffffffff, 0x0,
1035 0x353c, 0xffffffff, 0x0,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x400000,
1040 0x3538, 0xffffffff, 0x100010ff,
1041 0x353c, 0xffffffff, 0x0,
1042 0x353c, 0xffffffff, 0x0,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x120200,
1047 0x3538, 0xffffffff, 0x500010ff,
1048 0x353c, 0xffffffff, 0x0,
1049 0x353c, 0xffffffff, 0x0,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x1e1e16,
1054 0x3538, 0xffffffff, 0x600010ff,
1055 0x353c, 0xffffffff, 0x0,
1056 0x353c, 0xffffffff, 0x0,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x0,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x171f1e,
1061 0x3538, 0xffffffff, 0x700010ff,
1062 0x353c, 0xffffffff, 0x0,
1063 0x353c, 0xffffffff, 0x0,
1064 0x353c, 0xffffffff, 0x0,
1065 0x353c, 0xffffffff, 0x0,
1066 0x353c, 0xffffffff, 0x0,
1067 0x353c, 0xffffffff, 0x0,
1068 0x3538, 0xffffffff, 0x9ff,
1069 0x3500, 0xffffffff, 0x0,
1070 0x3504, 0xffffffff, 0x10000800,
1071 0x3504, 0xffffffff, 0xf,
1072 0x3504, 0xffffffff, 0xf,
1073 0x3500, 0xffffffff, 0x4,
1074 0x3504, 0xffffffff, 0x1000051e,
1075 0x3504, 0xffffffff, 0xffff,
1076 0x3504, 0xffffffff, 0xffff,
1077 0x3500, 0xffffffff, 0x8,
1078 0x3504, 0xffffffff, 0x80500,
1079 0x3500, 0xffffffff, 0x12,
1080 0x3504, 0xffffffff, 0x9050c,
1081 0x3500, 0xffffffff, 0x1d,
1082 0x3504, 0xffffffff, 0xb052c,
1083 0x3500, 0xffffffff, 0x2a,
1084 0x3504, 0xffffffff, 0x1053e,
1085 0x3500, 0xffffffff, 0x2d,
1086 0x3504, 0xffffffff, 0x10546,
1087 0x3500, 0xffffffff, 0x30,
1088 0x3504, 0xffffffff, 0xa054e,
1089 0x3500, 0xffffffff, 0x3c,
1090 0x3504, 0xffffffff, 0x1055f,
1091 0x3500, 0xffffffff, 0x3f,
1092 0x3504, 0xffffffff, 0x10567,
1093 0x3500, 0xffffffff, 0x42,
1094 0x3504, 0xffffffff, 0x1056f,
1095 0x3500, 0xffffffff, 0x45,
1096 0x3504, 0xffffffff, 0x10572,
1097 0x3500, 0xffffffff, 0x48,
1098 0x3504, 0xffffffff, 0x20575,
1099 0x3500, 0xffffffff, 0x4c,
1100 0x3504, 0xffffffff, 0x190801,
1101 0x3500, 0xffffffff, 0x67,
1102 0x3504, 0xffffffff, 0x1082a,
1103 0x3500, 0xffffffff, 0x6a,
1104 0x3504, 0xffffffff, 0x1b082d,
1105 0x3500, 0xffffffff, 0x87,
1106 0x3504, 0xffffffff, 0x310851,
1107 0x3500, 0xffffffff, 0xba,
1108 0x3504, 0xffffffff, 0x891,
1109 0x3500, 0xffffffff, 0xbc,
1110 0x3504, 0xffffffff, 0x893,
1111 0x3500, 0xffffffff, 0xbe,
1112 0x3504, 0xffffffff, 0x20895,
1113 0x3500, 0xffffffff, 0xc2,
1114 0x3504, 0xffffffff, 0x20899,
1115 0x3500, 0xffffffff, 0xc6,
1116 0x3504, 0xffffffff, 0x2089d,
1117 0x3500, 0xffffffff, 0xca,
1118 0x3504, 0xffffffff, 0x8a1,
1119 0x3500, 0xffffffff, 0xcc,
1120 0x3504, 0xffffffff, 0x8a3,
1121 0x3500, 0xffffffff, 0xce,
1122 0x3504, 0xffffffff, 0x308a5,
1123 0x3500, 0xffffffff, 0xd3,
1124 0x3504, 0xffffffff, 0x6d08cd,
1125 0x3500, 0xffffffff, 0x142,
1126 0x3504, 0xffffffff, 0x2000095a,
1127 0x3504, 0xffffffff, 0x1,
1128 0x3500, 0xffffffff, 0x144,
1129 0x3504, 0xffffffff, 0x301f095b,
1130 0x3500, 0xffffffff, 0x165,
1131 0x3504, 0xffffffff, 0xc094d,
1132 0x3500, 0xffffffff, 0x173,
1133 0x3504, 0xffffffff, 0xf096d,
1134 0x3500, 0xffffffff, 0x184,
1135 0x3504, 0xffffffff, 0x15097f,
1136 0x3500, 0xffffffff, 0x19b,
1137 0x3504, 0xffffffff, 0xc0998,
1138 0x3500, 0xffffffff, 0x1a9,
1139 0x3504, 0xffffffff, 0x409a7,
1140 0x3500, 0xffffffff, 0x1af,
1141 0x3504, 0xffffffff, 0xcdc,
1142 0x3500, 0xffffffff, 0x1b1,
1143 0x3504, 0xffffffff, 0x800,
1144 0x3508, 0xffffffff, 0x6c9b2000,
1145 0x3510, 0xfc00, 0x2000,
1146 0x3544, 0xffffffff, 0xfc0,
1147 0x28d4, 0x00000100, 0x100
1148};
1149
1150static void si_init_golden_registers(struct radeon_device *rdev)
1151{
1152 switch (rdev->family) {
1153 case CHIP_TAHITI:
1154 radeon_program_register_sequence(rdev,
1155 tahiti_golden_registers,
1156 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1157 radeon_program_register_sequence(rdev,
1158 tahiti_golden_rlc_registers,
1159 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1160 radeon_program_register_sequence(rdev,
1161 tahiti_mgcg_cgcg_init,
1162 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1163 radeon_program_register_sequence(rdev,
1164 tahiti_golden_registers2,
1165 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1166 break;
1167 case CHIP_PITCAIRN:
1168 radeon_program_register_sequence(rdev,
1169 pitcairn_golden_registers,
1170 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1171 radeon_program_register_sequence(rdev,
1172 pitcairn_golden_rlc_registers,
1173 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1174 radeon_program_register_sequence(rdev,
1175 pitcairn_mgcg_cgcg_init,
1176 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1177 break;
1178 case CHIP_VERDE:
1179 radeon_program_register_sequence(rdev,
1180 verde_golden_registers,
1181 (const u32)ARRAY_SIZE(verde_golden_registers));
1182 radeon_program_register_sequence(rdev,
1183 verde_golden_rlc_registers,
1184 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1185 radeon_program_register_sequence(rdev,
1186 verde_mgcg_cgcg_init,
1187 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1188 radeon_program_register_sequence(rdev,
1189 verde_pg_init,
1190 (const u32)ARRAY_SIZE(verde_pg_init));
1191 break;
1192 case CHIP_OLAND:
1193 radeon_program_register_sequence(rdev,
1194 oland_golden_registers,
1195 (const u32)ARRAY_SIZE(oland_golden_registers));
1196 radeon_program_register_sequence(rdev,
1197 oland_golden_rlc_registers,
1198 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1199 radeon_program_register_sequence(rdev,
1200 oland_mgcg_cgcg_init,
1201 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1202 break;
fffbdda4
AD		 1203 case CHIP_HAINAN:
1204 radeon_program_register_sequence(rdev,
1205 hainan_golden_registers,
1206 (const u32)ARRAY_SIZE(hainan_golden_registers));
1207 radeon_program_register_sequence(rdev,
1208 hainan_golden_registers2,
1209 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1210 radeon_program_register_sequence(rdev,
1211 hainan_mgcg_cgcg_init,
1212 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1213 break;
205996c0
AD		 1214 default:
1215 break;
1216 }
1217}
1218
454d2e2a
AD		 1219#define PCIE_BUS_CLK 10000
1220#define TCLK (PCIE_BUS_CLK / 10)
1221
1222/**
1223 * si_get_xclk - get the xclk
1224 *
1225 * @rdev: radeon_device pointer
1226 *
1227 * Returns the reference clock used by the gfx engine
1228 * (SI).
1229 */
1230u32 si_get_xclk(struct radeon_device *rdev)
1231{
1232 u32 reference_clock = rdev->clock.spll.reference_freq;
1233 u32 tmp;
1234
1235 tmp = RREG32(CG_CLKPIN_CNTL_2);
1236 if (tmp & MUX_TCLK_TO_XCLK)
1237 return TCLK;
1238
1239 tmp = RREG32(CG_CLKPIN_CNTL);
1240 if (tmp & XTALIN_DIVIDE)
1241 return reference_clock / 4;
1242
1243 return reference_clock;
1244}
1245
1bd47d2e
AD		 1246/* get temperature in millidegrees */
1247int si_get_temp(struct radeon_device *rdev)
1248{
1249 u32 temp;
1250 int actual_temp = 0;
1251
1252 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1253 CTF_TEMP_SHIFT;
1254
1255 if (temp & 0x200)
1256 actual_temp = 255;
1257 else
1258 actual_temp = temp & 0x1ff;
1259
1260 actual_temp = (actual_temp * 1000);
1261
1262 return actual_temp;
1263}
1264
8b074dd6
AD		 1265#define TAHITI_IO_MC_REGS_SIZE 36
1266
1267static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1268 {0x0000006f, 0x03044000},
1269 {0x00000070, 0x0480c018},
1270 {0x00000071, 0x00000040},
1271 {0x00000072, 0x01000000},
1272 {0x00000074, 0x000000ff},
1273 {0x00000075, 0x00143400},
1274 {0x00000076, 0x08ec0800},
1275 {0x00000077, 0x040000cc},
1276 {0x00000079, 0x00000000},
1277 {0x0000007a, 0x21000409},
1278 {0x0000007c, 0x00000000},
1279 {0x0000007d, 0xe8000000},
1280 {0x0000007e, 0x044408a8},
1281 {0x0000007f, 0x00000003},
1282 {0x00000080, 0x00000000},
1283 {0x00000081, 0x01000000},
1284 {0x00000082, 0x02000000},
1285 {0x00000083, 0x00000000},
1286 {0x00000084, 0xe3f3e4f4},
1287 {0x00000085, 0x00052024},
1288 {0x00000087, 0x00000000},
1289 {0x00000088, 0x66036603},
1290 {0x00000089, 0x01000000},
1291 {0x0000008b, 0x1c0a0000},
1292 {0x0000008c, 0xff010000},
1293 {0x0000008e, 0xffffefff},
1294 {0x0000008f, 0xfff3efff},
1295 {0x00000090, 0xfff3efbf},
1296 {0x00000094, 0x00101101},
1297 {0x00000095, 0x00000fff},
1298 {0x00000096, 0x00116fff},
1299 {0x00000097, 0x60010000},
1300 {0x00000098, 0x10010000},
1301 {0x00000099, 0x00006000},
1302 {0x0000009a, 0x00001000},
1303 {0x0000009f, 0x00a77400}
1304};
1305
1306static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1307 {0x0000006f, 0x03044000},
1308 {0x00000070, 0x0480c018},
1309 {0x00000071, 0x00000040},
1310 {0x00000072, 0x01000000},
1311 {0x00000074, 0x000000ff},
1312 {0x00000075, 0x00143400},
1313 {0x00000076, 0x08ec0800},
1314 {0x00000077, 0x040000cc},
1315 {0x00000079, 0x00000000},
1316 {0x0000007a, 0x21000409},
1317 {0x0000007c, 0x00000000},
1318 {0x0000007d, 0xe8000000},
1319 {0x0000007e, 0x044408a8},
1320 {0x0000007f, 0x00000003},
1321 {0x00000080, 0x00000000},
1322 {0x00000081, 0x01000000},
1323 {0x00000082, 0x02000000},
1324 {0x00000083, 0x00000000},
1325 {0x00000084, 0xe3f3e4f4},
1326 {0x00000085, 0x00052024},
1327 {0x00000087, 0x00000000},
1328 {0x00000088, 0x66036603},
1329 {0x00000089, 0x01000000},
1330 {0x0000008b, 0x1c0a0000},
1331 {0x0000008c, 0xff010000},
1332 {0x0000008e, 0xffffefff},
1333 {0x0000008f, 0xfff3efff},
1334 {0x00000090, 0xfff3efbf},
1335 {0x00000094, 0x00101101},
1336 {0x00000095, 0x00000fff},
1337 {0x00000096, 0x00116fff},
1338 {0x00000097, 0x60010000},
1339 {0x00000098, 0x10010000},
1340 {0x00000099, 0x00006000},
1341 {0x0000009a, 0x00001000},
1342 {0x0000009f, 0x00a47400}
1343};
1344
1345static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1346 {0x0000006f, 0x03044000},
1347 {0x00000070, 0x0480c018},
1348 {0x00000071, 0x00000040},
1349 {0x00000072, 0x01000000},
1350 {0x00000074, 0x000000ff},
1351 {0x00000075, 0x00143400},
1352 {0x00000076, 0x08ec0800},
1353 {0x00000077, 0x040000cc},
1354 {0x00000079, 0x00000000},
1355 {0x0000007a, 0x21000409},
1356 {0x0000007c, 0x00000000},
1357 {0x0000007d, 0xe8000000},
1358 {0x0000007e, 0x044408a8},
1359 {0x0000007f, 0x00000003},
1360 {0x00000080, 0x00000000},
1361 {0x00000081, 0x01000000},
1362 {0x00000082, 0x02000000},
1363 {0x00000083, 0x00000000},
1364 {0x00000084, 0xe3f3e4f4},
1365 {0x00000085, 0x00052024},
1366 {0x00000087, 0x00000000},
1367 {0x00000088, 0x66036603},
1368 {0x00000089, 0x01000000},
1369 {0x0000008b, 0x1c0a0000},
1370 {0x0000008c, 0xff010000},
1371 {0x0000008e, 0xffffefff},
1372 {0x0000008f, 0xfff3efff},
1373 {0x00000090, 0xfff3efbf},
1374 {0x00000094, 0x00101101},
1375 {0x00000095, 0x00000fff},
1376 {0x00000096, 0x00116fff},
1377 {0x00000097, 0x60010000},
1378 {0x00000098, 0x10010000},
1379 {0x00000099, 0x00006000},
1380 {0x0000009a, 0x00001000},
1381 {0x0000009f, 0x00a37400}
1382};
1383
bcc7f5d2
AD		 1384static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1385 {0x0000006f, 0x03044000},
1386 {0x00000070, 0x0480c018},
1387 {0x00000071, 0x00000040},
1388 {0x00000072, 0x01000000},
1389 {0x00000074, 0x000000ff},
1390 {0x00000075, 0x00143400},
1391 {0x00000076, 0x08ec0800},
1392 {0x00000077, 0x040000cc},
1393 {0x00000079, 0x00000000},
1394 {0x0000007a, 0x21000409},
1395 {0x0000007c, 0x00000000},
1396 {0x0000007d, 0xe8000000},
1397 {0x0000007e, 0x044408a8},
1398 {0x0000007f, 0x00000003},
1399 {0x00000080, 0x00000000},
1400 {0x00000081, 0x01000000},
1401 {0x00000082, 0x02000000},
1402 {0x00000083, 0x00000000},
1403 {0x00000084, 0xe3f3e4f4},
1404 {0x00000085, 0x00052024},
1405 {0x00000087, 0x00000000},
1406 {0x00000088, 0x66036603},
1407 {0x00000089, 0x01000000},
1408 {0x0000008b, 0x1c0a0000},
1409 {0x0000008c, 0xff010000},
1410 {0x0000008e, 0xffffefff},
1411 {0x0000008f, 0xfff3efff},
1412 {0x00000090, 0xfff3efbf},
1413 {0x00000094, 0x00101101},
1414 {0x00000095, 0x00000fff},
1415 {0x00000096, 0x00116fff},
1416 {0x00000097, 0x60010000},
1417 {0x00000098, 0x10010000},
1418 {0x00000099, 0x00006000},
1419 {0x0000009a, 0x00001000},
1420 {0x0000009f, 0x00a17730}
1421};
1422
c04c00b4
AD		 1423static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1424 {0x0000006f, 0x03044000},
1425 {0x00000070, 0x0480c018},
1426 {0x00000071, 0x00000040},
1427 {0x00000072, 0x01000000},
1428 {0x00000074, 0x000000ff},
1429 {0x00000075, 0x00143400},
1430 {0x00000076, 0x08ec0800},
1431 {0x00000077, 0x040000cc},
1432 {0x00000079, 0x00000000},
1433 {0x0000007a, 0x21000409},
1434 {0x0000007c, 0x00000000},
1435 {0x0000007d, 0xe8000000},
1436 {0x0000007e, 0x044408a8},
1437 {0x0000007f, 0x00000003},
1438 {0x00000080, 0x00000000},
1439 {0x00000081, 0x01000000},
1440 {0x00000082, 0x02000000},
1441 {0x00000083, 0x00000000},
1442 {0x00000084, 0xe3f3e4f4},
1443 {0x00000085, 0x00052024},
1444 {0x00000087, 0x00000000},
1445 {0x00000088, 0x66036603},
1446 {0x00000089, 0x01000000},
1447 {0x0000008b, 0x1c0a0000},
1448 {0x0000008c, 0xff010000},
1449 {0x0000008e, 0xffffefff},
1450 {0x0000008f, 0xfff3efff},
1451 {0x00000090, 0xfff3efbf},
1452 {0x00000094, 0x00101101},
1453 {0x00000095, 0x00000fff},
1454 {0x00000096, 0x00116fff},
1455 {0x00000097, 0x60010000},
1456 {0x00000098, 0x10010000},
1457 {0x00000099, 0x00006000},
1458 {0x0000009a, 0x00001000},
1459 {0x0000009f, 0x00a07730}
1460};
1461
8b074dd6
AD		 1462/* ucode loading */
1463static int si_mc_load_microcode(struct radeon_device *rdev)
1464{
1465 const __be32 *fw_data;
1466 u32 running, blackout = 0;
1467 u32 *io_mc_regs;
1468 int i, ucode_size, regs_size;
1469
1470 if (!rdev->mc_fw)
1471 return -EINVAL;
1472
1473 switch (rdev->family) {
1474 case CHIP_TAHITI:
1475 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1476 ucode_size = SI_MC_UCODE_SIZE;
1477 regs_size = TAHITI_IO_MC_REGS_SIZE;
1478 break;
1479 case CHIP_PITCAIRN:
1480 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1481 ucode_size = SI_MC_UCODE_SIZE;
1482 regs_size = TAHITI_IO_MC_REGS_SIZE;
1483 break;
1484 case CHIP_VERDE:
1485 default:
1486 io_mc_regs = (u32 *)&verde_io_mc_regs;
1487 ucode_size = SI_MC_UCODE_SIZE;
1488 regs_size = TAHITI_IO_MC_REGS_SIZE;
1489 break;
bcc7f5d2
AD		 1490 case CHIP_OLAND:
1491 io_mc_regs = (u32 *)&oland_io_mc_regs;
1492 ucode_size = OLAND_MC_UCODE_SIZE;
1493 regs_size = TAHITI_IO_MC_REGS_SIZE;
1494 break;
c04c00b4
AD		 1495 case CHIP_HAINAN:
1496 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1497 ucode_size = OLAND_MC_UCODE_SIZE;
1498 regs_size = TAHITI_IO_MC_REGS_SIZE;
1499 break;
8b074dd6
AD		 1500 }
1501
1502 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1503
1504 if (running == 0) {
1505 if (running) {
1506 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1507 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1508 }
1509
1510 /* reset the engine and set to writable */
1511 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1512 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1513
1514 /* load mc io regs */
1515 for (i = 0; i < regs_size; i++) {
1516 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1517 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1518 }
1519 /* load the MC ucode */
1520 fw_data = (const __be32 *)rdev->mc_fw->data;
1521 for (i = 0; i < ucode_size; i++)
1522 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1523
1524 /* put the engine back into the active state */
1525 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1526 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1527 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1528
1529 /* wait for training to complete */
1530 for (i = 0; i < rdev->usec_timeout; i++) {
1531 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1532 break;
1533 udelay(1);
1534 }
1535 for (i = 0; i < rdev->usec_timeout; i++) {
1536 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1537 break;
1538 udelay(1);
1539 }
1540
1541 if (running)
1542 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1543 }
1544
1545 return 0;
1546}
1547
0f0de06c
AD		 1548static int si_init_microcode(struct radeon_device *rdev)
1549{
0f0de06c
AD		 1550 const char *chip_name;
1551 const char *rlc_chip_name;
1552 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
a9e61410 1553 size_t smc_req_size;
0f0de06c
AD		 1554 char fw_name[30];
1555 int err;
1556
1557 DRM_DEBUG("\n");
1558
0f0de06c
AD		 1559 switch (rdev->family) {
1560 case CHIP_TAHITI:
1561 chip_name = "TAHITI";
1562 rlc_chip_name = "TAHITI";
1563 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1564 me_req_size = SI_PM4_UCODE_SIZE * 4;
1565 ce_req_size = SI_CE_UCODE_SIZE * 4;
1566 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1567 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1568 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1569 break;
1570 case CHIP_PITCAIRN:
1571 chip_name = "PITCAIRN";
1572 rlc_chip_name = "PITCAIRN";
1573 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1574 me_req_size = SI_PM4_UCODE_SIZE * 4;
1575 ce_req_size = SI_CE_UCODE_SIZE * 4;
1576 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1577 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1578 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1579 break;
1580 case CHIP_VERDE:
1581 chip_name = "VERDE";
1582 rlc_chip_name = "VERDE";
1583 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1584 me_req_size = SI_PM4_UCODE_SIZE * 4;
1585 ce_req_size = SI_CE_UCODE_SIZE * 4;
1586 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1587 mc_req_size = SI_MC_UCODE_SIZE * 4;
a9e61410 1588 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
0f0de06c 1589 break;
bcc7f5d2
AD		 1590 case CHIP_OLAND:
1591 chip_name = "OLAND";
1592 rlc_chip_name = "OLAND";
1593 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1594 me_req_size = SI_PM4_UCODE_SIZE * 4;
1595 ce_req_size = SI_CE_UCODE_SIZE * 4;
1596 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1597 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1598 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
bcc7f5d2 1599 break;
c04c00b4
AD		 1600 case CHIP_HAINAN:
1601 chip_name = "HAINAN";
1602 rlc_chip_name = "HAINAN";
1603 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1604 me_req_size = SI_PM4_UCODE_SIZE * 4;
1605 ce_req_size = SI_CE_UCODE_SIZE * 4;
1606 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1607 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1608 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
c04c00b4 1609 break;
0f0de06c
AD		 1610 default: BUG();
1611 }
1612
1613 DRM_INFO("Loading %s Microcode\n", chip_name);
1614
1615 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
0a168933 1616 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
0f0de06c
AD		 1617 if (err)
1618 goto out;
1619 if (rdev->pfp_fw->size != pfp_req_size) {
1620 printk(KERN_ERR
1621 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1622 rdev->pfp_fw->size, fw_name);
1623 err = -EINVAL;
1624 goto out;
1625 }
1626
1627 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
0a168933 1628 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
0f0de06c
AD		 1629 if (err)
1630 goto out;
1631 if (rdev->me_fw->size != me_req_size) {
1632 printk(KERN_ERR
1633 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1634 rdev->me_fw->size, fw_name);
1635 err = -EINVAL;
1636 }
1637
1638 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
0a168933 1639 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
0f0de06c
AD		 1640 if (err)
1641 goto out;
1642 if (rdev->ce_fw->size != ce_req_size) {
1643 printk(KERN_ERR
1644 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1645 rdev->ce_fw->size, fw_name);
1646 err = -EINVAL;
1647 }
1648
1649 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
0a168933 1650 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
0f0de06c
AD		 1651 if (err)
1652 goto out;
1653 if (rdev->rlc_fw->size != rlc_req_size) {
1654 printk(KERN_ERR
1655 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1656 rdev->rlc_fw->size, fw_name);
1657 err = -EINVAL;
1658 }
1659
1660 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
0a168933 1661 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
0f0de06c
AD		 1662 if (err)
1663 goto out;
1664 if (rdev->mc_fw->size != mc_req_size) {
1665 printk(KERN_ERR
1666 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1667 rdev->mc_fw->size, fw_name);
1668 err = -EINVAL;
1669 }
1670
a9e61410 1671 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
0a168933 1672 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
8a53fa23
AD		 1673 if (err) {
1674 printk(KERN_ERR
1675 "smc: error loading firmware \"%s\"\n",
1676 fw_name);
1677 release_firmware(rdev->smc_fw);
1678 rdev->smc_fw = NULL;
1679 } else if (rdev->smc_fw->size != smc_req_size) {
a9e61410
AD		 1680 printk(KERN_ERR
1681 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1682 rdev->smc_fw->size, fw_name);
1683 err = -EINVAL;
1684 }
1685
0f0de06c 1686out:
0f0de06c
AD		 1687 if (err) {
1688 if (err != -EINVAL)
1689 printk(KERN_ERR
1690 "si_cp: Failed to load firmware \"%s\"\n",
1691 fw_name);
1692 release_firmware(rdev->pfp_fw);
1693 rdev->pfp_fw = NULL;
1694 release_firmware(rdev->me_fw);
1695 rdev->me_fw = NULL;
1696 release_firmware(rdev->ce_fw);
1697 rdev->ce_fw = NULL;
1698 release_firmware(rdev->rlc_fw);
1699 rdev->rlc_fw = NULL;
1700 release_firmware(rdev->mc_fw);
1701 rdev->mc_fw = NULL;
a9e61410
AD		 1702 release_firmware(rdev->smc_fw);
1703 rdev->smc_fw = NULL;
0f0de06c
AD		 1704 }
1705 return err;
1706}
1707
43b3cd99
AD		 1708/* watermark setup */
1709static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1710 struct radeon_crtc *radeon_crtc,
1711 struct drm_display_mode *mode,
1712 struct drm_display_mode *other_mode)
1713{
290d2457
AD		 1714 u32 tmp, buffer_alloc, i;
1715 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
43b3cd99
AD		 1716 /*
1717 * Line Buffer Setup
1718 * There are 3 line buffers, each one shared by 2 display controllers.
1719 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1720 * the display controllers. The paritioning is done via one of four
1721 * preset allocations specified in bits 21:20:
1722 * 0 - half lb
1723 * 2 - whole lb, other crtc must be disabled
1724 */
1725 /* this can get tricky if we have two large displays on a paired group
1726 * of crtcs. Ideally for multiple large displays we'd assign them to
1727 * non-linked crtcs for maximum line buffer allocation.
1728 */
1729 if (radeon_crtc->base.enabled && mode) {
290d2457 1730 if (other_mode) {
43b3cd99 1731 tmp = 0; /* 1/2 */
290d2457
AD		 1732 buffer_alloc = 1;
1733 } else {
43b3cd99 1734 tmp = 2; /* whole */
290d2457
AD		 1735 buffer_alloc = 2;
1736 }
1737 } else {
43b3cd99 1738 tmp = 0;
290d2457
AD		 1739 buffer_alloc = 0;
1740 }
43b3cd99
AD		 1741
1742 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1743 DC_LB_MEMORY_CONFIG(tmp));
1744
290d2457
AD		 1745 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1746 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1747 for (i = 0; i < rdev->usec_timeout; i++) {
1748 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1749 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1750 break;
1751 udelay(1);
1752 }
1753
43b3cd99
AD		 1754 if (radeon_crtc->base.enabled && mode) {
1755 switch (tmp) {
1756 case 0:
1757 default:
1758 return 4096 * 2;
1759 case 2:
1760 return 8192 * 2;
1761 }
1762 }
1763
1764 /* controller not enabled, so no lb used */
1765 return 0;
1766}
1767
ca7db22b 1768static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 1769{
1770 u32 tmp = RREG32(MC_SHARED_CHMAP);
1771
1772 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1773 case 0:
1774 default:
1775 return 1;
1776 case 1:
1777 return 2;
1778 case 2:
1779 return 4;
1780 case 3:
1781 return 8;
1782 case 4:
1783 return 3;
1784 case 5:
1785 return 6;
1786 case 6:
1787 return 10;
1788 case 7:
1789 return 12;
1790 case 8:
1791 return 16;
1792 }
1793}
1794
1795struct dce6_wm_params {
1796 u32 dram_channels; /* number of dram channels */
1797 u32 yclk; /* bandwidth per dram data pin in kHz */
1798 u32 sclk; /* engine clock in kHz */
1799 u32 disp_clk; /* display clock in kHz */
1800 u32 src_width; /* viewport width */
1801 u32 active_time; /* active display time in ns */
1802 u32 blank_time; /* blank time in ns */
1803 bool interlaced; /* mode is interlaced */
1804 fixed20_12 vsc; /* vertical scale ratio */
1805 u32 num_heads; /* number of active crtcs */
1806 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1807 u32 lb_size; /* line buffer allocated to pipe */
1808 u32 vtaps; /* vertical scaler taps */
1809};
1810
1811static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1812{
1813 /* Calculate raw DRAM Bandwidth */
1814 fixed20_12 dram_efficiency; /* 0.7 */
1815 fixed20_12 yclk, dram_channels, bandwidth;
1816 fixed20_12 a;
1817
1818 a.full = dfixed_const(1000);
1819 yclk.full = dfixed_const(wm->yclk);
1820 yclk.full = dfixed_div(yclk, a);
1821 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1822 a.full = dfixed_const(10);
1823 dram_efficiency.full = dfixed_const(7);
1824 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1825 bandwidth.full = dfixed_mul(dram_channels, yclk);
1826 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1827
1828 return dfixed_trunc(bandwidth);
1829}
1830
1831static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1832{
1833 /* Calculate DRAM Bandwidth and the part allocated to display. */
1834 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1835 fixed20_12 yclk, dram_channels, bandwidth;
1836 fixed20_12 a;
1837
1838 a.full = dfixed_const(1000);
1839 yclk.full = dfixed_const(wm->yclk);
1840 yclk.full = dfixed_div(yclk, a);
1841 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1842 a.full = dfixed_const(10);
1843 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1844 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1845 bandwidth.full = dfixed_mul(dram_channels, yclk);
1846 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1847
1848 return dfixed_trunc(bandwidth);
1849}
1850
1851static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1852{
1853 /* Calculate the display Data return Bandwidth */
1854 fixed20_12 return_efficiency; /* 0.8 */
1855 fixed20_12 sclk, bandwidth;
1856 fixed20_12 a;
1857
1858 a.full = dfixed_const(1000);
1859 sclk.full = dfixed_const(wm->sclk);
1860 sclk.full = dfixed_div(sclk, a);
1861 a.full = dfixed_const(10);
1862 return_efficiency.full = dfixed_const(8);
1863 return_efficiency.full = dfixed_div(return_efficiency, a);
1864 a.full = dfixed_const(32);
1865 bandwidth.full = dfixed_mul(a, sclk);
1866 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1867
1868 return dfixed_trunc(bandwidth);
1869}
1870
1871static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1872{
1873 return 32;
1874}
1875
1876static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1877{
1878 /* Calculate the DMIF Request Bandwidth */
1879 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1880 fixed20_12 disp_clk, sclk, bandwidth;
1881 fixed20_12 a, b1, b2;
1882 u32 min_bandwidth;
1883
1884 a.full = dfixed_const(1000);
1885 disp_clk.full = dfixed_const(wm->disp_clk);
1886 disp_clk.full = dfixed_div(disp_clk, a);
1887 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1888 b1.full = dfixed_mul(a, disp_clk);
1889
1890 a.full = dfixed_const(1000);
1891 sclk.full = dfixed_const(wm->sclk);
1892 sclk.full = dfixed_div(sclk, a);
1893 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1894 b2.full = dfixed_mul(a, sclk);
1895
1896 a.full = dfixed_const(10);
1897 disp_clk_request_efficiency.full = dfixed_const(8);
1898 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1899
1900 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1901
1902 a.full = dfixed_const(min_bandwidth);
1903 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1904
1905 return dfixed_trunc(bandwidth);
1906}
1907
1908static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1909{
1910 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1911 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1912 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1913 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1914
1915 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1916}
1917
1918static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1919{
1920 /* Calculate the display mode Average Bandwidth
1921 * DisplayMode should contain the source and destination dimensions,
1922 * timing, etc.
1923 */
1924 fixed20_12 bpp;
1925 fixed20_12 line_time;
1926 fixed20_12 src_width;
1927 fixed20_12 bandwidth;
1928 fixed20_12 a;
1929
1930 a.full = dfixed_const(1000);
1931 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1932 line_time.full = dfixed_div(line_time, a);
1933 bpp.full = dfixed_const(wm->bytes_per_pixel);
1934 src_width.full = dfixed_const(wm->src_width);
1935 bandwidth.full = dfixed_mul(src_width, bpp);
1936 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1937 bandwidth.full = dfixed_div(bandwidth, line_time);
1938
1939 return dfixed_trunc(bandwidth);
1940}
1941
1942static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1943{
1944 /* First calcualte the latency in ns */
1945 u32 mc_latency = 2000; /* 2000 ns. */
1946 u32 available_bandwidth = dce6_available_bandwidth(wm);
1947 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1948 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1949 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1950 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1951 (wm->num_heads * cursor_line_pair_return_time);
1952 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1953 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1954 u32 tmp, dmif_size = 12288;
1955 fixed20_12 a, b, c;
1956
1957 if (wm->num_heads == 0)
1958 return 0;
1959
1960 a.full = dfixed_const(2);
1961 b.full = dfixed_const(1);
1962 if ((wm->vsc.full > a.full) ||
1963 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1964 (wm->vtaps >= 5) ||
1965 ((wm->vsc.full >= a.full) && wm->interlaced))
1966 max_src_lines_per_dst_line = 4;
1967 else
1968 max_src_lines_per_dst_line = 2;
1969
1970 a.full = dfixed_const(available_bandwidth);
1971 b.full = dfixed_const(wm->num_heads);
1972 a.full = dfixed_div(a, b);
1973
1974 b.full = dfixed_const(mc_latency + 512);
1975 c.full = dfixed_const(wm->disp_clk);
1976 b.full = dfixed_div(b, c);
1977
1978 c.full = dfixed_const(dmif_size);
1979 b.full = dfixed_div(c, b);
1980
1981 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1982
1983 b.full = dfixed_const(1000);
1984 c.full = dfixed_const(wm->disp_clk);
1985 b.full = dfixed_div(c, b);
1986 c.full = dfixed_const(wm->bytes_per_pixel);
1987 b.full = dfixed_mul(b, c);
1988
1989 lb_fill_bw = min(tmp, dfixed_trunc(b));
1990
1991 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1992 b.full = dfixed_const(1000);
1993 c.full = dfixed_const(lb_fill_bw);
1994 b.full = dfixed_div(c, b);
1995 a.full = dfixed_div(a, b);
1996 line_fill_time = dfixed_trunc(a);
1997
1998 if (line_fill_time < wm->active_time)
1999 return latency;
2000 else
2001 return latency + (line_fill_time - wm->active_time);
2002
2003}
2004
2005static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2006{
2007 if (dce6_average_bandwidth(wm) <=
2008 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2009 return true;
2010 else
2011 return false;
2012};
2013
2014static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2015{
2016 if (dce6_average_bandwidth(wm) <=
2017 (dce6_available_bandwidth(wm) / wm->num_heads))
2018 return true;
2019 else
2020 return false;
2021};
2022
2023static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2024{
2025 u32 lb_partitions = wm->lb_size / wm->src_width;
2026 u32 line_time = wm->active_time + wm->blank_time;
2027 u32 latency_tolerant_lines;
2028 u32 latency_hiding;
2029 fixed20_12 a;
2030
2031 a.full = dfixed_const(1);
2032 if (wm->vsc.full > a.full)
2033 latency_tolerant_lines = 1;
2034 else {
2035 if (lb_partitions <= (wm->vtaps + 1))
2036 latency_tolerant_lines = 1;
2037 else
2038 latency_tolerant_lines = 2;
2039 }
2040
2041 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2042
2043 if (dce6_latency_watermark(wm) <= latency_hiding)
2044 return true;
2045 else
2046 return false;
2047}
2048
2049static void dce6_program_watermarks(struct radeon_device *rdev,
2050 struct radeon_crtc *radeon_crtc,
2051 u32 lb_size, u32 num_heads)
2052{
2053 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 2054 struct dce6_wm_params wm_low, wm_high;
2055 u32 dram_channels;
43b3cd99
AD		 2056 u32 pixel_period;
2057 u32 line_time = 0;
2058 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2059 u32 priority_a_mark = 0, priority_b_mark = 0;
2060 u32 priority_a_cnt = PRIORITY_OFF;
2061 u32 priority_b_cnt = PRIORITY_OFF;
2062 u32 tmp, arb_control3;
2063 fixed20_12 a, b, c;
2064
2065 if (radeon_crtc->base.enabled && num_heads && mode) {
2066 pixel_period = 1000000 / (u32)mode->clock;
2067 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2068 priority_a_cnt = 0;
2069 priority_b_cnt = 0;
2070
ca7db22b 2071 if (rdev->family == CHIP_ARUBA)
c696e53f 2072 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 2073 else
c696e53f
AD		 2074 dram_channels = si_get_number_of_dram_channels(rdev);
2075
2076 /* watermark for high clocks */
2077 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2078 wm_high.yclk =
2079 radeon_dpm_get_mclk(rdev, false) * 10;
2080 wm_high.sclk =
2081 radeon_dpm_get_sclk(rdev, false) * 10;
2082 } else {
2083 wm_high.yclk = rdev->pm.current_mclk * 10;
2084 wm_high.sclk = rdev->pm.current_sclk * 10;
2085 }
2086
2087 wm_high.disp_clk = mode->clock;
2088 wm_high.src_width = mode->crtc_hdisplay;
2089 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2090 wm_high.blank_time = line_time - wm_high.active_time;
2091 wm_high.interlaced = false;
2092 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2093 wm_high.interlaced = true;
2094 wm_high.vsc = radeon_crtc->vsc;
2095 wm_high.vtaps = 1;
2096 if (radeon_crtc->rmx_type != RMX_OFF)
2097 wm_high.vtaps = 2;
2098 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2099 wm_high.lb_size = lb_size;
2100 wm_high.dram_channels = dram_channels;
2101 wm_high.num_heads = num_heads;
2102
2103 /* watermark for low clocks */
2104 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2105 wm_low.yclk =
2106 radeon_dpm_get_mclk(rdev, true) * 10;
2107 wm_low.sclk =
2108 radeon_dpm_get_sclk(rdev, true) * 10;
2109 } else {
2110 wm_low.yclk = rdev->pm.current_mclk * 10;
2111 wm_low.sclk = rdev->pm.current_sclk * 10;
2112 }
2113
2114 wm_low.disp_clk = mode->clock;
2115 wm_low.src_width = mode->crtc_hdisplay;
2116 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2117 wm_low.blank_time = line_time - wm_low.active_time;
2118 wm_low.interlaced = false;
2119 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2120 wm_low.interlaced = true;
2121 wm_low.vsc = radeon_crtc->vsc;
2122 wm_low.vtaps = 1;
2123 if (radeon_crtc->rmx_type != RMX_OFF)
2124 wm_low.vtaps = 2;
2125 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2126 wm_low.lb_size = lb_size;
2127 wm_low.dram_channels = dram_channels;
2128 wm_low.num_heads = num_heads;
43b3cd99
AD		 2129
2130 /* set for high clocks */
c696e53f 2131 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 2132 /* set for low clocks */
c696e53f 2133 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 2134
2135 /* possibly force display priority to high */
2136 /* should really do this at mode validation time... */
c696e53f
AD		 2137 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2138 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2139 !dce6_check_latency_hiding(&wm_high) ||
2140 (rdev->disp_priority == 2)) {
2141 DRM_DEBUG_KMS("force priority to high\n");
2142 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2143 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2144 }
2145 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2146 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2147 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 2148 (rdev->disp_priority == 2)) {
2149 DRM_DEBUG_KMS("force priority to high\n");
2150 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2151 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2152 }
2153
2154 a.full = dfixed_const(1000);
2155 b.full = dfixed_const(mode->clock);
2156 b.full = dfixed_div(b, a);
2157 c.full = dfixed_const(latency_watermark_a);
2158 c.full = dfixed_mul(c, b);
2159 c.full = dfixed_mul(c, radeon_crtc->hsc);
2160 c.full = dfixed_div(c, a);
2161 a.full = dfixed_const(16);
2162 c.full = dfixed_div(c, a);
2163 priority_a_mark = dfixed_trunc(c);
2164 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2165
2166 a.full = dfixed_const(1000);
2167 b.full = dfixed_const(mode->clock);
2168 b.full = dfixed_div(b, a);
2169 c.full = dfixed_const(latency_watermark_b);
2170 c.full = dfixed_mul(c, b);
2171 c.full = dfixed_mul(c, radeon_crtc->hsc);
2172 c.full = dfixed_div(c, a);
2173 a.full = dfixed_const(16);
2174 c.full = dfixed_div(c, a);
2175 priority_b_mark = dfixed_trunc(c);
2176 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2177 }
2178
2179 /* select wm A */
2180 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2181 tmp = arb_control3;
2182 tmp &= ~LATENCY_WATERMARK_MASK(3);
2183 tmp |= LATENCY_WATERMARK_MASK(1);
2184 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2185 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2186 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2187 LATENCY_HIGH_WATERMARK(line_time)));
2188 /* select wm B */
2189 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2190 tmp &= ~LATENCY_WATERMARK_MASK(3);
2191 tmp |= LATENCY_WATERMARK_MASK(2);
2192 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2193 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2194 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2195 LATENCY_HIGH_WATERMARK(line_time)));
2196 /* restore original selection */
2197 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2198
2199 /* write the priority marks */
2200 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2201 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2202
7178d2a6
AD		 2203 /* save values for DPM */
2204 radeon_crtc->line_time = line_time;
2205 radeon_crtc->wm_high = latency_watermark_a;
2206 radeon_crtc->wm_low = latency_watermark_b;
43b3cd99
AD		 2207}
2208
2209void dce6_bandwidth_update(struct radeon_device *rdev)
2210{
2211 struct drm_display_mode *mode0 = NULL;
2212 struct drm_display_mode *mode1 = NULL;
2213 u32 num_heads = 0, lb_size;
2214 int i;
2215
2216 radeon_update_display_priority(rdev);
2217
2218 for (i = 0; i < rdev->num_crtc; i++) {
2219 if (rdev->mode_info.crtcs[i]->base.enabled)
2220 num_heads++;
2221 }
2222 for (i = 0; i < rdev->num_crtc; i += 2) {
2223 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2224 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2225 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2226 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2227 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2228 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2229 }
2230}
2231
0a96d72b
AD		 2232/*
2233 * Core functions
2234 */
0a96d72b
AD		 2235static void si_tiling_mode_table_init(struct radeon_device *rdev)
2236{
2237 const u32 num_tile_mode_states = 32;
2238 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2239
2240 switch (rdev->config.si.mem_row_size_in_kb) {
2241 case 1:
2242 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2243 break;
2244 case 2:
2245 default:
2246 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2247 break;
2248 case 4:
2249 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2250 break;
2251 }
2252
2253 if ((rdev->family == CHIP_TAHITI) ||
2254 (rdev->family == CHIP_PITCAIRN)) {
2255 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2256 switch (reg_offset) {
2257 case 0: /* non-AA compressed depth or any compressed stencil */
2258 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2259 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2260 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2261 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2262 NUM_BANKS(ADDR_SURF_16_BANK) |
2263 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2264 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2265 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2266 break;
2267 case 1: /* 2xAA/4xAA compressed depth only */
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2269 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2270 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2271 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2272 NUM_BANKS(ADDR_SURF_16_BANK) |
2273 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2274 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2275 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2276 break;
2277 case 2: /* 8xAA compressed depth only */
2278 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2279 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2280 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2281 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2282 NUM_BANKS(ADDR_SURF_16_BANK) |
2283 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2284 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2285 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2286 break;
2287 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2288 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2289 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2290 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2291 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2292 NUM_BANKS(ADDR_SURF_16_BANK) |
2293 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2294 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2295 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2296 break;
2297 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2298 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2299 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2300 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2301 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2302 NUM_BANKS(ADDR_SURF_16_BANK) |
2303 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2304 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2305 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2306 break;
2307 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2308 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2309 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2310 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2311 TILE_SPLIT(split_equal_to_row_size) |
2312 NUM_BANKS(ADDR_SURF_16_BANK) |
2313 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2314 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2315 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2316 break;
2317 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2318 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2319 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2320 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2321 TILE_SPLIT(split_equal_to_row_size) |
2322 NUM_BANKS(ADDR_SURF_16_BANK) |
2323 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2324 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2325 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2326 break;
2327 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2328 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2329 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2330 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2331 TILE_SPLIT(split_equal_to_row_size) |
2332 NUM_BANKS(ADDR_SURF_16_BANK) |
2333 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2334 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2335 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2336 break;
2337 case 8: /* 1D and 1D Array Surfaces */
2338 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2339 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2340 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2341 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2342 NUM_BANKS(ADDR_SURF_16_BANK) |
2343 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2344 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2345 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2346 break;
2347 case 9: /* Displayable maps. */
2348 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2349 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2350 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2351 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2352 NUM_BANKS(ADDR_SURF_16_BANK) |
2353 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2354 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2355 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2356 break;
2357 case 10: /* Display 8bpp. */
2358 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2359 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2360 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2361 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2362 NUM_BANKS(ADDR_SURF_16_BANK) |
2363 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2364 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2365 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2366 break;
2367 case 11: /* Display 16bpp. */
2368 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2369 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2370 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2371 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2372 NUM_BANKS(ADDR_SURF_16_BANK) |
2373 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2376 break;
2377 case 12: /* Display 32bpp. */
2378 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2379 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2380 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2381 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2382 NUM_BANKS(ADDR_SURF_16_BANK) |
2383 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2384 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2385 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2386 break;
2387 case 13: /* Thin. */
2388 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2389 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2390 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2391 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2392 NUM_BANKS(ADDR_SURF_16_BANK) |
2393 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2394 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2395 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2396 break;
2397 case 14: /* Thin 8 bpp. */
2398 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2399 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2400 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2401 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2402 NUM_BANKS(ADDR_SURF_16_BANK) |
2403 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2404 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2405 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2406 break;
2407 case 15: /* Thin 16 bpp. */
2408 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2409 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2410 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2411 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2412 NUM_BANKS(ADDR_SURF_16_BANK) |
2413 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2414 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2415 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2416 break;
2417 case 16: /* Thin 32 bpp. */
2418 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2419 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2420 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2421 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2422 NUM_BANKS(ADDR_SURF_16_BANK) |
2423 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2424 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2425 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2426 break;
2427 case 17: /* Thin 64 bpp. */
2428 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2429 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2430 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2431 TILE_SPLIT(split_equal_to_row_size) |
2432 NUM_BANKS(ADDR_SURF_16_BANK) |
2433 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2434 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2435 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2436 break;
2437 case 21: /* 8 bpp PRT. */
2438 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2439 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2440 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2441 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2442 NUM_BANKS(ADDR_SURF_16_BANK) |
2443 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2444 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2445 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2446 break;
2447 case 22: /* 16 bpp PRT */
2448 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2449 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2450 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2451 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2452 NUM_BANKS(ADDR_SURF_16_BANK) |
2453 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2454 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2455 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2456 break;
2457 case 23: /* 32 bpp PRT */
2458 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2459 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2460 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2461 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2462 NUM_BANKS(ADDR_SURF_16_BANK) |
2463 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2464 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2465 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2466 break;
2467 case 24: /* 64 bpp PRT */
2468 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2469 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2470 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2471 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2472 NUM_BANKS(ADDR_SURF_16_BANK) |
2473 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2474 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2475 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2476 break;
2477 case 25: /* 128 bpp PRT */
2478 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2479 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2480 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2481 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2482 NUM_BANKS(ADDR_SURF_8_BANK) |
2483 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2484 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2485 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2486 break;
2487 default:
2488 gb_tile_moden = 0;
2489 break;
2490 }
64d7b8be 2491 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2492 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2493 }
d0ae7fcc 2494 } else if ((rdev->family == CHIP_VERDE) ||
8b02859d
AD		 2495 (rdev->family == CHIP_OLAND) ||
2496 (rdev->family == CHIP_HAINAN)) {
0a96d72b
AD		 2497 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2498 switch (reg_offset) {
2499 case 0: /* non-AA compressed depth or any compressed stencil */
2500 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2501 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2502 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2503 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2504 NUM_BANKS(ADDR_SURF_16_BANK) |
2505 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2506 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2507 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2508 break;
2509 case 1: /* 2xAA/4xAA compressed depth only */
2510 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2511 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2512 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2513 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2514 NUM_BANKS(ADDR_SURF_16_BANK) |
2515 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2516 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2517 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2518 break;
2519 case 2: /* 8xAA compressed depth only */
2520 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2521 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2522 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2523 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2524 NUM_BANKS(ADDR_SURF_16_BANK) |
2525 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2526 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2527 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2528 break;
2529 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2530 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2531 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2532 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2533 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2534 NUM_BANKS(ADDR_SURF_16_BANK) |
2535 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2536 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2537 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2538 break;
2539 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2540 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2541 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2542 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2543 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2544 NUM_BANKS(ADDR_SURF_16_BANK) |
2545 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2546 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2547 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2548 break;
2549 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2550 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2551 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2552 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2553 TILE_SPLIT(split_equal_to_row_size) |
2554 NUM_BANKS(ADDR_SURF_16_BANK) |
2555 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2556 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2557 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2558 break;
2559 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2560 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2561 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2562 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2563 TILE_SPLIT(split_equal_to_row_size) |
2564 NUM_BANKS(ADDR_SURF_16_BANK) |
2565 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2566 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2567 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2568 break;
2569 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2570 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2571 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2572 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2573 TILE_SPLIT(split_equal_to_row_size) |
2574 NUM_BANKS(ADDR_SURF_16_BANK) |
2575 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2576 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2577 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2578 break;
2579 case 8: /* 1D and 1D Array Surfaces */
2580 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2581 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2582 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2583 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2584 NUM_BANKS(ADDR_SURF_16_BANK) |
2585 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2586 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2587 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2588 break;
2589 case 9: /* Displayable maps. */
2590 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2591 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2592 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2593 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2594 NUM_BANKS(ADDR_SURF_16_BANK) |
2595 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2596 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2597 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2598 break;
2599 case 10: /* Display 8bpp. */
2600 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2601 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2602 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2603 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2604 NUM_BANKS(ADDR_SURF_16_BANK) |
2605 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2606 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2607 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2608 break;
2609 case 11: /* Display 16bpp. */
2610 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2611 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2612 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2613 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2614 NUM_BANKS(ADDR_SURF_16_BANK) |
2615 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2616 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2617 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2618 break;
2619 case 12: /* Display 32bpp. */
2620 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2621 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2622 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2623 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2624 NUM_BANKS(ADDR_SURF_16_BANK) |
2625 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2626 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2627 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2628 break;
2629 case 13: /* Thin. */
2630 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2631 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2632 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2633 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2634 NUM_BANKS(ADDR_SURF_16_BANK) |
2635 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2636 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2637 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2638 break;
2639 case 14: /* Thin 8 bpp. */
2640 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2641 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2642 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2643 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2644 NUM_BANKS(ADDR_SURF_16_BANK) |
2645 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2646 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2647 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2648 break;
2649 case 15: /* Thin 16 bpp. */
2650 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2651 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2652 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2653 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2654 NUM_BANKS(ADDR_SURF_16_BANK) |
2655 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2656 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2657 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2658 break;
2659 case 16: /* Thin 32 bpp. */
2660 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2661 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2662 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2663 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2664 NUM_BANKS(ADDR_SURF_16_BANK) |
2665 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2666 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2667 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2668 break;
2669 case 17: /* Thin 64 bpp. */
2670 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2671 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2672 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2673 TILE_SPLIT(split_equal_to_row_size) |
2674 NUM_BANKS(ADDR_SURF_16_BANK) |
2675 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2676 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2677 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2678 break;
2679 case 21: /* 8 bpp PRT. */
2680 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2681 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2682 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2683 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2684 NUM_BANKS(ADDR_SURF_16_BANK) |
2685 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2686 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2687 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2688 break;
2689 case 22: /* 16 bpp PRT */
2690 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2691 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2692 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2693 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2694 NUM_BANKS(ADDR_SURF_16_BANK) |
2695 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2696 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2697 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2698 break;
2699 case 23: /* 32 bpp PRT */
2700 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2701 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2702 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2703 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2704 NUM_BANKS(ADDR_SURF_16_BANK) |
2705 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2706 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2707 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2708 break;
2709 case 24: /* 64 bpp PRT */
2710 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2711 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2712 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2713 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2714 NUM_BANKS(ADDR_SURF_16_BANK) |
2715 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2716 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2717 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2718 break;
2719 case 25: /* 128 bpp PRT */
2720 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2721 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2722 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2723 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2724 NUM_BANKS(ADDR_SURF_8_BANK) |
2725 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2726 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2727 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2728 break;
2729 default:
2730 gb_tile_moden = 0;
2731 break;
2732 }
64d7b8be 2733 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2734 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2735 }
2736 } else
2737 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2738}
2739
1a8ca750
AD		 2740static void si_select_se_sh(struct radeon_device *rdev,
2741 u32 se_num, u32 sh_num)
2742{
2743 u32 data = INSTANCE_BROADCAST_WRITES;
2744
2745 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2746 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2747 else if (se_num == 0xffffffff)
2748 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2749 else if (sh_num == 0xffffffff)
2750 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2751 else
2752 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2753 WREG32(GRBM_GFX_INDEX, data);
2754}
2755
2756static u32 si_create_bitmask(u32 bit_width)
2757{
2758 u32 i, mask = 0;
2759
2760 for (i = 0; i < bit_width; i++) {
2761 mask <<= 1;
2762 mask |= 1;
2763 }
2764 return mask;
2765}
2766
2767static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2768{
2769 u32 data, mask;
2770
2771 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2772 if (data & 1)
2773 data &= INACTIVE_CUS_MASK;
2774 else
2775 data = 0;
2776 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2777
2778 data >>= INACTIVE_CUS_SHIFT;
2779
2780 mask = si_create_bitmask(cu_per_sh);
2781
2782 return ~data & mask;
2783}
2784
2785static void si_setup_spi(struct radeon_device *rdev,
2786 u32 se_num, u32 sh_per_se,
2787 u32 cu_per_sh)
2788{
2789 int i, j, k;
2790 u32 data, mask, active_cu;
2791
2792 for (i = 0; i < se_num; i++) {
2793 for (j = 0; j < sh_per_se; j++) {
2794 si_select_se_sh(rdev, i, j);
2795 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2796 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2797
2798 mask = 1;
2799 for (k = 0; k < 16; k++) {
2800 mask <<= k;
2801 if (active_cu & mask) {
2802 data &= ~mask;
2803 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2804 break;
2805 }
2806 }
2807 }
2808 }
2809 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2810}
2811
2812static u32 si_get_rb_disabled(struct radeon_device *rdev,
2813 u32 max_rb_num, u32 se_num,
2814 u32 sh_per_se)
2815{
2816 u32 data, mask;
2817
2818 data = RREG32(CC_RB_BACKEND_DISABLE);
2819 if (data & 1)
2820 data &= BACKEND_DISABLE_MASK;
2821 else
2822 data = 0;
2823 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2824
2825 data >>= BACKEND_DISABLE_SHIFT;
2826
2827 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2828
2829 return data & mask;
2830}
2831
2832static void si_setup_rb(struct radeon_device *rdev,
2833 u32 se_num, u32 sh_per_se,
2834 u32 max_rb_num)
2835{
2836 int i, j;
2837 u32 data, mask;
2838 u32 disabled_rbs = 0;
2839 u32 enabled_rbs = 0;
2840
2841 for (i = 0; i < se_num; i++) {
2842 for (j = 0; j < sh_per_se; j++) {
2843 si_select_se_sh(rdev, i, j);
2844 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2845 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2846 }
2847 }
2848 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2849
2850 mask = 1;
2851 for (i = 0; i < max_rb_num; i++) {
2852 if (!(disabled_rbs & mask))
2853 enabled_rbs |= mask;
2854 mask <<= 1;
2855 }
2856
2857 for (i = 0; i < se_num; i++) {
2858 si_select_se_sh(rdev, i, 0xffffffff);
2859 data = 0;
2860 for (j = 0; j < sh_per_se; j++) {
2861 switch (enabled_rbs & 3) {
2862 case 1:
2863 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2864 break;
2865 case 2:
2866 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2867 break;
2868 case 3:
2869 default:
2870 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2871 break;
2872 }
2873 enabled_rbs >>= 2;
2874 }
2875 WREG32(PA_SC_RASTER_CONFIG, data);
2876 }
2877 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2878}
2879
0a96d72b
AD		 2880static void si_gpu_init(struct radeon_device *rdev)
2881{
0a96d72b
AD		 2882 u32 gb_addr_config = 0;
2883 u32 mc_shared_chmap, mc_arb_ramcfg;
0a96d72b 2884 u32 sx_debug_1;
0a96d72b
AD		 2885 u32 hdp_host_path_cntl;
2886 u32 tmp;
2887 int i, j;
2888
2889 switch (rdev->family) {
2890 case CHIP_TAHITI:
2891 rdev->config.si.max_shader_engines = 2;
0a96d72b 2892 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 2893 rdev->config.si.max_cu_per_sh = 8;
2894 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2895 rdev->config.si.max_backends_per_se = 4;
2896 rdev->config.si.max_texture_channel_caches = 12;
2897 rdev->config.si.max_gprs = 256;
2898 rdev->config.si.max_gs_threads = 32;
2899 rdev->config.si.max_hw_contexts = 8;
2900
2901 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2902 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2903 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2904 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2905 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2906 break;
2907 case CHIP_PITCAIRN:
2908 rdev->config.si.max_shader_engines = 2;
0a96d72b 2909 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 2910 rdev->config.si.max_cu_per_sh = 5;
2911 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2912 rdev->config.si.max_backends_per_se = 4;
2913 rdev->config.si.max_texture_channel_caches = 8;
2914 rdev->config.si.max_gprs = 256;
2915 rdev->config.si.max_gs_threads = 32;
2916 rdev->config.si.max_hw_contexts = 8;
2917
2918 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2919 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2920 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2921 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2922 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2923 break;
2924 case CHIP_VERDE:
2925 default:
2926 rdev->config.si.max_shader_engines = 1;
0a96d72b 2927 rdev->config.si.max_tile_pipes = 4;
468ef1a5 2928 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 2929 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2930 rdev->config.si.max_backends_per_se = 4;
2931 rdev->config.si.max_texture_channel_caches = 4;
2932 rdev->config.si.max_gprs = 256;
2933 rdev->config.si.max_gs_threads = 32;
2934 rdev->config.si.max_hw_contexts = 8;
2935
d0ae7fcc
AD		 2936 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2937 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2938 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2939 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2940 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2941 break;
2942 case CHIP_OLAND:
2943 rdev->config.si.max_shader_engines = 1;
2944 rdev->config.si.max_tile_pipes = 4;
2945 rdev->config.si.max_cu_per_sh = 6;
2946 rdev->config.si.max_sh_per_se = 1;
2947 rdev->config.si.max_backends_per_se = 2;
2948 rdev->config.si.max_texture_channel_caches = 4;
2949 rdev->config.si.max_gprs = 256;
2950 rdev->config.si.max_gs_threads = 16;
2951 rdev->config.si.max_hw_contexts = 8;
2952
0a96d72b
AD		 2953 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2954 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2955 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2956 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2957 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 2958 break;
8b02859d
AD		 2959 case CHIP_HAINAN:
2960 rdev->config.si.max_shader_engines = 1;
2961 rdev->config.si.max_tile_pipes = 4;
2962 rdev->config.si.max_cu_per_sh = 5;
2963 rdev->config.si.max_sh_per_se = 1;
2964 rdev->config.si.max_backends_per_se = 1;
2965 rdev->config.si.max_texture_channel_caches = 2;
2966 rdev->config.si.max_gprs = 256;
2967 rdev->config.si.max_gs_threads = 16;
2968 rdev->config.si.max_hw_contexts = 8;
2969
2970 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2971 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2972 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2973 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2974 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2975 break;
0a96d72b
AD		 2976 }
2977
2978 /* Initialize HDP */
2979 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2980 WREG32((0x2c14 + j), 0x00000000);
2981 WREG32((0x2c18 + j), 0x00000000);
2982 WREG32((0x2c1c + j), 0x00000000);
2983 WREG32((0x2c20 + j), 0x00000000);
2984 WREG32((0x2c24 + j), 0x00000000);
2985 }
2986
2987 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2988
2989 evergreen_fix_pci_max_read_req_size(rdev);
2990
2991 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2992
2993 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2994 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2995
0a96d72b 2996 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 2997 rdev->config.si.mem_max_burst_length_bytes = 256;
2998 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2999 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3000 if (rdev->config.si.mem_row_size_in_kb > 4)
3001 rdev->config.si.mem_row_size_in_kb = 4;
3002 /* XXX use MC settings? */
3003 rdev->config.si.shader_engine_tile_size = 32;
3004 rdev->config.si.num_gpus = 1;
3005 rdev->config.si.multi_gpu_tile_size = 64;
3006
1a8ca750
AD		 3007 /* fix up row size */
3008 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 3009 switch (rdev->config.si.mem_row_size_in_kb) {
3010 case 1:
3011 default:
3012 gb_addr_config |= ROW_SIZE(0);
3013 break;
3014 case 2:
3015 gb_addr_config |= ROW_SIZE(1);
3016 break;
3017 case 4:
3018 gb_addr_config |= ROW_SIZE(2);
3019 break;
3020 }
3021
0a96d72b
AD		 3022 /* setup tiling info dword. gb_addr_config is not adequate since it does
3023 * not have bank info, so create a custom tiling dword.
3024 * bits 3:0 num_pipes
3025 * bits 7:4 num_banks
3026 * bits 11:8 group_size
3027 * bits 15:12 row_size
3028 */
3029 rdev->config.si.tile_config = 0;
3030 switch (rdev->config.si.num_tile_pipes) {
3031 case 1:
3032 rdev->config.si.tile_config |= (0 << 0);
3033 break;
3034 case 2:
3035 rdev->config.si.tile_config |= (1 << 0);
3036 break;
3037 case 4:
3038 rdev->config.si.tile_config |= (2 << 0);
3039 break;
3040 case 8:
3041 default:
3042 /* XXX what about 12? */
3043 rdev->config.si.tile_config |= (3 << 0);
3044 break;
dca571a6
CK		 3045 }
3046 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3047 case 0: /* four banks */
1a8ca750 3048 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 3049 break;
3050 case 1: /* eight banks */
3051 rdev->config.si.tile_config |= 1 << 4;
3052 break;
3053 case 2: /* sixteen banks */
3054 default:
3055 rdev->config.si.tile_config |= 2 << 4;
3056 break;
3057 }
0a96d72b
AD		 3058 rdev->config.si.tile_config |=
3059 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3060 rdev->config.si.tile_config |=
3061 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3062
0a96d72b
AD		 3063 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3064 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 3065 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 3066 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 3067 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3068 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 3069 if (rdev->has_uvd) {
3070 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3071 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3072 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3073 }
0a96d72b 3074
1a8ca750 3075 si_tiling_mode_table_init(rdev);
0a96d72b 3076
1a8ca750
AD		 3077 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3078 rdev->config.si.max_sh_per_se,
3079 rdev->config.si.max_backends_per_se);
0a96d72b 3080
1a8ca750
AD		 3081 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3082 rdev->config.si.max_sh_per_se,
3083 rdev->config.si.max_cu_per_sh);
0a96d72b 3084
0a96d72b
AD		 3085
3086 /* set HW defaults for 3D engine */
3087 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3088 ROQ_IB2_START(0x2b)));
3089 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3090
3091 sx_debug_1 = RREG32(SX_DEBUG_1);
3092 WREG32(SX_DEBUG_1, sx_debug_1);
3093
3094 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3095
3096 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3097 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3098 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3099 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3100
3101 WREG32(VGT_NUM_INSTANCES, 1);
3102
3103 WREG32(CP_PERFMON_CNTL, 0);
3104
3105 WREG32(SQ_CONFIG, 0);
3106
3107 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3108 FORCE_EOV_MAX_REZ_CNT(255)));
3109
3110 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3111 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3112
3113 WREG32(VGT_GS_VERTEX_REUSE, 16);
3114 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3115
3116 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3117 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3118 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3119 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3120 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3121 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3122 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3123 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3124
3125 tmp = RREG32(HDP_MISC_CNTL);
3126 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3127 WREG32(HDP_MISC_CNTL, tmp);
3128
3129 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3130 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3131
3132 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3133
3134 udelay(50);
3135}
c476dde2 3136
2ece2e8b
AD		 3137/*
3138 * GPU scratch registers helpers function.
3139 */
3140static void si_scratch_init(struct radeon_device *rdev)
3141{
3142 int i;
3143
3144 rdev->scratch.num_reg = 7;
3145 rdev->scratch.reg_base = SCRATCH_REG0;
3146 for (i = 0; i < rdev->scratch.num_reg; i++) {
3147 rdev->scratch.free[i] = true;
3148 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3149 }
3150}
3151
3152void si_fence_ring_emit(struct radeon_device *rdev,
3153 struct radeon_fence *fence)
3154{
3155 struct radeon_ring *ring = &rdev->ring[fence->ring];
3156 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3157
3158 /* flush read cache over gart */
3159 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3160 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3161 radeon_ring_write(ring, 0);
3162 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3163 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3164 PACKET3_TC_ACTION_ENA |
3165 PACKET3_SH_KCACHE_ACTION_ENA |
3166 PACKET3_SH_ICACHE_ACTION_ENA);
3167 radeon_ring_write(ring, 0xFFFFFFFF);
3168 radeon_ring_write(ring, 0);
3169 radeon_ring_write(ring, 10); /* poll interval */
3170 /* EVENT_WRITE_EOP - flush caches, send int */
3171 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3172 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3173 radeon_ring_write(ring, addr & 0xffffffff);
3174 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3175 radeon_ring_write(ring, fence->seq);
3176 radeon_ring_write(ring, 0);
3177}
3178
3179/*
3180 * IB stuff
3181 */
3182void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3183{
876dc9f3 3184 struct radeon_ring *ring = &rdev->ring[ib->ring];
2ece2e8b
AD		 3185 u32 header;
3186
a85a7da4
AD		 3187 if (ib->is_const_ib) {
3188 /* set switch buffer packet before const IB */
3189 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3190 radeon_ring_write(ring, 0);
45df6803 3191
2ece2e8b 3192 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 3193 } else {
89d35807 3194 u32 next_rptr;
a85a7da4 3195 if (ring->rptr_save_reg) {
89d35807 3196 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 3197 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3198 radeon_ring_write(ring, ((ring->rptr_save_reg -
3199 PACKET3_SET_CONFIG_REG_START) >> 2));
3200 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3201 } else if (rdev->wb.enabled) {
3202 next_rptr = ring->wptr + 5 + 4 + 8;
3203 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3204 radeon_ring_write(ring, (1 << 8));
3205 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3206 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3207 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 3208 }
3209
2ece2e8b 3210 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 3211 }
2ece2e8b
AD		 3212
3213 radeon_ring_write(ring, header);
3214 radeon_ring_write(ring,
3215#ifdef __BIG_ENDIAN
3216 (2 << 0) |
3217#endif
3218 (ib->gpu_addr & 0xFFFFFFFC));
3219 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4bf3dd92
CK		 3220 radeon_ring_write(ring, ib->length_dw |
3221 (ib->vm ? (ib->vm->id << 24) : 0));
2ece2e8b 3222
a85a7da4
AD		 3223 if (!ib->is_const_ib) {
3224 /* flush read cache over gart for this vmid */
3225 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3226 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 3227 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
a85a7da4
AD		 3228 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3229 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3230 PACKET3_TC_ACTION_ENA |
3231 PACKET3_SH_KCACHE_ACTION_ENA |
3232 PACKET3_SH_ICACHE_ACTION_ENA);
3233 radeon_ring_write(ring, 0xFFFFFFFF);
3234 radeon_ring_write(ring, 0);
3235 radeon_ring_write(ring, 10); /* poll interval */
3236 }
2ece2e8b
AD		 3237}
3238
48c0c902
AD		 3239/*
3240 * CP.
3241 */
3242static void si_cp_enable(struct radeon_device *rdev, bool enable)
3243{
3244 if (enable)
3245 WREG32(CP_ME_CNTL, 0);
3246 else {
3247 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3248 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3249 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 3250 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3251 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3252 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 3253 }
3254 udelay(50);
3255}
3256
3257static int si_cp_load_microcode(struct radeon_device *rdev)
3258{
3259 const __be32 *fw_data;
3260 int i;
3261
3262 if (!rdev->me_fw || !rdev->pfp_fw)
3263 return -EINVAL;
3264
3265 si_cp_enable(rdev, false);
3266
3267 /* PFP */
3268 fw_data = (const __be32 *)rdev->pfp_fw->data;
3269 WREG32(CP_PFP_UCODE_ADDR, 0);
3270 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3271 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3272 WREG32(CP_PFP_UCODE_ADDR, 0);
3273
3274 /* CE */
3275 fw_data = (const __be32 *)rdev->ce_fw->data;
3276 WREG32(CP_CE_UCODE_ADDR, 0);
3277 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3278 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3279 WREG32(CP_CE_UCODE_ADDR, 0);
3280
3281 /* ME */
3282 fw_data = (const __be32 *)rdev->me_fw->data;
3283 WREG32(CP_ME_RAM_WADDR, 0);
3284 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3285 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3286 WREG32(CP_ME_RAM_WADDR, 0);
3287
3288 WREG32(CP_PFP_UCODE_ADDR, 0);
3289 WREG32(CP_CE_UCODE_ADDR, 0);
3290 WREG32(CP_ME_RAM_WADDR, 0);
3291 WREG32(CP_ME_RAM_RADDR, 0);
3292 return 0;
3293}
3294
3295static int si_cp_start(struct radeon_device *rdev)
3296{
3297 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3298 int r, i;
3299
3300 r = radeon_ring_lock(rdev, ring, 7 + 4);
3301 if (r) {
3302 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3303 return r;
3304 }
3305 /* init the CP */
3306 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3307 radeon_ring_write(ring, 0x1);
3308 radeon_ring_write(ring, 0x0);
3309 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3310 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3311 radeon_ring_write(ring, 0);
3312 radeon_ring_write(ring, 0);
3313
3314 /* init the CE partitions */
3315 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3316 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3317 radeon_ring_write(ring, 0xc000);
3318 radeon_ring_write(ring, 0xe000);
3319 radeon_ring_unlock_commit(rdev, ring);
3320
3321 si_cp_enable(rdev, true);
3322
3323 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3324 if (r) {
3325 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3326 return r;
3327 }
3328
3329 /* setup clear context state */
3330 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3331 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3332
3333 for (i = 0; i < si_default_size; i++)
3334 radeon_ring_write(ring, si_default_state[i]);
3335
3336 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3337 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3338
3339 /* set clear context state */
3340 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3341 radeon_ring_write(ring, 0);
3342
3343 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3344 radeon_ring_write(ring, 0x00000316);
3345 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3346 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3347
3348 radeon_ring_unlock_commit(rdev, ring);
3349
3350 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3351 ring = &rdev->ring[i];
3352 r = radeon_ring_lock(rdev, ring, 2);
3353
3354 /* clear the compute context state */
3355 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3356 radeon_ring_write(ring, 0);
3357
3358 radeon_ring_unlock_commit(rdev, ring);
3359 }
3360
3361 return 0;
3362}
3363
3364static void si_cp_fini(struct radeon_device *rdev)
3365{
45df6803 3366 struct radeon_ring *ring;
48c0c902 3367 si_cp_enable(rdev, false);
45df6803
CK		 3368
3369 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3370 radeon_ring_fini(rdev, ring);
3371 radeon_scratch_free(rdev, ring->rptr_save_reg);
3372
3373 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3374 radeon_ring_fini(rdev, ring);
3375 radeon_scratch_free(rdev, ring->rptr_save_reg);
3376
3377 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3378 radeon_ring_fini(rdev, ring);
3379 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3380}
3381
3382static int si_cp_resume(struct radeon_device *rdev)
3383{
3384 struct radeon_ring *ring;
3385 u32 tmp;
3386 u32 rb_bufsz;
3387 int r;
3388
48c0c902
AD		 3389 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3390 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3391
3392 /* Set the write pointer delay */
3393 WREG32(CP_RB_WPTR_DELAY, 0);
3394
3395 WREG32(CP_DEBUG, 0);
3396 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3397
3398 /* ring 0 - compute and gfx */
3399 /* Set ring buffer size */
3400 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3401 rb_bufsz = drm_order(ring->ring_size / 8);
3402 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3403#ifdef __BIG_ENDIAN
3404 tmp |= BUF_SWAP_32BIT;
3405#endif
3406 WREG32(CP_RB0_CNTL, tmp);
3407
3408 /* Initialize the ring buffer's read and write pointers */
3409 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3410 ring->wptr = 0;
3411 WREG32(CP_RB0_WPTR, ring->wptr);
3412
48fc7f7e 3413 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3414 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3415 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3416
3417 if (rdev->wb.enabled)
3418 WREG32(SCRATCH_UMSK, 0xff);
3419 else {
3420 tmp |= RB_NO_UPDATE;
3421 WREG32(SCRATCH_UMSK, 0);
3422 }
3423
3424 mdelay(1);
3425 WREG32(CP_RB0_CNTL, tmp);
3426
3427 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3428
3429 ring->rptr = RREG32(CP_RB0_RPTR);
3430
3431 /* ring1 - compute only */
3432 /* Set ring buffer size */
3433 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3434 rb_bufsz = drm_order(ring->ring_size / 8);
3435 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3436#ifdef __BIG_ENDIAN
3437 tmp |= BUF_SWAP_32BIT;
3438#endif
3439 WREG32(CP_RB1_CNTL, tmp);
3440
3441 /* Initialize the ring buffer's read and write pointers */
3442 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3443 ring->wptr = 0;
3444 WREG32(CP_RB1_WPTR, ring->wptr);
3445
48fc7f7e 3446 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3447 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3448 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3449
3450 mdelay(1);
3451 WREG32(CP_RB1_CNTL, tmp);
3452
3453 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3454
3455 ring->rptr = RREG32(CP_RB1_RPTR);
3456
3457 /* ring2 - compute only */
3458 /* Set ring buffer size */
3459 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3460 rb_bufsz = drm_order(ring->ring_size / 8);
3461 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3462#ifdef __BIG_ENDIAN
3463 tmp |= BUF_SWAP_32BIT;
3464#endif
3465 WREG32(CP_RB2_CNTL, tmp);
3466
3467 /* Initialize the ring buffer's read and write pointers */
3468 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3469 ring->wptr = 0;
3470 WREG32(CP_RB2_WPTR, ring->wptr);
3471
48fc7f7e 3472 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3473 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3474 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3475
3476 mdelay(1);
3477 WREG32(CP_RB2_CNTL, tmp);
3478
3479 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3480
3481 ring->rptr = RREG32(CP_RB2_RPTR);
3482
3483 /* start the rings */
3484 si_cp_start(rdev);
3485 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3486 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3487 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3488 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3489 if (r) {
3490 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3491 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3492 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3493 return r;
3494 }
3495 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3496 if (r) {
3497 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3498 }
3499 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3500 if (r) {
3501 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3502 }
3503
3504 return 0;
3505}
3506
2483b4ea 3507u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3508{
014bb209 3509 u32 reset_mask = 0;
1c534671 3510 u32 tmp;
06bc6df0 3511
014bb209
AD		 3512 /* GRBM_STATUS */
3513 tmp = RREG32(GRBM_STATUS);
3514 if (tmp & (PA_BUSY | SC_BUSY |
3515 BCI_BUSY | SX_BUSY |
3516 TA_BUSY | VGT_BUSY |
3517 DB_BUSY | CB_BUSY |
3518 GDS_BUSY | SPI_BUSY |
3519 IA_BUSY | IA_BUSY_NO_DMA))
3520 reset_mask |= RADEON_RESET_GFX;
3521
3522 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3523 CP_BUSY | CP_COHERENCY_BUSY))
3524 reset_mask |= RADEON_RESET_CP;
3525
3526 if (tmp & GRBM_EE_BUSY)
3527 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3528
3529 /* GRBM_STATUS2 */
3530 tmp = RREG32(GRBM_STATUS2);
3531 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3532 reset_mask |= RADEON_RESET_RLC;
3533
3534 /* DMA_STATUS_REG 0 */
3535 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3536 if (!(tmp & DMA_IDLE))
3537 reset_mask |= RADEON_RESET_DMA;
3538
3539 /* DMA_STATUS_REG 1 */
3540 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3541 if (!(tmp & DMA_IDLE))
3542 reset_mask |= RADEON_RESET_DMA1;
3543
3544 /* SRBM_STATUS2 */
3545 tmp = RREG32(SRBM_STATUS2);
3546 if (tmp & DMA_BUSY)
3547 reset_mask |= RADEON_RESET_DMA;
3548
3549 if (tmp & DMA1_BUSY)
3550 reset_mask |= RADEON_RESET_DMA1;
3551
3552 /* SRBM_STATUS */
3553 tmp = RREG32(SRBM_STATUS);
3554
3555 if (tmp & IH_BUSY)
3556 reset_mask |= RADEON_RESET_IH;
3557
3558 if (tmp & SEM_BUSY)
3559 reset_mask |= RADEON_RESET_SEM;
3560
3561 if (tmp & GRBM_RQ_PENDING)
3562 reset_mask |= RADEON_RESET_GRBM;
3563
3564 if (tmp & VMC_BUSY)
3565 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3566
014bb209
AD		 3567 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3568 MCC_BUSY | MCD_BUSY))
3569 reset_mask |= RADEON_RESET_MC;
3570
3571 if (evergreen_is_display_hung(rdev))
3572 reset_mask |= RADEON_RESET_DISPLAY;
3573
3574 /* VM_L2_STATUS */
3575 tmp = RREG32(VM_L2_STATUS);
3576 if (tmp & L2_BUSY)
3577 reset_mask |= RADEON_RESET_VMC;
3578
d808fc88
AD		 3579 /* Skip MC reset as it's mostly likely not hung, just busy */
3580 if (reset_mask & RADEON_RESET_MC) {
3581 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3582 reset_mask &= ~RADEON_RESET_MC;
3583 }
3584
014bb209
AD		 3585 return reset_mask;
3586}
3587
3588static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3589{
3590 struct evergreen_mc_save save;
3591 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3592 u32 tmp;
19fc42ed 3593
06bc6df0 3594 if (reset_mask == 0)
014bb209 3595 return;
06bc6df0
AD		 3596
3597 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3598
1c534671 3599 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3600 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3601 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3602 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3603 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3604
1c534671
AD		 3605 /* Disable CP parsing/prefetching */
3606 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3607
3608 if (reset_mask & RADEON_RESET_DMA) {
3609 /* dma0 */
3610 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3611 tmp &= ~DMA_RB_ENABLE;
3612 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3613 }
3614 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3615 /* dma1 */
3616 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3617 tmp &= ~DMA_RB_ENABLE;
3618 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3619 }
3620
f770d78a
AD		 3621 udelay(50);
3622
3623 evergreen_mc_stop(rdev, &save);
3624 if (evergreen_mc_wait_for_idle(rdev)) {
3625 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3626 }
3627
1c534671
AD		 3628 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3629 grbm_soft_reset = SOFT_RESET_CB |
3630 SOFT_RESET_DB |
3631 SOFT_RESET_GDS |
3632 SOFT_RESET_PA |
3633 SOFT_RESET_SC |
3634 SOFT_RESET_BCI |
3635 SOFT_RESET_SPI |
3636 SOFT_RESET_SX |
3637 SOFT_RESET_TC |
3638 SOFT_RESET_TA |
3639 SOFT_RESET_VGT |
3640 SOFT_RESET_IA;
3641 }
3642
3643 if (reset_mask & RADEON_RESET_CP) {
3644 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3645
3646 srbm_soft_reset |= SOFT_RESET_GRBM;
3647 }
06bc6df0
AD		 3648
3649 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3650 srbm_soft_reset |= SOFT_RESET_DMA;
3651
3652 if (reset_mask & RADEON_RESET_DMA1)
3653 srbm_soft_reset |= SOFT_RESET_DMA1;
3654
3655 if (reset_mask & RADEON_RESET_DISPLAY)
3656 srbm_soft_reset |= SOFT_RESET_DC;
3657
3658 if (reset_mask & RADEON_RESET_RLC)
3659 grbm_soft_reset |= SOFT_RESET_RLC;
3660
3661 if (reset_mask & RADEON_RESET_SEM)
3662 srbm_soft_reset |= SOFT_RESET_SEM;
3663
3664 if (reset_mask & RADEON_RESET_IH)
3665 srbm_soft_reset |= SOFT_RESET_IH;
3666
3667 if (reset_mask & RADEON_RESET_GRBM)
3668 srbm_soft_reset |= SOFT_RESET_GRBM;
3669
3670 if (reset_mask & RADEON_RESET_VMC)
3671 srbm_soft_reset |= SOFT_RESET_VMC;
3672
3673 if (reset_mask & RADEON_RESET_MC)
3674 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3675
3676 if (grbm_soft_reset) {
3677 tmp = RREG32(GRBM_SOFT_RESET);
3678 tmp |= grbm_soft_reset;
3679 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3680 WREG32(GRBM_SOFT_RESET, tmp);
3681 tmp = RREG32(GRBM_SOFT_RESET);
3682
3683 udelay(50);
3684
3685 tmp &= ~grbm_soft_reset;
3686 WREG32(GRBM_SOFT_RESET, tmp);
3687 tmp = RREG32(GRBM_SOFT_RESET);
3688 }
3689
3690 if (srbm_soft_reset) {
3691 tmp = RREG32(SRBM_SOFT_RESET);
3692 tmp |= srbm_soft_reset;
3693 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3694 WREG32(SRBM_SOFT_RESET, tmp);
3695 tmp = RREG32(SRBM_SOFT_RESET);
3696
3697 udelay(50);
3698
3699 tmp &= ~srbm_soft_reset;
3700 WREG32(SRBM_SOFT_RESET, tmp);
3701 tmp = RREG32(SRBM_SOFT_RESET);
3702 }
06bc6df0
AD		 3703
3704 /* Wait a little for things to settle down */
3705 udelay(50);
3706
c476dde2 3707 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3708 udelay(50);
3709
1c534671 3710 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3711}
3712
3713int si_asic_reset(struct radeon_device *rdev)
3714{
014bb209
AD		 3715 u32 reset_mask;
3716
3717 reset_mask = si_gpu_check_soft_reset(rdev);
3718
3719 if (reset_mask)
3720 r600_set_bios_scratch_engine_hung(rdev, true);
3721
3722 si_gpu_soft_reset(rdev, reset_mask);
3723
3724 reset_mask = si_gpu_check_soft_reset(rdev);
3725
3726 if (!reset_mask)
3727 r600_set_bios_scratch_engine_hung(rdev, false);
3728
3729 return 0;
c476dde2
AD		 3730}
3731
123bc183
AD		 3732/**
3733 * si_gfx_is_lockup - Check if the GFX engine is locked up
3734 *
3735 * @rdev: radeon_device pointer
3736 * @ring: radeon_ring structure holding ring information
3737 *
3738 * Check if the GFX engine is locked up.
3739 * Returns true if the engine appears to be locked up, false if not.
3740 */
3741bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3742{
3743 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3744
3745 if (!(reset_mask & (RADEON_RESET_GFX |
3746 RADEON_RESET_COMPUTE |
3747 RADEON_RESET_CP))) {
3748 radeon_ring_lockup_update(ring);
3749 return false;
3750 }
3751 /* force CP activities */
3752 radeon_ring_force_activity(rdev, ring);
3753 return radeon_ring_test_lockup(rdev, ring);
3754}
3755
d2800ee5
AD		 3756/* MC */
3757static void si_mc_program(struct radeon_device *rdev)
3758{
3759 struct evergreen_mc_save save;
3760 u32 tmp;
3761 int i, j;
3762
3763 /* Initialize HDP */
3764 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3765 WREG32((0x2c14 + j), 0x00000000);
3766 WREG32((0x2c18 + j), 0x00000000);
3767 WREG32((0x2c1c + j), 0x00000000);
3768 WREG32((0x2c20 + j), 0x00000000);
3769 WREG32((0x2c24 + j), 0x00000000);
3770 }
3771 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3772
3773 evergreen_mc_stop(rdev, &save);
3774 if (radeon_mc_wait_for_idle(rdev)) {
3775 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3776 }
5153550a
AD		 3777 if (!ASIC_IS_NODCE(rdev))
3778 /* Lockout access through VGA aperture*/
3779 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 3780 /* Update configuration */
3781 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3782 rdev->mc.vram_start >> 12);
3783 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3784 rdev->mc.vram_end >> 12);
3785 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3786 rdev->vram_scratch.gpu_addr >> 12);
3787 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3788 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3789 WREG32(MC_VM_FB_LOCATION, tmp);
3790 /* XXX double check these! */
3791 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3792 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3793 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3794 WREG32(MC_VM_AGP_BASE, 0);
3795 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3796 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3797 if (radeon_mc_wait_for_idle(rdev)) {
3798 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3799 }
3800 evergreen_mc_resume(rdev, &save);
5153550a
AD		 3801 if (!ASIC_IS_NODCE(rdev)) {
3802 /* we need to own VRAM, so turn off the VGA renderer here
3803 * to stop it overwriting our objects */
3804 rv515_vga_render_disable(rdev);
3805 }
d2800ee5
AD		 3806}
3807
1c49165d
AD		 3808void si_vram_gtt_location(struct radeon_device *rdev,
3809 struct radeon_mc *mc)
d2800ee5
AD		 3810{
3811 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3812 /* leave room for at least 1024M GTT */
3813 dev_warn(rdev->dev, "limiting VRAM\n");
3814 mc->real_vram_size = 0xFFC0000000ULL;
3815 mc->mc_vram_size = 0xFFC0000000ULL;
3816 }
9ed8b1f9 3817 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 3818 rdev->mc.gtt_base_align = 0;
9ed8b1f9 3819 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 3820}
3821
3822static int si_mc_init(struct radeon_device *rdev)
3823{
3824 u32 tmp;
3825 int chansize, numchan;
3826
3827 /* Get VRAM informations */
3828 rdev->mc.vram_is_ddr = true;
3829 tmp = RREG32(MC_ARB_RAMCFG);
3830 if (tmp & CHANSIZE_OVERRIDE) {
3831 chansize = 16;
3832 } else if (tmp & CHANSIZE_MASK) {
3833 chansize = 64;
3834 } else {
3835 chansize = 32;
3836 }
3837 tmp = RREG32(MC_SHARED_CHMAP);
3838 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3839 case 0:
3840 default:
3841 numchan = 1;
3842 break;
3843 case 1:
3844 numchan = 2;
3845 break;
3846 case 2:
3847 numchan = 4;
3848 break;
3849 case 3:
3850 numchan = 8;
3851 break;
3852 case 4:
3853 numchan = 3;
3854 break;
3855 case 5:
3856 numchan = 6;
3857 break;
3858 case 6:
3859 numchan = 10;
3860 break;
3861 case 7:
3862 numchan = 12;
3863 break;
3864 case 8:
3865 numchan = 16;
3866 break;
3867 }
3868 rdev->mc.vram_width = numchan * chansize;
3869 /* Could aper size report 0 ? */
3870 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3871 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3872 /* size in MB on si */
fc986034
NOS		 3873 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3874 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
d2800ee5
AD		 3875 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3876 si_vram_gtt_location(rdev, &rdev->mc);
3877 radeon_update_bandwidth_info(rdev);
3878
3879 return 0;
3880}
3881
3882/*
3883 * GART
3884 */
3885void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3886{
3887 /* flush hdp cache */
3888 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3889
3890 /* bits 0-15 are the VM contexts0-15 */
3891 WREG32(VM_INVALIDATE_REQUEST, 1);
3892}
3893
1109ca09 3894static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 3895{
3896 int r, i;
3897
3898 if (rdev->gart.robj == NULL) {
3899 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3900 return -EINVAL;
3901 }
3902 r = radeon_gart_table_vram_pin(rdev);
3903 if (r)
3904 return r;
3905 radeon_gart_restore(rdev);
3906 /* Setup TLB control */
3907 WREG32(MC_VM_MX_L1_TLB_CNTL,
3908 (0xA << 7) |
3909 ENABLE_L1_TLB |
3910 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3911 ENABLE_ADVANCED_DRIVER_MODEL |
3912 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3913 /* Setup L2 cache */
3914 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3915 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3916 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3917 EFFECTIVE_L2_QUEUE_SIZE(7) |
3918 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3919 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3920 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3921 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3922 /* setup context0 */
3923 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3924 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3925 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3926 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3927 (u32)(rdev->dummy_page.addr >> 12));
3928 WREG32(VM_CONTEXT0_CNTL2, 0);
3929 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3930 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3931
3932 WREG32(0x15D4, 0);
3933 WREG32(0x15D8, 0);
3934 WREG32(0x15DC, 0);
3935
3936 /* empty context1-15 */
d2800ee5
AD		 3937 /* set vm size, must be a multiple of 4 */
3938 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
c21b328e 3939 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
23d4f1f2
AD		 3940 /* Assign the pt base to something valid for now; the pts used for
3941 * the VMs are determined by the application and setup and assigned
3942 * on the fly in the vm part of radeon_gart.c
3943 */
d2800ee5
AD		 3944 for (i = 1; i < 16; i++) {
3945 if (i < 8)
3946 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3947 rdev->gart.table_addr >> 12);
3948 else
3949 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3950 rdev->gart.table_addr >> 12);
3951 }
3952
3953 /* enable context1-15 */
3954 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3955 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 3956 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 3957 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
ae133a11
CK		 3958 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3959 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3960 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3961 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3962 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3963 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3964 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3965 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3966 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3967 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3968 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3969 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 3970
3971 si_pcie_gart_tlb_flush(rdev);
3972 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3973 (unsigned)(rdev->mc.gtt_size >> 20),
3974 (unsigned long long)rdev->gart.table_addr);
3975 rdev->gart.ready = true;
3976 return 0;
3977}
3978
1109ca09 3979static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5
AD		 3980{
3981 /* Disable all tables */
3982 WREG32(VM_CONTEXT0_CNTL, 0);
3983 WREG32(VM_CONTEXT1_CNTL, 0);
3984 /* Setup TLB control */
3985 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3986 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3987 /* Setup L2 cache */
3988 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3989 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3990 EFFECTIVE_L2_QUEUE_SIZE(7) |
3991 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3992 WREG32(VM_L2_CNTL2, 0);
3993 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3994 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3995 radeon_gart_table_vram_unpin(rdev);
3996}
3997
1109ca09 3998static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 3999{
4000 si_pcie_gart_disable(rdev);
4001 radeon_gart_table_vram_free(rdev);
4002 radeon_gart_fini(rdev);
4003}
4004
498dd8b3
AD		 4005/* vm parser */
4006static bool si_vm_reg_valid(u32 reg)
4007{
4008 /* context regs are fine */
4009 if (reg >= 0x28000)
4010 return true;
4011
4012 /* check config regs */
4013 switch (reg) {
4014 case GRBM_GFX_INDEX:
f418b88a 4015 case CP_STRMOUT_CNTL:
498dd8b3
AD		 4016 case VGT_VTX_VECT_EJECT_REG:
4017 case VGT_CACHE_INVALIDATION:
4018 case VGT_ESGS_RING_SIZE:
4019 case VGT_GSVS_RING_SIZE:
4020 case VGT_GS_VERTEX_REUSE:
4021 case VGT_PRIMITIVE_TYPE:
4022 case VGT_INDEX_TYPE:
4023 case VGT_NUM_INDICES:
4024 case VGT_NUM_INSTANCES:
4025 case VGT_TF_RING_SIZE:
4026 case VGT_HS_OFFCHIP_PARAM:
4027 case VGT_TF_MEMORY_BASE:
4028 case PA_CL_ENHANCE:
4029 case PA_SU_LINE_STIPPLE_VALUE:
4030 case PA_SC_LINE_STIPPLE_STATE:
4031 case PA_SC_ENHANCE:
4032 case SQC_CACHES:
4033 case SPI_STATIC_THREAD_MGMT_1:
4034 case SPI_STATIC_THREAD_MGMT_2:
4035 case SPI_STATIC_THREAD_MGMT_3:
4036 case SPI_PS_MAX_WAVE_ID:
4037 case SPI_CONFIG_CNTL:
4038 case SPI_CONFIG_CNTL_1:
4039 case TA_CNTL_AUX:
4040 return true;
4041 default:
4042 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4043 return false;
4044 }
4045}
4046
4047static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4048 u32 *ib, struct radeon_cs_packet *pkt)
4049{
4050 switch (pkt->opcode) {
4051 case PACKET3_NOP:
4052 case PACKET3_SET_BASE:
4053 case PACKET3_SET_CE_DE_COUNTERS:
4054 case PACKET3_LOAD_CONST_RAM:
4055 case PACKET3_WRITE_CONST_RAM:
4056 case PACKET3_WRITE_CONST_RAM_OFFSET:
4057 case PACKET3_DUMP_CONST_RAM:
4058 case PACKET3_INCREMENT_CE_COUNTER:
4059 case PACKET3_WAIT_ON_DE_COUNTER:
4060 case PACKET3_CE_WRITE:
4061 break;
4062 default:
4063 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4064 return -EINVAL;
4065 }
4066 return 0;
4067}
4068
e5b9e750
TS		 4069static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4070{
4071 u32 start_reg, reg, i;
4072 u32 command = ib[idx + 4];
4073 u32 info = ib[idx + 1];
4074 u32 idx_value = ib[idx];
4075 if (command & PACKET3_CP_DMA_CMD_SAS) {
4076 /* src address space is register */
4077 if (((info & 0x60000000) >> 29) == 0) {
4078 start_reg = idx_value << 2;
4079 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4080 reg = start_reg;
4081 if (!si_vm_reg_valid(reg)) {
4082 DRM_ERROR("CP DMA Bad SRC register\n");
4083 return -EINVAL;
4084 }
4085 } else {
4086 for (i = 0; i < (command & 0x1fffff); i++) {
4087 reg = start_reg + (4 * i);
4088 if (!si_vm_reg_valid(reg)) {
4089 DRM_ERROR("CP DMA Bad SRC register\n");
4090 return -EINVAL;
4091 }
4092 }
4093 }
4094 }
4095 }
4096 if (command & PACKET3_CP_DMA_CMD_DAS) {
4097 /* dst address space is register */
4098 if (((info & 0x00300000) >> 20) == 0) {
4099 start_reg = ib[idx + 2];
4100 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4101 reg = start_reg;
4102 if (!si_vm_reg_valid(reg)) {
4103 DRM_ERROR("CP DMA Bad DST register\n");
4104 return -EINVAL;
4105 }
4106 } else {
4107 for (i = 0; i < (command & 0x1fffff); i++) {
4108 reg = start_reg + (4 * i);
4109 if (!si_vm_reg_valid(reg)) {
4110 DRM_ERROR("CP DMA Bad DST register\n");
4111 return -EINVAL;
4112 }
4113 }
4114 }
4115 }
4116 }
4117 return 0;
4118}
4119
498dd8b3
AD		 4120static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4121 u32 *ib, struct radeon_cs_packet *pkt)
4122{
e5b9e750 4123 int r;
498dd8b3
AD		 4124 u32 idx = pkt->idx + 1;
4125 u32 idx_value = ib[idx];
4126 u32 start_reg, end_reg, reg, i;
4127
4128 switch (pkt->opcode) {
4129 case PACKET3_NOP:
4130 case PACKET3_SET_BASE:
4131 case PACKET3_CLEAR_STATE:
4132 case PACKET3_INDEX_BUFFER_SIZE:
4133 case PACKET3_DISPATCH_DIRECT:
4134 case PACKET3_DISPATCH_INDIRECT:
4135 case PACKET3_ALLOC_GDS:
4136 case PACKET3_WRITE_GDS_RAM:
4137 case PACKET3_ATOMIC_GDS:
4138 case PACKET3_ATOMIC:
4139 case PACKET3_OCCLUSION_QUERY:
4140 case PACKET3_SET_PREDICATION:
4141 case PACKET3_COND_EXEC:
4142 case PACKET3_PRED_EXEC:
4143 case PACKET3_DRAW_INDIRECT:
4144 case PACKET3_DRAW_INDEX_INDIRECT:
4145 case PACKET3_INDEX_BASE:
4146 case PACKET3_DRAW_INDEX_2:
4147 case PACKET3_CONTEXT_CONTROL:
4148 case PACKET3_INDEX_TYPE:
4149 case PACKET3_DRAW_INDIRECT_MULTI:
4150 case PACKET3_DRAW_INDEX_AUTO:
4151 case PACKET3_DRAW_INDEX_IMMD:
4152 case PACKET3_NUM_INSTANCES:
4153 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4154 case PACKET3_STRMOUT_BUFFER_UPDATE:
4155 case PACKET3_DRAW_INDEX_OFFSET_2:
4156 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4157 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4158 case PACKET3_MPEG_INDEX:
4159 case PACKET3_WAIT_REG_MEM:
4160 case PACKET3_MEM_WRITE:
4161 case PACKET3_PFP_SYNC_ME:
4162 case PACKET3_SURFACE_SYNC:
4163 case PACKET3_EVENT_WRITE:
4164 case PACKET3_EVENT_WRITE_EOP:
4165 case PACKET3_EVENT_WRITE_EOS:
4166 case PACKET3_SET_CONTEXT_REG:
4167 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4168 case PACKET3_SET_SH_REG:
4169 case PACKET3_SET_SH_REG_OFFSET:
4170 case PACKET3_INCREMENT_DE_COUNTER:
4171 case PACKET3_WAIT_ON_CE_COUNTER:
4172 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4173 case PACKET3_ME_WRITE:
4174 break;
4175 case PACKET3_COPY_DATA:
4176 if ((idx_value & 0xf00) == 0) {
4177 reg = ib[idx + 3] * 4;
4178 if (!si_vm_reg_valid(reg))
4179 return -EINVAL;
4180 }
4181 break;
4182 case PACKET3_WRITE_DATA:
4183 if ((idx_value & 0xf00) == 0) {
4184 start_reg = ib[idx + 1] * 4;
4185 if (idx_value & 0x10000) {
4186 if (!si_vm_reg_valid(start_reg))
4187 return -EINVAL;
4188 } else {
4189 for (i = 0; i < (pkt->count - 2); i++) {
4190 reg = start_reg + (4 * i);
4191 if (!si_vm_reg_valid(reg))
4192 return -EINVAL;
4193 }
4194 }
4195 }
4196 break;
4197 case PACKET3_COND_WRITE:
4198 if (idx_value & 0x100) {
4199 reg = ib[idx + 5] * 4;
4200 if (!si_vm_reg_valid(reg))
4201 return -EINVAL;
4202 }
4203 break;
4204 case PACKET3_COPY_DW:
4205 if (idx_value & 0x2) {
4206 reg = ib[idx + 3] * 4;
4207 if (!si_vm_reg_valid(reg))
4208 return -EINVAL;
4209 }
4210 break;
4211 case PACKET3_SET_CONFIG_REG:
4212 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4213 end_reg = 4 * pkt->count + start_reg - 4;
4214 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4215 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4216 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4217 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4218 return -EINVAL;
4219 }
4220 for (i = 0; i < pkt->count; i++) {
4221 reg = start_reg + (4 * i);
4222 if (!si_vm_reg_valid(reg))
4223 return -EINVAL;
4224 }
4225 break;
5aa709be 4226 case PACKET3_CP_DMA:
e5b9e750
TS		 4227 r = si_vm_packet3_cp_dma_check(ib, idx);
4228 if (r)
4229 return r;
5aa709be 4230 break;
498dd8b3
AD		 4231 default:
4232 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4233 return -EINVAL;
4234 }
4235 return 0;
4236}
4237
4238static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4239 u32 *ib, struct radeon_cs_packet *pkt)
4240{
e5b9e750 4241 int r;
498dd8b3
AD		 4242 u32 idx = pkt->idx + 1;
4243 u32 idx_value = ib[idx];
4244 u32 start_reg, reg, i;
4245
4246 switch (pkt->opcode) {
4247 case PACKET3_NOP:
4248 case PACKET3_SET_BASE:
4249 case PACKET3_CLEAR_STATE:
4250 case PACKET3_DISPATCH_DIRECT:
4251 case PACKET3_DISPATCH_INDIRECT:
4252 case PACKET3_ALLOC_GDS:
4253 case PACKET3_WRITE_GDS_RAM:
4254 case PACKET3_ATOMIC_GDS:
4255 case PACKET3_ATOMIC:
4256 case PACKET3_OCCLUSION_QUERY:
4257 case PACKET3_SET_PREDICATION:
4258 case PACKET3_COND_EXEC:
4259 case PACKET3_PRED_EXEC:
4260 case PACKET3_CONTEXT_CONTROL:
4261 case PACKET3_STRMOUT_BUFFER_UPDATE:
4262 case PACKET3_WAIT_REG_MEM:
4263 case PACKET3_MEM_WRITE:
4264 case PACKET3_PFP_SYNC_ME:
4265 case PACKET3_SURFACE_SYNC:
4266 case PACKET3_EVENT_WRITE:
4267 case PACKET3_EVENT_WRITE_EOP:
4268 case PACKET3_EVENT_WRITE_EOS:
4269 case PACKET3_SET_CONTEXT_REG:
4270 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4271 case PACKET3_SET_SH_REG:
4272 case PACKET3_SET_SH_REG_OFFSET:
4273 case PACKET3_INCREMENT_DE_COUNTER:
4274 case PACKET3_WAIT_ON_CE_COUNTER:
4275 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4276 case PACKET3_ME_WRITE:
4277 break;
4278 case PACKET3_COPY_DATA:
4279 if ((idx_value & 0xf00) == 0) {
4280 reg = ib[idx + 3] * 4;
4281 if (!si_vm_reg_valid(reg))
4282 return -EINVAL;
4283 }
4284 break;
4285 case PACKET3_WRITE_DATA:
4286 if ((idx_value & 0xf00) == 0) {
4287 start_reg = ib[idx + 1] * 4;
4288 if (idx_value & 0x10000) {
4289 if (!si_vm_reg_valid(start_reg))
4290 return -EINVAL;
4291 } else {
4292 for (i = 0; i < (pkt->count - 2); i++) {
4293 reg = start_reg + (4 * i);
4294 if (!si_vm_reg_valid(reg))
4295 return -EINVAL;
4296 }
4297 }
4298 }
4299 break;
4300 case PACKET3_COND_WRITE:
4301 if (idx_value & 0x100) {
4302 reg = ib[idx + 5] * 4;
4303 if (!si_vm_reg_valid(reg))
4304 return -EINVAL;
4305 }
4306 break;
4307 case PACKET3_COPY_DW:
4308 if (idx_value & 0x2) {
4309 reg = ib[idx + 3] * 4;
4310 if (!si_vm_reg_valid(reg))
4311 return -EINVAL;
4312 }
4313 break;
e5b9e750
TS		 4314 case PACKET3_CP_DMA:
4315 r = si_vm_packet3_cp_dma_check(ib, idx);
4316 if (r)
4317 return r;
4318 break;
498dd8b3
AD		 4319 default:
4320 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4321 return -EINVAL;
4322 }
4323 return 0;
4324}
4325
4326int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4327{
4328 int ret = 0;
4329 u32 idx = 0;
4330 struct radeon_cs_packet pkt;
4331
4332 do {
4333 pkt.idx = idx;
4e872ae2
IH		 4334 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4335 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4336 pkt.one_reg_wr = 0;
4337 switch (pkt.type) {
4e872ae2 4338 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 4339 dev_err(rdev->dev, "Packet0 not allowed!\n");
4340 ret = -EINVAL;
4341 break;
4e872ae2 4342 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4343 idx += 1;
4344 break;
4e872ae2
IH		 4345 case RADEON_PACKET_TYPE3:
4346 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4347 if (ib->is_const_ib)
4348 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4349 else {
876dc9f3 4350 switch (ib->ring) {
498dd8b3
AD		 4351 case RADEON_RING_TYPE_GFX_INDEX:
4352 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4353 break;
4354 case CAYMAN_RING_TYPE_CP1_INDEX:
4355 case CAYMAN_RING_TYPE_CP2_INDEX:
4356 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4357 break;
4358 default:
876dc9f3 4359 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4360 ret = -EINVAL;
4361 break;
4362 }
4363 }
4364 idx += pkt.count + 2;
4365 break;
4366 default:
4367 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4368 ret = -EINVAL;
4369 break;
4370 }
4371 if (ret)
4372 break;
4373 } while (idx < ib->length_dw);
4374
4375 return ret;
4376}
4377
d2800ee5
AD		 4378/*
4379 * vm
4380 */
4381int si_vm_init(struct radeon_device *rdev)
4382{
4383 /* number of VMs */
4384 rdev->vm_manager.nvm = 16;
4385 /* base offset of vram pages */
4386 rdev->vm_manager.vram_base_offset = 0;
4387
4388 return 0;
4389}
4390
4391void si_vm_fini(struct radeon_device *rdev)
4392{
4393}
4394
fbf6dc7a
AD		 4395/**
4396 * si_vm_decode_fault - print human readable fault info
4397 *
4398 * @rdev: radeon_device pointer
4399 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4400 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4401 *
4402 * Print human readable fault information (SI).
4403 */
4404static void si_vm_decode_fault(struct radeon_device *rdev,
4405 u32 status, u32 addr)
4406{
4407 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4408 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4409 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4410 char *block;
4411
4412 if (rdev->family == CHIP_TAHITI) {
4413 switch (mc_id) {
4414 case 160:
4415 case 144:
4416 case 96:
4417 case 80:
4418 case 224:
4419 case 208:
4420 case 32:
4421 case 16:
4422 block = "CB";
4423 break;
4424 case 161:
4425 case 145:
4426 case 97:
4427 case 81:
4428 case 225:
4429 case 209:
4430 case 33:
4431 case 17:
4432 block = "CB_FMASK";
4433 break;
4434 case 162:
4435 case 146:
4436 case 98:
4437 case 82:
4438 case 226:
4439 case 210:
4440 case 34:
4441 case 18:
4442 block = "CB_CMASK";
4443 break;
4444 case 163:
4445 case 147:
4446 case 99:
4447 case 83:
4448 case 227:
4449 case 211:
4450 case 35:
4451 case 19:
4452 block = "CB_IMMED";
4453 break;
4454 case 164:
4455 case 148:
4456 case 100:
4457 case 84:
4458 case 228:
4459 case 212:
4460 case 36:
4461 case 20:
4462 block = "DB";
4463 break;
4464 case 165:
4465 case 149:
4466 case 101:
4467 case 85:
4468 case 229:
4469 case 213:
4470 case 37:
4471 case 21:
4472 block = "DB_HTILE";
4473 break;
4474 case 167:
4475 case 151:
4476 case 103:
4477 case 87:
4478 case 231:
4479 case 215:
4480 case 39:
4481 case 23:
4482 block = "DB_STEN";
4483 break;
4484 case 72:
4485 case 68:
4486 case 64:
4487 case 8:
4488 case 4:
4489 case 0:
4490 case 136:
4491 case 132:
4492 case 128:
4493 case 200:
4494 case 196:
4495 case 192:
4496 block = "TC";
4497 break;
4498 case 112:
4499 case 48:
4500 block = "CP";
4501 break;
4502 case 49:
4503 case 177:
4504 case 50:
4505 case 178:
4506 block = "SH";
4507 break;
4508 case 53:
4509 case 190:
4510 block = "VGT";
4511 break;
4512 case 117:
4513 block = "IH";
4514 break;
4515 case 51:
4516 case 115:
4517 block = "RLC";
4518 break;
4519 case 119:
4520 case 183:
4521 block = "DMA0";
4522 break;
4523 case 61:
4524 block = "DMA1";
4525 break;
4526 case 248:
4527 case 120:
4528 block = "HDP";
4529 break;
4530 default:
4531 block = "unknown";
4532 break;
4533 }
4534 } else {
4535 switch (mc_id) {
4536 case 32:
4537 case 16:
4538 case 96:
4539 case 80:
4540 case 160:
4541 case 144:
4542 case 224:
4543 case 208:
4544 block = "CB";
4545 break;
4546 case 33:
4547 case 17:
4548 case 97:
4549 case 81:
4550 case 161:
4551 case 145:
4552 case 225:
4553 case 209:
4554 block = "CB_FMASK";
4555 break;
4556 case 34:
4557 case 18:
4558 case 98:
4559 case 82:
4560 case 162:
4561 case 146:
4562 case 226:
4563 case 210:
4564 block = "CB_CMASK";
4565 break;
4566 case 35:
4567 case 19:
4568 case 99:
4569 case 83:
4570 case 163:
4571 case 147:
4572 case 227:
4573 case 211:
4574 block = "CB_IMMED";
4575 break;
4576 case 36:
4577 case 20:
4578 case 100:
4579 case 84:
4580 case 164:
4581 case 148:
4582 case 228:
4583 case 212:
4584 block = "DB";
4585 break;
4586 case 37:
4587 case 21:
4588 case 101:
4589 case 85:
4590 case 165:
4591 case 149:
4592 case 229:
4593 case 213:
4594 block = "DB_HTILE";
4595 break;
4596 case 39:
4597 case 23:
4598 case 103:
4599 case 87:
4600 case 167:
4601 case 151:
4602 case 231:
4603 case 215:
4604 block = "DB_STEN";
4605 break;
4606 case 72:
4607 case 68:
4608 case 8:
4609 case 4:
4610 case 136:
4611 case 132:
4612 case 200:
4613 case 196:
4614 block = "TC";
4615 break;
4616 case 112:
4617 case 48:
4618 block = "CP";
4619 break;
4620 case 49:
4621 case 177:
4622 case 50:
4623 case 178:
4624 block = "SH";
4625 break;
4626 case 53:
4627 block = "VGT";
4628 break;
4629 case 117:
4630 block = "IH";
4631 break;
4632 case 51:
4633 case 115:
4634 block = "RLC";
4635 break;
4636 case 119:
4637 case 183:
4638 block = "DMA0";
4639 break;
4640 case 61:
4641 block = "DMA1";
4642 break;
4643 case 248:
4644 case 120:
4645 block = "HDP";
4646 break;
4647 default:
4648 block = "unknown";
4649 break;
4650 }
4651 }
4652
4653 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4654 protections, vmid, addr,
4655 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4656 block, mc_id);
4657}
4658
82ffd92b
AD		 4659/**
4660 * si_vm_set_page - update the page tables using the CP
4661 *
4662 * @rdev: radeon_device pointer
43f1214a 4663 * @ib: indirect buffer to fill with commands
82ffd92b
AD		 4664 * @pe: addr of the page entry
4665 * @addr: dst addr to write into pe
4666 * @count: number of page entries to update
4667 * @incr: increase next addr by incr bytes
4668 * @flags: access flags
4669 *
43f1214a 4670 * Update the page tables using the CP (SI).
82ffd92b 4671 */
43f1214a
AD		 4672void si_vm_set_page(struct radeon_device *rdev,
4673 struct radeon_ib *ib,
4674 uint64_t pe,
82ffd92b
AD		 4675 uint64_t addr, unsigned count,
4676 uint32_t incr, uint32_t flags)
d2800ee5 4677{
82ffd92b 4678 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
deab48f1
AD		 4679 uint64_t value;
4680 unsigned ndw;
4681
4682 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4683 while (count) {
4684 ndw = 2 + count * 2;
4685 if (ndw > 0x3FFE)
4686 ndw = 0x3FFE;
4687
43f1214a
AD		 4688 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4689 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4690 WRITE_DATA_DST_SEL(1));
4691 ib->ptr[ib->length_dw++] = pe;
4692 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
deab48f1
AD		 4693 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4694 if (flags & RADEON_VM_PAGE_SYSTEM) {
4695 value = radeon_vm_map_gart(rdev, addr);
4696 value &= 0xFFFFFFFFFFFFF000ULL;
4697 } else if (flags & RADEON_VM_PAGE_VALID) {
4698 value = addr;
4699 } else {
4700 value = 0;
4701 }
4702 addr += incr;
4703 value |= r600_flags;
43f1214a
AD		 4704 ib->ptr[ib->length_dw++] = value;
4705 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4706 }
4707 }
4708 } else {
4709 /* DMA */
2483b4ea 4710 si_dma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
82ffd92b 4711 }
d2800ee5
AD		 4712}
4713
498522b4 4714void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
d2800ee5 4715{
498522b4 4716 struct radeon_ring *ring = &rdev->ring[ridx];
d2800ee5 4717
ee60e29f 4718 if (vm == NULL)
d2800ee5
AD		 4719 return;
4720
76c44f2c
AD		 4721 /* write new base address */
4722 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4723 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4724 WRITE_DATA_DST_SEL(0)));
4725
ee60e29f 4726 if (vm->id < 8) {
76c44f2c
AD		 4727 radeon_ring_write(ring,
4728 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
ee60e29f 4729 } else {
76c44f2c
AD		 4730 radeon_ring_write(ring,
4731 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
ee60e29f 4732 }
76c44f2c 4733 radeon_ring_write(ring, 0);
fa87e62d 4734 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 4735
d2800ee5 4736 /* flush hdp cache */
76c44f2c
AD		 4737 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4738 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4739 WRITE_DATA_DST_SEL(0)));
4740 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4741 radeon_ring_write(ring, 0);
ee60e29f
CK		 4742 radeon_ring_write(ring, 0x1);
4743
d2800ee5 4744 /* bits 0-15 are the VM contexts0-15 */
76c44f2c
AD		 4745 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4746 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4747 WRITE_DATA_DST_SEL(0)));
4748 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4749 radeon_ring_write(ring, 0);
498522b4 4750 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 4751
4752 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4753 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4754 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 4755}
4756
f8f84ac5
AD		 4757/*
4758 * Power and clock gating
4759 */
4760static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4761{
4762 int i;
4763
4764 for (i = 0; i < rdev->usec_timeout; i++) {
4765 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4766 break;
4767 udelay(1);
4768 }
4769
4770 for (i = 0; i < rdev->usec_timeout; i++) {
4771 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4772 break;
4773 udelay(1);
4774 }
4775}
4776
4777static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4778 bool enable)
4779{
4780 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4781 u32 mask;
4782 int i;
4783
4784 if (enable)
4785 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4786 else
4787 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4788 WREG32(CP_INT_CNTL_RING0, tmp);
4789
4790 if (!enable) {
4791 /* read a gfx register */
4792 tmp = RREG32(DB_DEPTH_INFO);
4793
4794 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4795 for (i = 0; i < rdev->usec_timeout; i++) {
4796 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4797 break;
4798 udelay(1);
4799 }
4800 }
4801}
4802
4803static void si_set_uvd_dcm(struct radeon_device *rdev,
4804 bool sw_mode)
4805{
4806 u32 tmp, tmp2;
4807
4808 tmp = RREG32(UVD_CGC_CTRL);
4809 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4810 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4811
4812 if (sw_mode) {
4813 tmp &= ~0x7ffff800;
4814 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4815 } else {
4816 tmp |= 0x7ffff800;
4817 tmp2 = 0;
4818 }
4819
4820 WREG32(UVD_CGC_CTRL, tmp);
4821 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4822}
4823
22c775ce 4824void si_init_uvd_internal_cg(struct radeon_device *rdev)
f8f84ac5
AD		 4825{
4826 bool hw_mode = true;
4827
4828 if (hw_mode) {
4829 si_set_uvd_dcm(rdev, false);
4830 } else {
4831 u32 tmp = RREG32(UVD_CGC_CTRL);
4832 tmp &= ~DCM;
4833 WREG32(UVD_CGC_CTRL, tmp);
4834 }
4835}
4836
4837static u32 si_halt_rlc(struct radeon_device *rdev)
4838{
4839 u32 data, orig;
4840
4841 orig = data = RREG32(RLC_CNTL);
4842
4843 if (data & RLC_ENABLE) {
4844 data &= ~RLC_ENABLE;
4845 WREG32(RLC_CNTL, data);
4846
4847 si_wait_for_rlc_serdes(rdev);
4848 }
4849
4850 return orig;
4851}
4852
4853static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4854{
4855 u32 tmp;
4856
4857 tmp = RREG32(RLC_CNTL);
4858 if (tmp != rlc)
4859 WREG32(RLC_CNTL, rlc);
4860}
4861
4862static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4863{
4864 u32 data, orig;
4865
4866 orig = data = RREG32(DMA_PG);
e16866ec 4867 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
f8f84ac5
AD		 4868 data |= PG_CNTL_ENABLE;
4869 else
4870 data &= ~PG_CNTL_ENABLE;
4871 if (orig != data)
4872 WREG32(DMA_PG, data);
4873}
4874
4875static void si_init_dma_pg(struct radeon_device *rdev)
4876{
4877 u32 tmp;
4878
4879 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4880 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4881
4882 for (tmp = 0; tmp < 5; tmp++)
4883 WREG32(DMA_PGFSM_WRITE, 0);
4884}
4885
4886static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4887 bool enable)
4888{
4889 u32 tmp;
4890
e16866ec 4891 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_CG)) {
f8f84ac5
AD		 4892 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4893 WREG32(RLC_TTOP_D, tmp);
4894
4895 tmp = RREG32(RLC_PG_CNTL);
4896 tmp |= GFX_PG_ENABLE;
4897 WREG32(RLC_PG_CNTL, tmp);
4898
4899 tmp = RREG32(RLC_AUTO_PG_CTRL);
4900 tmp |= AUTO_PG_EN;
4901 WREG32(RLC_AUTO_PG_CTRL, tmp);
4902 } else {
4903 tmp = RREG32(RLC_AUTO_PG_CTRL);
4904 tmp &= ~AUTO_PG_EN;
4905 WREG32(RLC_AUTO_PG_CTRL, tmp);
4906
4907 tmp = RREG32(DB_RENDER_CONTROL);
4908 }
4909}
4910
4911static void si_init_gfx_cgpg(struct radeon_device *rdev)
4912{
4913 u32 tmp;
4914
4915 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4916
4917 tmp = RREG32(RLC_PG_CNTL);
4918 tmp |= GFX_PG_SRC;
4919 WREG32(RLC_PG_CNTL, tmp);
4920
4921 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4922
4923 tmp = RREG32(RLC_AUTO_PG_CTRL);
4924
4925 tmp &= ~GRBM_REG_SGIT_MASK;
4926 tmp |= GRBM_REG_SGIT(0x700);
4927 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
4928 WREG32(RLC_AUTO_PG_CTRL, tmp);
4929}
4930
ba19031a 4931static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
f8f84ac5
AD		 4932{
4933 u32 mask = 0, tmp, tmp1;
4934 int i;
4935
4936 si_select_se_sh(rdev, se, sh);
4937 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
4938 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
4939 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4940
4941 tmp &= 0xffff0000;
4942
4943 tmp |= tmp1;
4944 tmp >>= 16;
4945
4946 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
4947 mask <<= 1;
4948 mask |= 1;
4949 }
4950
4951 return (~tmp) & mask;
4952}
4953
4954static void si_init_ao_cu_mask(struct radeon_device *rdev)
4955{
4956 u32 i, j, k, active_cu_number = 0;
4957 u32 mask, counter, cu_bitmap;
4958 u32 tmp = 0;
4959
4960 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
4961 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
4962 mask = 1;
4963 cu_bitmap = 0;
4964 counter = 0;
4965 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
ba19031a 4966 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
f8f84ac5
AD		 4967 if (counter < 2)
4968 cu_bitmap |= mask;
4969 counter++;
4970 }
4971 mask <<= 1;
4972 }
4973
4974 active_cu_number += counter;
4975 tmp |= (cu_bitmap << (i * 16 + j * 8));
4976 }
4977 }
4978
4979 WREG32(RLC_PG_AO_CU_MASK, tmp);
4980
4981 tmp = RREG32(RLC_MAX_PG_CU);
4982 tmp &= ~MAX_PU_CU_MASK;
4983 tmp |= MAX_PU_CU(active_cu_number);
4984 WREG32(RLC_MAX_PG_CU, tmp);
4985}
4986
4987static void si_enable_cgcg(struct radeon_device *rdev,
4988 bool enable)
4989{
4990 u32 data, orig, tmp;
4991
4992 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
4993
e16866ec 4994 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5594a558
AD		 4995 si_enable_gui_idle_interrupt(rdev, true);
4996
f8f84ac5
AD		 4997 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
4998
4999 tmp = si_halt_rlc(rdev);
5000
5001 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5002 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5003 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5004
5005 si_wait_for_rlc_serdes(rdev);
5006
5007 si_update_rlc(rdev, tmp);
5008
5009 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5010
5011 data |= CGCG_EN | CGLS_EN;
5012 } else {
5594a558
AD		 5013 si_enable_gui_idle_interrupt(rdev, false);
5014
f8f84ac5
AD		 5015 RREG32(CB_CGTT_SCLK_CTRL);
5016 RREG32(CB_CGTT_SCLK_CTRL);
5017 RREG32(CB_CGTT_SCLK_CTRL);
5018 RREG32(CB_CGTT_SCLK_CTRL);
5019
5020 data &= ~(CGCG_EN | CGLS_EN);
5021 }
5022
5023 if (orig != data)
5024 WREG32(RLC_CGCG_CGLS_CTRL, data);
5025}
5026
5027static void si_enable_mgcg(struct radeon_device *rdev,
5028 bool enable)
5029{
5030 u32 data, orig, tmp = 0;
5031
e16866ec 5032 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
f8f84ac5
AD		 5033 orig = data = RREG32(CGTS_SM_CTRL_REG);
5034 data = 0x96940200;
5035 if (orig != data)
5036 WREG32(CGTS_SM_CTRL_REG, data);
5037
e16866ec
AD		 5038 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5039 orig = data = RREG32(CP_MEM_SLP_CNTL);
5040 data |= CP_MEM_LS_EN;
5041 if (orig != data)
5042 WREG32(CP_MEM_SLP_CNTL, data);
5043 }
f8f84ac5
AD		 5044
5045 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5046 data &= 0xffffffc0;
5047 if (orig != data)
5048 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5049
5050 tmp = si_halt_rlc(rdev);
5051
5052 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5053 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5054 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5055
5056 si_update_rlc(rdev, tmp);
5057 } else {
5058 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5059 data |= 0x00000003;
5060 if (orig != data)
5061 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5062
5063 data = RREG32(CP_MEM_SLP_CNTL);
5064 if (data & CP_MEM_LS_EN) {
5065 data &= ~CP_MEM_LS_EN;
5066 WREG32(CP_MEM_SLP_CNTL, data);
5067 }
5068 orig = data = RREG32(CGTS_SM_CTRL_REG);
5069 data |= LS_OVERRIDE | OVERRIDE;
5070 if (orig != data)
5071 WREG32(CGTS_SM_CTRL_REG, data);
5072
5073 tmp = si_halt_rlc(rdev);
5074
5075 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5076 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5077 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5078
5079 si_update_rlc(rdev, tmp);
5080 }
5081}
5082
5083static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5084 bool enable)
5085{
5086 u32 orig, data, tmp;
5087
e16866ec 5088 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
f8f84ac5
AD		 5089 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5090 tmp |= 0x3fff;
5091 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5092
5093 orig = data = RREG32(UVD_CGC_CTRL);
5094 data |= DCM;
5095 if (orig != data)
5096 WREG32(UVD_CGC_CTRL, data);
5097
5098 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5099 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5100 } else {
5101 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5102 tmp &= ~0x3fff;
5103 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5104
5105 orig = data = RREG32(UVD_CGC_CTRL);
5106 data &= ~DCM;
5107 if (orig != data)
5108 WREG32(UVD_CGC_CTRL, data);
5109
5110 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5111 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5112 }
5113}
5114
5115static const u32 mc_cg_registers[] =
5116{
5117 MC_HUB_MISC_HUB_CG,
5118 MC_HUB_MISC_SIP_CG,
5119 MC_HUB_MISC_VM_CG,
5120 MC_XPB_CLK_GAT,
5121 ATC_MISC_CG,
5122 MC_CITF_MISC_WR_CG,
5123 MC_CITF_MISC_RD_CG,
5124 MC_CITF_MISC_VM_CG,
5125 VM_L2_CG,
5126};
5127
5128static void si_enable_mc_ls(struct radeon_device *rdev,
5129 bool enable)
5130{
5131 int i;
5132 u32 orig, data;
5133
5134 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5135 orig = data = RREG32(mc_cg_registers[i]);
e16866ec 5136 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
f8f84ac5
AD		 5137 data |= MC_LS_ENABLE;
5138 else
5139 data &= ~MC_LS_ENABLE;
5140 if (data != orig)
5141 WREG32(mc_cg_registers[i], data);
5142 }
5143}
5144
e16866ec
AD		 5145static void si_enable_mc_mgcg(struct radeon_device *rdev,
5146 bool enable)
5147{
5148 int i;
5149 u32 orig, data;
5150
5151 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5152 orig = data = RREG32(mc_cg_registers[i]);
5153 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5154 data |= MC_CG_ENABLE;
5155 else
5156 data &= ~MC_CG_ENABLE;
5157 if (data != orig)
5158 WREG32(mc_cg_registers[i], data);
5159 }
5160}
5161
5162static void si_enable_dma_mgcg(struct radeon_device *rdev,
5163 bool enable)
5164{
5165 u32 orig, data, offset;
5166 int i;
5167
5168 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5169 for (i = 0; i < 2; i++) {
5170 if (i == 0)
5171 offset = DMA0_REGISTER_OFFSET;
5172 else
5173 offset = DMA1_REGISTER_OFFSET;
5174 orig = data = RREG32(DMA_POWER_CNTL + offset);
5175 data &= ~MEM_POWER_OVERRIDE;
5176 if (data != orig)
5177 WREG32(DMA_POWER_CNTL + offset, data);
5178 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5179 }
5180 } else {
5181 for (i = 0; i < 2; i++) {
5182 if (i == 0)
5183 offset = DMA0_REGISTER_OFFSET;
5184 else
5185 offset = DMA1_REGISTER_OFFSET;
5186 orig = data = RREG32(DMA_POWER_CNTL + offset);
5187 data |= MEM_POWER_OVERRIDE;
5188 if (data != orig)
5189 WREG32(DMA_POWER_CNTL + offset, data);
5190
5191 orig = data = RREG32(DMA_CLK_CTRL + offset);
5192 data = 0xff000000;
5193 if (data != orig)
5194 WREG32(DMA_CLK_CTRL + offset, data);
5195 }
5196 }
5197}
5198
5199static void si_enable_bif_mgls(struct radeon_device *rdev,
5200 bool enable)
5201{
5202 u32 orig, data;
5203
5204 orig = data = RREG32_PCIE(PCIE_CNTL2);
5205
5206 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5207 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5208 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5209 else
5210 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5211 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5212
5213 if (orig != data)
5214 WREG32_PCIE(PCIE_CNTL2, data);
5215}
5216
5217static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5218 bool enable)
5219{
5220 u32 orig, data;
5221
5222 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5223
5224 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5225 data &= ~CLOCK_GATING_DIS;
5226 else
5227 data |= CLOCK_GATING_DIS;
5228
5229 if (orig != data)
5230 WREG32(HDP_HOST_PATH_CNTL, data);
5231}
5232
5233static void si_enable_hdp_ls(struct radeon_device *rdev,
5234 bool enable)
5235{
5236 u32 orig, data;
5237
5238 orig = data = RREG32(HDP_MEM_POWER_LS);
5239
5240 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5241 data |= HDP_LS_ENABLE;
5242 else
5243 data &= ~HDP_LS_ENABLE;
5244
5245 if (orig != data)
5246 WREG32(HDP_MEM_POWER_LS, data);
5247}
5248
5249void si_update_cg(struct radeon_device *rdev,
5250 u32 block, bool enable)
5251{
5252 if (block & RADEON_CG_BLOCK_GFX) {
5253 /* order matters! */
5254 if (enable) {
5255 si_enable_mgcg(rdev, true);
5256 si_enable_cgcg(rdev, true);
5257 } else {
5258 si_enable_cgcg(rdev, false);
5259 si_enable_mgcg(rdev, false);
5260 }
5261 }
5262
5263 if (block & RADEON_CG_BLOCK_MC) {
5264 si_enable_mc_mgcg(rdev, enable);
5265 si_enable_mc_ls(rdev, enable);
5266 }
5267
5268 if (block & RADEON_CG_BLOCK_SDMA) {
5269 si_enable_dma_mgcg(rdev, enable);
5270 }
5271
5272 if (block & RADEON_CG_BLOCK_BIF) {
5273 si_enable_bif_mgls(rdev, enable);
5274 }
5275
5276 if (block & RADEON_CG_BLOCK_UVD) {
5277 if (rdev->has_uvd) {
5278 si_enable_uvd_mgcg(rdev, enable);
5279 }
5280 }
5281
5282 if (block & RADEON_CG_BLOCK_HDP) {
5283 si_enable_hdp_mgcg(rdev, enable);
5284 si_enable_hdp_ls(rdev, enable);
5285 }
5286}
f8f84ac5
AD		 5287
5288static void si_init_cg(struct radeon_device *rdev)
5289{
e16866ec
AD		 5290 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5291 RADEON_CG_BLOCK_MC |
5292 RADEON_CG_BLOCK_SDMA |
5293 RADEON_CG_BLOCK_BIF |
5294 RADEON_CG_BLOCK_HDP), true);
b2d70917 5295 if (rdev->has_uvd) {
e16866ec 5296 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
f8f84ac5
AD		 5297 si_init_uvd_internal_cg(rdev);
5298 }
5299}
5300
5301static void si_fini_cg(struct radeon_device *rdev)
5302{
0116e1ef 5303 if (rdev->has_uvd) {
e16866ec 5304 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
0116e1ef 5305 }
e16866ec
AD		 5306 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5307 RADEON_CG_BLOCK_MC |
5308 RADEON_CG_BLOCK_SDMA |
5309 RADEON_CG_BLOCK_BIF |
5310 RADEON_CG_BLOCK_HDP), false);
5311}
5312
59a82d0e
AD		 5313u32 si_get_csb_size(struct radeon_device *rdev)
5314{
5315 u32 count = 0;
5316 const struct cs_section_def *sect = NULL;
5317 const struct cs_extent_def *ext = NULL;
5318
5319 if (rdev->rlc.cs_data == NULL)
5320 return 0;
5321
5322 /* begin clear state */
5323 count += 2;
5324 /* context control state */
5325 count += 3;
5326
5327 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5328 for (ext = sect->section; ext->extent != NULL; ++ext) {
5329 if (sect->id == SECT_CONTEXT)
5330 count += 2 + ext->reg_count;
5331 else
5332 return 0;
5333 }
5334 }
5335 /* pa_sc_raster_config */
5336 count += 3;
5337 /* end clear state */
5338 count += 2;
5339 /* clear state */
5340 count += 2;
5341
5342 return count;
5343}
5344
5345void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5346{
5347 u32 count = 0, i;
5348 const struct cs_section_def *sect = NULL;
5349 const struct cs_extent_def *ext = NULL;
5350
5351 if (rdev->rlc.cs_data == NULL)
5352 return;
5353 if (buffer == NULL)
5354 return;
5355
5356 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5357 buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE;
5358
5359 buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1);
5360 buffer[count++] = 0x80000000;
5361 buffer[count++] = 0x80000000;
5362
5363 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5364 for (ext = sect->section; ext->extent != NULL; ++ext) {
5365 if (sect->id == SECT_CONTEXT) {
5366 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count);
5367 buffer[count++] = ext->reg_index - 0xa000;
5368 for (i = 0; i < ext->reg_count; i++)
5369 buffer[count++] = ext->extent[i];
5370 } else {
5371 return;
5372 }
5373 }
5374 }
5375
5376 buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 1);
5377 buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START;
5378 switch (rdev->family) {
5379 case CHIP_TAHITI:
5380 case CHIP_PITCAIRN:
5381 buffer[count++] = 0x2a00126a;
5382 break;
5383 case CHIP_VERDE:
5384 buffer[count++] = 0x0000124a;
5385 break;
5386 case CHIP_OLAND:
5387 buffer[count++] = 0x00000082;
5388 break;
5389 case CHIP_HAINAN:
5390 buffer[count++] = 0x00000000;
5391 break;
5392 default:
5393 buffer[count++] = 0x00000000;
5394 break;
5395 }
5396
5397 buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
5398 buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE;
5399
5400 buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0);
5401 buffer[count++] = 0;
5402}
5403
f8f84ac5
AD		 5404static void si_init_pg(struct radeon_device *rdev)
5405{
0116e1ef
AD		 5406 if (rdev->pg_flags) {
5407 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5408 si_init_dma_pg(rdev);
0116e1ef 5409 }
f8f84ac5 5410 si_init_ao_cu_mask(rdev);
0116e1ef
AD		 5411 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_CG) {
5412 si_init_gfx_cgpg(rdev);
0116e1ef 5413 }
ca6ebb39
AD		 5414 si_enable_dma_pg(rdev, true);
5415 si_enable_gfx_cgpg(rdev, true);
f8f84ac5
AD		 5416 } else {
5417 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5418 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5419 }
5420}
5421
5422static void si_fini_pg(struct radeon_device *rdev)
5423{
0116e1ef 5424 if (rdev->pg_flags) {
ca6ebb39
AD		 5425 si_enable_dma_pg(rdev, false);
5426 si_enable_gfx_cgpg(rdev, false);
f8f84ac5
AD		 5427 }
5428}
5429
347e7592
AD		 5430/*
5431 * RLC
5432 */
866d83de 5433void si_rlc_reset(struct radeon_device *rdev)
d719cef3 5434{
f8f84ac5 5435 u32 tmp = RREG32(GRBM_SOFT_RESET);
d719cef3 5436
f8f84ac5
AD		 5437 tmp |= SOFT_RESET_RLC;
5438 WREG32(GRBM_SOFT_RESET, tmp);
5439 udelay(50);
5440 tmp &= ~SOFT_RESET_RLC;
5441 WREG32(GRBM_SOFT_RESET, tmp);
5442 udelay(50);
d719cef3
AD		 5443}
5444
347e7592
AD		 5445static void si_rlc_stop(struct radeon_device *rdev)
5446{
5447 WREG32(RLC_CNTL, 0);
d719cef3
AD		 5448
5449 si_enable_gui_idle_interrupt(rdev, false);
5450
5451 si_wait_for_rlc_serdes(rdev);
347e7592
AD		 5452}
5453
5454static void si_rlc_start(struct radeon_device *rdev)
5455{
5456 WREG32(RLC_CNTL, RLC_ENABLE);
d719cef3
AD		 5457
5458 si_enable_gui_idle_interrupt(rdev, true);
5459
5460 udelay(50);
5461}
5462
5463static bool si_lbpw_supported(struct radeon_device *rdev)
5464{
5465 u32 tmp;
5466
5467 /* Enable LBPW only for DDR3 */
5468 tmp = RREG32(MC_SEQ_MISC0);
5469 if ((tmp & 0xF0000000) == 0xB0000000)
5470 return true;
5471 return false;
5472}
5473
5474static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5475{
5476 u32 tmp;
5477
5478 tmp = RREG32(RLC_LB_CNTL);
5479 if (enable)
5480 tmp |= LOAD_BALANCE_ENABLE;
5481 else
5482 tmp &= ~LOAD_BALANCE_ENABLE;
5483 WREG32(RLC_LB_CNTL, tmp);
5484
5485 if (!enable) {
5486 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5487 WREG32(SPI_LB_CU_MASK, 0x00ff);
5488 }
347e7592
AD		 5489}
5490
5491static int si_rlc_resume(struct radeon_device *rdev)
5492{
5493 u32 i;
5494 const __be32 *fw_data;
5495
5496 if (!rdev->rlc_fw)
5497 return -EINVAL;
5498
5499 si_rlc_stop(rdev);
5500
f8f84ac5
AD		 5501 si_rlc_reset(rdev);
5502
5503 si_init_pg(rdev);
5504
5505 si_init_cg(rdev);
5506
347e7592
AD		 5507 WREG32(RLC_RL_BASE, 0);
5508 WREG32(RLC_RL_SIZE, 0);
5509 WREG32(RLC_LB_CNTL, 0);
5510 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5511 WREG32(RLC_LB_CNTR_INIT, 0);
d719cef3 5512 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
347e7592 5513
347e7592
AD		 5514 WREG32(RLC_MC_CNTL, 0);
5515 WREG32(RLC_UCODE_CNTL, 0);
5516
5517 fw_data = (const __be32 *)rdev->rlc_fw->data;
5518 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5519 WREG32(RLC_UCODE_ADDR, i);
5520 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5521 }
5522 WREG32(RLC_UCODE_ADDR, 0);
5523
d719cef3
AD		 5524 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5525
347e7592
AD		 5526 si_rlc_start(rdev);
5527
5528 return 0;
5529}
5530
25a857fb
AD		 5531static void si_enable_interrupts(struct radeon_device *rdev)
5532{
5533 u32 ih_cntl = RREG32(IH_CNTL);
5534 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5535
5536 ih_cntl |= ENABLE_INTR;
5537 ih_rb_cntl |= IH_RB_ENABLE;
5538 WREG32(IH_CNTL, ih_cntl);
5539 WREG32(IH_RB_CNTL, ih_rb_cntl);
5540 rdev->ih.enabled = true;
5541}
5542
5543static void si_disable_interrupts(struct radeon_device *rdev)
5544{
5545 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5546 u32 ih_cntl = RREG32(IH_CNTL);
5547
5548 ih_rb_cntl &= ~IH_RB_ENABLE;
5549 ih_cntl &= ~ENABLE_INTR;
5550 WREG32(IH_RB_CNTL, ih_rb_cntl);
5551 WREG32(IH_CNTL, ih_cntl);
5552 /* set rptr, wptr to 0 */
5553 WREG32(IH_RB_RPTR, 0);
5554 WREG32(IH_RB_WPTR, 0);
5555 rdev->ih.enabled = false;
25a857fb
AD		 5556 rdev->ih.rptr = 0;
5557}
5558
5559static void si_disable_interrupt_state(struct radeon_device *rdev)
5560{
5561 u32 tmp;
5562
5563 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5564 WREG32(CP_INT_CNTL_RING1, 0);
5565 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 5566 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5567 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5568 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5569 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 5570 WREG32(GRBM_INT_CNTL, 0);
5153550a
AD		 5571 if (rdev->num_crtc >= 2) {
5572 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5573 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5574 }
25a857fb
AD		 5575 if (rdev->num_crtc >= 4) {
5576 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5577 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5578 }
5579 if (rdev->num_crtc >= 6) {
5580 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5581 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5582 }
5583
5153550a
AD		 5584 if (rdev->num_crtc >= 2) {
5585 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5586 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5587 }
25a857fb
AD		 5588 if (rdev->num_crtc >= 4) {
5589 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5590 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5591 }
5592 if (rdev->num_crtc >= 6) {
5593 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5594 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5595 }
5596
5153550a
AD		 5597 if (!ASIC_IS_NODCE(rdev)) {
5598 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
5599
5600 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5601 WREG32(DC_HPD1_INT_CONTROL, tmp);
5602 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5603 WREG32(DC_HPD2_INT_CONTROL, tmp);
5604 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5605 WREG32(DC_HPD3_INT_CONTROL, tmp);
5606 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5607 WREG32(DC_HPD4_INT_CONTROL, tmp);
5608 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5609 WREG32(DC_HPD5_INT_CONTROL, tmp);
5610 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5611 WREG32(DC_HPD6_INT_CONTROL, tmp);
5612 }
25a857fb
AD		 5613}
5614
5615static int si_irq_init(struct radeon_device *rdev)
5616{
5617 int ret = 0;
5618 int rb_bufsz;
5619 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5620
5621 /* allocate ring */
5622 ret = r600_ih_ring_alloc(rdev);
5623 if (ret)
5624 return ret;
5625
5626 /* disable irqs */
5627 si_disable_interrupts(rdev);
5628
5629 /* init rlc */
5630 ret = si_rlc_resume(rdev);
5631 if (ret) {
5632 r600_ih_ring_fini(rdev);
5633 return ret;
5634 }
5635
5636 /* setup interrupt control */
5637 /* set dummy read address to ring address */
5638 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5639 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5640 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5641 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5642 */
5643 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5644 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5645 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5646 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5647
5648 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5649 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
5650
5651 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5652 IH_WPTR_OVERFLOW_CLEAR |
5653 (rb_bufsz << 1));
5654
5655 if (rdev->wb.enabled)
5656 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5657
5658 /* set the writeback address whether it's enabled or not */
5659 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5660 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5661
5662 WREG32(IH_RB_CNTL, ih_rb_cntl);
5663
5664 /* set rptr, wptr to 0 */
5665 WREG32(IH_RB_RPTR, 0);
5666 WREG32(IH_RB_WPTR, 0);
5667
5668 /* Default settings for IH_CNTL (disabled at first) */
5669 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5670 /* RPTR_REARM only works if msi's are enabled */
5671 if (rdev->msi_enabled)
5672 ih_cntl |= RPTR_REARM;
5673 WREG32(IH_CNTL, ih_cntl);
5674
5675 /* force the active interrupt state to all disabled */
5676 si_disable_interrupt_state(rdev);
5677
2099810f
DA		 5678 pci_set_master(rdev->pdev);
5679
25a857fb
AD		 5680 /* enable irqs */
5681 si_enable_interrupts(rdev);
5682
5683 return ret;
5684}
5685
5686int si_irq_set(struct radeon_device *rdev)
5687{
5688 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
5689 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5690 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5153550a 5691 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
25a857fb
AD		 5692 u32 grbm_int_cntl = 0;
5693 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
8c5fd7ef 5694 u32 dma_cntl, dma_cntl1;
a9e61410 5695 u32 thermal_int = 0;
25a857fb
AD		 5696
5697 if (!rdev->irq.installed) {
5698 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5699 return -EINVAL;
5700 }
5701 /* don't enable anything if the ih is disabled */
5702 if (!rdev->ih.enabled) {
5703 si_disable_interrupts(rdev);
5704 /* force the active interrupt state to all disabled */
5705 si_disable_interrupt_state(rdev);
5706 return 0;
5707 }
5708
5153550a
AD		 5709 if (!ASIC_IS_NODCE(rdev)) {
5710 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5711 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5712 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5713 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5714 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5715 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5716 }
25a857fb 5717
8c5fd7ef
AD		 5718 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5719 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5720
a9e61410
AD		 5721 thermal_int = RREG32(CG_THERMAL_INT) &
5722 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5723
25a857fb 5724 /* enable CP interrupts on all rings */
736fc37f 5725 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 5726 DRM_DEBUG("si_irq_set: sw int gfx\n");
5727 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5728 }
736fc37f 5729 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 5730 DRM_DEBUG("si_irq_set: sw int cp1\n");
5731 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5732 }
736fc37f 5733 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 5734 DRM_DEBUG("si_irq_set: sw int cp2\n");
5735 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5736 }
8c5fd7ef
AD		 5737 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5738 DRM_DEBUG("si_irq_set: sw int dma\n");
5739 dma_cntl |= TRAP_ENABLE;
5740 }
5741
5742 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5743 DRM_DEBUG("si_irq_set: sw int dma1\n");
5744 dma_cntl1 |= TRAP_ENABLE;
5745 }
25a857fb 5746 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 5747 atomic_read(&rdev->irq.pflip[0])) {
25a857fb
AD		 5748 DRM_DEBUG("si_irq_set: vblank 0\n");
5749 crtc1 |= VBLANK_INT_MASK;
5750 }
5751 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 5752 atomic_read(&rdev->irq.pflip[1])) {
25a857fb
AD		 5753 DRM_DEBUG("si_irq_set: vblank 1\n");
5754 crtc2 |= VBLANK_INT_MASK;
5755 }
5756 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 5757 atomic_read(&rdev->irq.pflip[2])) {
25a857fb
AD		 5758 DRM_DEBUG("si_irq_set: vblank 2\n");
5759 crtc3 |= VBLANK_INT_MASK;
5760 }
5761 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 5762 atomic_read(&rdev->irq.pflip[3])) {
25a857fb
AD		 5763 DRM_DEBUG("si_irq_set: vblank 3\n");
5764 crtc4 |= VBLANK_INT_MASK;
5765 }
5766 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 5767 atomic_read(&rdev->irq.pflip[4])) {
25a857fb
AD		 5768 DRM_DEBUG("si_irq_set: vblank 4\n");
5769 crtc5 |= VBLANK_INT_MASK;
5770 }
5771 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 5772 atomic_read(&rdev->irq.pflip[5])) {
25a857fb
AD		 5773 DRM_DEBUG("si_irq_set: vblank 5\n");
5774 crtc6 |= VBLANK_INT_MASK;
5775 }
5776 if (rdev->irq.hpd[0]) {
5777 DRM_DEBUG("si_irq_set: hpd 1\n");
5778 hpd1 |= DC_HPDx_INT_EN;
5779 }
5780 if (rdev->irq.hpd[1]) {
5781 DRM_DEBUG("si_irq_set: hpd 2\n");
5782 hpd2 |= DC_HPDx_INT_EN;
5783 }
5784 if (rdev->irq.hpd[2]) {
5785 DRM_DEBUG("si_irq_set: hpd 3\n");
5786 hpd3 |= DC_HPDx_INT_EN;
5787 }
5788 if (rdev->irq.hpd[3]) {
5789 DRM_DEBUG("si_irq_set: hpd 4\n");
5790 hpd4 |= DC_HPDx_INT_EN;
5791 }
5792 if (rdev->irq.hpd[4]) {
5793 DRM_DEBUG("si_irq_set: hpd 5\n");
5794 hpd5 |= DC_HPDx_INT_EN;
5795 }
5796 if (rdev->irq.hpd[5]) {
5797 DRM_DEBUG("si_irq_set: hpd 6\n");
5798 hpd6 |= DC_HPDx_INT_EN;
5799 }
25a857fb
AD		 5800
5801 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5802 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5803 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5804
8c5fd7ef
AD		 5805 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5806 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5807
25a857fb
AD		 5808 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5809
a9e61410
AD		 5810 if (rdev->irq.dpm_thermal) {
5811 DRM_DEBUG("dpm thermal\n");
5812 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5813 }
5814
5153550a
AD		 5815 if (rdev->num_crtc >= 2) {
5816 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5817 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5818 }
25a857fb
AD		 5819 if (rdev->num_crtc >= 4) {
5820 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5821 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5822 }
5823 if (rdev->num_crtc >= 6) {
5824 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5825 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5826 }
5827
5153550a
AD		 5828 if (rdev->num_crtc >= 2) {
5829 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5830 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5831 }
25a857fb
AD		 5832 if (rdev->num_crtc >= 4) {
5833 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5834 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5835 }
5836 if (rdev->num_crtc >= 6) {
5837 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5838 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5839 }
5840
5153550a
AD		 5841 if (!ASIC_IS_NODCE(rdev)) {
5842 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5843 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5844 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5845 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5846 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5847 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5848 }
25a857fb 5849
a9e61410
AD		 5850 WREG32(CG_THERMAL_INT, thermal_int);
5851
25a857fb
AD		 5852 return 0;
5853}
5854
5855static inline void si_irq_ack(struct radeon_device *rdev)
5856{
5857 u32 tmp;
5858
5153550a
AD		 5859 if (ASIC_IS_NODCE(rdev))
5860 return;
5861
25a857fb
AD		 5862 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5863 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5864 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5865 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5866 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5867 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5868 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5869 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5870 if (rdev->num_crtc >= 4) {
5871 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5872 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5873 }
5874 if (rdev->num_crtc >= 6) {
5875 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5876 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5877 }
5878
5879 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5880 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5881 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5882 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5883 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5884 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5885 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5886 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5887 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5888 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5889 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5890 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5891
5892 if (rdev->num_crtc >= 4) {
5893 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5894 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5895 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5896 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5897 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5898 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5899 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5900 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5901 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5902 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5903 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5904 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5905 }
5906
5907 if (rdev->num_crtc >= 6) {
5908 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5909 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5910 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5911 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5912 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5913 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5914 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5915 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5916 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5917 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5918 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5919 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5920 }
5921
5922 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5923 tmp = RREG32(DC_HPD1_INT_CONTROL);
5924 tmp |= DC_HPDx_INT_ACK;
5925 WREG32(DC_HPD1_INT_CONTROL, tmp);
5926 }
5927 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5928 tmp = RREG32(DC_HPD2_INT_CONTROL);
5929 tmp |= DC_HPDx_INT_ACK;
5930 WREG32(DC_HPD2_INT_CONTROL, tmp);
5931 }
5932 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5933 tmp = RREG32(DC_HPD3_INT_CONTROL);
5934 tmp |= DC_HPDx_INT_ACK;
5935 WREG32(DC_HPD3_INT_CONTROL, tmp);
5936 }
5937 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5938 tmp = RREG32(DC_HPD4_INT_CONTROL);
5939 tmp |= DC_HPDx_INT_ACK;
5940 WREG32(DC_HPD4_INT_CONTROL, tmp);
5941 }
5942 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5943 tmp = RREG32(DC_HPD5_INT_CONTROL);
5944 tmp |= DC_HPDx_INT_ACK;
5945 WREG32(DC_HPD5_INT_CONTROL, tmp);
5946 }
5947 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5948 tmp = RREG32(DC_HPD5_INT_CONTROL);
5949 tmp |= DC_HPDx_INT_ACK;
5950 WREG32(DC_HPD6_INT_CONTROL, tmp);
5951 }
5952}
5953
5954static void si_irq_disable(struct radeon_device *rdev)
5955{
5956 si_disable_interrupts(rdev);
5957 /* Wait and acknowledge irq */
5958 mdelay(1);
5959 si_irq_ack(rdev);
5960 si_disable_interrupt_state(rdev);
5961}
5962
5963static void si_irq_suspend(struct radeon_device *rdev)
5964{
5965 si_irq_disable(rdev);
5966 si_rlc_stop(rdev);
5967}
5968
9b136d51
AD		 5969static void si_irq_fini(struct radeon_device *rdev)
5970{
5971 si_irq_suspend(rdev);
5972 r600_ih_ring_fini(rdev);
5973}
5974
25a857fb
AD		 5975static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
5976{
5977 u32 wptr, tmp;
5978
5979 if (rdev->wb.enabled)
5980 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
5981 else
5982 wptr = RREG32(IH_RB_WPTR);
5983
5984 if (wptr & RB_OVERFLOW) {
5985 /* When a ring buffer overflow happen start parsing interrupt
5986 * from the last not overwritten vector (wptr + 16). Hopefully
5987 * this should allow us to catchup.
5988 */
5989 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
5990 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
5991 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
5992 tmp = RREG32(IH_RB_CNTL);
5993 tmp |= IH_WPTR_OVERFLOW_CLEAR;
5994 WREG32(IH_RB_CNTL, tmp);
5995 }
5996 return (wptr & rdev->ih.ptr_mask);
5997}
5998
5999/* SI IV Ring
6000 * Each IV ring entry is 128 bits:
6001 * [7:0] - interrupt source id
6002 * [31:8] - reserved
6003 * [59:32] - interrupt source data
6004 * [63:60] - reserved
6005 * [71:64] - RINGID
6006 * [79:72] - VMID
6007 * [127:80] - reserved
6008 */
6009int si_irq_process(struct radeon_device *rdev)
6010{
6011 u32 wptr;
6012 u32 rptr;
6013 u32 src_id, src_data, ring_id;
6014 u32 ring_index;
25a857fb 6015 bool queue_hotplug = false;
a9e61410 6016 bool queue_thermal = false;
fbf6dc7a 6017 u32 status, addr;
25a857fb
AD		 6018
6019 if (!rdev->ih.enabled || rdev->shutdown)
6020 return IRQ_NONE;
6021
6022 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 6023
6024restart_ih:
6025 /* is somebody else already processing irqs? */
6026 if (atomic_xchg(&rdev->ih.lock, 1))
6027 return IRQ_NONE;
6028
25a857fb
AD		 6029 rptr = rdev->ih.rptr;
6030 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6031
25a857fb
AD		 6032 /* Order reading of wptr vs. reading of IH ring data */
6033 rmb();
6034
6035 /* display interrupts */
6036 si_irq_ack(rdev);
6037
25a857fb
AD		 6038 while (rptr != wptr) {
6039 /* wptr/rptr are in bytes! */
6040 ring_index = rptr / 4;
6041 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6042 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6043 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6044
6045 switch (src_id) {
6046 case 1: /* D1 vblank/vline */
6047 switch (src_data) {
6048 case 0: /* D1 vblank */
6049 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6050 if (rdev->irq.crtc_vblank_int[0]) {
6051 drm_handle_vblank(rdev->ddev, 0);
6052 rdev->pm.vblank_sync = true;
6053 wake_up(&rdev->irq.vblank_queue);
6054 }
736fc37f 6055 if (atomic_read(&rdev->irq.pflip[0]))
25a857fb
AD		 6056 radeon_crtc_handle_flip(rdev, 0);
6057 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6058 DRM_DEBUG("IH: D1 vblank\n");
6059 }
6060 break;
6061 case 1: /* D1 vline */
6062 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6063 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6064 DRM_DEBUG("IH: D1 vline\n");
6065 }
6066 break;
6067 default:
6068 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6069 break;
6070 }
6071 break;
6072 case 2: /* D2 vblank/vline */
6073 switch (src_data) {
6074 case 0: /* D2 vblank */
6075 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6076 if (rdev->irq.crtc_vblank_int[1]) {
6077 drm_handle_vblank(rdev->ddev, 1);
6078 rdev->pm.vblank_sync = true;
6079 wake_up(&rdev->irq.vblank_queue);
6080 }
736fc37f 6081 if (atomic_read(&rdev->irq.pflip[1]))
25a857fb
AD		 6082 radeon_crtc_handle_flip(rdev, 1);
6083 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6084 DRM_DEBUG("IH: D2 vblank\n");
6085 }
6086 break;
6087 case 1: /* D2 vline */
6088 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6089 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6090 DRM_DEBUG("IH: D2 vline\n");
6091 }
6092 break;
6093 default:
6094 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6095 break;
6096 }
6097 break;
6098 case 3: /* D3 vblank/vline */
6099 switch (src_data) {
6100 case 0: /* D3 vblank */
6101 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6102 if (rdev->irq.crtc_vblank_int[2]) {
6103 drm_handle_vblank(rdev->ddev, 2);
6104 rdev->pm.vblank_sync = true;
6105 wake_up(&rdev->irq.vblank_queue);
6106 }
736fc37f 6107 if (atomic_read(&rdev->irq.pflip[2]))
25a857fb
AD		 6108 radeon_crtc_handle_flip(rdev, 2);
6109 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6110 DRM_DEBUG("IH: D3 vblank\n");
6111 }
6112 break;
6113 case 1: /* D3 vline */
6114 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6115 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6116 DRM_DEBUG("IH: D3 vline\n");
6117 }
6118 break;
6119 default:
6120 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6121 break;
6122 }
6123 break;
6124 case 4: /* D4 vblank/vline */
6125 switch (src_data) {
6126 case 0: /* D4 vblank */
6127 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6128 if (rdev->irq.crtc_vblank_int[3]) {
6129 drm_handle_vblank(rdev->ddev, 3);
6130 rdev->pm.vblank_sync = true;
6131 wake_up(&rdev->irq.vblank_queue);
6132 }
736fc37f 6133 if (atomic_read(&rdev->irq.pflip[3]))
25a857fb
AD		 6134 radeon_crtc_handle_flip(rdev, 3);
6135 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6136 DRM_DEBUG("IH: D4 vblank\n");
6137 }
6138 break;
6139 case 1: /* D4 vline */
6140 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6141 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6142 DRM_DEBUG("IH: D4 vline\n");
6143 }
6144 break;
6145 default:
6146 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6147 break;
6148 }
6149 break;
6150 case 5: /* D5 vblank/vline */
6151 switch (src_data) {
6152 case 0: /* D5 vblank */
6153 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6154 if (rdev->irq.crtc_vblank_int[4]) {
6155 drm_handle_vblank(rdev->ddev, 4);
6156 rdev->pm.vblank_sync = true;
6157 wake_up(&rdev->irq.vblank_queue);
6158 }
736fc37f 6159 if (atomic_read(&rdev->irq.pflip[4]))
25a857fb
AD		 6160 radeon_crtc_handle_flip(rdev, 4);
6161 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6162 DRM_DEBUG("IH: D5 vblank\n");
6163 }
6164 break;
6165 case 1: /* D5 vline */
6166 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6167 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6168 DRM_DEBUG("IH: D5 vline\n");
6169 }
6170 break;
6171 default:
6172 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6173 break;
6174 }
6175 break;
6176 case 6: /* D6 vblank/vline */
6177 switch (src_data) {
6178 case 0: /* D6 vblank */
6179 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6180 if (rdev->irq.crtc_vblank_int[5]) {
6181 drm_handle_vblank(rdev->ddev, 5);
6182 rdev->pm.vblank_sync = true;
6183 wake_up(&rdev->irq.vblank_queue);
6184 }
736fc37f 6185 if (atomic_read(&rdev->irq.pflip[5]))
25a857fb
AD		 6186 radeon_crtc_handle_flip(rdev, 5);
6187 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6188 DRM_DEBUG("IH: D6 vblank\n");
6189 }
6190 break;
6191 case 1: /* D6 vline */
6192 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6193 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6194 DRM_DEBUG("IH: D6 vline\n");
6195 }
6196 break;
6197 default:
6198 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6199 break;
6200 }
6201 break;
6202 case 42: /* HPD hotplug */
6203 switch (src_data) {
6204 case 0:
6205 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6206 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6207 queue_hotplug = true;
6208 DRM_DEBUG("IH: HPD1\n");
6209 }
6210 break;
6211 case 1:
6212 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6213 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6214 queue_hotplug = true;
6215 DRM_DEBUG("IH: HPD2\n");
6216 }
6217 break;
6218 case 2:
6219 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6220 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6221 queue_hotplug = true;
6222 DRM_DEBUG("IH: HPD3\n");
6223 }
6224 break;
6225 case 3:
6226 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6227 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6228 queue_hotplug = true;
6229 DRM_DEBUG("IH: HPD4\n");
6230 }
6231 break;
6232 case 4:
6233 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6234 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6235 queue_hotplug = true;
6236 DRM_DEBUG("IH: HPD5\n");
6237 }
6238 break;
6239 case 5:
6240 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6241 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6242 queue_hotplug = true;
6243 DRM_DEBUG("IH: HPD6\n");
6244 }
6245 break;
6246 default:
6247 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6248 break;
6249 }
6250 break;
ae133a11
CK		 6251 case 146:
6252 case 147:
fbf6dc7a
AD		 6253 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6254 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
ae133a11
CK		 6255 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6256 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
fbf6dc7a 6257 addr);
ae133a11 6258 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
fbf6dc7a
AD		 6259 status);
6260 si_vm_decode_fault(rdev, status, addr);
ae133a11
CK		 6261 /* reset addr and status */
6262 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6263 break;
25a857fb
AD		 6264 case 176: /* RINGID0 CP_INT */
6265 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6266 break;
6267 case 177: /* RINGID1 CP_INT */
6268 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6269 break;
6270 case 178: /* RINGID2 CP_INT */
6271 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6272 break;
6273 case 181: /* CP EOP event */
6274 DRM_DEBUG("IH: CP EOP\n");
6275 switch (ring_id) {
6276 case 0:
6277 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6278 break;
6279 case 1:
6280 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6281 break;
6282 case 2:
6283 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6284 break;
6285 }
6286 break;
8c5fd7ef
AD		 6287 case 224: /* DMA trap event */
6288 DRM_DEBUG("IH: DMA trap\n");
6289 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6290 break;
a9e61410
AD		 6291 case 230: /* thermal low to high */
6292 DRM_DEBUG("IH: thermal low to high\n");
6293 rdev->pm.dpm.thermal.high_to_low = false;
6294 queue_thermal = true;
6295 break;
6296 case 231: /* thermal high to low */
6297 DRM_DEBUG("IH: thermal high to low\n");
6298 rdev->pm.dpm.thermal.high_to_low = true;
6299 queue_thermal = true;
6300 break;
25a857fb
AD		 6301 case 233: /* GUI IDLE */
6302 DRM_DEBUG("IH: GUI idle\n");
25a857fb 6303 break;
8c5fd7ef
AD		 6304 case 244: /* DMA trap event */
6305 DRM_DEBUG("IH: DMA1 trap\n");
6306 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6307 break;
25a857fb
AD		 6308 default:
6309 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6310 break;
6311 }
6312
6313 /* wptr/rptr are in bytes! */
6314 rptr += 16;
6315 rptr &= rdev->ih.ptr_mask;
6316 }
25a857fb
AD		 6317 if (queue_hotplug)
6318 schedule_work(&rdev->hotplug_work);
a9e61410
AD		 6319 if (queue_thermal && rdev->pm.dpm_enabled)
6320 schedule_work(&rdev->pm.dpm.thermal.work);
25a857fb
AD		 6321 rdev->ih.rptr = rptr;
6322 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 6323 atomic_set(&rdev->ih.lock, 0);
6324
6325 /* make sure wptr hasn't changed while processing */
6326 wptr = si_get_ih_wptr(rdev);
6327 if (wptr != rptr)
6328 goto restart_ih;
6329
25a857fb
AD		 6330 return IRQ_HANDLED;
6331}
6332
9b136d51
AD		 6333/*
6334 * startup/shutdown callbacks
6335 */
6336static int si_startup(struct radeon_device *rdev)
6337{
6338 struct radeon_ring *ring;
6339 int r;
6340
b9d305df
AD		 6341 /* enable pcie gen2/3 link */
6342 si_pcie_gen3_enable(rdev);
e0bcf165
AD		 6343 /* enable aspm */
6344 si_program_aspm(rdev);
b9d305df 6345
6fab3feb
AD		 6346 si_mc_program(rdev);
6347
9b136d51
AD		 6348 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6349 !rdev->rlc_fw || !rdev->mc_fw) {
6350 r = si_init_microcode(rdev);
6351 if (r) {
6352 DRM_ERROR("Failed to load firmware!\n");
6353 return r;
6354 }
6355 }
6356
6357 r = si_mc_load_microcode(rdev);
6358 if (r) {
6359 DRM_ERROR("Failed to load MC firmware!\n");
6360 return r;
6361 }
6362
6363 r = r600_vram_scratch_init(rdev);
6364 if (r)
6365 return r;
6366
9b136d51
AD		 6367 r = si_pcie_gart_enable(rdev);
6368 if (r)
6369 return r;
6370 si_gpu_init(rdev);
6371
9b136d51 6372 /* allocate rlc buffers */
1fd11777
AD		 6373 if (rdev->family == CHIP_VERDE) {
6374 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6375 rdev->rlc.reg_list_size =
6376 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6377 }
6378 rdev->rlc.cs_data = si_cs_data;
6379 r = sumo_rlc_init(rdev);
9b136d51
AD		 6380 if (r) {
6381 DRM_ERROR("Failed to init rlc BOs!\n");
6382 return r;
6383 }
6384
6385 /* allocate wb buffer */
6386 r = radeon_wb_init(rdev);
6387 if (r)
6388 return r;
6389
6390 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6391 if (r) {
6392 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6393 return r;
6394 }
6395
6396 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6397 if (r) {
6398 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6399 return r;
6400 }
6401
6402 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6403 if (r) {
6404 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6405 return r;
6406 }
6407
8c5fd7ef
AD		 6408 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6409 if (r) {
6410 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6411 return r;
6412 }
6413
6414 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6415 if (r) {
6416 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6417 return r;
6418 }
6419
1df0d523 6420 if (rdev->has_uvd) {
e409b128 6421 r = uvd_v2_2_resume(rdev);
1df0d523
AD		 6422 if (!r) {
6423 r = radeon_fence_driver_start_ring(rdev,
6424 R600_RING_TYPE_UVD_INDEX);
6425 if (r)
6426 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6427 }
f2ba57b5 6428 if (r)
1df0d523 6429 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
f2ba57b5 6430 }
f2ba57b5 6431
9b136d51 6432 /* Enable IRQ */
e49f3959
AH		 6433 if (!rdev->irq.installed) {
6434 r = radeon_irq_kms_init(rdev);
6435 if (r)
6436 return r;
6437 }
6438
9b136d51
AD		 6439 r = si_irq_init(rdev);
6440 if (r) {
6441 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6442 radeon_irq_kms_fini(rdev);
6443 return r;
6444 }
6445 si_irq_set(rdev);
6446
6447 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6448 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6449 CP_RB0_RPTR, CP_RB0_WPTR,
2e1e6dad 6450 RADEON_CP_PACKET2);
9b136d51
AD		 6451 if (r)
6452 return r;
6453
6454 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6455 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6456 CP_RB1_RPTR, CP_RB1_WPTR,
2e1e6dad 6457 RADEON_CP_PACKET2);
9b136d51
AD		 6458 if (r)
6459 return r;
6460
6461 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6462 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6463 CP_RB2_RPTR, CP_RB2_WPTR,
2e1e6dad 6464 RADEON_CP_PACKET2);
9b136d51
AD		 6465 if (r)
6466 return r;
6467
8c5fd7ef
AD		 6468 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6469 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6470 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
6471 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
2e1e6dad 6472 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6473 if (r)
6474 return r;
6475
6476 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6477 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6478 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
6479 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
2e1e6dad 6480 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6481 if (r)
6482 return r;
6483
9b136d51
AD		 6484 r = si_cp_load_microcode(rdev);
6485 if (r)
6486 return r;
6487 r = si_cp_resume(rdev);
6488 if (r)
6489 return r;
6490
8c5fd7ef
AD		 6491 r = cayman_dma_resume(rdev);
6492 if (r)
6493 return r;
6494
1df0d523
AD		 6495 if (rdev->has_uvd) {
6496 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6497 if (ring->ring_size) {
02c9f7fa 6498 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
1df0d523 6499 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
2e1e6dad 6500 RADEON_CP_PACKET2);
1df0d523 6501 if (!r)
e409b128 6502 r = uvd_v1_0_init(rdev);
1df0d523
AD		 6503 if (r)
6504 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6505 }
f2ba57b5
CK		 6506 }
6507
2898c348
CK		 6508 r = radeon_ib_pool_init(rdev);
6509 if (r) {
6510 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 6511 return r;
2898c348 6512 }
9b136d51 6513
c6105f24
CK		 6514 r = radeon_vm_manager_init(rdev);
6515 if (r) {
6516 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 6517 return r;
c6105f24 6518 }
9b136d51 6519
b530602f
AD		 6520 r = dce6_audio_init(rdev);
6521 if (r)
6522 return r;
6523
9b136d51
AD		 6524 return 0;
6525}
6526
6527int si_resume(struct radeon_device *rdev)
6528{
6529 int r;
6530
6531 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6532 * posting will perform necessary task to bring back GPU into good
6533 * shape.
6534 */
6535 /* post card */
6536 atom_asic_init(rdev->mode_info.atom_context);
6537
205996c0
AD		 6538 /* init golden registers */
6539 si_init_golden_registers(rdev);
6540
9b136d51
AD		 6541 rdev->accel_working = true;
6542 r = si_startup(rdev);
6543 if (r) {
6544 DRM_ERROR("si startup failed on resume\n");
6545 rdev->accel_working = false;
6546 return r;
6547 }
6548
6549 return r;
6550
6551}
6552
6553int si_suspend(struct radeon_device *rdev)
6554{
b530602f 6555 dce6_audio_fini(rdev);
fa3daf9a 6556 radeon_vm_manager_fini(rdev);
9b136d51 6557 si_cp_enable(rdev, false);
8c5fd7ef 6558 cayman_dma_stop(rdev);
1df0d523 6559 if (rdev->has_uvd) {
e409b128 6560 uvd_v1_0_fini(rdev);
1df0d523
AD		 6561 radeon_uvd_suspend(rdev);
6562 }
e16866ec
AD		 6563 si_fini_pg(rdev);
6564 si_fini_cg(rdev);
9b136d51
AD		 6565 si_irq_suspend(rdev);
6566 radeon_wb_disable(rdev);
6567 si_pcie_gart_disable(rdev);
6568 return 0;
6569}
6570
6571/* Plan is to move initialization in that function and use
6572 * helper function so that radeon_device_init pretty much
6573 * do nothing more than calling asic specific function. This
6574 * should also allow to remove a bunch of callback function
6575 * like vram_info.
6576 */
6577int si_init(struct radeon_device *rdev)
6578{
6579 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6580 int r;
6581
9b136d51
AD		 6582 /* Read BIOS */
6583 if (!radeon_get_bios(rdev)) {
6584 if (ASIC_IS_AVIVO(rdev))
6585 return -EINVAL;
6586 }
6587 /* Must be an ATOMBIOS */
6588 if (!rdev->is_atom_bios) {
6589 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6590 return -EINVAL;
6591 }
6592 r = radeon_atombios_init(rdev);
6593 if (r)
6594 return r;
6595
6596 /* Post card if necessary */
6597 if (!radeon_card_posted(rdev)) {
6598 if (!rdev->bios) {
6599 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6600 return -EINVAL;
6601 }
6602 DRM_INFO("GPU not posted. posting now...\n");
6603 atom_asic_init(rdev->mode_info.atom_context);
6604 }
205996c0
AD		 6605 /* init golden registers */
6606 si_init_golden_registers(rdev);
9b136d51
AD		 6607 /* Initialize scratch registers */
6608 si_scratch_init(rdev);
6609 /* Initialize surface registers */
6610 radeon_surface_init(rdev);
6611 /* Initialize clocks */
6612 radeon_get_clock_info(rdev->ddev);
6613
6614 /* Fence driver */
6615 r = radeon_fence_driver_init(rdev);
6616 if (r)
6617 return r;
6618
6619 /* initialize memory controller */
6620 r = si_mc_init(rdev);
6621 if (r)
6622 return r;
6623 /* Memory manager */
6624 r = radeon_bo_init(rdev);
6625 if (r)
6626 return r;
6627
9b136d51
AD		 6628 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6629 ring->ring_obj = NULL;
6630 r600_ring_init(rdev, ring, 1024 * 1024);
6631
6632 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6633 ring->ring_obj = NULL;
6634 r600_ring_init(rdev, ring, 1024 * 1024);
6635
6636 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6637 ring->ring_obj = NULL;
6638 r600_ring_init(rdev, ring, 1024 * 1024);
6639
8c5fd7ef
AD		 6640 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6641 ring->ring_obj = NULL;
6642 r600_ring_init(rdev, ring, 64 * 1024);
6643
6644 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6645 ring->ring_obj = NULL;
6646 r600_ring_init(rdev, ring, 64 * 1024);
6647
1df0d523
AD		 6648 if (rdev->has_uvd) {
6649 r = radeon_uvd_init(rdev);
6650 if (!r) {
6651 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6652 ring->ring_obj = NULL;
6653 r600_ring_init(rdev, ring, 4096);
6654 }
f2ba57b5
CK		 6655 }
6656
9b136d51
AD		 6657 rdev->ih.ring_obj = NULL;
6658 r600_ih_ring_init(rdev, 64 * 1024);
6659
6660 r = r600_pcie_gart_init(rdev);
6661 if (r)
6662 return r;
6663
9b136d51 6664 rdev->accel_working = true;
9b136d51
AD		 6665 r = si_startup(rdev);
6666 if (r) {
6667 dev_err(rdev->dev, "disabling GPU acceleration\n");
6668 si_cp_fini(rdev);
8c5fd7ef 6669 cayman_dma_fini(rdev);
9b136d51 6670 si_irq_fini(rdev);
1fd11777 6671 sumo_rlc_fini(rdev);
9b136d51 6672 radeon_wb_fini(rdev);
2898c348 6673 radeon_ib_pool_fini(rdev);
9b136d51
AD		 6674 radeon_vm_manager_fini(rdev);
6675 radeon_irq_kms_fini(rdev);
6676 si_pcie_gart_fini(rdev);
6677 rdev->accel_working = false;
6678 }
6679
6680 /* Don't start up if the MC ucode is missing.
6681 * The default clocks and voltages before the MC ucode
6682 * is loaded are not suffient for advanced operations.
6683 */
6684 if (!rdev->mc_fw) {
6685 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6686 return -EINVAL;
6687 }
6688
6689 return 0;
6690}
6691
6692void si_fini(struct radeon_device *rdev)
6693{
9b136d51 6694 si_cp_fini(rdev);
8c5fd7ef 6695 cayman_dma_fini(rdev);
e16866ec
AD		 6696 si_fini_pg(rdev);
6697 si_fini_cg(rdev);
9b136d51 6698 si_irq_fini(rdev);
1fd11777 6699 sumo_rlc_fini(rdev);
9b136d51
AD		 6700 radeon_wb_fini(rdev);
6701 radeon_vm_manager_fini(rdev);
2898c348 6702 radeon_ib_pool_fini(rdev);
9b136d51 6703 radeon_irq_kms_fini(rdev);
2858c00d 6704 if (rdev->has_uvd) {
e409b128 6705 uvd_v1_0_fini(rdev);
1df0d523 6706 radeon_uvd_fini(rdev);
2858c00d 6707 }
9b136d51
AD		 6708 si_pcie_gart_fini(rdev);
6709 r600_vram_scratch_fini(rdev);
6710 radeon_gem_fini(rdev);
9b136d51
AD		 6711 radeon_fence_driver_fini(rdev);
6712 radeon_bo_fini(rdev);
6713 radeon_atombios_fini(rdev);
6714 kfree(rdev->bios);
6715 rdev->bios = NULL;
6716}
6717
6759a0a7 6718/**
d0418894 6719 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 6720 *
6721 * @rdev: radeon_device pointer
6722 *
6723 * Fetches a GPU clock counter snapshot (SI).
6724 * Returns the 64 bit clock counter snapshot.
6725 */
d0418894 6726uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 6727{
6728 uint64_t clock;
6729
6730 mutex_lock(&rdev->gpu_clock_mutex);
6731 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6732 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6733 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6734 mutex_unlock(&rdev->gpu_clock_mutex);
6735 return clock;
6736}
2539eb02 6737
2539eb02
CK		 6738int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6739{
facd112d 6740 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 6741 int r;
6742
4ed10835
CK		 6743 /* bypass vclk and dclk with bclk */
6744 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6745 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6746 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6747
6748 /* put PLL in bypass mode */
6749 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6750
6751 if (!vclk || !dclk) {
6752 /* keep the Bypass mode, put PLL to sleep */
6753 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6754 return 0;
6755 }
6756
facd112d
CK		 6757 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6758 16384, 0x03FFFFFF, 0, 128, 5,
6759 &fb_div, &vclk_div, &dclk_div);
6760 if (r)
6761 return r;
2539eb02
CK		 6762
6763 /* set RESET_ANTI_MUX to 0 */
6764 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6765
6766 /* set VCO_MODE to 1 */
6767 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6768
6769 /* toggle UPLL_SLEEP to 1 then back to 0 */
6770 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6771 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6772
6773 /* deassert UPLL_RESET */
6774 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6775
6776 mdelay(1);
6777
facd112d 6778 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6779 if (r)
6780 return r;
6781
6782 /* assert UPLL_RESET again */
6783 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6784
6785 /* disable spread spectrum. */
6786 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6787
6788 /* set feedback divider */
facd112d 6789 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 6790
6791 /* set ref divider to 0 */
6792 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6793
facd112d 6794 if (fb_div < 307200)
2539eb02
CK		 6795 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6796 else
6797 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6798
6799 /* set PDIV_A and PDIV_B */
6800 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 6801 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 6802 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6803
6804 /* give the PLL some time to settle */
6805 mdelay(15);
6806
6807 /* deassert PLL_RESET */
6808 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6809
6810 mdelay(15);
6811
6812 /* switch from bypass mode to normal mode */
6813 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6814
facd112d 6815 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6816 if (r)
6817 return r;
6818
6819 /* switch VCLK and DCLK selection */
6820 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6821 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6822 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6823
6824 mdelay(100);
6825
6826 return 0;
6827}
b9d305df
AD		 6828
6829static void si_pcie_gen3_enable(struct radeon_device *rdev)
6830{
6831 struct pci_dev *root = rdev->pdev->bus->self;
6832 int bridge_pos, gpu_pos;
6833 u32 speed_cntl, mask, current_data_rate;
6834 int ret, i;
6835 u16 tmp16;
6836
6837 if (radeon_pcie_gen2 == 0)
6838 return;
6839
6840 if (rdev->flags & RADEON_IS_IGP)
6841 return;
6842
6843 if (!(rdev->flags & RADEON_IS_PCIE))
6844 return;
6845
6846 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6847 if (ret != 0)
6848 return;
6849
6850 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6851 return;
6852
6853 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6854 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6855 LC_CURRENT_DATA_RATE_SHIFT;
6856 if (mask & DRM_PCIE_SPEED_80) {
6857 if (current_data_rate == 2) {
6858 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6859 return;
6860 }
6861 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6862 } else if (mask & DRM_PCIE_SPEED_50) {
6863 if (current_data_rate == 1) {
6864 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6865 return;
6866 }
6867 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6868 }
6869
6870 bridge_pos = pci_pcie_cap(root);
6871 if (!bridge_pos)
6872 return;
6873
6874 gpu_pos = pci_pcie_cap(rdev->pdev);
6875 if (!gpu_pos)
6876 return;
6877
6878 if (mask & DRM_PCIE_SPEED_80) {
6879 /* re-try equalization if gen3 is not already enabled */
6880 if (current_data_rate != 2) {
6881 u16 bridge_cfg, gpu_cfg;
6882 u16 bridge_cfg2, gpu_cfg2;
6883 u32 max_lw, current_lw, tmp;
6884
6885 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6886 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6887
6888 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6889 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6890
6891 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6892 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6893
6894 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6895 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6896 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6897
6898 if (current_lw < max_lw) {
6899 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6900 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6901 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6902 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6903 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6904 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6905 }
6906 }
6907
6908 for (i = 0; i < 10; i++) {
6909 /* check status */
6910 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6911 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6912 break;
6913
6914 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6915 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6916
6917 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6918 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6919
6920 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6921 tmp |= LC_SET_QUIESCE;
6922 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6923
6924 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6925 tmp |= LC_REDO_EQ;
6926 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6927
6928 mdelay(100);
6929
6930 /* linkctl */
6931 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6932 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6933 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6934 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6935
6936 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
6937 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6938 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
6939 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6940
6941 /* linkctl2 */
6942 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
6943 tmp16 &= ~((1 << 4) | (7 << 9));
6944 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
6945 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
6946
6947 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6948 tmp16 &= ~((1 << 4) | (7 << 9));
6949 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
6950 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6951
6952 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6953 tmp &= ~LC_SET_QUIESCE;
6954 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6955 }
6956 }
6957 }
6958
6959 /* set the link speed */
6960 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
6961 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
6962 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6963
6964 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6965 tmp16 &= ~0xf;
6966 if (mask & DRM_PCIE_SPEED_80)
6967 tmp16 |= 3; /* gen3 */
6968 else if (mask & DRM_PCIE_SPEED_50)
6969 tmp16 |= 2; /* gen2 */
6970 else
6971 tmp16 |= 1; /* gen1 */
6972 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6973
6974 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6975 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
6976 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6977
6978 for (i = 0; i < rdev->usec_timeout; i++) {
6979 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6980 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
6981 break;
6982 udelay(1);
6983 }
6984}
6985
e0bcf165
AD		 6986static void si_program_aspm(struct radeon_device *rdev)
6987{
6988 u32 data, orig;
6989 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
6990 bool disable_clkreq = false;
6991
1294d4a3
AD		 6992 if (radeon_aspm == 0)
6993 return;
6994
e0bcf165
AD		 6995 if (!(rdev->flags & RADEON_IS_PCIE))
6996 return;
6997
6998 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6999 data &= ~LC_XMIT_N_FTS_MASK;
7000 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7001 if (orig != data)
7002 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7003
7004 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7005 data |= LC_GO_TO_RECOVERY;
7006 if (orig != data)
7007 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7008
7009 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7010 data |= P_IGNORE_EDB_ERR;
7011 if (orig != data)
7012 WREG32_PCIE(PCIE_P_CNTL, data);
7013
7014 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7015 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7016 data |= LC_PMI_TO_L1_DIS;
7017 if (!disable_l0s)
7018 data |= LC_L0S_INACTIVITY(7);
7019
7020 if (!disable_l1) {
7021 data |= LC_L1_INACTIVITY(7);
7022 data &= ~LC_PMI_TO_L1_DIS;
7023 if (orig != data)
7024 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7025
7026 if (!disable_plloff_in_l1) {
7027 bool clk_req_support;
7028
7029 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7030 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7031 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7032 if (orig != data)
7033 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7034
7035 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7036 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7037 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7038 if (orig != data)
7039 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7040
7041 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7042 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7043 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7044 if (orig != data)
7045 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7046
7047 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7048 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7049 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7050 if (orig != data)
7051 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7052
7053 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7054 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7055 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7056 if (orig != data)
7057 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7058
7059 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7060 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7061 if (orig != data)
7062 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7063
7064 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7065 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7066 if (orig != data)
7067 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7068
7069 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7070 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7071 if (orig != data)
7072 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7073
7074 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7075 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7076 if (orig != data)
7077 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7078
7079 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7080 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7081 if (orig != data)
7082 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7083
7084 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7085 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7086 if (orig != data)
7087 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7088
7089 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7090 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7091 if (orig != data)
7092 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7093 }
7094 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7095 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7096 data |= LC_DYN_LANES_PWR_STATE(3);
7097 if (orig != data)
7098 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7099
7100 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7101 data &= ~LS2_EXIT_TIME_MASK;
7102 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7103 data |= LS2_EXIT_TIME(5);
7104 if (orig != data)
7105 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7106
7107 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7108 data &= ~LS2_EXIT_TIME_MASK;
7109 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7110 data |= LS2_EXIT_TIME(5);
7111 if (orig != data)
7112 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7113
7114 if (!disable_clkreq) {
7115 struct pci_dev *root = rdev->pdev->bus->self;
7116 u32 lnkcap;
7117
7118 clk_req_support = false;
7119 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7120 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7121 clk_req_support = true;
7122 } else {
7123 clk_req_support = false;
7124 }
7125
7126 if (clk_req_support) {
7127 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7128 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7129 if (orig != data)
7130 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7131
7132 orig = data = RREG32(THM_CLK_CNTL);
7133 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7134 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7135 if (orig != data)
7136 WREG32(THM_CLK_CNTL, data);
7137
7138 orig = data = RREG32(MISC_CLK_CNTL);
7139 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7140 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7141 if (orig != data)
7142 WREG32(MISC_CLK_CNTL, data);
7143
7144 orig = data = RREG32(CG_CLKPIN_CNTL);
7145 data &= ~BCLK_AS_XCLK;
7146 if (orig != data)
7147 WREG32(CG_CLKPIN_CNTL, data);
7148
7149 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7150 data &= ~FORCE_BIF_REFCLK_EN;
7151 if (orig != data)
7152 WREG32(CG_CLKPIN_CNTL_2, data);
7153
7154 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7155 data &= ~MPLL_CLKOUT_SEL_MASK;
7156 data |= MPLL_CLKOUT_SEL(4);
7157 if (orig != data)
7158 WREG32(MPLL_BYPASSCLK_SEL, data);
7159
7160 orig = data = RREG32(SPLL_CNTL_MODE);
7161 data &= ~SPLL_REFCLK_SEL_MASK;
7162 if (orig != data)
7163 WREG32(SPLL_CNTL_MODE, data);
7164 }
7165 }
7166 } else {
7167 if (orig != data)
7168 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7169 }
7170
7171 orig = data = RREG32_PCIE(PCIE_CNTL2);
7172 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7173 if (orig != data)
7174 WREG32_PCIE(PCIE_CNTL2, data);
7175
7176 if (!disable_l0s) {
7177 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7178 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7179 data = RREG32_PCIE(PCIE_LC_STATUS1);
7180 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7181 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7182 data &= ~LC_L0S_INACTIVITY_MASK;
7183 if (orig != data)
7184 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7185 }
7186 }
7187 }
7188}
Linux 6.x block layer and io_uring tree(s)