projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: switch SI to use radeon_ucode.h
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
0f0de06c
AD		 24#include <linux/firmware.h>
25#include <linux/platform_device.h>
26#include <linux/slab.h>
27#include <linux/module.h>
760285e7 28#include <drm/drmP.h>
43b3cd99
AD		 29#include "radeon.h"
30#include "radeon_asic.h"
760285e7 31#include <drm/radeon_drm.h>
43b3cd99
AD		 32#include "sid.h"
33#include "atom.h"
48c0c902 34#include "si_blit_shaders.h"
bd8cd539 35#include "clearstate_si.h"
a0ceada6 36#include "radeon_ucode.h"
43b3cd99 37
0f0de06c
AD		 38
39MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
40MODULE_FIRMWARE("radeon/TAHITI_me.bin");
41MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
42MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
43MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
44MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
45MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
46MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
47MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
48MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
49MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
50MODULE_FIRMWARE("radeon/VERDE_me.bin");
51MODULE_FIRMWARE("radeon/VERDE_ce.bin");
52MODULE_FIRMWARE("radeon/VERDE_mc.bin");
53MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
bcc7f5d2
AD		 54MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
55MODULE_FIRMWARE("radeon/OLAND_me.bin");
56MODULE_FIRMWARE("radeon/OLAND_ce.bin");
57MODULE_FIRMWARE("radeon/OLAND_mc.bin");
58MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
c04c00b4
AD		 59MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
60MODULE_FIRMWARE("radeon/HAINAN_me.bin");
61MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
62MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
63MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
0f0de06c 64
b9d305df 65static void si_pcie_gen3_enable(struct radeon_device *rdev);
e0bcf165 66static void si_program_aspm(struct radeon_device *rdev);
25a857fb
AD		 67extern int r600_ih_ring_alloc(struct radeon_device *rdev);
68extern void r600_ih_ring_fini(struct radeon_device *rdev);
0a96d72b 69extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
c476dde2
AD		 70extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
71extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
ca7db22b 72extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
1c534671 73extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
014bb209 74extern bool evergreen_is_display_hung(struct radeon_device *rdev);
0a96d72b 75
6d8cf000
AD		 76static const u32 verde_rlc_save_restore_register_list[] =
77{
78 (0x8000 << 16) | (0x98f4 >> 2),
79 0x00000000,
80 (0x8040 << 16) | (0x98f4 >> 2),
81 0x00000000,
82 (0x8000 << 16) | (0xe80 >> 2),
83 0x00000000,
84 (0x8040 << 16) | (0xe80 >> 2),
85 0x00000000,
86 (0x8000 << 16) | (0x89bc >> 2),
87 0x00000000,
88 (0x8040 << 16) | (0x89bc >> 2),
89 0x00000000,
90 (0x8000 << 16) | (0x8c1c >> 2),
91 0x00000000,
92 (0x8040 << 16) | (0x8c1c >> 2),
93 0x00000000,
94 (0x9c00 << 16) | (0x98f0 >> 2),
95 0x00000000,
96 (0x9c00 << 16) | (0xe7c >> 2),
97 0x00000000,
98 (0x8000 << 16) | (0x9148 >> 2),
99 0x00000000,
100 (0x8040 << 16) | (0x9148 >> 2),
101 0x00000000,
102 (0x9c00 << 16) | (0x9150 >> 2),
103 0x00000000,
104 (0x9c00 << 16) | (0x897c >> 2),
105 0x00000000,
106 (0x9c00 << 16) | (0x8d8c >> 2),
107 0x00000000,
108 (0x9c00 << 16) | (0xac54 >> 2),
109 0X00000000,
110 0x3,
111 (0x9c00 << 16) | (0x98f8 >> 2),
112 0x00000000,
113 (0x9c00 << 16) | (0x9910 >> 2),
114 0x00000000,
115 (0x9c00 << 16) | (0x9914 >> 2),
116 0x00000000,
117 (0x9c00 << 16) | (0x9918 >> 2),
118 0x00000000,
119 (0x9c00 << 16) | (0x991c >> 2),
120 0x00000000,
121 (0x9c00 << 16) | (0x9920 >> 2),
122 0x00000000,
123 (0x9c00 << 16) | (0x9924 >> 2),
124 0x00000000,
125 (0x9c00 << 16) | (0x9928 >> 2),
126 0x00000000,
127 (0x9c00 << 16) | (0x992c >> 2),
128 0x00000000,
129 (0x9c00 << 16) | (0x9930 >> 2),
130 0x00000000,
131 (0x9c00 << 16) | (0x9934 >> 2),
132 0x00000000,
133 (0x9c00 << 16) | (0x9938 >> 2),
134 0x00000000,
135 (0x9c00 << 16) | (0x993c >> 2),
136 0x00000000,
137 (0x9c00 << 16) | (0x9940 >> 2),
138 0x00000000,
139 (0x9c00 << 16) | (0x9944 >> 2),
140 0x00000000,
141 (0x9c00 << 16) | (0x9948 >> 2),
142 0x00000000,
143 (0x9c00 << 16) | (0x994c >> 2),
144 0x00000000,
145 (0x9c00 << 16) | (0x9950 >> 2),
146 0x00000000,
147 (0x9c00 << 16) | (0x9954 >> 2),
148 0x00000000,
149 (0x9c00 << 16) | (0x9958 >> 2),
150 0x00000000,
151 (0x9c00 << 16) | (0x995c >> 2),
152 0x00000000,
153 (0x9c00 << 16) | (0x9960 >> 2),
154 0x00000000,
155 (0x9c00 << 16) | (0x9964 >> 2),
156 0x00000000,
157 (0x9c00 << 16) | (0x9968 >> 2),
158 0x00000000,
159 (0x9c00 << 16) | (0x996c >> 2),
160 0x00000000,
161 (0x9c00 << 16) | (0x9970 >> 2),
162 0x00000000,
163 (0x9c00 << 16) | (0x9974 >> 2),
164 0x00000000,
165 (0x9c00 << 16) | (0x9978 >> 2),
166 0x00000000,
167 (0x9c00 << 16) | (0x997c >> 2),
168 0x00000000,
169 (0x9c00 << 16) | (0x9980 >> 2),
170 0x00000000,
171 (0x9c00 << 16) | (0x9984 >> 2),
172 0x00000000,
173 (0x9c00 << 16) | (0x9988 >> 2),
174 0x00000000,
175 (0x9c00 << 16) | (0x998c >> 2),
176 0x00000000,
177 (0x9c00 << 16) | (0x8c00 >> 2),
178 0x00000000,
179 (0x9c00 << 16) | (0x8c14 >> 2),
180 0x00000000,
181 (0x9c00 << 16) | (0x8c04 >> 2),
182 0x00000000,
183 (0x9c00 << 16) | (0x8c08 >> 2),
184 0x00000000,
185 (0x8000 << 16) | (0x9b7c >> 2),
186 0x00000000,
187 (0x8040 << 16) | (0x9b7c >> 2),
188 0x00000000,
189 (0x8000 << 16) | (0xe84 >> 2),
190 0x00000000,
191 (0x8040 << 16) | (0xe84 >> 2),
192 0x00000000,
193 (0x8000 << 16) | (0x89c0 >> 2),
194 0x00000000,
195 (0x8040 << 16) | (0x89c0 >> 2),
196 0x00000000,
197 (0x8000 << 16) | (0x914c >> 2),
198 0x00000000,
199 (0x8040 << 16) | (0x914c >> 2),
200 0x00000000,
201 (0x8000 << 16) | (0x8c20 >> 2),
202 0x00000000,
203 (0x8040 << 16) | (0x8c20 >> 2),
204 0x00000000,
205 (0x8000 << 16) | (0x9354 >> 2),
206 0x00000000,
207 (0x8040 << 16) | (0x9354 >> 2),
208 0x00000000,
209 (0x9c00 << 16) | (0x9060 >> 2),
210 0x00000000,
211 (0x9c00 << 16) | (0x9364 >> 2),
212 0x00000000,
213 (0x9c00 << 16) | (0x9100 >> 2),
214 0x00000000,
215 (0x9c00 << 16) | (0x913c >> 2),
216 0x00000000,
217 (0x8000 << 16) | (0x90e0 >> 2),
218 0x00000000,
219 (0x8000 << 16) | (0x90e4 >> 2),
220 0x00000000,
221 (0x8000 << 16) | (0x90e8 >> 2),
222 0x00000000,
223 (0x8040 << 16) | (0x90e0 >> 2),
224 0x00000000,
225 (0x8040 << 16) | (0x90e4 >> 2),
226 0x00000000,
227 (0x8040 << 16) | (0x90e8 >> 2),
228 0x00000000,
229 (0x9c00 << 16) | (0x8bcc >> 2),
230 0x00000000,
231 (0x9c00 << 16) | (0x8b24 >> 2),
232 0x00000000,
233 (0x9c00 << 16) | (0x88c4 >> 2),
234 0x00000000,
235 (0x9c00 << 16) | (0x8e50 >> 2),
236 0x00000000,
237 (0x9c00 << 16) | (0x8c0c >> 2),
238 0x00000000,
239 (0x9c00 << 16) | (0x8e58 >> 2),
240 0x00000000,
241 (0x9c00 << 16) | (0x8e5c >> 2),
242 0x00000000,
243 (0x9c00 << 16) | (0x9508 >> 2),
244 0x00000000,
245 (0x9c00 << 16) | (0x950c >> 2),
246 0x00000000,
247 (0x9c00 << 16) | (0x9494 >> 2),
248 0x00000000,
249 (0x9c00 << 16) | (0xac0c >> 2),
250 0x00000000,
251 (0x9c00 << 16) | (0xac10 >> 2),
252 0x00000000,
253 (0x9c00 << 16) | (0xac14 >> 2),
254 0x00000000,
255 (0x9c00 << 16) | (0xae00 >> 2),
256 0x00000000,
257 (0x9c00 << 16) | (0xac08 >> 2),
258 0x00000000,
259 (0x9c00 << 16) | (0x88d4 >> 2),
260 0x00000000,
261 (0x9c00 << 16) | (0x88c8 >> 2),
262 0x00000000,
263 (0x9c00 << 16) | (0x88cc >> 2),
264 0x00000000,
265 (0x9c00 << 16) | (0x89b0 >> 2),
266 0x00000000,
267 (0x9c00 << 16) | (0x8b10 >> 2),
268 0x00000000,
269 (0x9c00 << 16) | (0x8a14 >> 2),
270 0x00000000,
271 (0x9c00 << 16) | (0x9830 >> 2),
272 0x00000000,
273 (0x9c00 << 16) | (0x9834 >> 2),
274 0x00000000,
275 (0x9c00 << 16) | (0x9838 >> 2),
276 0x00000000,
277 (0x9c00 << 16) | (0x9a10 >> 2),
278 0x00000000,
279 (0x8000 << 16) | (0x9870 >> 2),
280 0x00000000,
281 (0x8000 << 16) | (0x9874 >> 2),
282 0x00000000,
283 (0x8001 << 16) | (0x9870 >> 2),
284 0x00000000,
285 (0x8001 << 16) | (0x9874 >> 2),
286 0x00000000,
287 (0x8040 << 16) | (0x9870 >> 2),
288 0x00000000,
289 (0x8040 << 16) | (0x9874 >> 2),
290 0x00000000,
291 (0x8041 << 16) | (0x9870 >> 2),
292 0x00000000,
293 (0x8041 << 16) | (0x9874 >> 2),
294 0x00000000,
295 0x00000000
296};
297
205996c0
AD		 298static const u32 tahiti_golden_rlc_registers[] =
299{
300 0xc424, 0xffffffff, 0x00601005,
301 0xc47c, 0xffffffff, 0x10104040,
302 0xc488, 0xffffffff, 0x0100000a,
303 0xc314, 0xffffffff, 0x00000800,
304 0xc30c, 0xffffffff, 0x800000f4,
305 0xf4a8, 0xffffffff, 0x00000000
306};
307
308static const u32 tahiti_golden_registers[] =
309{
310 0x9a10, 0x00010000, 0x00018208,
311 0x9830, 0xffffffff, 0x00000000,
312 0x9834, 0xf00fffff, 0x00000400,
313 0x9838, 0x0002021c, 0x00020200,
314 0xc78, 0x00000080, 0x00000000,
315 0xd030, 0x000300c0, 0x00800040,
316 0xd830, 0x000300c0, 0x00800040,
317 0x5bb0, 0x000000f0, 0x00000070,
318 0x5bc0, 0x00200000, 0x50100000,
319 0x7030, 0x31000311, 0x00000011,
320 0x277c, 0x00000003, 0x000007ff,
321 0x240c, 0x000007ff, 0x00000000,
322 0x8a14, 0xf000001f, 0x00000007,
323 0x8b24, 0xffffffff, 0x00ffffff,
324 0x8b10, 0x0000ff0f, 0x00000000,
325 0x28a4c, 0x07ffffff, 0x4e000000,
326 0x28350, 0x3f3f3fff, 0x2a00126a,
327 0x30, 0x000000ff, 0x0040,
328 0x34, 0x00000040, 0x00004040,
329 0x9100, 0x07ffffff, 0x03000000,
330 0x8e88, 0x01ff1f3f, 0x00000000,
331 0x8e84, 0x01ff1f3f, 0x00000000,
332 0x9060, 0x0000007f, 0x00000020,
333 0x9508, 0x00010000, 0x00010000,
334 0xac14, 0x00000200, 0x000002fb,
335 0xac10, 0xffffffff, 0x0000543b,
336 0xac0c, 0xffffffff, 0xa9210876,
337 0x88d0, 0xffffffff, 0x000fff40,
338 0x88d4, 0x0000001f, 0x00000010,
339 0x1410, 0x20000000, 0x20fffed8,
340 0x15c0, 0x000c0fc0, 0x000c0400
341};
342
343static const u32 tahiti_golden_registers2[] =
344{
345 0xc64, 0x00000001, 0x00000001
346};
347
348static const u32 pitcairn_golden_rlc_registers[] =
349{
350 0xc424, 0xffffffff, 0x00601004,
351 0xc47c, 0xffffffff, 0x10102020,
352 0xc488, 0xffffffff, 0x01000020,
353 0xc314, 0xffffffff, 0x00000800,
354 0xc30c, 0xffffffff, 0x800000a4
355};
356
357static const u32 pitcairn_golden_registers[] =
358{
359 0x9a10, 0x00010000, 0x00018208,
360 0x9830, 0xffffffff, 0x00000000,
361 0x9834, 0xf00fffff, 0x00000400,
362 0x9838, 0x0002021c, 0x00020200,
363 0xc78, 0x00000080, 0x00000000,
364 0xd030, 0x000300c0, 0x00800040,
365 0xd830, 0x000300c0, 0x00800040,
366 0x5bb0, 0x000000f0, 0x00000070,
367 0x5bc0, 0x00200000, 0x50100000,
368 0x7030, 0x31000311, 0x00000011,
369 0x2ae4, 0x00073ffe, 0x000022a2,
370 0x240c, 0x000007ff, 0x00000000,
371 0x8a14, 0xf000001f, 0x00000007,
372 0x8b24, 0xffffffff, 0x00ffffff,
373 0x8b10, 0x0000ff0f, 0x00000000,
374 0x28a4c, 0x07ffffff, 0x4e000000,
375 0x28350, 0x3f3f3fff, 0x2a00126a,
376 0x30, 0x000000ff, 0x0040,
377 0x34, 0x00000040, 0x00004040,
378 0x9100, 0x07ffffff, 0x03000000,
379 0x9060, 0x0000007f, 0x00000020,
380 0x9508, 0x00010000, 0x00010000,
381 0xac14, 0x000003ff, 0x000000f7,
382 0xac10, 0xffffffff, 0x00000000,
383 0xac0c, 0xffffffff, 0x32761054,
384 0x88d4, 0x0000001f, 0x00000010,
385 0x15c0, 0x000c0fc0, 0x000c0400
386};
387
388static const u32 verde_golden_rlc_registers[] =
389{
390 0xc424, 0xffffffff, 0x033f1005,
391 0xc47c, 0xffffffff, 0x10808020,
392 0xc488, 0xffffffff, 0x00800008,
393 0xc314, 0xffffffff, 0x00001000,
394 0xc30c, 0xffffffff, 0x80010014
395};
396
397static const u32 verde_golden_registers[] =
398{
399 0x9a10, 0x00010000, 0x00018208,
400 0x9830, 0xffffffff, 0x00000000,
401 0x9834, 0xf00fffff, 0x00000400,
402 0x9838, 0x0002021c, 0x00020200,
403 0xc78, 0x00000080, 0x00000000,
404 0xd030, 0x000300c0, 0x00800040,
405 0xd030, 0x000300c0, 0x00800040,
406 0xd830, 0x000300c0, 0x00800040,
407 0xd830, 0x000300c0, 0x00800040,
408 0x5bb0, 0x000000f0, 0x00000070,
409 0x5bc0, 0x00200000, 0x50100000,
410 0x7030, 0x31000311, 0x00000011,
411 0x2ae4, 0x00073ffe, 0x000022a2,
412 0x2ae4, 0x00073ffe, 0x000022a2,
413 0x2ae4, 0x00073ffe, 0x000022a2,
414 0x240c, 0x000007ff, 0x00000000,
415 0x240c, 0x000007ff, 0x00000000,
416 0x240c, 0x000007ff, 0x00000000,
417 0x8a14, 0xf000001f, 0x00000007,
418 0x8a14, 0xf000001f, 0x00000007,
419 0x8a14, 0xf000001f, 0x00000007,
420 0x8b24, 0xffffffff, 0x00ffffff,
421 0x8b10, 0x0000ff0f, 0x00000000,
422 0x28a4c, 0x07ffffff, 0x4e000000,
423 0x28350, 0x3f3f3fff, 0x0000124a,
424 0x28350, 0x3f3f3fff, 0x0000124a,
425 0x28350, 0x3f3f3fff, 0x0000124a,
426 0x30, 0x000000ff, 0x0040,
427 0x34, 0x00000040, 0x00004040,
428 0x9100, 0x07ffffff, 0x03000000,
429 0x9100, 0x07ffffff, 0x03000000,
430 0x8e88, 0x01ff1f3f, 0x00000000,
431 0x8e88, 0x01ff1f3f, 0x00000000,
432 0x8e88, 0x01ff1f3f, 0x00000000,
433 0x8e84, 0x01ff1f3f, 0x00000000,
434 0x8e84, 0x01ff1f3f, 0x00000000,
435 0x8e84, 0x01ff1f3f, 0x00000000,
436 0x9060, 0x0000007f, 0x00000020,
437 0x9508, 0x00010000, 0x00010000,
438 0xac14, 0x000003ff, 0x00000003,
439 0xac14, 0x000003ff, 0x00000003,
440 0xac14, 0x000003ff, 0x00000003,
441 0xac10, 0xffffffff, 0x00000000,
442 0xac10, 0xffffffff, 0x00000000,
443 0xac10, 0xffffffff, 0x00000000,
444 0xac0c, 0xffffffff, 0x00001032,
445 0xac0c, 0xffffffff, 0x00001032,
446 0xac0c, 0xffffffff, 0x00001032,
447 0x88d4, 0x0000001f, 0x00000010,
448 0x88d4, 0x0000001f, 0x00000010,
449 0x88d4, 0x0000001f, 0x00000010,
450 0x15c0, 0x000c0fc0, 0x000c0400
451};
452
453static const u32 oland_golden_rlc_registers[] =
454{
455 0xc424, 0xffffffff, 0x00601005,
456 0xc47c, 0xffffffff, 0x10104040,
457 0xc488, 0xffffffff, 0x0100000a,
458 0xc314, 0xffffffff, 0x00000800,
459 0xc30c, 0xffffffff, 0x800000f4
460};
461
462static const u32 oland_golden_registers[] =
463{
464 0x9a10, 0x00010000, 0x00018208,
465 0x9830, 0xffffffff, 0x00000000,
466 0x9834, 0xf00fffff, 0x00000400,
467 0x9838, 0x0002021c, 0x00020200,
468 0xc78, 0x00000080, 0x00000000,
469 0xd030, 0x000300c0, 0x00800040,
470 0xd830, 0x000300c0, 0x00800040,
471 0x5bb0, 0x000000f0, 0x00000070,
472 0x5bc0, 0x00200000, 0x50100000,
473 0x7030, 0x31000311, 0x00000011,
474 0x2ae4, 0x00073ffe, 0x000022a2,
475 0x240c, 0x000007ff, 0x00000000,
476 0x8a14, 0xf000001f, 0x00000007,
477 0x8b24, 0xffffffff, 0x00ffffff,
478 0x8b10, 0x0000ff0f, 0x00000000,
479 0x28a4c, 0x07ffffff, 0x4e000000,
480 0x28350, 0x3f3f3fff, 0x00000082,
481 0x30, 0x000000ff, 0x0040,
482 0x34, 0x00000040, 0x00004040,
483 0x9100, 0x07ffffff, 0x03000000,
484 0x9060, 0x0000007f, 0x00000020,
485 0x9508, 0x00010000, 0x00010000,
486 0xac14, 0x000003ff, 0x000000f3,
487 0xac10, 0xffffffff, 0x00000000,
488 0xac0c, 0xffffffff, 0x00003210,
489 0x88d4, 0x0000001f, 0x00000010,
490 0x15c0, 0x000c0fc0, 0x000c0400
491};
492
fffbdda4
AD		 493static const u32 hainan_golden_registers[] =
494{
495 0x9a10, 0x00010000, 0x00018208,
496 0x9830, 0xffffffff, 0x00000000,
497 0x9834, 0xf00fffff, 0x00000400,
498 0x9838, 0x0002021c, 0x00020200,
499 0xd0c0, 0xff000fff, 0x00000100,
500 0xd030, 0x000300c0, 0x00800040,
501 0xd8c0, 0xff000fff, 0x00000100,
502 0xd830, 0x000300c0, 0x00800040,
503 0x2ae4, 0x00073ffe, 0x000022a2,
504 0x240c, 0x000007ff, 0x00000000,
505 0x8a14, 0xf000001f, 0x00000007,
506 0x8b24, 0xffffffff, 0x00ffffff,
507 0x8b10, 0x0000ff0f, 0x00000000,
508 0x28a4c, 0x07ffffff, 0x4e000000,
509 0x28350, 0x3f3f3fff, 0x00000000,
510 0x30, 0x000000ff, 0x0040,
511 0x34, 0x00000040, 0x00004040,
512 0x9100, 0x03e00000, 0x03600000,
513 0x9060, 0x0000007f, 0x00000020,
514 0x9508, 0x00010000, 0x00010000,
515 0xac14, 0x000003ff, 0x000000f1,
516 0xac10, 0xffffffff, 0x00000000,
517 0xac0c, 0xffffffff, 0x00003210,
518 0x88d4, 0x0000001f, 0x00000010,
519 0x15c0, 0x000c0fc0, 0x000c0400
520};
521
522static const u32 hainan_golden_registers2[] =
523{
524 0x98f8, 0xffffffff, 0x02010001
525};
526
205996c0
AD		 527static const u32 tahiti_mgcg_cgcg_init[] =
528{
529 0xc400, 0xffffffff, 0xfffffffc,
530 0x802c, 0xffffffff, 0xe0000000,
531 0x9a60, 0xffffffff, 0x00000100,
532 0x92a4, 0xffffffff, 0x00000100,
533 0xc164, 0xffffffff, 0x00000100,
534 0x9774, 0xffffffff, 0x00000100,
535 0x8984, 0xffffffff, 0x06000100,
536 0x8a18, 0xffffffff, 0x00000100,
537 0x92a0, 0xffffffff, 0x00000100,
538 0xc380, 0xffffffff, 0x00000100,
539 0x8b28, 0xffffffff, 0x00000100,
540 0x9144, 0xffffffff, 0x00000100,
541 0x8d88, 0xffffffff, 0x00000100,
542 0x8d8c, 0xffffffff, 0x00000100,
543 0x9030, 0xffffffff, 0x00000100,
544 0x9034, 0xffffffff, 0x00000100,
545 0x9038, 0xffffffff, 0x00000100,
546 0x903c, 0xffffffff, 0x00000100,
547 0xad80, 0xffffffff, 0x00000100,
548 0xac54, 0xffffffff, 0x00000100,
549 0x897c, 0xffffffff, 0x06000100,
550 0x9868, 0xffffffff, 0x00000100,
551 0x9510, 0xffffffff, 0x00000100,
552 0xaf04, 0xffffffff, 0x00000100,
553 0xae04, 0xffffffff, 0x00000100,
554 0x949c, 0xffffffff, 0x00000100,
555 0x802c, 0xffffffff, 0xe0000000,
556 0x9160, 0xffffffff, 0x00010000,
557 0x9164, 0xffffffff, 0x00030002,
558 0x9168, 0xffffffff, 0x00040007,
559 0x916c, 0xffffffff, 0x00060005,
560 0x9170, 0xffffffff, 0x00090008,
561 0x9174, 0xffffffff, 0x00020001,
562 0x9178, 0xffffffff, 0x00040003,
563 0x917c, 0xffffffff, 0x00000007,
564 0x9180, 0xffffffff, 0x00060005,
565 0x9184, 0xffffffff, 0x00090008,
566 0x9188, 0xffffffff, 0x00030002,
567 0x918c, 0xffffffff, 0x00050004,
568 0x9190, 0xffffffff, 0x00000008,
569 0x9194, 0xffffffff, 0x00070006,
570 0x9198, 0xffffffff, 0x000a0009,
571 0x919c, 0xffffffff, 0x00040003,
572 0x91a0, 0xffffffff, 0x00060005,
573 0x91a4, 0xffffffff, 0x00000009,
574 0x91a8, 0xffffffff, 0x00080007,
575 0x91ac, 0xffffffff, 0x000b000a,
576 0x91b0, 0xffffffff, 0x00050004,
577 0x91b4, 0xffffffff, 0x00070006,
578 0x91b8, 0xffffffff, 0x0008000b,
579 0x91bc, 0xffffffff, 0x000a0009,
580 0x91c0, 0xffffffff, 0x000d000c,
581 0x91c4, 0xffffffff, 0x00060005,
582 0x91c8, 0xffffffff, 0x00080007,
583 0x91cc, 0xffffffff, 0x0000000b,
584 0x91d0, 0xffffffff, 0x000a0009,
585 0x91d4, 0xffffffff, 0x000d000c,
586 0x91d8, 0xffffffff, 0x00070006,
587 0x91dc, 0xffffffff, 0x00090008,
588 0x91e0, 0xffffffff, 0x0000000c,
589 0x91e4, 0xffffffff, 0x000b000a,
590 0x91e8, 0xffffffff, 0x000e000d,
591 0x91ec, 0xffffffff, 0x00080007,
592 0x91f0, 0xffffffff, 0x000a0009,
593 0x91f4, 0xffffffff, 0x0000000d,
594 0x91f8, 0xffffffff, 0x000c000b,
595 0x91fc, 0xffffffff, 0x000f000e,
596 0x9200, 0xffffffff, 0x00090008,
597 0x9204, 0xffffffff, 0x000b000a,
598 0x9208, 0xffffffff, 0x000c000f,
599 0x920c, 0xffffffff, 0x000e000d,
600 0x9210, 0xffffffff, 0x00110010,
601 0x9214, 0xffffffff, 0x000a0009,
602 0x9218, 0xffffffff, 0x000c000b,
603 0x921c, 0xffffffff, 0x0000000f,
604 0x9220, 0xffffffff, 0x000e000d,
605 0x9224, 0xffffffff, 0x00110010,
606 0x9228, 0xffffffff, 0x000b000a,
607 0x922c, 0xffffffff, 0x000d000c,
608 0x9230, 0xffffffff, 0x00000010,
609 0x9234, 0xffffffff, 0x000f000e,
610 0x9238, 0xffffffff, 0x00120011,
611 0x923c, 0xffffffff, 0x000c000b,
612 0x9240, 0xffffffff, 0x000e000d,
613 0x9244, 0xffffffff, 0x00000011,
614 0x9248, 0xffffffff, 0x0010000f,
615 0x924c, 0xffffffff, 0x00130012,
616 0x9250, 0xffffffff, 0x000d000c,
617 0x9254, 0xffffffff, 0x000f000e,
618 0x9258, 0xffffffff, 0x00100013,
619 0x925c, 0xffffffff, 0x00120011,
620 0x9260, 0xffffffff, 0x00150014,
621 0x9264, 0xffffffff, 0x000e000d,
622 0x9268, 0xffffffff, 0x0010000f,
623 0x926c, 0xffffffff, 0x00000013,
624 0x9270, 0xffffffff, 0x00120011,
625 0x9274, 0xffffffff, 0x00150014,
626 0x9278, 0xffffffff, 0x000f000e,
627 0x927c, 0xffffffff, 0x00110010,
628 0x9280, 0xffffffff, 0x00000014,
629 0x9284, 0xffffffff, 0x00130012,
630 0x9288, 0xffffffff, 0x00160015,
631 0x928c, 0xffffffff, 0x0010000f,
632 0x9290, 0xffffffff, 0x00120011,
633 0x9294, 0xffffffff, 0x00000015,
634 0x9298, 0xffffffff, 0x00140013,
635 0x929c, 0xffffffff, 0x00170016,
636 0x9150, 0xffffffff, 0x96940200,
637 0x8708, 0xffffffff, 0x00900100,
638 0xc478, 0xffffffff, 0x00000080,
639 0xc404, 0xffffffff, 0x0020003f,
640 0x30, 0xffffffff, 0x0000001c,
641 0x34, 0x000f0000, 0x000f0000,
642 0x160c, 0xffffffff, 0x00000100,
643 0x1024, 0xffffffff, 0x00000100,
644 0x102c, 0x00000101, 0x00000000,
645 0x20a8, 0xffffffff, 0x00000104,
646 0x264c, 0x000c0000, 0x000c0000,
647 0x2648, 0x000c0000, 0x000c0000,
648 0x55e4, 0xff000fff, 0x00000100,
649 0x55e8, 0x00000001, 0x00000001,
650 0x2f50, 0x00000001, 0x00000001,
651 0x30cc, 0xc0000fff, 0x00000104,
652 0xc1e4, 0x00000001, 0x00000001,
653 0xd0c0, 0xfffffff0, 0x00000100,
654 0xd8c0, 0xfffffff0, 0x00000100
655};
656
657static const u32 pitcairn_mgcg_cgcg_init[] =
658{
659 0xc400, 0xffffffff, 0xfffffffc,
660 0x802c, 0xffffffff, 0xe0000000,
661 0x9a60, 0xffffffff, 0x00000100,
662 0x92a4, 0xffffffff, 0x00000100,
663 0xc164, 0xffffffff, 0x00000100,
664 0x9774, 0xffffffff, 0x00000100,
665 0x8984, 0xffffffff, 0x06000100,
666 0x8a18, 0xffffffff, 0x00000100,
667 0x92a0, 0xffffffff, 0x00000100,
668 0xc380, 0xffffffff, 0x00000100,
669 0x8b28, 0xffffffff, 0x00000100,
670 0x9144, 0xffffffff, 0x00000100,
671 0x8d88, 0xffffffff, 0x00000100,
672 0x8d8c, 0xffffffff, 0x00000100,
673 0x9030, 0xffffffff, 0x00000100,
674 0x9034, 0xffffffff, 0x00000100,
675 0x9038, 0xffffffff, 0x00000100,
676 0x903c, 0xffffffff, 0x00000100,
677 0xad80, 0xffffffff, 0x00000100,
678 0xac54, 0xffffffff, 0x00000100,
679 0x897c, 0xffffffff, 0x06000100,
680 0x9868, 0xffffffff, 0x00000100,
681 0x9510, 0xffffffff, 0x00000100,
682 0xaf04, 0xffffffff, 0x00000100,
683 0xae04, 0xffffffff, 0x00000100,
684 0x949c, 0xffffffff, 0x00000100,
685 0x802c, 0xffffffff, 0xe0000000,
686 0x9160, 0xffffffff, 0x00010000,
687 0x9164, 0xffffffff, 0x00030002,
688 0x9168, 0xffffffff, 0x00040007,
689 0x916c, 0xffffffff, 0x00060005,
690 0x9170, 0xffffffff, 0x00090008,
691 0x9174, 0xffffffff, 0x00020001,
692 0x9178, 0xffffffff, 0x00040003,
693 0x917c, 0xffffffff, 0x00000007,
694 0x9180, 0xffffffff, 0x00060005,
695 0x9184, 0xffffffff, 0x00090008,
696 0x9188, 0xffffffff, 0x00030002,
697 0x918c, 0xffffffff, 0x00050004,
698 0x9190, 0xffffffff, 0x00000008,
699 0x9194, 0xffffffff, 0x00070006,
700 0x9198, 0xffffffff, 0x000a0009,
701 0x919c, 0xffffffff, 0x00040003,
702 0x91a0, 0xffffffff, 0x00060005,
703 0x91a4, 0xffffffff, 0x00000009,
704 0x91a8, 0xffffffff, 0x00080007,
705 0x91ac, 0xffffffff, 0x000b000a,
706 0x91b0, 0xffffffff, 0x00050004,
707 0x91b4, 0xffffffff, 0x00070006,
708 0x91b8, 0xffffffff, 0x0008000b,
709 0x91bc, 0xffffffff, 0x000a0009,
710 0x91c0, 0xffffffff, 0x000d000c,
711 0x9200, 0xffffffff, 0x00090008,
712 0x9204, 0xffffffff, 0x000b000a,
713 0x9208, 0xffffffff, 0x000c000f,
714 0x920c, 0xffffffff, 0x000e000d,
715 0x9210, 0xffffffff, 0x00110010,
716 0x9214, 0xffffffff, 0x000a0009,
717 0x9218, 0xffffffff, 0x000c000b,
718 0x921c, 0xffffffff, 0x0000000f,
719 0x9220, 0xffffffff, 0x000e000d,
720 0x9224, 0xffffffff, 0x00110010,
721 0x9228, 0xffffffff, 0x000b000a,
722 0x922c, 0xffffffff, 0x000d000c,
723 0x9230, 0xffffffff, 0x00000010,
724 0x9234, 0xffffffff, 0x000f000e,
725 0x9238, 0xffffffff, 0x00120011,
726 0x923c, 0xffffffff, 0x000c000b,
727 0x9240, 0xffffffff, 0x000e000d,
728 0x9244, 0xffffffff, 0x00000011,
729 0x9248, 0xffffffff, 0x0010000f,
730 0x924c, 0xffffffff, 0x00130012,
731 0x9250, 0xffffffff, 0x000d000c,
732 0x9254, 0xffffffff, 0x000f000e,
733 0x9258, 0xffffffff, 0x00100013,
734 0x925c, 0xffffffff, 0x00120011,
735 0x9260, 0xffffffff, 0x00150014,
736 0x9150, 0xffffffff, 0x96940200,
737 0x8708, 0xffffffff, 0x00900100,
738 0xc478, 0xffffffff, 0x00000080,
739 0xc404, 0xffffffff, 0x0020003f,
740 0x30, 0xffffffff, 0x0000001c,
741 0x34, 0x000f0000, 0x000f0000,
742 0x160c, 0xffffffff, 0x00000100,
743 0x1024, 0xffffffff, 0x00000100,
744 0x102c, 0x00000101, 0x00000000,
745 0x20a8, 0xffffffff, 0x00000104,
746 0x55e4, 0xff000fff, 0x00000100,
747 0x55e8, 0x00000001, 0x00000001,
748 0x2f50, 0x00000001, 0x00000001,
749 0x30cc, 0xc0000fff, 0x00000104,
750 0xc1e4, 0x00000001, 0x00000001,
751 0xd0c0, 0xfffffff0, 0x00000100,
752 0xd8c0, 0xfffffff0, 0x00000100
753};
754
755static const u32 verde_mgcg_cgcg_init[] =
756{
757 0xc400, 0xffffffff, 0xfffffffc,
758 0x802c, 0xffffffff, 0xe0000000,
759 0x9a60, 0xffffffff, 0x00000100,
760 0x92a4, 0xffffffff, 0x00000100,
761 0xc164, 0xffffffff, 0x00000100,
762 0x9774, 0xffffffff, 0x00000100,
763 0x8984, 0xffffffff, 0x06000100,
764 0x8a18, 0xffffffff, 0x00000100,
765 0x92a0, 0xffffffff, 0x00000100,
766 0xc380, 0xffffffff, 0x00000100,
767 0x8b28, 0xffffffff, 0x00000100,
768 0x9144, 0xffffffff, 0x00000100,
769 0x8d88, 0xffffffff, 0x00000100,
770 0x8d8c, 0xffffffff, 0x00000100,
771 0x9030, 0xffffffff, 0x00000100,
772 0x9034, 0xffffffff, 0x00000100,
773 0x9038, 0xffffffff, 0x00000100,
774 0x903c, 0xffffffff, 0x00000100,
775 0xad80, 0xffffffff, 0x00000100,
776 0xac54, 0xffffffff, 0x00000100,
777 0x897c, 0xffffffff, 0x06000100,
778 0x9868, 0xffffffff, 0x00000100,
779 0x9510, 0xffffffff, 0x00000100,
780 0xaf04, 0xffffffff, 0x00000100,
781 0xae04, 0xffffffff, 0x00000100,
782 0x949c, 0xffffffff, 0x00000100,
783 0x802c, 0xffffffff, 0xe0000000,
784 0x9160, 0xffffffff, 0x00010000,
785 0x9164, 0xffffffff, 0x00030002,
786 0x9168, 0xffffffff, 0x00040007,
787 0x916c, 0xffffffff, 0x00060005,
788 0x9170, 0xffffffff, 0x00090008,
789 0x9174, 0xffffffff, 0x00020001,
790 0x9178, 0xffffffff, 0x00040003,
791 0x917c, 0xffffffff, 0x00000007,
792 0x9180, 0xffffffff, 0x00060005,
793 0x9184, 0xffffffff, 0x00090008,
794 0x9188, 0xffffffff, 0x00030002,
795 0x918c, 0xffffffff, 0x00050004,
796 0x9190, 0xffffffff, 0x00000008,
797 0x9194, 0xffffffff, 0x00070006,
798 0x9198, 0xffffffff, 0x000a0009,
799 0x919c, 0xffffffff, 0x00040003,
800 0x91a0, 0xffffffff, 0x00060005,
801 0x91a4, 0xffffffff, 0x00000009,
802 0x91a8, 0xffffffff, 0x00080007,
803 0x91ac, 0xffffffff, 0x000b000a,
804 0x91b0, 0xffffffff, 0x00050004,
805 0x91b4, 0xffffffff, 0x00070006,
806 0x91b8, 0xffffffff, 0x0008000b,
807 0x91bc, 0xffffffff, 0x000a0009,
808 0x91c0, 0xffffffff, 0x000d000c,
809 0x9200, 0xffffffff, 0x00090008,
810 0x9204, 0xffffffff, 0x000b000a,
811 0x9208, 0xffffffff, 0x000c000f,
812 0x920c, 0xffffffff, 0x000e000d,
813 0x9210, 0xffffffff, 0x00110010,
814 0x9214, 0xffffffff, 0x000a0009,
815 0x9218, 0xffffffff, 0x000c000b,
816 0x921c, 0xffffffff, 0x0000000f,
817 0x9220, 0xffffffff, 0x000e000d,
818 0x9224, 0xffffffff, 0x00110010,
819 0x9228, 0xffffffff, 0x000b000a,
820 0x922c, 0xffffffff, 0x000d000c,
821 0x9230, 0xffffffff, 0x00000010,
822 0x9234, 0xffffffff, 0x000f000e,
823 0x9238, 0xffffffff, 0x00120011,
824 0x923c, 0xffffffff, 0x000c000b,
825 0x9240, 0xffffffff, 0x000e000d,
826 0x9244, 0xffffffff, 0x00000011,
827 0x9248, 0xffffffff, 0x0010000f,
828 0x924c, 0xffffffff, 0x00130012,
829 0x9250, 0xffffffff, 0x000d000c,
830 0x9254, 0xffffffff, 0x000f000e,
831 0x9258, 0xffffffff, 0x00100013,
832 0x925c, 0xffffffff, 0x00120011,
833 0x9260, 0xffffffff, 0x00150014,
834 0x9150, 0xffffffff, 0x96940200,
835 0x8708, 0xffffffff, 0x00900100,
836 0xc478, 0xffffffff, 0x00000080,
837 0xc404, 0xffffffff, 0x0020003f,
838 0x30, 0xffffffff, 0x0000001c,
839 0x34, 0x000f0000, 0x000f0000,
840 0x160c, 0xffffffff, 0x00000100,
841 0x1024, 0xffffffff, 0x00000100,
842 0x102c, 0x00000101, 0x00000000,
843 0x20a8, 0xffffffff, 0x00000104,
844 0x264c, 0x000c0000, 0x000c0000,
845 0x2648, 0x000c0000, 0x000c0000,
846 0x55e4, 0xff000fff, 0x00000100,
847 0x55e8, 0x00000001, 0x00000001,
848 0x2f50, 0x00000001, 0x00000001,
849 0x30cc, 0xc0000fff, 0x00000104,
850 0xc1e4, 0x00000001, 0x00000001,
851 0xd0c0, 0xfffffff0, 0x00000100,
852 0xd8c0, 0xfffffff0, 0x00000100
853};
854
855static const u32 oland_mgcg_cgcg_init[] =
856{
857 0xc400, 0xffffffff, 0xfffffffc,
858 0x802c, 0xffffffff, 0xe0000000,
859 0x9a60, 0xffffffff, 0x00000100,
860 0x92a4, 0xffffffff, 0x00000100,
861 0xc164, 0xffffffff, 0x00000100,
862 0x9774, 0xffffffff, 0x00000100,
863 0x8984, 0xffffffff, 0x06000100,
864 0x8a18, 0xffffffff, 0x00000100,
865 0x92a0, 0xffffffff, 0x00000100,
866 0xc380, 0xffffffff, 0x00000100,
867 0x8b28, 0xffffffff, 0x00000100,
868 0x9144, 0xffffffff, 0x00000100,
869 0x8d88, 0xffffffff, 0x00000100,
870 0x8d8c, 0xffffffff, 0x00000100,
871 0x9030, 0xffffffff, 0x00000100,
872 0x9034, 0xffffffff, 0x00000100,
873 0x9038, 0xffffffff, 0x00000100,
874 0x903c, 0xffffffff, 0x00000100,
875 0xad80, 0xffffffff, 0x00000100,
876 0xac54, 0xffffffff, 0x00000100,
877 0x897c, 0xffffffff, 0x06000100,
878 0x9868, 0xffffffff, 0x00000100,
879 0x9510, 0xffffffff, 0x00000100,
880 0xaf04, 0xffffffff, 0x00000100,
881 0xae04, 0xffffffff, 0x00000100,
882 0x949c, 0xffffffff, 0x00000100,
883 0x802c, 0xffffffff, 0xe0000000,
884 0x9160, 0xffffffff, 0x00010000,
885 0x9164, 0xffffffff, 0x00030002,
886 0x9168, 0xffffffff, 0x00040007,
887 0x916c, 0xffffffff, 0x00060005,
888 0x9170, 0xffffffff, 0x00090008,
889 0x9174, 0xffffffff, 0x00020001,
890 0x9178, 0xffffffff, 0x00040003,
891 0x917c, 0xffffffff, 0x00000007,
892 0x9180, 0xffffffff, 0x00060005,
893 0x9184, 0xffffffff, 0x00090008,
894 0x9188, 0xffffffff, 0x00030002,
895 0x918c, 0xffffffff, 0x00050004,
896 0x9190, 0xffffffff, 0x00000008,
897 0x9194, 0xffffffff, 0x00070006,
898 0x9198, 0xffffffff, 0x000a0009,
899 0x919c, 0xffffffff, 0x00040003,
900 0x91a0, 0xffffffff, 0x00060005,
901 0x91a4, 0xffffffff, 0x00000009,
902 0x91a8, 0xffffffff, 0x00080007,
903 0x91ac, 0xffffffff, 0x000b000a,
904 0x91b0, 0xffffffff, 0x00050004,
905 0x91b4, 0xffffffff, 0x00070006,
906 0x91b8, 0xffffffff, 0x0008000b,
907 0x91bc, 0xffffffff, 0x000a0009,
908 0x91c0, 0xffffffff, 0x000d000c,
909 0x91c4, 0xffffffff, 0x00060005,
910 0x91c8, 0xffffffff, 0x00080007,
911 0x91cc, 0xffffffff, 0x0000000b,
912 0x91d0, 0xffffffff, 0x000a0009,
913 0x91d4, 0xffffffff, 0x000d000c,
914 0x9150, 0xffffffff, 0x96940200,
915 0x8708, 0xffffffff, 0x00900100,
916 0xc478, 0xffffffff, 0x00000080,
917 0xc404, 0xffffffff, 0x0020003f,
918 0x30, 0xffffffff, 0x0000001c,
919 0x34, 0x000f0000, 0x000f0000,
920 0x160c, 0xffffffff, 0x00000100,
921 0x1024, 0xffffffff, 0x00000100,
922 0x102c, 0x00000101, 0x00000000,
923 0x20a8, 0xffffffff, 0x00000104,
924 0x264c, 0x000c0000, 0x000c0000,
925 0x2648, 0x000c0000, 0x000c0000,
926 0x55e4, 0xff000fff, 0x00000100,
927 0x55e8, 0x00000001, 0x00000001,
928 0x2f50, 0x00000001, 0x00000001,
929 0x30cc, 0xc0000fff, 0x00000104,
930 0xc1e4, 0x00000001, 0x00000001,
931 0xd0c0, 0xfffffff0, 0x00000100,
932 0xd8c0, 0xfffffff0, 0x00000100
933};
934
fffbdda4
AD		 935static const u32 hainan_mgcg_cgcg_init[] =
936{
937 0xc400, 0xffffffff, 0xfffffffc,
938 0x802c, 0xffffffff, 0xe0000000,
939 0x9a60, 0xffffffff, 0x00000100,
940 0x92a4, 0xffffffff, 0x00000100,
941 0xc164, 0xffffffff, 0x00000100,
942 0x9774, 0xffffffff, 0x00000100,
943 0x8984, 0xffffffff, 0x06000100,
944 0x8a18, 0xffffffff, 0x00000100,
945 0x92a0, 0xffffffff, 0x00000100,
946 0xc380, 0xffffffff, 0x00000100,
947 0x8b28, 0xffffffff, 0x00000100,
948 0x9144, 0xffffffff, 0x00000100,
949 0x8d88, 0xffffffff, 0x00000100,
950 0x8d8c, 0xffffffff, 0x00000100,
951 0x9030, 0xffffffff, 0x00000100,
952 0x9034, 0xffffffff, 0x00000100,
953 0x9038, 0xffffffff, 0x00000100,
954 0x903c, 0xffffffff, 0x00000100,
955 0xad80, 0xffffffff, 0x00000100,
956 0xac54, 0xffffffff, 0x00000100,
957 0x897c, 0xffffffff, 0x06000100,
958 0x9868, 0xffffffff, 0x00000100,
959 0x9510, 0xffffffff, 0x00000100,
960 0xaf04, 0xffffffff, 0x00000100,
961 0xae04, 0xffffffff, 0x00000100,
962 0x949c, 0xffffffff, 0x00000100,
963 0x802c, 0xffffffff, 0xe0000000,
964 0x9160, 0xffffffff, 0x00010000,
965 0x9164, 0xffffffff, 0x00030002,
966 0x9168, 0xffffffff, 0x00040007,
967 0x916c, 0xffffffff, 0x00060005,
968 0x9170, 0xffffffff, 0x00090008,
969 0x9174, 0xffffffff, 0x00020001,
970 0x9178, 0xffffffff, 0x00040003,
971 0x917c, 0xffffffff, 0x00000007,
972 0x9180, 0xffffffff, 0x00060005,
973 0x9184, 0xffffffff, 0x00090008,
974 0x9188, 0xffffffff, 0x00030002,
975 0x918c, 0xffffffff, 0x00050004,
976 0x9190, 0xffffffff, 0x00000008,
977 0x9194, 0xffffffff, 0x00070006,
978 0x9198, 0xffffffff, 0x000a0009,
979 0x919c, 0xffffffff, 0x00040003,
980 0x91a0, 0xffffffff, 0x00060005,
981 0x91a4, 0xffffffff, 0x00000009,
982 0x91a8, 0xffffffff, 0x00080007,
983 0x91ac, 0xffffffff, 0x000b000a,
984 0x91b0, 0xffffffff, 0x00050004,
985 0x91b4, 0xffffffff, 0x00070006,
986 0x91b8, 0xffffffff, 0x0008000b,
987 0x91bc, 0xffffffff, 0x000a0009,
988 0x91c0, 0xffffffff, 0x000d000c,
989 0x91c4, 0xffffffff, 0x00060005,
990 0x91c8, 0xffffffff, 0x00080007,
991 0x91cc, 0xffffffff, 0x0000000b,
992 0x91d0, 0xffffffff, 0x000a0009,
993 0x91d4, 0xffffffff, 0x000d000c,
994 0x9150, 0xffffffff, 0x96940200,
995 0x8708, 0xffffffff, 0x00900100,
996 0xc478, 0xffffffff, 0x00000080,
997 0xc404, 0xffffffff, 0x0020003f,
998 0x30, 0xffffffff, 0x0000001c,
999 0x34, 0x000f0000, 0x000f0000,
1000 0x160c, 0xffffffff, 0x00000100,
1001 0x1024, 0xffffffff, 0x00000100,
1002 0x20a8, 0xffffffff, 0x00000104,
1003 0x264c, 0x000c0000, 0x000c0000,
1004 0x2648, 0x000c0000, 0x000c0000,
1005 0x2f50, 0x00000001, 0x00000001,
1006 0x30cc, 0xc0000fff, 0x00000104,
1007 0xc1e4, 0x00000001, 0x00000001,
1008 0xd0c0, 0xfffffff0, 0x00000100,
1009 0xd8c0, 0xfffffff0, 0x00000100
1010};
1011
205996c0
AD		 1012static u32 verde_pg_init[] =
1013{
1014 0x353c, 0xffffffff, 0x40000,
1015 0x3538, 0xffffffff, 0x200010ff,
1016 0x353c, 0xffffffff, 0x0,
1017 0x353c, 0xffffffff, 0x0,
1018 0x353c, 0xffffffff, 0x0,
1019 0x353c, 0xffffffff, 0x0,
1020 0x353c, 0xffffffff, 0x0,
1021 0x353c, 0xffffffff, 0x7007,
1022 0x3538, 0xffffffff, 0x300010ff,
1023 0x353c, 0xffffffff, 0x0,
1024 0x353c, 0xffffffff, 0x0,
1025 0x353c, 0xffffffff, 0x0,
1026 0x353c, 0xffffffff, 0x0,
1027 0x353c, 0xffffffff, 0x0,
1028 0x353c, 0xffffffff, 0x400000,
1029 0x3538, 0xffffffff, 0x100010ff,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x0,
1034 0x353c, 0xffffffff, 0x0,
1035 0x353c, 0xffffffff, 0x120200,
1036 0x3538, 0xffffffff, 0x500010ff,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x0,
1041 0x353c, 0xffffffff, 0x0,
1042 0x353c, 0xffffffff, 0x1e1e16,
1043 0x3538, 0xffffffff, 0x600010ff,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x0,
1048 0x353c, 0xffffffff, 0x0,
1049 0x353c, 0xffffffff, 0x171f1e,
1050 0x3538, 0xffffffff, 0x700010ff,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x0,
1055 0x353c, 0xffffffff, 0x0,
1056 0x353c, 0xffffffff, 0x0,
1057 0x3538, 0xffffffff, 0x9ff,
1058 0x3500, 0xffffffff, 0x0,
1059 0x3504, 0xffffffff, 0x10000800,
1060 0x3504, 0xffffffff, 0xf,
1061 0x3504, 0xffffffff, 0xf,
1062 0x3500, 0xffffffff, 0x4,
1063 0x3504, 0xffffffff, 0x1000051e,
1064 0x3504, 0xffffffff, 0xffff,
1065 0x3504, 0xffffffff, 0xffff,
1066 0x3500, 0xffffffff, 0x8,
1067 0x3504, 0xffffffff, 0x80500,
1068 0x3500, 0xffffffff, 0x12,
1069 0x3504, 0xffffffff, 0x9050c,
1070 0x3500, 0xffffffff, 0x1d,
1071 0x3504, 0xffffffff, 0xb052c,
1072 0x3500, 0xffffffff, 0x2a,
1073 0x3504, 0xffffffff, 0x1053e,
1074 0x3500, 0xffffffff, 0x2d,
1075 0x3504, 0xffffffff, 0x10546,
1076 0x3500, 0xffffffff, 0x30,
1077 0x3504, 0xffffffff, 0xa054e,
1078 0x3500, 0xffffffff, 0x3c,
1079 0x3504, 0xffffffff, 0x1055f,
1080 0x3500, 0xffffffff, 0x3f,
1081 0x3504, 0xffffffff, 0x10567,
1082 0x3500, 0xffffffff, 0x42,
1083 0x3504, 0xffffffff, 0x1056f,
1084 0x3500, 0xffffffff, 0x45,
1085 0x3504, 0xffffffff, 0x10572,
1086 0x3500, 0xffffffff, 0x48,
1087 0x3504, 0xffffffff, 0x20575,
1088 0x3500, 0xffffffff, 0x4c,
1089 0x3504, 0xffffffff, 0x190801,
1090 0x3500, 0xffffffff, 0x67,
1091 0x3504, 0xffffffff, 0x1082a,
1092 0x3500, 0xffffffff, 0x6a,
1093 0x3504, 0xffffffff, 0x1b082d,
1094 0x3500, 0xffffffff, 0x87,
1095 0x3504, 0xffffffff, 0x310851,
1096 0x3500, 0xffffffff, 0xba,
1097 0x3504, 0xffffffff, 0x891,
1098 0x3500, 0xffffffff, 0xbc,
1099 0x3504, 0xffffffff, 0x893,
1100 0x3500, 0xffffffff, 0xbe,
1101 0x3504, 0xffffffff, 0x20895,
1102 0x3500, 0xffffffff, 0xc2,
1103 0x3504, 0xffffffff, 0x20899,
1104 0x3500, 0xffffffff, 0xc6,
1105 0x3504, 0xffffffff, 0x2089d,
1106 0x3500, 0xffffffff, 0xca,
1107 0x3504, 0xffffffff, 0x8a1,
1108 0x3500, 0xffffffff, 0xcc,
1109 0x3504, 0xffffffff, 0x8a3,
1110 0x3500, 0xffffffff, 0xce,
1111 0x3504, 0xffffffff, 0x308a5,
1112 0x3500, 0xffffffff, 0xd3,
1113 0x3504, 0xffffffff, 0x6d08cd,
1114 0x3500, 0xffffffff, 0x142,
1115 0x3504, 0xffffffff, 0x2000095a,
1116 0x3504, 0xffffffff, 0x1,
1117 0x3500, 0xffffffff, 0x144,
1118 0x3504, 0xffffffff, 0x301f095b,
1119 0x3500, 0xffffffff, 0x165,
1120 0x3504, 0xffffffff, 0xc094d,
1121 0x3500, 0xffffffff, 0x173,
1122 0x3504, 0xffffffff, 0xf096d,
1123 0x3500, 0xffffffff, 0x184,
1124 0x3504, 0xffffffff, 0x15097f,
1125 0x3500, 0xffffffff, 0x19b,
1126 0x3504, 0xffffffff, 0xc0998,
1127 0x3500, 0xffffffff, 0x1a9,
1128 0x3504, 0xffffffff, 0x409a7,
1129 0x3500, 0xffffffff, 0x1af,
1130 0x3504, 0xffffffff, 0xcdc,
1131 0x3500, 0xffffffff, 0x1b1,
1132 0x3504, 0xffffffff, 0x800,
1133 0x3508, 0xffffffff, 0x6c9b2000,
1134 0x3510, 0xfc00, 0x2000,
1135 0x3544, 0xffffffff, 0xfc0,
1136 0x28d4, 0x00000100, 0x100
1137};
1138
1139static void si_init_golden_registers(struct radeon_device *rdev)
1140{
1141 switch (rdev->family) {
1142 case CHIP_TAHITI:
1143 radeon_program_register_sequence(rdev,
1144 tahiti_golden_registers,
1145 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1146 radeon_program_register_sequence(rdev,
1147 tahiti_golden_rlc_registers,
1148 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1149 radeon_program_register_sequence(rdev,
1150 tahiti_mgcg_cgcg_init,
1151 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1152 radeon_program_register_sequence(rdev,
1153 tahiti_golden_registers2,
1154 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1155 break;
1156 case CHIP_PITCAIRN:
1157 radeon_program_register_sequence(rdev,
1158 pitcairn_golden_registers,
1159 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1160 radeon_program_register_sequence(rdev,
1161 pitcairn_golden_rlc_registers,
1162 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1163 radeon_program_register_sequence(rdev,
1164 pitcairn_mgcg_cgcg_init,
1165 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1166 break;
1167 case CHIP_VERDE:
1168 radeon_program_register_sequence(rdev,
1169 verde_golden_registers,
1170 (const u32)ARRAY_SIZE(verde_golden_registers));
1171 radeon_program_register_sequence(rdev,
1172 verde_golden_rlc_registers,
1173 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1174 radeon_program_register_sequence(rdev,
1175 verde_mgcg_cgcg_init,
1176 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1177 radeon_program_register_sequence(rdev,
1178 verde_pg_init,
1179 (const u32)ARRAY_SIZE(verde_pg_init));
1180 break;
1181 case CHIP_OLAND:
1182 radeon_program_register_sequence(rdev,
1183 oland_golden_registers,
1184 (const u32)ARRAY_SIZE(oland_golden_registers));
1185 radeon_program_register_sequence(rdev,
1186 oland_golden_rlc_registers,
1187 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1188 radeon_program_register_sequence(rdev,
1189 oland_mgcg_cgcg_init,
1190 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1191 break;
fffbdda4
AD		 1192 case CHIP_HAINAN:
1193 radeon_program_register_sequence(rdev,
1194 hainan_golden_registers,
1195 (const u32)ARRAY_SIZE(hainan_golden_registers));
1196 radeon_program_register_sequence(rdev,
1197 hainan_golden_registers2,
1198 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1199 radeon_program_register_sequence(rdev,
1200 hainan_mgcg_cgcg_init,
1201 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1202 break;
205996c0
AD		 1203 default:
1204 break;
1205 }
1206}
1207
454d2e2a
AD		 1208#define PCIE_BUS_CLK 10000
1209#define TCLK (PCIE_BUS_CLK / 10)
1210
1211/**
1212 * si_get_xclk - get the xclk
1213 *
1214 * @rdev: radeon_device pointer
1215 *
1216 * Returns the reference clock used by the gfx engine
1217 * (SI).
1218 */
1219u32 si_get_xclk(struct radeon_device *rdev)
1220{
1221 u32 reference_clock = rdev->clock.spll.reference_freq;
1222 u32 tmp;
1223
1224 tmp = RREG32(CG_CLKPIN_CNTL_2);
1225 if (tmp & MUX_TCLK_TO_XCLK)
1226 return TCLK;
1227
1228 tmp = RREG32(CG_CLKPIN_CNTL);
1229 if (tmp & XTALIN_DIVIDE)
1230 return reference_clock / 4;
1231
1232 return reference_clock;
1233}
1234
1bd47d2e
AD		 1235/* get temperature in millidegrees */
1236int si_get_temp(struct radeon_device *rdev)
1237{
1238 u32 temp;
1239 int actual_temp = 0;
1240
1241 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1242 CTF_TEMP_SHIFT;
1243
1244 if (temp & 0x200)
1245 actual_temp = 255;
1246 else
1247 actual_temp = temp & 0x1ff;
1248
1249 actual_temp = (actual_temp * 1000);
1250
1251 return actual_temp;
1252}
1253
8b074dd6
AD		 1254#define TAHITI_IO_MC_REGS_SIZE 36
1255
1256static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1257 {0x0000006f, 0x03044000},
1258 {0x00000070, 0x0480c018},
1259 {0x00000071, 0x00000040},
1260 {0x00000072, 0x01000000},
1261 {0x00000074, 0x000000ff},
1262 {0x00000075, 0x00143400},
1263 {0x00000076, 0x08ec0800},
1264 {0x00000077, 0x040000cc},
1265 {0x00000079, 0x00000000},
1266 {0x0000007a, 0x21000409},
1267 {0x0000007c, 0x00000000},
1268 {0x0000007d, 0xe8000000},
1269 {0x0000007e, 0x044408a8},
1270 {0x0000007f, 0x00000003},
1271 {0x00000080, 0x00000000},
1272 {0x00000081, 0x01000000},
1273 {0x00000082, 0x02000000},
1274 {0x00000083, 0x00000000},
1275 {0x00000084, 0xe3f3e4f4},
1276 {0x00000085, 0x00052024},
1277 {0x00000087, 0x00000000},
1278 {0x00000088, 0x66036603},
1279 {0x00000089, 0x01000000},
1280 {0x0000008b, 0x1c0a0000},
1281 {0x0000008c, 0xff010000},
1282 {0x0000008e, 0xffffefff},
1283 {0x0000008f, 0xfff3efff},
1284 {0x00000090, 0xfff3efbf},
1285 {0x00000094, 0x00101101},
1286 {0x00000095, 0x00000fff},
1287 {0x00000096, 0x00116fff},
1288 {0x00000097, 0x60010000},
1289 {0x00000098, 0x10010000},
1290 {0x00000099, 0x00006000},
1291 {0x0000009a, 0x00001000},
1292 {0x0000009f, 0x00a77400}
1293};
1294
1295static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1296 {0x0000006f, 0x03044000},
1297 {0x00000070, 0x0480c018},
1298 {0x00000071, 0x00000040},
1299 {0x00000072, 0x01000000},
1300 {0x00000074, 0x000000ff},
1301 {0x00000075, 0x00143400},
1302 {0x00000076, 0x08ec0800},
1303 {0x00000077, 0x040000cc},
1304 {0x00000079, 0x00000000},
1305 {0x0000007a, 0x21000409},
1306 {0x0000007c, 0x00000000},
1307 {0x0000007d, 0xe8000000},
1308 {0x0000007e, 0x044408a8},
1309 {0x0000007f, 0x00000003},
1310 {0x00000080, 0x00000000},
1311 {0x00000081, 0x01000000},
1312 {0x00000082, 0x02000000},
1313 {0x00000083, 0x00000000},
1314 {0x00000084, 0xe3f3e4f4},
1315 {0x00000085, 0x00052024},
1316 {0x00000087, 0x00000000},
1317 {0x00000088, 0x66036603},
1318 {0x00000089, 0x01000000},
1319 {0x0000008b, 0x1c0a0000},
1320 {0x0000008c, 0xff010000},
1321 {0x0000008e, 0xffffefff},
1322 {0x0000008f, 0xfff3efff},
1323 {0x00000090, 0xfff3efbf},
1324 {0x00000094, 0x00101101},
1325 {0x00000095, 0x00000fff},
1326 {0x00000096, 0x00116fff},
1327 {0x00000097, 0x60010000},
1328 {0x00000098, 0x10010000},
1329 {0x00000099, 0x00006000},
1330 {0x0000009a, 0x00001000},
1331 {0x0000009f, 0x00a47400}
1332};
1333
1334static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1335 {0x0000006f, 0x03044000},
1336 {0x00000070, 0x0480c018},
1337 {0x00000071, 0x00000040},
1338 {0x00000072, 0x01000000},
1339 {0x00000074, 0x000000ff},
1340 {0x00000075, 0x00143400},
1341 {0x00000076, 0x08ec0800},
1342 {0x00000077, 0x040000cc},
1343 {0x00000079, 0x00000000},
1344 {0x0000007a, 0x21000409},
1345 {0x0000007c, 0x00000000},
1346 {0x0000007d, 0xe8000000},
1347 {0x0000007e, 0x044408a8},
1348 {0x0000007f, 0x00000003},
1349 {0x00000080, 0x00000000},
1350 {0x00000081, 0x01000000},
1351 {0x00000082, 0x02000000},
1352 {0x00000083, 0x00000000},
1353 {0x00000084, 0xe3f3e4f4},
1354 {0x00000085, 0x00052024},
1355 {0x00000087, 0x00000000},
1356 {0x00000088, 0x66036603},
1357 {0x00000089, 0x01000000},
1358 {0x0000008b, 0x1c0a0000},
1359 {0x0000008c, 0xff010000},
1360 {0x0000008e, 0xffffefff},
1361 {0x0000008f, 0xfff3efff},
1362 {0x00000090, 0xfff3efbf},
1363 {0x00000094, 0x00101101},
1364 {0x00000095, 0x00000fff},
1365 {0x00000096, 0x00116fff},
1366 {0x00000097, 0x60010000},
1367 {0x00000098, 0x10010000},
1368 {0x00000099, 0x00006000},
1369 {0x0000009a, 0x00001000},
1370 {0x0000009f, 0x00a37400}
1371};
1372
bcc7f5d2
AD		 1373static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1374 {0x0000006f, 0x03044000},
1375 {0x00000070, 0x0480c018},
1376 {0x00000071, 0x00000040},
1377 {0x00000072, 0x01000000},
1378 {0x00000074, 0x000000ff},
1379 {0x00000075, 0x00143400},
1380 {0x00000076, 0x08ec0800},
1381 {0x00000077, 0x040000cc},
1382 {0x00000079, 0x00000000},
1383 {0x0000007a, 0x21000409},
1384 {0x0000007c, 0x00000000},
1385 {0x0000007d, 0xe8000000},
1386 {0x0000007e, 0x044408a8},
1387 {0x0000007f, 0x00000003},
1388 {0x00000080, 0x00000000},
1389 {0x00000081, 0x01000000},
1390 {0x00000082, 0x02000000},
1391 {0x00000083, 0x00000000},
1392 {0x00000084, 0xe3f3e4f4},
1393 {0x00000085, 0x00052024},
1394 {0x00000087, 0x00000000},
1395 {0x00000088, 0x66036603},
1396 {0x00000089, 0x01000000},
1397 {0x0000008b, 0x1c0a0000},
1398 {0x0000008c, 0xff010000},
1399 {0x0000008e, 0xffffefff},
1400 {0x0000008f, 0xfff3efff},
1401 {0x00000090, 0xfff3efbf},
1402 {0x00000094, 0x00101101},
1403 {0x00000095, 0x00000fff},
1404 {0x00000096, 0x00116fff},
1405 {0x00000097, 0x60010000},
1406 {0x00000098, 0x10010000},
1407 {0x00000099, 0x00006000},
1408 {0x0000009a, 0x00001000},
1409 {0x0000009f, 0x00a17730}
1410};
1411
c04c00b4
AD		 1412static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1413 {0x0000006f, 0x03044000},
1414 {0x00000070, 0x0480c018},
1415 {0x00000071, 0x00000040},
1416 {0x00000072, 0x01000000},
1417 {0x00000074, 0x000000ff},
1418 {0x00000075, 0x00143400},
1419 {0x00000076, 0x08ec0800},
1420 {0x00000077, 0x040000cc},
1421 {0x00000079, 0x00000000},
1422 {0x0000007a, 0x21000409},
1423 {0x0000007c, 0x00000000},
1424 {0x0000007d, 0xe8000000},
1425 {0x0000007e, 0x044408a8},
1426 {0x0000007f, 0x00000003},
1427 {0x00000080, 0x00000000},
1428 {0x00000081, 0x01000000},
1429 {0x00000082, 0x02000000},
1430 {0x00000083, 0x00000000},
1431 {0x00000084, 0xe3f3e4f4},
1432 {0x00000085, 0x00052024},
1433 {0x00000087, 0x00000000},
1434 {0x00000088, 0x66036603},
1435 {0x00000089, 0x01000000},
1436 {0x0000008b, 0x1c0a0000},
1437 {0x0000008c, 0xff010000},
1438 {0x0000008e, 0xffffefff},
1439 {0x0000008f, 0xfff3efff},
1440 {0x00000090, 0xfff3efbf},
1441 {0x00000094, 0x00101101},
1442 {0x00000095, 0x00000fff},
1443 {0x00000096, 0x00116fff},
1444 {0x00000097, 0x60010000},
1445 {0x00000098, 0x10010000},
1446 {0x00000099, 0x00006000},
1447 {0x0000009a, 0x00001000},
1448 {0x0000009f, 0x00a07730}
1449};
1450
8b074dd6
AD		 1451/* ucode loading */
1452static int si_mc_load_microcode(struct radeon_device *rdev)
1453{
1454 const __be32 *fw_data;
1455 u32 running, blackout = 0;
1456 u32 *io_mc_regs;
1457 int i, ucode_size, regs_size;
1458
1459 if (!rdev->mc_fw)
1460 return -EINVAL;
1461
1462 switch (rdev->family) {
1463 case CHIP_TAHITI:
1464 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1465 ucode_size = SI_MC_UCODE_SIZE;
1466 regs_size = TAHITI_IO_MC_REGS_SIZE;
1467 break;
1468 case CHIP_PITCAIRN:
1469 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1470 ucode_size = SI_MC_UCODE_SIZE;
1471 regs_size = TAHITI_IO_MC_REGS_SIZE;
1472 break;
1473 case CHIP_VERDE:
1474 default:
1475 io_mc_regs = (u32 *)&verde_io_mc_regs;
1476 ucode_size = SI_MC_UCODE_SIZE;
1477 regs_size = TAHITI_IO_MC_REGS_SIZE;
1478 break;
bcc7f5d2
AD		 1479 case CHIP_OLAND:
1480 io_mc_regs = (u32 *)&oland_io_mc_regs;
1481 ucode_size = OLAND_MC_UCODE_SIZE;
1482 regs_size = TAHITI_IO_MC_REGS_SIZE;
1483 break;
c04c00b4
AD		 1484 case CHIP_HAINAN:
1485 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1486 ucode_size = OLAND_MC_UCODE_SIZE;
1487 regs_size = TAHITI_IO_MC_REGS_SIZE;
1488 break;
8b074dd6
AD		 1489 }
1490
1491 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1492
1493 if (running == 0) {
1494 if (running) {
1495 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1496 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1497 }
1498
1499 /* reset the engine and set to writable */
1500 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1501 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1502
1503 /* load mc io regs */
1504 for (i = 0; i < regs_size; i++) {
1505 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1506 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1507 }
1508 /* load the MC ucode */
1509 fw_data = (const __be32 *)rdev->mc_fw->data;
1510 for (i = 0; i < ucode_size; i++)
1511 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1512
1513 /* put the engine back into the active state */
1514 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1515 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1516 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1517
1518 /* wait for training to complete */
1519 for (i = 0; i < rdev->usec_timeout; i++) {
1520 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1521 break;
1522 udelay(1);
1523 }
1524 for (i = 0; i < rdev->usec_timeout; i++) {
1525 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1526 break;
1527 udelay(1);
1528 }
1529
1530 if (running)
1531 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1532 }
1533
1534 return 0;
1535}
1536
0f0de06c
AD		 1537static int si_init_microcode(struct radeon_device *rdev)
1538{
1539 struct platform_device *pdev;
1540 const char *chip_name;
1541 const char *rlc_chip_name;
1542 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1543 char fw_name[30];
1544 int err;
1545
1546 DRM_DEBUG("\n");
1547
1548 pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1549 err = IS_ERR(pdev);
1550 if (err) {
1551 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1552 return -EINVAL;
1553 }
1554
1555 switch (rdev->family) {
1556 case CHIP_TAHITI:
1557 chip_name = "TAHITI";
1558 rlc_chip_name = "TAHITI";
1559 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1560 me_req_size = SI_PM4_UCODE_SIZE * 4;
1561 ce_req_size = SI_CE_UCODE_SIZE * 4;
1562 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1563 mc_req_size = SI_MC_UCODE_SIZE * 4;
1564 break;
1565 case CHIP_PITCAIRN:
1566 chip_name = "PITCAIRN";
1567 rlc_chip_name = "PITCAIRN";
1568 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1569 me_req_size = SI_PM4_UCODE_SIZE * 4;
1570 ce_req_size = SI_CE_UCODE_SIZE * 4;
1571 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1572 mc_req_size = SI_MC_UCODE_SIZE * 4;
1573 break;
1574 case CHIP_VERDE:
1575 chip_name = "VERDE";
1576 rlc_chip_name = "VERDE";
1577 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1578 me_req_size = SI_PM4_UCODE_SIZE * 4;
1579 ce_req_size = SI_CE_UCODE_SIZE * 4;
1580 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1581 mc_req_size = SI_MC_UCODE_SIZE * 4;
1582 break;
bcc7f5d2
AD		 1583 case CHIP_OLAND:
1584 chip_name = "OLAND";
1585 rlc_chip_name = "OLAND";
1586 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1587 me_req_size = SI_PM4_UCODE_SIZE * 4;
1588 ce_req_size = SI_CE_UCODE_SIZE * 4;
1589 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1590 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1591 break;
c04c00b4
AD		 1592 case CHIP_HAINAN:
1593 chip_name = "HAINAN";
1594 rlc_chip_name = "HAINAN";
1595 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1596 me_req_size = SI_PM4_UCODE_SIZE * 4;
1597 ce_req_size = SI_CE_UCODE_SIZE * 4;
1598 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1599 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1600 break;
0f0de06c
AD		 1601 default: BUG();
1602 }
1603
1604 DRM_INFO("Loading %s Microcode\n", chip_name);
1605
1606 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1607 err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1608 if (err)
1609 goto out;
1610 if (rdev->pfp_fw->size != pfp_req_size) {
1611 printk(KERN_ERR
1612 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1613 rdev->pfp_fw->size, fw_name);
1614 err = -EINVAL;
1615 goto out;
1616 }
1617
1618 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1619 err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1620 if (err)
1621 goto out;
1622 if (rdev->me_fw->size != me_req_size) {
1623 printk(KERN_ERR
1624 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1625 rdev->me_fw->size, fw_name);
1626 err = -EINVAL;
1627 }
1628
1629 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1630 err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
1631 if (err)
1632 goto out;
1633 if (rdev->ce_fw->size != ce_req_size) {
1634 printk(KERN_ERR
1635 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1636 rdev->ce_fw->size, fw_name);
1637 err = -EINVAL;
1638 }
1639
1640 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1641 err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1642 if (err)
1643 goto out;
1644 if (rdev->rlc_fw->size != rlc_req_size) {
1645 printk(KERN_ERR
1646 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1647 rdev->rlc_fw->size, fw_name);
1648 err = -EINVAL;
1649 }
1650
1651 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1652 err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
1653 if (err)
1654 goto out;
1655 if (rdev->mc_fw->size != mc_req_size) {
1656 printk(KERN_ERR
1657 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1658 rdev->mc_fw->size, fw_name);
1659 err = -EINVAL;
1660 }
1661
1662out:
1663 platform_device_unregister(pdev);
1664
1665 if (err) {
1666 if (err != -EINVAL)
1667 printk(KERN_ERR
1668 "si_cp: Failed to load firmware \"%s\"\n",
1669 fw_name);
1670 release_firmware(rdev->pfp_fw);
1671 rdev->pfp_fw = NULL;
1672 release_firmware(rdev->me_fw);
1673 rdev->me_fw = NULL;
1674 release_firmware(rdev->ce_fw);
1675 rdev->ce_fw = NULL;
1676 release_firmware(rdev->rlc_fw);
1677 rdev->rlc_fw = NULL;
1678 release_firmware(rdev->mc_fw);
1679 rdev->mc_fw = NULL;
1680 }
1681 return err;
1682}
1683
43b3cd99
AD		 1684/* watermark setup */
1685static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1686 struct radeon_crtc *radeon_crtc,
1687 struct drm_display_mode *mode,
1688 struct drm_display_mode *other_mode)
1689{
1690 u32 tmp;
1691 /*
1692 * Line Buffer Setup
1693 * There are 3 line buffers, each one shared by 2 display controllers.
1694 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1695 * the display controllers. The paritioning is done via one of four
1696 * preset allocations specified in bits 21:20:
1697 * 0 - half lb
1698 * 2 - whole lb, other crtc must be disabled
1699 */
1700 /* this can get tricky if we have two large displays on a paired group
1701 * of crtcs. Ideally for multiple large displays we'd assign them to
1702 * non-linked crtcs for maximum line buffer allocation.
1703 */
1704 if (radeon_crtc->base.enabled && mode) {
1705 if (other_mode)
1706 tmp = 0; /* 1/2 */
1707 else
1708 tmp = 2; /* whole */
1709 } else
1710 tmp = 0;
1711
1712 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1713 DC_LB_MEMORY_CONFIG(tmp));
1714
1715 if (radeon_crtc->base.enabled && mode) {
1716 switch (tmp) {
1717 case 0:
1718 default:
1719 return 4096 * 2;
1720 case 2:
1721 return 8192 * 2;
1722 }
1723 }
1724
1725 /* controller not enabled, so no lb used */
1726 return 0;
1727}
1728
ca7db22b 1729static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 1730{
1731 u32 tmp = RREG32(MC_SHARED_CHMAP);
1732
1733 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1734 case 0:
1735 default:
1736 return 1;
1737 case 1:
1738 return 2;
1739 case 2:
1740 return 4;
1741 case 3:
1742 return 8;
1743 case 4:
1744 return 3;
1745 case 5:
1746 return 6;
1747 case 6:
1748 return 10;
1749 case 7:
1750 return 12;
1751 case 8:
1752 return 16;
1753 }
1754}
1755
1756struct dce6_wm_params {
1757 u32 dram_channels; /* number of dram channels */
1758 u32 yclk; /* bandwidth per dram data pin in kHz */
1759 u32 sclk; /* engine clock in kHz */
1760 u32 disp_clk; /* display clock in kHz */
1761 u32 src_width; /* viewport width */
1762 u32 active_time; /* active display time in ns */
1763 u32 blank_time; /* blank time in ns */
1764 bool interlaced; /* mode is interlaced */
1765 fixed20_12 vsc; /* vertical scale ratio */
1766 u32 num_heads; /* number of active crtcs */
1767 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1768 u32 lb_size; /* line buffer allocated to pipe */
1769 u32 vtaps; /* vertical scaler taps */
1770};
1771
1772static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1773{
1774 /* Calculate raw DRAM Bandwidth */
1775 fixed20_12 dram_efficiency; /* 0.7 */
1776 fixed20_12 yclk, dram_channels, bandwidth;
1777 fixed20_12 a;
1778
1779 a.full = dfixed_const(1000);
1780 yclk.full = dfixed_const(wm->yclk);
1781 yclk.full = dfixed_div(yclk, a);
1782 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1783 a.full = dfixed_const(10);
1784 dram_efficiency.full = dfixed_const(7);
1785 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1786 bandwidth.full = dfixed_mul(dram_channels, yclk);
1787 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1788
1789 return dfixed_trunc(bandwidth);
1790}
1791
1792static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1793{
1794 /* Calculate DRAM Bandwidth and the part allocated to display. */
1795 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1796 fixed20_12 yclk, dram_channels, bandwidth;
1797 fixed20_12 a;
1798
1799 a.full = dfixed_const(1000);
1800 yclk.full = dfixed_const(wm->yclk);
1801 yclk.full = dfixed_div(yclk, a);
1802 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1803 a.full = dfixed_const(10);
1804 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1805 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1806 bandwidth.full = dfixed_mul(dram_channels, yclk);
1807 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1808
1809 return dfixed_trunc(bandwidth);
1810}
1811
1812static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1813{
1814 /* Calculate the display Data return Bandwidth */
1815 fixed20_12 return_efficiency; /* 0.8 */
1816 fixed20_12 sclk, bandwidth;
1817 fixed20_12 a;
1818
1819 a.full = dfixed_const(1000);
1820 sclk.full = dfixed_const(wm->sclk);
1821 sclk.full = dfixed_div(sclk, a);
1822 a.full = dfixed_const(10);
1823 return_efficiency.full = dfixed_const(8);
1824 return_efficiency.full = dfixed_div(return_efficiency, a);
1825 a.full = dfixed_const(32);
1826 bandwidth.full = dfixed_mul(a, sclk);
1827 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1828
1829 return dfixed_trunc(bandwidth);
1830}
1831
1832static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1833{
1834 return 32;
1835}
1836
1837static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1838{
1839 /* Calculate the DMIF Request Bandwidth */
1840 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1841 fixed20_12 disp_clk, sclk, bandwidth;
1842 fixed20_12 a, b1, b2;
1843 u32 min_bandwidth;
1844
1845 a.full = dfixed_const(1000);
1846 disp_clk.full = dfixed_const(wm->disp_clk);
1847 disp_clk.full = dfixed_div(disp_clk, a);
1848 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1849 b1.full = dfixed_mul(a, disp_clk);
1850
1851 a.full = dfixed_const(1000);
1852 sclk.full = dfixed_const(wm->sclk);
1853 sclk.full = dfixed_div(sclk, a);
1854 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1855 b2.full = dfixed_mul(a, sclk);
1856
1857 a.full = dfixed_const(10);
1858 disp_clk_request_efficiency.full = dfixed_const(8);
1859 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1860
1861 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1862
1863 a.full = dfixed_const(min_bandwidth);
1864 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1865
1866 return dfixed_trunc(bandwidth);
1867}
1868
1869static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1870{
1871 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1872 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1873 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1874 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1875
1876 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1877}
1878
1879static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1880{
1881 /* Calculate the display mode Average Bandwidth
1882 * DisplayMode should contain the source and destination dimensions,
1883 * timing, etc.
1884 */
1885 fixed20_12 bpp;
1886 fixed20_12 line_time;
1887 fixed20_12 src_width;
1888 fixed20_12 bandwidth;
1889 fixed20_12 a;
1890
1891 a.full = dfixed_const(1000);
1892 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1893 line_time.full = dfixed_div(line_time, a);
1894 bpp.full = dfixed_const(wm->bytes_per_pixel);
1895 src_width.full = dfixed_const(wm->src_width);
1896 bandwidth.full = dfixed_mul(src_width, bpp);
1897 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1898 bandwidth.full = dfixed_div(bandwidth, line_time);
1899
1900 return dfixed_trunc(bandwidth);
1901}
1902
1903static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1904{
1905 /* First calcualte the latency in ns */
1906 u32 mc_latency = 2000; /* 2000 ns. */
1907 u32 available_bandwidth = dce6_available_bandwidth(wm);
1908 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1909 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1910 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1911 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1912 (wm->num_heads * cursor_line_pair_return_time);
1913 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1914 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1915 u32 tmp, dmif_size = 12288;
1916 fixed20_12 a, b, c;
1917
1918 if (wm->num_heads == 0)
1919 return 0;
1920
1921 a.full = dfixed_const(2);
1922 b.full = dfixed_const(1);
1923 if ((wm->vsc.full > a.full) ||
1924 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1925 (wm->vtaps >= 5) ||
1926 ((wm->vsc.full >= a.full) && wm->interlaced))
1927 max_src_lines_per_dst_line = 4;
1928 else
1929 max_src_lines_per_dst_line = 2;
1930
1931 a.full = dfixed_const(available_bandwidth);
1932 b.full = dfixed_const(wm->num_heads);
1933 a.full = dfixed_div(a, b);
1934
1935 b.full = dfixed_const(mc_latency + 512);
1936 c.full = dfixed_const(wm->disp_clk);
1937 b.full = dfixed_div(b, c);
1938
1939 c.full = dfixed_const(dmif_size);
1940 b.full = dfixed_div(c, b);
1941
1942 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1943
1944 b.full = dfixed_const(1000);
1945 c.full = dfixed_const(wm->disp_clk);
1946 b.full = dfixed_div(c, b);
1947 c.full = dfixed_const(wm->bytes_per_pixel);
1948 b.full = dfixed_mul(b, c);
1949
1950 lb_fill_bw = min(tmp, dfixed_trunc(b));
1951
1952 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1953 b.full = dfixed_const(1000);
1954 c.full = dfixed_const(lb_fill_bw);
1955 b.full = dfixed_div(c, b);
1956 a.full = dfixed_div(a, b);
1957 line_fill_time = dfixed_trunc(a);
1958
1959 if (line_fill_time < wm->active_time)
1960 return latency;
1961 else
1962 return latency + (line_fill_time - wm->active_time);
1963
1964}
1965
1966static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1967{
1968 if (dce6_average_bandwidth(wm) <=
1969 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
1970 return true;
1971 else
1972 return false;
1973};
1974
1975static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
1976{
1977 if (dce6_average_bandwidth(wm) <=
1978 (dce6_available_bandwidth(wm) / wm->num_heads))
1979 return true;
1980 else
1981 return false;
1982};
1983
1984static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
1985{
1986 u32 lb_partitions = wm->lb_size / wm->src_width;
1987 u32 line_time = wm->active_time + wm->blank_time;
1988 u32 latency_tolerant_lines;
1989 u32 latency_hiding;
1990 fixed20_12 a;
1991
1992 a.full = dfixed_const(1);
1993 if (wm->vsc.full > a.full)
1994 latency_tolerant_lines = 1;
1995 else {
1996 if (lb_partitions <= (wm->vtaps + 1))
1997 latency_tolerant_lines = 1;
1998 else
1999 latency_tolerant_lines = 2;
2000 }
2001
2002 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2003
2004 if (dce6_latency_watermark(wm) <= latency_hiding)
2005 return true;
2006 else
2007 return false;
2008}
2009
2010static void dce6_program_watermarks(struct radeon_device *rdev,
2011 struct radeon_crtc *radeon_crtc,
2012 u32 lb_size, u32 num_heads)
2013{
2014 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 2015 struct dce6_wm_params wm_low, wm_high;
2016 u32 dram_channels;
43b3cd99
AD		 2017 u32 pixel_period;
2018 u32 line_time = 0;
2019 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2020 u32 priority_a_mark = 0, priority_b_mark = 0;
2021 u32 priority_a_cnt = PRIORITY_OFF;
2022 u32 priority_b_cnt = PRIORITY_OFF;
2023 u32 tmp, arb_control3;
2024 fixed20_12 a, b, c;
2025
2026 if (radeon_crtc->base.enabled && num_heads && mode) {
2027 pixel_period = 1000000 / (u32)mode->clock;
2028 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2029 priority_a_cnt = 0;
2030 priority_b_cnt = 0;
2031
ca7db22b 2032 if (rdev->family == CHIP_ARUBA)
c696e53f 2033 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 2034 else
c696e53f
AD		 2035 dram_channels = si_get_number_of_dram_channels(rdev);
2036
2037 /* watermark for high clocks */
2038 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2039 wm_high.yclk =
2040 radeon_dpm_get_mclk(rdev, false) * 10;
2041 wm_high.sclk =
2042 radeon_dpm_get_sclk(rdev, false) * 10;
2043 } else {
2044 wm_high.yclk = rdev->pm.current_mclk * 10;
2045 wm_high.sclk = rdev->pm.current_sclk * 10;
2046 }
2047
2048 wm_high.disp_clk = mode->clock;
2049 wm_high.src_width = mode->crtc_hdisplay;
2050 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2051 wm_high.blank_time = line_time - wm_high.active_time;
2052 wm_high.interlaced = false;
2053 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2054 wm_high.interlaced = true;
2055 wm_high.vsc = radeon_crtc->vsc;
2056 wm_high.vtaps = 1;
2057 if (radeon_crtc->rmx_type != RMX_OFF)
2058 wm_high.vtaps = 2;
2059 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2060 wm_high.lb_size = lb_size;
2061 wm_high.dram_channels = dram_channels;
2062 wm_high.num_heads = num_heads;
2063
2064 /* watermark for low clocks */
2065 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2066 wm_low.yclk =
2067 radeon_dpm_get_mclk(rdev, true) * 10;
2068 wm_low.sclk =
2069 radeon_dpm_get_sclk(rdev, true) * 10;
2070 } else {
2071 wm_low.yclk = rdev->pm.current_mclk * 10;
2072 wm_low.sclk = rdev->pm.current_sclk * 10;
2073 }
2074
2075 wm_low.disp_clk = mode->clock;
2076 wm_low.src_width = mode->crtc_hdisplay;
2077 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2078 wm_low.blank_time = line_time - wm_low.active_time;
2079 wm_low.interlaced = false;
2080 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2081 wm_low.interlaced = true;
2082 wm_low.vsc = radeon_crtc->vsc;
2083 wm_low.vtaps = 1;
2084 if (radeon_crtc->rmx_type != RMX_OFF)
2085 wm_low.vtaps = 2;
2086 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2087 wm_low.lb_size = lb_size;
2088 wm_low.dram_channels = dram_channels;
2089 wm_low.num_heads = num_heads;
43b3cd99
AD		 2090
2091 /* set for high clocks */
c696e53f 2092 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 2093 /* set for low clocks */
c696e53f 2094 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 2095
2096 /* possibly force display priority to high */
2097 /* should really do this at mode validation time... */
c696e53f
AD		 2098 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2099 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2100 !dce6_check_latency_hiding(&wm_high) ||
2101 (rdev->disp_priority == 2)) {
2102 DRM_DEBUG_KMS("force priority to high\n");
2103 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2104 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2105 }
2106 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2107 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2108 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 2109 (rdev->disp_priority == 2)) {
2110 DRM_DEBUG_KMS("force priority to high\n");
2111 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2112 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2113 }
2114
2115 a.full = dfixed_const(1000);
2116 b.full = dfixed_const(mode->clock);
2117 b.full = dfixed_div(b, a);
2118 c.full = dfixed_const(latency_watermark_a);
2119 c.full = dfixed_mul(c, b);
2120 c.full = dfixed_mul(c, radeon_crtc->hsc);
2121 c.full = dfixed_div(c, a);
2122 a.full = dfixed_const(16);
2123 c.full = dfixed_div(c, a);
2124 priority_a_mark = dfixed_trunc(c);
2125 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2126
2127 a.full = dfixed_const(1000);
2128 b.full = dfixed_const(mode->clock);
2129 b.full = dfixed_div(b, a);
2130 c.full = dfixed_const(latency_watermark_b);
2131 c.full = dfixed_mul(c, b);
2132 c.full = dfixed_mul(c, radeon_crtc->hsc);
2133 c.full = dfixed_div(c, a);
2134 a.full = dfixed_const(16);
2135 c.full = dfixed_div(c, a);
2136 priority_b_mark = dfixed_trunc(c);
2137 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2138 }
2139
2140 /* select wm A */
2141 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2142 tmp = arb_control3;
2143 tmp &= ~LATENCY_WATERMARK_MASK(3);
2144 tmp |= LATENCY_WATERMARK_MASK(1);
2145 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2146 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2147 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2148 LATENCY_HIGH_WATERMARK(line_time)));
2149 /* select wm B */
2150 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2151 tmp &= ~LATENCY_WATERMARK_MASK(3);
2152 tmp |= LATENCY_WATERMARK_MASK(2);
2153 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2154 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2155 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2156 LATENCY_HIGH_WATERMARK(line_time)));
2157 /* restore original selection */
2158 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2159
2160 /* write the priority marks */
2161 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2162 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2163
7178d2a6
AD		 2164 /* save values for DPM */
2165 radeon_crtc->line_time = line_time;
2166 radeon_crtc->wm_high = latency_watermark_a;
2167 radeon_crtc->wm_low = latency_watermark_b;
43b3cd99
AD		 2168}
2169
2170void dce6_bandwidth_update(struct radeon_device *rdev)
2171{
2172 struct drm_display_mode *mode0 = NULL;
2173 struct drm_display_mode *mode1 = NULL;
2174 u32 num_heads = 0, lb_size;
2175 int i;
2176
2177 radeon_update_display_priority(rdev);
2178
2179 for (i = 0; i < rdev->num_crtc; i++) {
2180 if (rdev->mode_info.crtcs[i]->base.enabled)
2181 num_heads++;
2182 }
2183 for (i = 0; i < rdev->num_crtc; i += 2) {
2184 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2185 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2186 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2187 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2188 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2189 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2190 }
2191}
2192
0a96d72b
AD		 2193/*
2194 * Core functions
2195 */
0a96d72b
AD		 2196static void si_tiling_mode_table_init(struct radeon_device *rdev)
2197{
2198 const u32 num_tile_mode_states = 32;
2199 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2200
2201 switch (rdev->config.si.mem_row_size_in_kb) {
2202 case 1:
2203 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2204 break;
2205 case 2:
2206 default:
2207 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2208 break;
2209 case 4:
2210 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2211 break;
2212 }
2213
2214 if ((rdev->family == CHIP_TAHITI) ||
2215 (rdev->family == CHIP_PITCAIRN)) {
2216 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2217 switch (reg_offset) {
2218 case 0: /* non-AA compressed depth or any compressed stencil */
2219 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2220 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2221 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2222 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2223 NUM_BANKS(ADDR_SURF_16_BANK) |
2224 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2225 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2226 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2227 break;
2228 case 1: /* 2xAA/4xAA compressed depth only */
2229 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2230 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2231 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2232 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2233 NUM_BANKS(ADDR_SURF_16_BANK) |
2234 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2235 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2236 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2237 break;
2238 case 2: /* 8xAA compressed depth only */
2239 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2240 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2241 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2242 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2243 NUM_BANKS(ADDR_SURF_16_BANK) |
2244 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2245 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2246 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2247 break;
2248 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2249 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2250 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2251 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2252 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2253 NUM_BANKS(ADDR_SURF_16_BANK) |
2254 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2255 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2256 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2257 break;
2258 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2259 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2260 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2261 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2262 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2263 NUM_BANKS(ADDR_SURF_16_BANK) |
2264 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2265 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2266 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2267 break;
2268 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2269 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2270 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2271 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2272 TILE_SPLIT(split_equal_to_row_size) |
2273 NUM_BANKS(ADDR_SURF_16_BANK) |
2274 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2275 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2276 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2277 break;
2278 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2279 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2280 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2281 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2282 TILE_SPLIT(split_equal_to_row_size) |
2283 NUM_BANKS(ADDR_SURF_16_BANK) |
2284 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2285 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2286 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2287 break;
2288 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2289 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2290 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2291 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2292 TILE_SPLIT(split_equal_to_row_size) |
2293 NUM_BANKS(ADDR_SURF_16_BANK) |
2294 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2295 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2296 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2297 break;
2298 case 8: /* 1D and 1D Array Surfaces */
2299 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2300 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2301 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2302 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2303 NUM_BANKS(ADDR_SURF_16_BANK) |
2304 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2305 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2306 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2307 break;
2308 case 9: /* Displayable maps. */
2309 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2310 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2311 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2312 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2313 NUM_BANKS(ADDR_SURF_16_BANK) |
2314 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2315 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2316 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2317 break;
2318 case 10: /* Display 8bpp. */
2319 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2320 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2321 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2322 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2323 NUM_BANKS(ADDR_SURF_16_BANK) |
2324 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2325 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2326 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2327 break;
2328 case 11: /* Display 16bpp. */
2329 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2330 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2331 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2332 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2333 NUM_BANKS(ADDR_SURF_16_BANK) |
2334 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2335 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2336 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2337 break;
2338 case 12: /* Display 32bpp. */
2339 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2340 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2341 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2342 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2343 NUM_BANKS(ADDR_SURF_16_BANK) |
2344 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2345 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2346 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2347 break;
2348 case 13: /* Thin. */
2349 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2350 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2351 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2352 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2353 NUM_BANKS(ADDR_SURF_16_BANK) |
2354 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2355 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2356 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2357 break;
2358 case 14: /* Thin 8 bpp. */
2359 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2360 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2361 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2362 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2363 NUM_BANKS(ADDR_SURF_16_BANK) |
2364 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2365 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2366 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2367 break;
2368 case 15: /* Thin 16 bpp. */
2369 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2370 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2371 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2372 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2373 NUM_BANKS(ADDR_SURF_16_BANK) |
2374 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2375 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2376 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2377 break;
2378 case 16: /* Thin 32 bpp. */
2379 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2380 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2381 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2382 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2383 NUM_BANKS(ADDR_SURF_16_BANK) |
2384 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2385 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2386 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2387 break;
2388 case 17: /* Thin 64 bpp. */
2389 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2390 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2391 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2392 TILE_SPLIT(split_equal_to_row_size) |
2393 NUM_BANKS(ADDR_SURF_16_BANK) |
2394 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2395 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2396 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2397 break;
2398 case 21: /* 8 bpp PRT. */
2399 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2400 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2401 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2402 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2403 NUM_BANKS(ADDR_SURF_16_BANK) |
2404 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2405 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2406 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2407 break;
2408 case 22: /* 16 bpp PRT */
2409 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2410 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2411 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2412 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2413 NUM_BANKS(ADDR_SURF_16_BANK) |
2414 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2415 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2416 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2417 break;
2418 case 23: /* 32 bpp PRT */
2419 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2420 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2421 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2422 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2423 NUM_BANKS(ADDR_SURF_16_BANK) |
2424 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2425 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2426 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2427 break;
2428 case 24: /* 64 bpp PRT */
2429 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2430 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2431 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2432 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2433 NUM_BANKS(ADDR_SURF_16_BANK) |
2434 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2435 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2436 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2437 break;
2438 case 25: /* 128 bpp PRT */
2439 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2440 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2441 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2442 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2443 NUM_BANKS(ADDR_SURF_8_BANK) |
2444 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2445 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2446 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2447 break;
2448 default:
2449 gb_tile_moden = 0;
2450 break;
2451 }
64d7b8be 2452 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2453 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2454 }
d0ae7fcc 2455 } else if ((rdev->family == CHIP_VERDE) ||
8b02859d
AD		 2456 (rdev->family == CHIP_OLAND) ||
2457 (rdev->family == CHIP_HAINAN)) {
0a96d72b
AD		 2458 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2459 switch (reg_offset) {
2460 case 0: /* non-AA compressed depth or any compressed stencil */
2461 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2462 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2463 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2464 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2465 NUM_BANKS(ADDR_SURF_16_BANK) |
2466 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2467 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2468 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2469 break;
2470 case 1: /* 2xAA/4xAA compressed depth only */
2471 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2472 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2473 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2474 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2475 NUM_BANKS(ADDR_SURF_16_BANK) |
2476 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2478 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2479 break;
2480 case 2: /* 8xAA compressed depth only */
2481 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2482 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2483 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2484 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2485 NUM_BANKS(ADDR_SURF_16_BANK) |
2486 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2487 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2488 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2489 break;
2490 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2491 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2492 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2493 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2494 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2495 NUM_BANKS(ADDR_SURF_16_BANK) |
2496 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2497 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2498 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2499 break;
2500 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2501 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2502 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2503 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2504 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2505 NUM_BANKS(ADDR_SURF_16_BANK) |
2506 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2507 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2508 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2509 break;
2510 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2511 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2512 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2513 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2514 TILE_SPLIT(split_equal_to_row_size) |
2515 NUM_BANKS(ADDR_SURF_16_BANK) |
2516 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2517 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2518 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2519 break;
2520 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2521 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2522 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2523 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2524 TILE_SPLIT(split_equal_to_row_size) |
2525 NUM_BANKS(ADDR_SURF_16_BANK) |
2526 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2527 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2528 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2529 break;
2530 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2531 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2532 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2533 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2534 TILE_SPLIT(split_equal_to_row_size) |
2535 NUM_BANKS(ADDR_SURF_16_BANK) |
2536 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2537 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2538 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2539 break;
2540 case 8: /* 1D and 1D Array Surfaces */
2541 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2542 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2543 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2544 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2545 NUM_BANKS(ADDR_SURF_16_BANK) |
2546 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2547 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2548 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2549 break;
2550 case 9: /* Displayable maps. */
2551 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2552 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2553 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2554 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2555 NUM_BANKS(ADDR_SURF_16_BANK) |
2556 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2557 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2558 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2559 break;
2560 case 10: /* Display 8bpp. */
2561 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2562 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2563 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2564 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2565 NUM_BANKS(ADDR_SURF_16_BANK) |
2566 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2567 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2568 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2569 break;
2570 case 11: /* Display 16bpp. */
2571 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2572 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2573 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2574 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2575 NUM_BANKS(ADDR_SURF_16_BANK) |
2576 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2577 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2578 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2579 break;
2580 case 12: /* Display 32bpp. */
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2583 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2584 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2585 NUM_BANKS(ADDR_SURF_16_BANK) |
2586 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2587 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2588 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2589 break;
2590 case 13: /* Thin. */
2591 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2592 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2593 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2594 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2595 NUM_BANKS(ADDR_SURF_16_BANK) |
2596 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2597 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2598 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2599 break;
2600 case 14: /* Thin 8 bpp. */
2601 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2602 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2603 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2604 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2605 NUM_BANKS(ADDR_SURF_16_BANK) |
2606 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2607 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2608 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2609 break;
2610 case 15: /* Thin 16 bpp. */
2611 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2612 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2613 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2614 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2615 NUM_BANKS(ADDR_SURF_16_BANK) |
2616 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2617 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2618 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2619 break;
2620 case 16: /* Thin 32 bpp. */
2621 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2622 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2623 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2624 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2625 NUM_BANKS(ADDR_SURF_16_BANK) |
2626 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2627 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2628 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2629 break;
2630 case 17: /* Thin 64 bpp. */
2631 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2632 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2633 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2634 TILE_SPLIT(split_equal_to_row_size) |
2635 NUM_BANKS(ADDR_SURF_16_BANK) |
2636 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2637 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2638 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2639 break;
2640 case 21: /* 8 bpp PRT. */
2641 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2643 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2644 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2645 NUM_BANKS(ADDR_SURF_16_BANK) |
2646 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2647 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2648 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2649 break;
2650 case 22: /* 16 bpp PRT */
2651 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2652 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2653 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2654 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2655 NUM_BANKS(ADDR_SURF_16_BANK) |
2656 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2657 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2658 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2659 break;
2660 case 23: /* 32 bpp PRT */
2661 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2662 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2663 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2664 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2665 NUM_BANKS(ADDR_SURF_16_BANK) |
2666 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2667 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2668 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2669 break;
2670 case 24: /* 64 bpp PRT */
2671 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2672 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2673 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2674 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2675 NUM_BANKS(ADDR_SURF_16_BANK) |
2676 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2677 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2678 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2679 break;
2680 case 25: /* 128 bpp PRT */
2681 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2682 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2683 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2684 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2685 NUM_BANKS(ADDR_SURF_8_BANK) |
2686 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2687 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2688 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2689 break;
2690 default:
2691 gb_tile_moden = 0;
2692 break;
2693 }
64d7b8be 2694 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2695 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2696 }
2697 } else
2698 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2699}
2700
1a8ca750
AD		 2701static void si_select_se_sh(struct radeon_device *rdev,
2702 u32 se_num, u32 sh_num)
2703{
2704 u32 data = INSTANCE_BROADCAST_WRITES;
2705
2706 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2707 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2708 else if (se_num == 0xffffffff)
2709 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2710 else if (sh_num == 0xffffffff)
2711 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2712 else
2713 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2714 WREG32(GRBM_GFX_INDEX, data);
2715}
2716
2717static u32 si_create_bitmask(u32 bit_width)
2718{
2719 u32 i, mask = 0;
2720
2721 for (i = 0; i < bit_width; i++) {
2722 mask <<= 1;
2723 mask |= 1;
2724 }
2725 return mask;
2726}
2727
2728static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2729{
2730 u32 data, mask;
2731
2732 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2733 if (data & 1)
2734 data &= INACTIVE_CUS_MASK;
2735 else
2736 data = 0;
2737 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2738
2739 data >>= INACTIVE_CUS_SHIFT;
2740
2741 mask = si_create_bitmask(cu_per_sh);
2742
2743 return ~data & mask;
2744}
2745
2746static void si_setup_spi(struct radeon_device *rdev,
2747 u32 se_num, u32 sh_per_se,
2748 u32 cu_per_sh)
2749{
2750 int i, j, k;
2751 u32 data, mask, active_cu;
2752
2753 for (i = 0; i < se_num; i++) {
2754 for (j = 0; j < sh_per_se; j++) {
2755 si_select_se_sh(rdev, i, j);
2756 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2757 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2758
2759 mask = 1;
2760 for (k = 0; k < 16; k++) {
2761 mask <<= k;
2762 if (active_cu & mask) {
2763 data &= ~mask;
2764 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2765 break;
2766 }
2767 }
2768 }
2769 }
2770 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2771}
2772
2773static u32 si_get_rb_disabled(struct radeon_device *rdev,
2774 u32 max_rb_num, u32 se_num,
2775 u32 sh_per_se)
2776{
2777 u32 data, mask;
2778
2779 data = RREG32(CC_RB_BACKEND_DISABLE);
2780 if (data & 1)
2781 data &= BACKEND_DISABLE_MASK;
2782 else
2783 data = 0;
2784 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2785
2786 data >>= BACKEND_DISABLE_SHIFT;
2787
2788 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2789
2790 return data & mask;
2791}
2792
2793static void si_setup_rb(struct radeon_device *rdev,
2794 u32 se_num, u32 sh_per_se,
2795 u32 max_rb_num)
2796{
2797 int i, j;
2798 u32 data, mask;
2799 u32 disabled_rbs = 0;
2800 u32 enabled_rbs = 0;
2801
2802 for (i = 0; i < se_num; i++) {
2803 for (j = 0; j < sh_per_se; j++) {
2804 si_select_se_sh(rdev, i, j);
2805 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2806 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2807 }
2808 }
2809 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2810
2811 mask = 1;
2812 for (i = 0; i < max_rb_num; i++) {
2813 if (!(disabled_rbs & mask))
2814 enabled_rbs |= mask;
2815 mask <<= 1;
2816 }
2817
2818 for (i = 0; i < se_num; i++) {
2819 si_select_se_sh(rdev, i, 0xffffffff);
2820 data = 0;
2821 for (j = 0; j < sh_per_se; j++) {
2822 switch (enabled_rbs & 3) {
2823 case 1:
2824 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2825 break;
2826 case 2:
2827 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2828 break;
2829 case 3:
2830 default:
2831 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2832 break;
2833 }
2834 enabled_rbs >>= 2;
2835 }
2836 WREG32(PA_SC_RASTER_CONFIG, data);
2837 }
2838 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2839}
2840
0a96d72b
AD		 2841static void si_gpu_init(struct radeon_device *rdev)
2842{
0a96d72b
AD		 2843 u32 gb_addr_config = 0;
2844 u32 mc_shared_chmap, mc_arb_ramcfg;
0a96d72b 2845 u32 sx_debug_1;
0a96d72b
AD		 2846 u32 hdp_host_path_cntl;
2847 u32 tmp;
2848 int i, j;
2849
2850 switch (rdev->family) {
2851 case CHIP_TAHITI:
2852 rdev->config.si.max_shader_engines = 2;
0a96d72b 2853 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 2854 rdev->config.si.max_cu_per_sh = 8;
2855 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2856 rdev->config.si.max_backends_per_se = 4;
2857 rdev->config.si.max_texture_channel_caches = 12;
2858 rdev->config.si.max_gprs = 256;
2859 rdev->config.si.max_gs_threads = 32;
2860 rdev->config.si.max_hw_contexts = 8;
2861
2862 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2863 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2864 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2865 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2866 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2867 break;
2868 case CHIP_PITCAIRN:
2869 rdev->config.si.max_shader_engines = 2;
0a96d72b 2870 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 2871 rdev->config.si.max_cu_per_sh = 5;
2872 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2873 rdev->config.si.max_backends_per_se = 4;
2874 rdev->config.si.max_texture_channel_caches = 8;
2875 rdev->config.si.max_gprs = 256;
2876 rdev->config.si.max_gs_threads = 32;
2877 rdev->config.si.max_hw_contexts = 8;
2878
2879 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2880 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2881 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2882 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2883 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2884 break;
2885 case CHIP_VERDE:
2886 default:
2887 rdev->config.si.max_shader_engines = 1;
0a96d72b 2888 rdev->config.si.max_tile_pipes = 4;
468ef1a5 2889 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 2890 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2891 rdev->config.si.max_backends_per_se = 4;
2892 rdev->config.si.max_texture_channel_caches = 4;
2893 rdev->config.si.max_gprs = 256;
2894 rdev->config.si.max_gs_threads = 32;
2895 rdev->config.si.max_hw_contexts = 8;
2896
d0ae7fcc
AD		 2897 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2898 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2899 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2900 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2901 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2902 break;
2903 case CHIP_OLAND:
2904 rdev->config.si.max_shader_engines = 1;
2905 rdev->config.si.max_tile_pipes = 4;
2906 rdev->config.si.max_cu_per_sh = 6;
2907 rdev->config.si.max_sh_per_se = 1;
2908 rdev->config.si.max_backends_per_se = 2;
2909 rdev->config.si.max_texture_channel_caches = 4;
2910 rdev->config.si.max_gprs = 256;
2911 rdev->config.si.max_gs_threads = 16;
2912 rdev->config.si.max_hw_contexts = 8;
2913
0a96d72b
AD		 2914 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2915 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2916 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2917 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2918 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 2919 break;
8b02859d
AD		 2920 case CHIP_HAINAN:
2921 rdev->config.si.max_shader_engines = 1;
2922 rdev->config.si.max_tile_pipes = 4;
2923 rdev->config.si.max_cu_per_sh = 5;
2924 rdev->config.si.max_sh_per_se = 1;
2925 rdev->config.si.max_backends_per_se = 1;
2926 rdev->config.si.max_texture_channel_caches = 2;
2927 rdev->config.si.max_gprs = 256;
2928 rdev->config.si.max_gs_threads = 16;
2929 rdev->config.si.max_hw_contexts = 8;
2930
2931 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2932 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2933 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2934 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2935 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2936 break;
0a96d72b
AD		 2937 }
2938
2939 /* Initialize HDP */
2940 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2941 WREG32((0x2c14 + j), 0x00000000);
2942 WREG32((0x2c18 + j), 0x00000000);
2943 WREG32((0x2c1c + j), 0x00000000);
2944 WREG32((0x2c20 + j), 0x00000000);
2945 WREG32((0x2c24 + j), 0x00000000);
2946 }
2947
2948 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2949
2950 evergreen_fix_pci_max_read_req_size(rdev);
2951
2952 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2953
2954 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2955 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2956
0a96d72b 2957 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 2958 rdev->config.si.mem_max_burst_length_bytes = 256;
2959 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2960 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2961 if (rdev->config.si.mem_row_size_in_kb > 4)
2962 rdev->config.si.mem_row_size_in_kb = 4;
2963 /* XXX use MC settings? */
2964 rdev->config.si.shader_engine_tile_size = 32;
2965 rdev->config.si.num_gpus = 1;
2966 rdev->config.si.multi_gpu_tile_size = 64;
2967
1a8ca750
AD		 2968 /* fix up row size */
2969 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 2970 switch (rdev->config.si.mem_row_size_in_kb) {
2971 case 1:
2972 default:
2973 gb_addr_config |= ROW_SIZE(0);
2974 break;
2975 case 2:
2976 gb_addr_config |= ROW_SIZE(1);
2977 break;
2978 case 4:
2979 gb_addr_config |= ROW_SIZE(2);
2980 break;
2981 }
2982
0a96d72b
AD		 2983 /* setup tiling info dword. gb_addr_config is not adequate since it does
2984 * not have bank info, so create a custom tiling dword.
2985 * bits 3:0 num_pipes
2986 * bits 7:4 num_banks
2987 * bits 11:8 group_size
2988 * bits 15:12 row_size
2989 */
2990 rdev->config.si.tile_config = 0;
2991 switch (rdev->config.si.num_tile_pipes) {
2992 case 1:
2993 rdev->config.si.tile_config |= (0 << 0);
2994 break;
2995 case 2:
2996 rdev->config.si.tile_config |= (1 << 0);
2997 break;
2998 case 4:
2999 rdev->config.si.tile_config |= (2 << 0);
3000 break;
3001 case 8:
3002 default:
3003 /* XXX what about 12? */
3004 rdev->config.si.tile_config |= (3 << 0);
3005 break;
dca571a6
CK		 3006 }
3007 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3008 case 0: /* four banks */
1a8ca750 3009 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 3010 break;
3011 case 1: /* eight banks */
3012 rdev->config.si.tile_config |= 1 << 4;
3013 break;
3014 case 2: /* sixteen banks */
3015 default:
3016 rdev->config.si.tile_config |= 2 << 4;
3017 break;
3018 }
0a96d72b
AD		 3019 rdev->config.si.tile_config |=
3020 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3021 rdev->config.si.tile_config |=
3022 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3023
0a96d72b
AD		 3024 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3025 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 3026 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 3027 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 3028 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3029 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 3030 if (rdev->has_uvd) {
3031 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3032 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3033 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3034 }
0a96d72b 3035
1a8ca750 3036 si_tiling_mode_table_init(rdev);
0a96d72b 3037
1a8ca750
AD		 3038 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3039 rdev->config.si.max_sh_per_se,
3040 rdev->config.si.max_backends_per_se);
0a96d72b 3041
1a8ca750
AD		 3042 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3043 rdev->config.si.max_sh_per_se,
3044 rdev->config.si.max_cu_per_sh);
0a96d72b 3045
0a96d72b
AD		 3046
3047 /* set HW defaults for 3D engine */
3048 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3049 ROQ_IB2_START(0x2b)));
3050 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3051
3052 sx_debug_1 = RREG32(SX_DEBUG_1);
3053 WREG32(SX_DEBUG_1, sx_debug_1);
3054
3055 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3056
3057 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3058 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3059 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3060 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3061
3062 WREG32(VGT_NUM_INSTANCES, 1);
3063
3064 WREG32(CP_PERFMON_CNTL, 0);
3065
3066 WREG32(SQ_CONFIG, 0);
3067
3068 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3069 FORCE_EOV_MAX_REZ_CNT(255)));
3070
3071 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3072 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3073
3074 WREG32(VGT_GS_VERTEX_REUSE, 16);
3075 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3076
3077 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3078 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3079 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3080 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3081 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3082 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3083 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3084 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3085
3086 tmp = RREG32(HDP_MISC_CNTL);
3087 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3088 WREG32(HDP_MISC_CNTL, tmp);
3089
3090 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3091 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3092
3093 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3094
3095 udelay(50);
3096}
c476dde2 3097
2ece2e8b
AD		 3098/*
3099 * GPU scratch registers helpers function.
3100 */
3101static void si_scratch_init(struct radeon_device *rdev)
3102{
3103 int i;
3104
3105 rdev->scratch.num_reg = 7;
3106 rdev->scratch.reg_base = SCRATCH_REG0;
3107 for (i = 0; i < rdev->scratch.num_reg; i++) {
3108 rdev->scratch.free[i] = true;
3109 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3110 }
3111}
3112
3113void si_fence_ring_emit(struct radeon_device *rdev,
3114 struct radeon_fence *fence)
3115{
3116 struct radeon_ring *ring = &rdev->ring[fence->ring];
3117 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3118
3119 /* flush read cache over gart */
3120 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3121 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3122 radeon_ring_write(ring, 0);
3123 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3124 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3125 PACKET3_TC_ACTION_ENA |
3126 PACKET3_SH_KCACHE_ACTION_ENA |
3127 PACKET3_SH_ICACHE_ACTION_ENA);
3128 radeon_ring_write(ring, 0xFFFFFFFF);
3129 radeon_ring_write(ring, 0);
3130 radeon_ring_write(ring, 10); /* poll interval */
3131 /* EVENT_WRITE_EOP - flush caches, send int */
3132 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3133 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3134 radeon_ring_write(ring, addr & 0xffffffff);
3135 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3136 radeon_ring_write(ring, fence->seq);
3137 radeon_ring_write(ring, 0);
3138}
3139
3140/*
3141 * IB stuff
3142 */
3143void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3144{
876dc9f3 3145 struct radeon_ring *ring = &rdev->ring[ib->ring];
2ece2e8b
AD		 3146 u32 header;
3147
a85a7da4
AD		 3148 if (ib->is_const_ib) {
3149 /* set switch buffer packet before const IB */
3150 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3151 radeon_ring_write(ring, 0);
45df6803 3152
2ece2e8b 3153 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 3154 } else {
89d35807 3155 u32 next_rptr;
a85a7da4 3156 if (ring->rptr_save_reg) {
89d35807 3157 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 3158 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3159 radeon_ring_write(ring, ((ring->rptr_save_reg -
3160 PACKET3_SET_CONFIG_REG_START) >> 2));
3161 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3162 } else if (rdev->wb.enabled) {
3163 next_rptr = ring->wptr + 5 + 4 + 8;
3164 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3165 radeon_ring_write(ring, (1 << 8));
3166 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3167 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3168 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 3169 }
3170
2ece2e8b 3171 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 3172 }
2ece2e8b
AD		 3173
3174 radeon_ring_write(ring, header);
3175 radeon_ring_write(ring,
3176#ifdef __BIG_ENDIAN
3177 (2 << 0) |
3178#endif
3179 (ib->gpu_addr & 0xFFFFFFFC));
3180 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4bf3dd92
CK		 3181 radeon_ring_write(ring, ib->length_dw |
3182 (ib->vm ? (ib->vm->id << 24) : 0));
2ece2e8b 3183
a85a7da4
AD		 3184 if (!ib->is_const_ib) {
3185 /* flush read cache over gart for this vmid */
3186 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3187 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 3188 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
a85a7da4
AD		 3189 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3190 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3191 PACKET3_TC_ACTION_ENA |
3192 PACKET3_SH_KCACHE_ACTION_ENA |
3193 PACKET3_SH_ICACHE_ACTION_ENA);
3194 radeon_ring_write(ring, 0xFFFFFFFF);
3195 radeon_ring_write(ring, 0);
3196 radeon_ring_write(ring, 10); /* poll interval */
3197 }
2ece2e8b
AD		 3198}
3199
48c0c902
AD		 3200/*
3201 * CP.
3202 */
3203static void si_cp_enable(struct radeon_device *rdev, bool enable)
3204{
3205 if (enable)
3206 WREG32(CP_ME_CNTL, 0);
3207 else {
3208 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3209 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3210 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 3211 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3212 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3213 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 3214 }
3215 udelay(50);
3216}
3217
3218static int si_cp_load_microcode(struct radeon_device *rdev)
3219{
3220 const __be32 *fw_data;
3221 int i;
3222
3223 if (!rdev->me_fw || !rdev->pfp_fw)
3224 return -EINVAL;
3225
3226 si_cp_enable(rdev, false);
3227
3228 /* PFP */
3229 fw_data = (const __be32 *)rdev->pfp_fw->data;
3230 WREG32(CP_PFP_UCODE_ADDR, 0);
3231 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3232 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3233 WREG32(CP_PFP_UCODE_ADDR, 0);
3234
3235 /* CE */
3236 fw_data = (const __be32 *)rdev->ce_fw->data;
3237 WREG32(CP_CE_UCODE_ADDR, 0);
3238 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3239 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3240 WREG32(CP_CE_UCODE_ADDR, 0);
3241
3242 /* ME */
3243 fw_data = (const __be32 *)rdev->me_fw->data;
3244 WREG32(CP_ME_RAM_WADDR, 0);
3245 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3246 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3247 WREG32(CP_ME_RAM_WADDR, 0);
3248
3249 WREG32(CP_PFP_UCODE_ADDR, 0);
3250 WREG32(CP_CE_UCODE_ADDR, 0);
3251 WREG32(CP_ME_RAM_WADDR, 0);
3252 WREG32(CP_ME_RAM_RADDR, 0);
3253 return 0;
3254}
3255
3256static int si_cp_start(struct radeon_device *rdev)
3257{
3258 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3259 int r, i;
3260
3261 r = radeon_ring_lock(rdev, ring, 7 + 4);
3262 if (r) {
3263 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3264 return r;
3265 }
3266 /* init the CP */
3267 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3268 radeon_ring_write(ring, 0x1);
3269 radeon_ring_write(ring, 0x0);
3270 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3271 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3272 radeon_ring_write(ring, 0);
3273 radeon_ring_write(ring, 0);
3274
3275 /* init the CE partitions */
3276 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3277 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3278 radeon_ring_write(ring, 0xc000);
3279 radeon_ring_write(ring, 0xe000);
3280 radeon_ring_unlock_commit(rdev, ring);
3281
3282 si_cp_enable(rdev, true);
3283
3284 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3285 if (r) {
3286 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3287 return r;
3288 }
3289
3290 /* setup clear context state */
3291 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3292 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3293
3294 for (i = 0; i < si_default_size; i++)
3295 radeon_ring_write(ring, si_default_state[i]);
3296
3297 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3298 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3299
3300 /* set clear context state */
3301 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3302 radeon_ring_write(ring, 0);
3303
3304 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3305 radeon_ring_write(ring, 0x00000316);
3306 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3307 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3308
3309 radeon_ring_unlock_commit(rdev, ring);
3310
3311 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3312 ring = &rdev->ring[i];
3313 r = radeon_ring_lock(rdev, ring, 2);
3314
3315 /* clear the compute context state */
3316 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3317 radeon_ring_write(ring, 0);
3318
3319 radeon_ring_unlock_commit(rdev, ring);
3320 }
3321
3322 return 0;
3323}
3324
3325static void si_cp_fini(struct radeon_device *rdev)
3326{
45df6803 3327 struct radeon_ring *ring;
48c0c902 3328 si_cp_enable(rdev, false);
45df6803
CK		 3329
3330 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3331 radeon_ring_fini(rdev, ring);
3332 radeon_scratch_free(rdev, ring->rptr_save_reg);
3333
3334 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3335 radeon_ring_fini(rdev, ring);
3336 radeon_scratch_free(rdev, ring->rptr_save_reg);
3337
3338 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3339 radeon_ring_fini(rdev, ring);
3340 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3341}
3342
3343static int si_cp_resume(struct radeon_device *rdev)
3344{
3345 struct radeon_ring *ring;
3346 u32 tmp;
3347 u32 rb_bufsz;
3348 int r;
3349
3350 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3351 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3352 SOFT_RESET_PA |
3353 SOFT_RESET_VGT |
3354 SOFT_RESET_SPI |
3355 SOFT_RESET_SX));
3356 RREG32(GRBM_SOFT_RESET);
3357 mdelay(15);
3358 WREG32(GRBM_SOFT_RESET, 0);
3359 RREG32(GRBM_SOFT_RESET);
3360
3361 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3362 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3363
3364 /* Set the write pointer delay */
3365 WREG32(CP_RB_WPTR_DELAY, 0);
3366
3367 WREG32(CP_DEBUG, 0);
3368 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3369
3370 /* ring 0 - compute and gfx */
3371 /* Set ring buffer size */
3372 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3373 rb_bufsz = drm_order(ring->ring_size / 8);
3374 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3375#ifdef __BIG_ENDIAN
3376 tmp |= BUF_SWAP_32BIT;
3377#endif
3378 WREG32(CP_RB0_CNTL, tmp);
3379
3380 /* Initialize the ring buffer's read and write pointers */
3381 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3382 ring->wptr = 0;
3383 WREG32(CP_RB0_WPTR, ring->wptr);
3384
48fc7f7e 3385 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3386 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3387 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3388
3389 if (rdev->wb.enabled)
3390 WREG32(SCRATCH_UMSK, 0xff);
3391 else {
3392 tmp |= RB_NO_UPDATE;
3393 WREG32(SCRATCH_UMSK, 0);
3394 }
3395
3396 mdelay(1);
3397 WREG32(CP_RB0_CNTL, tmp);
3398
3399 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3400
3401 ring->rptr = RREG32(CP_RB0_RPTR);
3402
3403 /* ring1 - compute only */
3404 /* Set ring buffer size */
3405 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3406 rb_bufsz = drm_order(ring->ring_size / 8);
3407 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3408#ifdef __BIG_ENDIAN
3409 tmp |= BUF_SWAP_32BIT;
3410#endif
3411 WREG32(CP_RB1_CNTL, tmp);
3412
3413 /* Initialize the ring buffer's read and write pointers */
3414 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3415 ring->wptr = 0;
3416 WREG32(CP_RB1_WPTR, ring->wptr);
3417
48fc7f7e 3418 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3419 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3420 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3421
3422 mdelay(1);
3423 WREG32(CP_RB1_CNTL, tmp);
3424
3425 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3426
3427 ring->rptr = RREG32(CP_RB1_RPTR);
3428
3429 /* ring2 - compute only */
3430 /* Set ring buffer size */
3431 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3432 rb_bufsz = drm_order(ring->ring_size / 8);
3433 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3434#ifdef __BIG_ENDIAN
3435 tmp |= BUF_SWAP_32BIT;
3436#endif
3437 WREG32(CP_RB2_CNTL, tmp);
3438
3439 /* Initialize the ring buffer's read and write pointers */
3440 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3441 ring->wptr = 0;
3442 WREG32(CP_RB2_WPTR, ring->wptr);
3443
48fc7f7e 3444 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3445 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3446 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3447
3448 mdelay(1);
3449 WREG32(CP_RB2_CNTL, tmp);
3450
3451 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3452
3453 ring->rptr = RREG32(CP_RB2_RPTR);
3454
3455 /* start the rings */
3456 si_cp_start(rdev);
3457 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3458 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3459 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3460 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3461 if (r) {
3462 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3463 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3464 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3465 return r;
3466 }
3467 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3468 if (r) {
3469 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3470 }
3471 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3472 if (r) {
3473 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3474 }
3475
3476 return 0;
3477}
3478
014bb209 3479static u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3480{
014bb209 3481 u32 reset_mask = 0;
1c534671 3482 u32 tmp;
06bc6df0 3483
014bb209
AD		 3484 /* GRBM_STATUS */
3485 tmp = RREG32(GRBM_STATUS);
3486 if (tmp & (PA_BUSY | SC_BUSY |
3487 BCI_BUSY | SX_BUSY |
3488 TA_BUSY | VGT_BUSY |
3489 DB_BUSY | CB_BUSY |
3490 GDS_BUSY | SPI_BUSY |
3491 IA_BUSY | IA_BUSY_NO_DMA))
3492 reset_mask |= RADEON_RESET_GFX;
3493
3494 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3495 CP_BUSY | CP_COHERENCY_BUSY))
3496 reset_mask |= RADEON_RESET_CP;
3497
3498 if (tmp & GRBM_EE_BUSY)
3499 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3500
3501 /* GRBM_STATUS2 */
3502 tmp = RREG32(GRBM_STATUS2);
3503 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3504 reset_mask |= RADEON_RESET_RLC;
3505
3506 /* DMA_STATUS_REG 0 */
3507 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3508 if (!(tmp & DMA_IDLE))
3509 reset_mask |= RADEON_RESET_DMA;
3510
3511 /* DMA_STATUS_REG 1 */
3512 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3513 if (!(tmp & DMA_IDLE))
3514 reset_mask |= RADEON_RESET_DMA1;
3515
3516 /* SRBM_STATUS2 */
3517 tmp = RREG32(SRBM_STATUS2);
3518 if (tmp & DMA_BUSY)
3519 reset_mask |= RADEON_RESET_DMA;
3520
3521 if (tmp & DMA1_BUSY)
3522 reset_mask |= RADEON_RESET_DMA1;
3523
3524 /* SRBM_STATUS */
3525 tmp = RREG32(SRBM_STATUS);
3526
3527 if (tmp & IH_BUSY)
3528 reset_mask |= RADEON_RESET_IH;
3529
3530 if (tmp & SEM_BUSY)
3531 reset_mask |= RADEON_RESET_SEM;
3532
3533 if (tmp & GRBM_RQ_PENDING)
3534 reset_mask |= RADEON_RESET_GRBM;
3535
3536 if (tmp & VMC_BUSY)
3537 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3538
014bb209
AD		 3539 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3540 MCC_BUSY | MCD_BUSY))
3541 reset_mask |= RADEON_RESET_MC;
3542
3543 if (evergreen_is_display_hung(rdev))
3544 reset_mask |= RADEON_RESET_DISPLAY;
3545
3546 /* VM_L2_STATUS */
3547 tmp = RREG32(VM_L2_STATUS);
3548 if (tmp & L2_BUSY)
3549 reset_mask |= RADEON_RESET_VMC;
3550
d808fc88
AD		 3551 /* Skip MC reset as it's mostly likely not hung, just busy */
3552 if (reset_mask & RADEON_RESET_MC) {
3553 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3554 reset_mask &= ~RADEON_RESET_MC;
3555 }
3556
014bb209
AD		 3557 return reset_mask;
3558}
3559
3560static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3561{
3562 struct evergreen_mc_save save;
3563 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3564 u32 tmp;
19fc42ed 3565
06bc6df0 3566 if (reset_mask == 0)
014bb209 3567 return;
06bc6df0
AD		 3568
3569 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3570
1c534671 3571 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3572 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3573 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3574 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3575 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3576
1c534671
AD		 3577 /* Disable CP parsing/prefetching */
3578 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3579
3580 if (reset_mask & RADEON_RESET_DMA) {
3581 /* dma0 */
3582 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3583 tmp &= ~DMA_RB_ENABLE;
3584 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3585 }
3586 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3587 /* dma1 */
3588 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3589 tmp &= ~DMA_RB_ENABLE;
3590 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3591 }
3592
f770d78a
AD		 3593 udelay(50);
3594
3595 evergreen_mc_stop(rdev, &save);
3596 if (evergreen_mc_wait_for_idle(rdev)) {
3597 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3598 }
3599
1c534671
AD		 3600 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3601 grbm_soft_reset = SOFT_RESET_CB |
3602 SOFT_RESET_DB |
3603 SOFT_RESET_GDS |
3604 SOFT_RESET_PA |
3605 SOFT_RESET_SC |
3606 SOFT_RESET_BCI |
3607 SOFT_RESET_SPI |
3608 SOFT_RESET_SX |
3609 SOFT_RESET_TC |
3610 SOFT_RESET_TA |
3611 SOFT_RESET_VGT |
3612 SOFT_RESET_IA;
3613 }
3614
3615 if (reset_mask & RADEON_RESET_CP) {
3616 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3617
3618 srbm_soft_reset |= SOFT_RESET_GRBM;
3619 }
06bc6df0
AD		 3620
3621 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3622 srbm_soft_reset |= SOFT_RESET_DMA;
3623
3624 if (reset_mask & RADEON_RESET_DMA1)
3625 srbm_soft_reset |= SOFT_RESET_DMA1;
3626
3627 if (reset_mask & RADEON_RESET_DISPLAY)
3628 srbm_soft_reset |= SOFT_RESET_DC;
3629
3630 if (reset_mask & RADEON_RESET_RLC)
3631 grbm_soft_reset |= SOFT_RESET_RLC;
3632
3633 if (reset_mask & RADEON_RESET_SEM)
3634 srbm_soft_reset |= SOFT_RESET_SEM;
3635
3636 if (reset_mask & RADEON_RESET_IH)
3637 srbm_soft_reset |= SOFT_RESET_IH;
3638
3639 if (reset_mask & RADEON_RESET_GRBM)
3640 srbm_soft_reset |= SOFT_RESET_GRBM;
3641
3642 if (reset_mask & RADEON_RESET_VMC)
3643 srbm_soft_reset |= SOFT_RESET_VMC;
3644
3645 if (reset_mask & RADEON_RESET_MC)
3646 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3647
3648 if (grbm_soft_reset) {
3649 tmp = RREG32(GRBM_SOFT_RESET);
3650 tmp |= grbm_soft_reset;
3651 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3652 WREG32(GRBM_SOFT_RESET, tmp);
3653 tmp = RREG32(GRBM_SOFT_RESET);
3654
3655 udelay(50);
3656
3657 tmp &= ~grbm_soft_reset;
3658 WREG32(GRBM_SOFT_RESET, tmp);
3659 tmp = RREG32(GRBM_SOFT_RESET);
3660 }
3661
3662 if (srbm_soft_reset) {
3663 tmp = RREG32(SRBM_SOFT_RESET);
3664 tmp |= srbm_soft_reset;
3665 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3666 WREG32(SRBM_SOFT_RESET, tmp);
3667 tmp = RREG32(SRBM_SOFT_RESET);
3668
3669 udelay(50);
3670
3671 tmp &= ~srbm_soft_reset;
3672 WREG32(SRBM_SOFT_RESET, tmp);
3673 tmp = RREG32(SRBM_SOFT_RESET);
3674 }
06bc6df0
AD		 3675
3676 /* Wait a little for things to settle down */
3677 udelay(50);
3678
c476dde2 3679 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3680 udelay(50);
3681
1c534671 3682 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3683}
3684
3685int si_asic_reset(struct radeon_device *rdev)
3686{
014bb209
AD		 3687 u32 reset_mask;
3688
3689 reset_mask = si_gpu_check_soft_reset(rdev);
3690
3691 if (reset_mask)
3692 r600_set_bios_scratch_engine_hung(rdev, true);
3693
3694 si_gpu_soft_reset(rdev, reset_mask);
3695
3696 reset_mask = si_gpu_check_soft_reset(rdev);
3697
3698 if (!reset_mask)
3699 r600_set_bios_scratch_engine_hung(rdev, false);
3700
3701 return 0;
c476dde2
AD		 3702}
3703
123bc183
AD		 3704/**
3705 * si_gfx_is_lockup - Check if the GFX engine is locked up
3706 *
3707 * @rdev: radeon_device pointer
3708 * @ring: radeon_ring structure holding ring information
3709 *
3710 * Check if the GFX engine is locked up.
3711 * Returns true if the engine appears to be locked up, false if not.
3712 */
3713bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3714{
3715 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3716
3717 if (!(reset_mask & (RADEON_RESET_GFX |
3718 RADEON_RESET_COMPUTE |
3719 RADEON_RESET_CP))) {
3720 radeon_ring_lockup_update(ring);
3721 return false;
3722 }
3723 /* force CP activities */
3724 radeon_ring_force_activity(rdev, ring);
3725 return radeon_ring_test_lockup(rdev, ring);
3726}
3727
3728/**
3729 * si_dma_is_lockup - Check if the DMA engine is locked up
3730 *
3731 * @rdev: radeon_device pointer
3732 * @ring: radeon_ring structure holding ring information
3733 *
3734 * Check if the async DMA engine is locked up.
3735 * Returns true if the engine appears to be locked up, false if not.
3736 */
3737bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3738{
3739 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3740 u32 mask;
3741
3742 if (ring->idx == R600_RING_TYPE_DMA_INDEX)
3743 mask = RADEON_RESET_DMA;
3744 else
3745 mask = RADEON_RESET_DMA1;
3746
3747 if (!(reset_mask & mask)) {
3748 radeon_ring_lockup_update(ring);
3749 return false;
3750 }
3751 /* force ring activities */
3752 radeon_ring_force_activity(rdev, ring);
3753 return radeon_ring_test_lockup(rdev, ring);
3754}
3755
d2800ee5
AD		 3756/* MC */
3757static void si_mc_program(struct radeon_device *rdev)
3758{
3759 struct evergreen_mc_save save;
3760 u32 tmp;
3761 int i, j;
3762
3763 /* Initialize HDP */
3764 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3765 WREG32((0x2c14 + j), 0x00000000);
3766 WREG32((0x2c18 + j), 0x00000000);
3767 WREG32((0x2c1c + j), 0x00000000);
3768 WREG32((0x2c20 + j), 0x00000000);
3769 WREG32((0x2c24 + j), 0x00000000);
3770 }
3771 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3772
3773 evergreen_mc_stop(rdev, &save);
3774 if (radeon_mc_wait_for_idle(rdev)) {
3775 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3776 }
5153550a
AD		 3777 if (!ASIC_IS_NODCE(rdev))
3778 /* Lockout access through VGA aperture*/
3779 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 3780 /* Update configuration */
3781 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3782 rdev->mc.vram_start >> 12);
3783 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3784 rdev->mc.vram_end >> 12);
3785 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3786 rdev->vram_scratch.gpu_addr >> 12);
3787 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3788 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3789 WREG32(MC_VM_FB_LOCATION, tmp);
3790 /* XXX double check these! */
3791 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3792 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3793 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3794 WREG32(MC_VM_AGP_BASE, 0);
3795 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3796 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3797 if (radeon_mc_wait_for_idle(rdev)) {
3798 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3799 }
3800 evergreen_mc_resume(rdev, &save);
5153550a
AD		 3801 if (!ASIC_IS_NODCE(rdev)) {
3802 /* we need to own VRAM, so turn off the VGA renderer here
3803 * to stop it overwriting our objects */
3804 rv515_vga_render_disable(rdev);
3805 }
d2800ee5
AD		 3806}
3807
1c49165d
AD		 3808void si_vram_gtt_location(struct radeon_device *rdev,
3809 struct radeon_mc *mc)
d2800ee5
AD		 3810{
3811 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3812 /* leave room for at least 1024M GTT */
3813 dev_warn(rdev->dev, "limiting VRAM\n");
3814 mc->real_vram_size = 0xFFC0000000ULL;
3815 mc->mc_vram_size = 0xFFC0000000ULL;
3816 }
9ed8b1f9 3817 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 3818 rdev->mc.gtt_base_align = 0;
9ed8b1f9 3819 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 3820}
3821
3822static int si_mc_init(struct radeon_device *rdev)
3823{
3824 u32 tmp;
3825 int chansize, numchan;
3826
3827 /* Get VRAM informations */
3828 rdev->mc.vram_is_ddr = true;
3829 tmp = RREG32(MC_ARB_RAMCFG);
3830 if (tmp & CHANSIZE_OVERRIDE) {
3831 chansize = 16;
3832 } else if (tmp & CHANSIZE_MASK) {
3833 chansize = 64;
3834 } else {
3835 chansize = 32;
3836 }
3837 tmp = RREG32(MC_SHARED_CHMAP);
3838 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3839 case 0:
3840 default:
3841 numchan = 1;
3842 break;
3843 case 1:
3844 numchan = 2;
3845 break;
3846 case 2:
3847 numchan = 4;
3848 break;
3849 case 3:
3850 numchan = 8;
3851 break;
3852 case 4:
3853 numchan = 3;
3854 break;
3855 case 5:
3856 numchan = 6;
3857 break;
3858 case 6:
3859 numchan = 10;
3860 break;
3861 case 7:
3862 numchan = 12;
3863 break;
3864 case 8:
3865 numchan = 16;
3866 break;
3867 }
3868 rdev->mc.vram_width = numchan * chansize;
3869 /* Could aper size report 0 ? */
3870 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3871 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3872 /* size in MB on si */
fc986034
NOS		 3873 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3874 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
d2800ee5
AD		 3875 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3876 si_vram_gtt_location(rdev, &rdev->mc);
3877 radeon_update_bandwidth_info(rdev);
3878
3879 return 0;
3880}
3881
3882/*
3883 * GART
3884 */
3885void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3886{
3887 /* flush hdp cache */
3888 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3889
3890 /* bits 0-15 are the VM contexts0-15 */
3891 WREG32(VM_INVALIDATE_REQUEST, 1);
3892}
3893
1109ca09 3894static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 3895{
3896 int r, i;
3897
3898 if (rdev->gart.robj == NULL) {
3899 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3900 return -EINVAL;
3901 }
3902 r = radeon_gart_table_vram_pin(rdev);
3903 if (r)
3904 return r;
3905 radeon_gart_restore(rdev);
3906 /* Setup TLB control */
3907 WREG32(MC_VM_MX_L1_TLB_CNTL,
3908 (0xA << 7) |
3909 ENABLE_L1_TLB |
3910 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3911 ENABLE_ADVANCED_DRIVER_MODEL |
3912 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3913 /* Setup L2 cache */
3914 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3915 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3916 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3917 EFFECTIVE_L2_QUEUE_SIZE(7) |
3918 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3919 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3920 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3921 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3922 /* setup context0 */
3923 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3924 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3925 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3926 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3927 (u32)(rdev->dummy_page.addr >> 12));
3928 WREG32(VM_CONTEXT0_CNTL2, 0);
3929 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3930 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3931
3932 WREG32(0x15D4, 0);
3933 WREG32(0x15D8, 0);
3934 WREG32(0x15DC, 0);
3935
3936 /* empty context1-15 */
d2800ee5
AD		 3937 /* set vm size, must be a multiple of 4 */
3938 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
c21b328e 3939 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
23d4f1f2
AD		 3940 /* Assign the pt base to something valid for now; the pts used for
3941 * the VMs are determined by the application and setup and assigned
3942 * on the fly in the vm part of radeon_gart.c
3943 */
d2800ee5
AD		 3944 for (i = 1; i < 16; i++) {
3945 if (i < 8)
3946 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3947 rdev->gart.table_addr >> 12);
3948 else
3949 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3950 rdev->gart.table_addr >> 12);
3951 }
3952
3953 /* enable context1-15 */
3954 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3955 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 3956 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 3957 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
ae133a11
CK		 3958 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3959 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3960 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3961 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3962 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3963 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3964 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3965 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3966 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3967 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3968 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3969 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 3970
3971 si_pcie_gart_tlb_flush(rdev);
3972 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3973 (unsigned)(rdev->mc.gtt_size >> 20),
3974 (unsigned long long)rdev->gart.table_addr);
3975 rdev->gart.ready = true;
3976 return 0;
3977}
3978
1109ca09 3979static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5
AD		 3980{
3981 /* Disable all tables */
3982 WREG32(VM_CONTEXT0_CNTL, 0);
3983 WREG32(VM_CONTEXT1_CNTL, 0);
3984 /* Setup TLB control */
3985 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3986 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3987 /* Setup L2 cache */
3988 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3989 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3990 EFFECTIVE_L2_QUEUE_SIZE(7) |
3991 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3992 WREG32(VM_L2_CNTL2, 0);
3993 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3994 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3995 radeon_gart_table_vram_unpin(rdev);
3996}
3997
1109ca09 3998static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 3999{
4000 si_pcie_gart_disable(rdev);
4001 radeon_gart_table_vram_free(rdev);
4002 radeon_gart_fini(rdev);
4003}
4004
498dd8b3
AD		 4005/* vm parser */
4006static bool si_vm_reg_valid(u32 reg)
4007{
4008 /* context regs are fine */
4009 if (reg >= 0x28000)
4010 return true;
4011
4012 /* check config regs */
4013 switch (reg) {
4014 case GRBM_GFX_INDEX:
f418b88a 4015 case CP_STRMOUT_CNTL:
498dd8b3
AD		 4016 case VGT_VTX_VECT_EJECT_REG:
4017 case VGT_CACHE_INVALIDATION:
4018 case VGT_ESGS_RING_SIZE:
4019 case VGT_GSVS_RING_SIZE:
4020 case VGT_GS_VERTEX_REUSE:
4021 case VGT_PRIMITIVE_TYPE:
4022 case VGT_INDEX_TYPE:
4023 case VGT_NUM_INDICES:
4024 case VGT_NUM_INSTANCES:
4025 case VGT_TF_RING_SIZE:
4026 case VGT_HS_OFFCHIP_PARAM:
4027 case VGT_TF_MEMORY_BASE:
4028 case PA_CL_ENHANCE:
4029 case PA_SU_LINE_STIPPLE_VALUE:
4030 case PA_SC_LINE_STIPPLE_STATE:
4031 case PA_SC_ENHANCE:
4032 case SQC_CACHES:
4033 case SPI_STATIC_THREAD_MGMT_1:
4034 case SPI_STATIC_THREAD_MGMT_2:
4035 case SPI_STATIC_THREAD_MGMT_3:
4036 case SPI_PS_MAX_WAVE_ID:
4037 case SPI_CONFIG_CNTL:
4038 case SPI_CONFIG_CNTL_1:
4039 case TA_CNTL_AUX:
4040 return true;
4041 default:
4042 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4043 return false;
4044 }
4045}
4046
4047static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4048 u32 *ib, struct radeon_cs_packet *pkt)
4049{
4050 switch (pkt->opcode) {
4051 case PACKET3_NOP:
4052 case PACKET3_SET_BASE:
4053 case PACKET3_SET_CE_DE_COUNTERS:
4054 case PACKET3_LOAD_CONST_RAM:
4055 case PACKET3_WRITE_CONST_RAM:
4056 case PACKET3_WRITE_CONST_RAM_OFFSET:
4057 case PACKET3_DUMP_CONST_RAM:
4058 case PACKET3_INCREMENT_CE_COUNTER:
4059 case PACKET3_WAIT_ON_DE_COUNTER:
4060 case PACKET3_CE_WRITE:
4061 break;
4062 default:
4063 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4064 return -EINVAL;
4065 }
4066 return 0;
4067}
4068
4069static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4070 u32 *ib, struct radeon_cs_packet *pkt)
4071{
4072 u32 idx = pkt->idx + 1;
4073 u32 idx_value = ib[idx];
4074 u32 start_reg, end_reg, reg, i;
5aa709be 4075 u32 command, info;
498dd8b3
AD		 4076
4077 switch (pkt->opcode) {
4078 case PACKET3_NOP:
4079 case PACKET3_SET_BASE:
4080 case PACKET3_CLEAR_STATE:
4081 case PACKET3_INDEX_BUFFER_SIZE:
4082 case PACKET3_DISPATCH_DIRECT:
4083 case PACKET3_DISPATCH_INDIRECT:
4084 case PACKET3_ALLOC_GDS:
4085 case PACKET3_WRITE_GDS_RAM:
4086 case PACKET3_ATOMIC_GDS:
4087 case PACKET3_ATOMIC:
4088 case PACKET3_OCCLUSION_QUERY:
4089 case PACKET3_SET_PREDICATION:
4090 case PACKET3_COND_EXEC:
4091 case PACKET3_PRED_EXEC:
4092 case PACKET3_DRAW_INDIRECT:
4093 case PACKET3_DRAW_INDEX_INDIRECT:
4094 case PACKET3_INDEX_BASE:
4095 case PACKET3_DRAW_INDEX_2:
4096 case PACKET3_CONTEXT_CONTROL:
4097 case PACKET3_INDEX_TYPE:
4098 case PACKET3_DRAW_INDIRECT_MULTI:
4099 case PACKET3_DRAW_INDEX_AUTO:
4100 case PACKET3_DRAW_INDEX_IMMD:
4101 case PACKET3_NUM_INSTANCES:
4102 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4103 case PACKET3_STRMOUT_BUFFER_UPDATE:
4104 case PACKET3_DRAW_INDEX_OFFSET_2:
4105 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4106 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4107 case PACKET3_MPEG_INDEX:
4108 case PACKET3_WAIT_REG_MEM:
4109 case PACKET3_MEM_WRITE:
4110 case PACKET3_PFP_SYNC_ME:
4111 case PACKET3_SURFACE_SYNC:
4112 case PACKET3_EVENT_WRITE:
4113 case PACKET3_EVENT_WRITE_EOP:
4114 case PACKET3_EVENT_WRITE_EOS:
4115 case PACKET3_SET_CONTEXT_REG:
4116 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4117 case PACKET3_SET_SH_REG:
4118 case PACKET3_SET_SH_REG_OFFSET:
4119 case PACKET3_INCREMENT_DE_COUNTER:
4120 case PACKET3_WAIT_ON_CE_COUNTER:
4121 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4122 case PACKET3_ME_WRITE:
4123 break;
4124 case PACKET3_COPY_DATA:
4125 if ((idx_value & 0xf00) == 0) {
4126 reg = ib[idx + 3] * 4;
4127 if (!si_vm_reg_valid(reg))
4128 return -EINVAL;
4129 }
4130 break;
4131 case PACKET3_WRITE_DATA:
4132 if ((idx_value & 0xf00) == 0) {
4133 start_reg = ib[idx + 1] * 4;
4134 if (idx_value & 0x10000) {
4135 if (!si_vm_reg_valid(start_reg))
4136 return -EINVAL;
4137 } else {
4138 for (i = 0; i < (pkt->count - 2); i++) {
4139 reg = start_reg + (4 * i);
4140 if (!si_vm_reg_valid(reg))
4141 return -EINVAL;
4142 }
4143 }
4144 }
4145 break;
4146 case PACKET3_COND_WRITE:
4147 if (idx_value & 0x100) {
4148 reg = ib[idx + 5] * 4;
4149 if (!si_vm_reg_valid(reg))
4150 return -EINVAL;
4151 }
4152 break;
4153 case PACKET3_COPY_DW:
4154 if (idx_value & 0x2) {
4155 reg = ib[idx + 3] * 4;
4156 if (!si_vm_reg_valid(reg))
4157 return -EINVAL;
4158 }
4159 break;
4160 case PACKET3_SET_CONFIG_REG:
4161 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4162 end_reg = 4 * pkt->count + start_reg - 4;
4163 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4164 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4165 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4166 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4167 return -EINVAL;
4168 }
4169 for (i = 0; i < pkt->count; i++) {
4170 reg = start_reg + (4 * i);
4171 if (!si_vm_reg_valid(reg))
4172 return -EINVAL;
4173 }
4174 break;
5aa709be
AD		 4175 case PACKET3_CP_DMA:
4176 command = ib[idx + 4];
4177 info = ib[idx + 1];
4178 if (command & PACKET3_CP_DMA_CMD_SAS) {
4179 /* src address space is register */
4180 if (((info & 0x60000000) >> 29) == 0) {
4181 start_reg = idx_value << 2;
4182 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4183 reg = start_reg;
4184 if (!si_vm_reg_valid(reg)) {
4185 DRM_ERROR("CP DMA Bad SRC register\n");
4186 return -EINVAL;
4187 }
4188 } else {
4189 for (i = 0; i < (command & 0x1fffff); i++) {
4190 reg = start_reg + (4 * i);
4191 if (!si_vm_reg_valid(reg)) {
4192 DRM_ERROR("CP DMA Bad SRC register\n");
4193 return -EINVAL;
4194 }
4195 }
4196 }
4197 }
4198 }
4199 if (command & PACKET3_CP_DMA_CMD_DAS) {
4200 /* dst address space is register */
4201 if (((info & 0x00300000) >> 20) == 0) {
4202 start_reg = ib[idx + 2];
4203 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4204 reg = start_reg;
4205 if (!si_vm_reg_valid(reg)) {
4206 DRM_ERROR("CP DMA Bad DST register\n");
4207 return -EINVAL;
4208 }
4209 } else {
4210 for (i = 0; i < (command & 0x1fffff); i++) {
4211 reg = start_reg + (4 * i);
4212 if (!si_vm_reg_valid(reg)) {
4213 DRM_ERROR("CP DMA Bad DST register\n");
4214 return -EINVAL;
4215 }
4216 }
4217 }
4218 }
4219 }
4220 break;
498dd8b3
AD		 4221 default:
4222 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4223 return -EINVAL;
4224 }
4225 return 0;
4226}
4227
4228static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4229 u32 *ib, struct radeon_cs_packet *pkt)
4230{
4231 u32 idx = pkt->idx + 1;
4232 u32 idx_value = ib[idx];
4233 u32 start_reg, reg, i;
4234
4235 switch (pkt->opcode) {
4236 case PACKET3_NOP:
4237 case PACKET3_SET_BASE:
4238 case PACKET3_CLEAR_STATE:
4239 case PACKET3_DISPATCH_DIRECT:
4240 case PACKET3_DISPATCH_INDIRECT:
4241 case PACKET3_ALLOC_GDS:
4242 case PACKET3_WRITE_GDS_RAM:
4243 case PACKET3_ATOMIC_GDS:
4244 case PACKET3_ATOMIC:
4245 case PACKET3_OCCLUSION_QUERY:
4246 case PACKET3_SET_PREDICATION:
4247 case PACKET3_COND_EXEC:
4248 case PACKET3_PRED_EXEC:
4249 case PACKET3_CONTEXT_CONTROL:
4250 case PACKET3_STRMOUT_BUFFER_UPDATE:
4251 case PACKET3_WAIT_REG_MEM:
4252 case PACKET3_MEM_WRITE:
4253 case PACKET3_PFP_SYNC_ME:
4254 case PACKET3_SURFACE_SYNC:
4255 case PACKET3_EVENT_WRITE:
4256 case PACKET3_EVENT_WRITE_EOP:
4257 case PACKET3_EVENT_WRITE_EOS:
4258 case PACKET3_SET_CONTEXT_REG:
4259 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4260 case PACKET3_SET_SH_REG:
4261 case PACKET3_SET_SH_REG_OFFSET:
4262 case PACKET3_INCREMENT_DE_COUNTER:
4263 case PACKET3_WAIT_ON_CE_COUNTER:
4264 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4265 case PACKET3_ME_WRITE:
4266 break;
4267 case PACKET3_COPY_DATA:
4268 if ((idx_value & 0xf00) == 0) {
4269 reg = ib[idx + 3] * 4;
4270 if (!si_vm_reg_valid(reg))
4271 return -EINVAL;
4272 }
4273 break;
4274 case PACKET3_WRITE_DATA:
4275 if ((idx_value & 0xf00) == 0) {
4276 start_reg = ib[idx + 1] * 4;
4277 if (idx_value & 0x10000) {
4278 if (!si_vm_reg_valid(start_reg))
4279 return -EINVAL;
4280 } else {
4281 for (i = 0; i < (pkt->count - 2); i++) {
4282 reg = start_reg + (4 * i);
4283 if (!si_vm_reg_valid(reg))
4284 return -EINVAL;
4285 }
4286 }
4287 }
4288 break;
4289 case PACKET3_COND_WRITE:
4290 if (idx_value & 0x100) {
4291 reg = ib[idx + 5] * 4;
4292 if (!si_vm_reg_valid(reg))
4293 return -EINVAL;
4294 }
4295 break;
4296 case PACKET3_COPY_DW:
4297 if (idx_value & 0x2) {
4298 reg = ib[idx + 3] * 4;
4299 if (!si_vm_reg_valid(reg))
4300 return -EINVAL;
4301 }
4302 break;
4303 default:
4304 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4305 return -EINVAL;
4306 }
4307 return 0;
4308}
4309
4310int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4311{
4312 int ret = 0;
4313 u32 idx = 0;
4314 struct radeon_cs_packet pkt;
4315
4316 do {
4317 pkt.idx = idx;
4e872ae2
IH		 4318 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4319 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4320 pkt.one_reg_wr = 0;
4321 switch (pkt.type) {
4e872ae2 4322 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 4323 dev_err(rdev->dev, "Packet0 not allowed!\n");
4324 ret = -EINVAL;
4325 break;
4e872ae2 4326 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4327 idx += 1;
4328 break;
4e872ae2
IH		 4329 case RADEON_PACKET_TYPE3:
4330 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4331 if (ib->is_const_ib)
4332 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4333 else {
876dc9f3 4334 switch (ib->ring) {
498dd8b3
AD		 4335 case RADEON_RING_TYPE_GFX_INDEX:
4336 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4337 break;
4338 case CAYMAN_RING_TYPE_CP1_INDEX:
4339 case CAYMAN_RING_TYPE_CP2_INDEX:
4340 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4341 break;
4342 default:
876dc9f3 4343 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4344 ret = -EINVAL;
4345 break;
4346 }
4347 }
4348 idx += pkt.count + 2;
4349 break;
4350 default:
4351 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4352 ret = -EINVAL;
4353 break;
4354 }
4355 if (ret)
4356 break;
4357 } while (idx < ib->length_dw);
4358
4359 return ret;
4360}
4361
d2800ee5
AD		 4362/*
4363 * vm
4364 */
4365int si_vm_init(struct radeon_device *rdev)
4366{
4367 /* number of VMs */
4368 rdev->vm_manager.nvm = 16;
4369 /* base offset of vram pages */
4370 rdev->vm_manager.vram_base_offset = 0;
4371
4372 return 0;
4373}
4374
4375void si_vm_fini(struct radeon_device *rdev)
4376{
4377}
4378
82ffd92b
AD		 4379/**
4380 * si_vm_set_page - update the page tables using the CP
4381 *
4382 * @rdev: radeon_device pointer
43f1214a 4383 * @ib: indirect buffer to fill with commands
82ffd92b
AD		 4384 * @pe: addr of the page entry
4385 * @addr: dst addr to write into pe
4386 * @count: number of page entries to update
4387 * @incr: increase next addr by incr bytes
4388 * @flags: access flags
4389 *
43f1214a 4390 * Update the page tables using the CP (SI).
82ffd92b 4391 */
43f1214a
AD		 4392void si_vm_set_page(struct radeon_device *rdev,
4393 struct radeon_ib *ib,
4394 uint64_t pe,
82ffd92b
AD		 4395 uint64_t addr, unsigned count,
4396 uint32_t incr, uint32_t flags)
d2800ee5 4397{
82ffd92b 4398 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
deab48f1
AD		 4399 uint64_t value;
4400 unsigned ndw;
4401
4402 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4403 while (count) {
4404 ndw = 2 + count * 2;
4405 if (ndw > 0x3FFE)
4406 ndw = 0x3FFE;
4407
43f1214a
AD		 4408 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4409 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4410 WRITE_DATA_DST_SEL(1));
4411 ib->ptr[ib->length_dw++] = pe;
4412 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
deab48f1
AD		 4413 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4414 if (flags & RADEON_VM_PAGE_SYSTEM) {
4415 value = radeon_vm_map_gart(rdev, addr);
4416 value &= 0xFFFFFFFFFFFFF000ULL;
4417 } else if (flags & RADEON_VM_PAGE_VALID) {
4418 value = addr;
4419 } else {
4420 value = 0;
4421 }
4422 addr += incr;
4423 value |= r600_flags;
43f1214a
AD		 4424 ib->ptr[ib->length_dw++] = value;
4425 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4426 }
4427 }
4428 } else {
4429 /* DMA */
4430 if (flags & RADEON_VM_PAGE_SYSTEM) {
4431 while (count) {
4432 ndw = count * 2;
4433 if (ndw > 0xFFFFE)
4434 ndw = 0xFFFFE;
4435
4436 /* for non-physically contiguous pages (system) */
43f1214a
AD		 4437 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
4438 ib->ptr[ib->length_dw++] = pe;
4439 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
deab48f1
AD		 4440 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
4441 if (flags & RADEON_VM_PAGE_SYSTEM) {
4442 value = radeon_vm_map_gart(rdev, addr);
4443 value &= 0xFFFFFFFFFFFFF000ULL;
4444 } else if (flags & RADEON_VM_PAGE_VALID) {
4445 value = addr;
4446 } else {
4447 value = 0;
4448 }
4449 addr += incr;
4450 value |= r600_flags;
43f1214a
AD		 4451 ib->ptr[ib->length_dw++] = value;
4452 ib->ptr[ib->length_dw++] = upper_32_bits(value);
deab48f1
AD		 4453 }
4454 }
4455 } else {
4456 while (count) {
4457 ndw = count * 2;
4458 if (ndw > 0xFFFFE)
4459 ndw = 0xFFFFE;
4460
4461 if (flags & RADEON_VM_PAGE_VALID)
4462 value = addr;
4463 else
4464 value = 0;
4465 /* for physically contiguous pages (vram) */
43f1214a
AD		 4466 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
4467 ib->ptr[ib->length_dw++] = pe; /* dst addr */
4468 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
4469 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
4470 ib->ptr[ib->length_dw++] = 0;
4471 ib->ptr[ib->length_dw++] = value; /* value */
4472 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4473 ib->ptr[ib->length_dw++] = incr; /* increment size */
4474 ib->ptr[ib->length_dw++] = 0;
deab48f1
AD		 4475 pe += ndw * 4;
4476 addr += (ndw / 2) * incr;
4477 count -= ndw / 2;
4478 }
d7025d89 4479 }
43f1214a
AD		 4480 while (ib->length_dw & 0x7)
4481 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
82ffd92b 4482 }
d2800ee5
AD		 4483}
4484
498522b4 4485void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
d2800ee5 4486{
498522b4 4487 struct radeon_ring *ring = &rdev->ring[ridx];
d2800ee5 4488
ee60e29f 4489 if (vm == NULL)
d2800ee5
AD		 4490 return;
4491
76c44f2c
AD		 4492 /* write new base address */
4493 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4494 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4495 WRITE_DATA_DST_SEL(0)));
4496
ee60e29f 4497 if (vm->id < 8) {
76c44f2c
AD		 4498 radeon_ring_write(ring,
4499 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
ee60e29f 4500 } else {
76c44f2c
AD		 4501 radeon_ring_write(ring,
4502 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
ee60e29f 4503 }
76c44f2c 4504 radeon_ring_write(ring, 0);
fa87e62d 4505 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 4506
d2800ee5 4507 /* flush hdp cache */
76c44f2c
AD		 4508 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4509 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4510 WRITE_DATA_DST_SEL(0)));
4511 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4512 radeon_ring_write(ring, 0);
ee60e29f
CK		 4513 radeon_ring_write(ring, 0x1);
4514
d2800ee5 4515 /* bits 0-15 are the VM contexts0-15 */
76c44f2c
AD		 4516 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4517 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4518 WRITE_DATA_DST_SEL(0)));
4519 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4520 radeon_ring_write(ring, 0);
498522b4 4521 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 4522
4523 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4524 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4525 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 4526}
4527
8c5fd7ef
AD		 4528void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4529{
4530 struct radeon_ring *ring = &rdev->ring[ridx];
4531
4532 if (vm == NULL)
4533 return;
4534
4535 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4536 if (vm->id < 8) {
4537 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
4538 } else {
4539 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2));
4540 }
4541 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4542
4543 /* flush hdp cache */
4544 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4545 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4546 radeon_ring_write(ring, 1);
4547
4548 /* bits 0-7 are the VM contexts0-7 */
4549 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4550 radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
4551 radeon_ring_write(ring, 1 << vm->id);
4552}
4553
f8f84ac5
AD		 4554/*
4555 * Power and clock gating
4556 */
4557static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4558{
4559 int i;
4560
4561 for (i = 0; i < rdev->usec_timeout; i++) {
4562 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4563 break;
4564 udelay(1);
4565 }
4566
4567 for (i = 0; i < rdev->usec_timeout; i++) {
4568 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4569 break;
4570 udelay(1);
4571 }
4572}
4573
4574static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4575 bool enable)
4576{
4577 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4578 u32 mask;
4579 int i;
4580
4581 if (enable)
4582 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4583 else
4584 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4585 WREG32(CP_INT_CNTL_RING0, tmp);
4586
4587 if (!enable) {
4588 /* read a gfx register */
4589 tmp = RREG32(DB_DEPTH_INFO);
4590
4591 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4592 for (i = 0; i < rdev->usec_timeout; i++) {
4593 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4594 break;
4595 udelay(1);
4596 }
4597 }
4598}
4599
4600static void si_set_uvd_dcm(struct radeon_device *rdev,
4601 bool sw_mode)
4602{
4603 u32 tmp, tmp2;
4604
4605 tmp = RREG32(UVD_CGC_CTRL);
4606 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4607 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4608
4609 if (sw_mode) {
4610 tmp &= ~0x7ffff800;
4611 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4612 } else {
4613 tmp |= 0x7ffff800;
4614 tmp2 = 0;
4615 }
4616
4617 WREG32(UVD_CGC_CTRL, tmp);
4618 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4619}
4620
4621static void si_init_uvd_internal_cg(struct radeon_device *rdev)
4622{
4623 bool hw_mode = true;
4624
4625 if (hw_mode) {
4626 si_set_uvd_dcm(rdev, false);
4627 } else {
4628 u32 tmp = RREG32(UVD_CGC_CTRL);
4629 tmp &= ~DCM;
4630 WREG32(UVD_CGC_CTRL, tmp);
4631 }
4632}
4633
4634static u32 si_halt_rlc(struct radeon_device *rdev)
4635{
4636 u32 data, orig;
4637
4638 orig = data = RREG32(RLC_CNTL);
4639
4640 if (data & RLC_ENABLE) {
4641 data &= ~RLC_ENABLE;
4642 WREG32(RLC_CNTL, data);
4643
4644 si_wait_for_rlc_serdes(rdev);
4645 }
4646
4647 return orig;
4648}
4649
4650static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4651{
4652 u32 tmp;
4653
4654 tmp = RREG32(RLC_CNTL);
4655 if (tmp != rlc)
4656 WREG32(RLC_CNTL, rlc);
4657}
4658
4659static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4660{
4661 u32 data, orig;
4662
4663 orig = data = RREG32(DMA_PG);
4664 if (enable)
4665 data |= PG_CNTL_ENABLE;
4666 else
4667 data &= ~PG_CNTL_ENABLE;
4668 if (orig != data)
4669 WREG32(DMA_PG, data);
4670}
4671
4672static void si_init_dma_pg(struct radeon_device *rdev)
4673{
4674 u32 tmp;
4675
4676 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4677 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4678
4679 for (tmp = 0; tmp < 5; tmp++)
4680 WREG32(DMA_PGFSM_WRITE, 0);
4681}
4682
4683static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4684 bool enable)
4685{
4686 u32 tmp;
4687
4688 if (enable) {
4689 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4690 WREG32(RLC_TTOP_D, tmp);
4691
4692 tmp = RREG32(RLC_PG_CNTL);
4693 tmp |= GFX_PG_ENABLE;
4694 WREG32(RLC_PG_CNTL, tmp);
4695
4696 tmp = RREG32(RLC_AUTO_PG_CTRL);
4697 tmp |= AUTO_PG_EN;
4698 WREG32(RLC_AUTO_PG_CTRL, tmp);
4699 } else {
4700 tmp = RREG32(RLC_AUTO_PG_CTRL);
4701 tmp &= ~AUTO_PG_EN;
4702 WREG32(RLC_AUTO_PG_CTRL, tmp);
4703
4704 tmp = RREG32(DB_RENDER_CONTROL);
4705 }
4706}
4707
4708static void si_init_gfx_cgpg(struct radeon_device *rdev)
4709{
4710 u32 tmp;
4711
4712 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4713
4714 tmp = RREG32(RLC_PG_CNTL);
4715 tmp |= GFX_PG_SRC;
4716 WREG32(RLC_PG_CNTL, tmp);
4717
4718 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4719
4720 tmp = RREG32(RLC_AUTO_PG_CTRL);
4721
4722 tmp &= ~GRBM_REG_SGIT_MASK;
4723 tmp |= GRBM_REG_SGIT(0x700);
4724 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
4725 WREG32(RLC_AUTO_PG_CTRL, tmp);
4726}
4727
4728static u32 get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
4729{
4730 u32 mask = 0, tmp, tmp1;
4731 int i;
4732
4733 si_select_se_sh(rdev, se, sh);
4734 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
4735 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
4736 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4737
4738 tmp &= 0xffff0000;
4739
4740 tmp |= tmp1;
4741 tmp >>= 16;
4742
4743 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
4744 mask <<= 1;
4745 mask |= 1;
4746 }
4747
4748 return (~tmp) & mask;
4749}
4750
4751static void si_init_ao_cu_mask(struct radeon_device *rdev)
4752{
4753 u32 i, j, k, active_cu_number = 0;
4754 u32 mask, counter, cu_bitmap;
4755 u32 tmp = 0;
4756
4757 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
4758 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
4759 mask = 1;
4760 cu_bitmap = 0;
4761 counter = 0;
4762 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
4763 if (get_cu_active_bitmap(rdev, i, j) & mask) {
4764 if (counter < 2)
4765 cu_bitmap |= mask;
4766 counter++;
4767 }
4768 mask <<= 1;
4769 }
4770
4771 active_cu_number += counter;
4772 tmp |= (cu_bitmap << (i * 16 + j * 8));
4773 }
4774 }
4775
4776 WREG32(RLC_PG_AO_CU_MASK, tmp);
4777
4778 tmp = RREG32(RLC_MAX_PG_CU);
4779 tmp &= ~MAX_PU_CU_MASK;
4780 tmp |= MAX_PU_CU(active_cu_number);
4781 WREG32(RLC_MAX_PG_CU, tmp);
4782}
4783
4784static void si_enable_cgcg(struct radeon_device *rdev,
4785 bool enable)
4786{
4787 u32 data, orig, tmp;
4788
4789 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
4790
4791 si_enable_gui_idle_interrupt(rdev, enable);
4792
4793 if (enable) {
4794 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
4795
4796 tmp = si_halt_rlc(rdev);
4797
4798 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4799 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4800 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
4801
4802 si_wait_for_rlc_serdes(rdev);
4803
4804 si_update_rlc(rdev, tmp);
4805
4806 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
4807
4808 data |= CGCG_EN | CGLS_EN;
4809 } else {
4810 RREG32(CB_CGTT_SCLK_CTRL);
4811 RREG32(CB_CGTT_SCLK_CTRL);
4812 RREG32(CB_CGTT_SCLK_CTRL);
4813 RREG32(CB_CGTT_SCLK_CTRL);
4814
4815 data &= ~(CGCG_EN | CGLS_EN);
4816 }
4817
4818 if (orig != data)
4819 WREG32(RLC_CGCG_CGLS_CTRL, data);
4820}
4821
4822static void si_enable_mgcg(struct radeon_device *rdev,
4823 bool enable)
4824{
4825 u32 data, orig, tmp = 0;
4826
4827 if (enable) {
4828 orig = data = RREG32(CGTS_SM_CTRL_REG);
4829 data = 0x96940200;
4830 if (orig != data)
4831 WREG32(CGTS_SM_CTRL_REG, data);
4832
4833 orig = data = RREG32(CP_MEM_SLP_CNTL);
4834 data |= CP_MEM_LS_EN;
4835 if (orig != data)
4836 WREG32(CP_MEM_SLP_CNTL, data);
4837
4838 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4839 data &= 0xffffffc0;
4840 if (orig != data)
4841 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
4842
4843 tmp = si_halt_rlc(rdev);
4844
4845 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4846 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4847 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
4848
4849 si_update_rlc(rdev, tmp);
4850 } else {
4851 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4852 data |= 0x00000003;
4853 if (orig != data)
4854 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
4855
4856 data = RREG32(CP_MEM_SLP_CNTL);
4857 if (data & CP_MEM_LS_EN) {
4858 data &= ~CP_MEM_LS_EN;
4859 WREG32(CP_MEM_SLP_CNTL, data);
4860 }
4861 orig = data = RREG32(CGTS_SM_CTRL_REG);
4862 data |= LS_OVERRIDE | OVERRIDE;
4863 if (orig != data)
4864 WREG32(CGTS_SM_CTRL_REG, data);
4865
4866 tmp = si_halt_rlc(rdev);
4867
4868 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4869 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4870 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
4871
4872 si_update_rlc(rdev, tmp);
4873 }
4874}
4875
4876static void si_enable_uvd_mgcg(struct radeon_device *rdev,
4877 bool enable)
4878{
4879 u32 orig, data, tmp;
4880
4881 if (enable) {
4882 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
4883 tmp |= 0x3fff;
4884 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
4885
4886 orig = data = RREG32(UVD_CGC_CTRL);
4887 data |= DCM;
4888 if (orig != data)
4889 WREG32(UVD_CGC_CTRL, data);
4890
4891 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
4892 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
4893 } else {
4894 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
4895 tmp &= ~0x3fff;
4896 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
4897
4898 orig = data = RREG32(UVD_CGC_CTRL);
4899 data &= ~DCM;
4900 if (orig != data)
4901 WREG32(UVD_CGC_CTRL, data);
4902
4903 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
4904 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
4905 }
4906}
4907
4908static const u32 mc_cg_registers[] =
4909{
4910 MC_HUB_MISC_HUB_CG,
4911 MC_HUB_MISC_SIP_CG,
4912 MC_HUB_MISC_VM_CG,
4913 MC_XPB_CLK_GAT,
4914 ATC_MISC_CG,
4915 MC_CITF_MISC_WR_CG,
4916 MC_CITF_MISC_RD_CG,
4917 MC_CITF_MISC_VM_CG,
4918 VM_L2_CG,
4919};
4920
4921static void si_enable_mc_ls(struct radeon_device *rdev,
4922 bool enable)
4923{
4924 int i;
4925 u32 orig, data;
4926
4927 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
4928 orig = data = RREG32(mc_cg_registers[i]);
4929 if (enable)
4930 data |= MC_LS_ENABLE;
4931 else
4932 data &= ~MC_LS_ENABLE;
4933 if (data != orig)
4934 WREG32(mc_cg_registers[i], data);
4935 }
4936}
4937
4938
4939static void si_init_cg(struct radeon_device *rdev)
4940{
4941 bool has_uvd = true;
4942
4943 si_enable_mgcg(rdev, true);
4944 si_enable_cgcg(rdev, true);
4945 /* disable MC LS on Tahiti */
4946 if (rdev->family == CHIP_TAHITI)
4947 si_enable_mc_ls(rdev, false);
4948 if (has_uvd) {
4949 si_enable_uvd_mgcg(rdev, true);
4950 si_init_uvd_internal_cg(rdev);
4951 }
4952}
4953
4954static void si_fini_cg(struct radeon_device *rdev)
4955{
4956 bool has_uvd = true;
4957
4958 if (has_uvd)
4959 si_enable_uvd_mgcg(rdev, false);
4960 si_enable_cgcg(rdev, false);
4961 si_enable_mgcg(rdev, false);
4962}
4963
4964static void si_init_pg(struct radeon_device *rdev)
4965{
4966 bool has_pg = false;
4967
4968 /* only cape verde supports PG */
4969 if (rdev->family == CHIP_VERDE)
4970 has_pg = true;
4971
4972 if (has_pg) {
4973 si_init_ao_cu_mask(rdev);
4974 si_init_dma_pg(rdev);
4975 si_enable_dma_pg(rdev, true);
4976 si_init_gfx_cgpg(rdev);
4977 si_enable_gfx_cgpg(rdev, true);
4978 } else {
4979 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4980 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4981 }
4982}
4983
4984static void si_fini_pg(struct radeon_device *rdev)
4985{
4986 bool has_pg = false;
4987
4988 /* only cape verde supports PG */
4989 if (rdev->family == CHIP_VERDE)
4990 has_pg = true;
4991
4992 if (has_pg) {
4993 si_enable_dma_pg(rdev, false);
4994 si_enable_gfx_cgpg(rdev, false);
4995 }
4996}
4997
347e7592
AD		 4998/*
4999 * RLC
5000 */
c420c745 5001void si_rlc_fini(struct radeon_device *rdev)
347e7592
AD		 5002{
5003 int r;
5004
5005 /* save restore block */
5006 if (rdev->rlc.save_restore_obj) {
5007 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
5008 if (unlikely(r != 0))
5009 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
5010 radeon_bo_unpin(rdev->rlc.save_restore_obj);
5011 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
5012
5013 radeon_bo_unref(&rdev->rlc.save_restore_obj);
5014 rdev->rlc.save_restore_obj = NULL;
5015 }
5016
5017 /* clear state block */
5018 if (rdev->rlc.clear_state_obj) {
5019 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
5020 if (unlikely(r != 0))
5021 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
5022 radeon_bo_unpin(rdev->rlc.clear_state_obj);
5023 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
5024
5025 radeon_bo_unref(&rdev->rlc.clear_state_obj);
5026 rdev->rlc.clear_state_obj = NULL;
5027 }
5028}
5029
bd8cd539
AD		 5030#define RLC_CLEAR_STATE_END_MARKER 0x00000001
5031
c420c745 5032int si_rlc_init(struct radeon_device *rdev)
347e7592 5033{
6d8cf000 5034 volatile u32 *dst_ptr;
bd8cd539
AD		 5035 u32 dws, data, i, j, k, reg_num;
5036 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
5037 u64 reg_list_mc_addr;
5038 const struct cs_section_def *cs_data = si_cs_data;
5039 int r;
347e7592
AD		 5040
5041 /* save restore block */
5042 if (rdev->rlc.save_restore_obj == NULL) {
5043 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
40f5cf99
AD		 5044 RADEON_GEM_DOMAIN_VRAM, NULL,
5045 &rdev->rlc.save_restore_obj);
347e7592
AD		 5046 if (r) {
5047 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
5048 return r;
5049 }
5050 }
5051
5052 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
5053 if (unlikely(r != 0)) {
5054 si_rlc_fini(rdev);
5055 return r;
5056 }
5057 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
5058 &rdev->rlc.save_restore_gpu_addr);
5059 if (r) {
6d8cf000 5060 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
347e7592
AD		 5061 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
5062 si_rlc_fini(rdev);
5063 return r;
5064 }
5065
6d8cf000
AD		 5066 if (rdev->family == CHIP_VERDE) {
5067 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
5068 if (r) {
5069 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
5070 si_rlc_fini(rdev);
5071 return r;
5072 }
5073 /* write the sr buffer */
5074 dst_ptr = rdev->rlc.sr_ptr;
5075 for (i = 0; i < ARRAY_SIZE(verde_rlc_save_restore_register_list); i++) {
5076 dst_ptr[i] = verde_rlc_save_restore_register_list[i];
5077 }
5078 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
5079 }
5080 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
5081
347e7592 5082 /* clear state block */
bd8cd539
AD		 5083 reg_list_num = 0;
5084 dws = 0;
5085 for (i = 0; cs_data[i].section != NULL; i++) {
5086 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
5087 reg_list_num++;
5088 dws += cs_data[i].section[j].reg_count;
5089 }
5090 }
5091 reg_list_blk_index = (3 * reg_list_num + 2);
5092 dws += reg_list_blk_index;
5093
347e7592 5094 if (rdev->rlc.clear_state_obj == NULL) {
bd8cd539
AD		 5095 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
5096 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
347e7592
AD		 5097 if (r) {
5098 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
5099 si_rlc_fini(rdev);
5100 return r;
5101 }
5102 }
5103 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
5104 if (unlikely(r != 0)) {
5105 si_rlc_fini(rdev);
5106 return r;
5107 }
5108 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
5109 &rdev->rlc.clear_state_gpu_addr);
5110 if (r) {
bd8cd539
AD		 5111
5112 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
347e7592
AD		 5113 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
5114 si_rlc_fini(rdev);
5115 return r;
5116 }
bd8cd539
AD		 5117 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
5118 if (r) {
5119 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
5120 si_rlc_fini(rdev);
5121 return r;
5122 }
5123 /* set up the cs buffer */
5124 dst_ptr = rdev->rlc.cs_ptr;
5125 reg_list_hdr_blk_index = 0;
5126 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
5127 data = upper_32_bits(reg_list_mc_addr);
5128 dst_ptr[reg_list_hdr_blk_index] = data;
5129 reg_list_hdr_blk_index++;
5130 for (i = 0; cs_data[i].section != NULL; i++) {
5131 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
5132 reg_num = cs_data[i].section[j].reg_count;
5133 data = reg_list_mc_addr & 0xffffffff;
5134 dst_ptr[reg_list_hdr_blk_index] = data;
5135 reg_list_hdr_blk_index++;
5136
5137 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
5138 dst_ptr[reg_list_hdr_blk_index] = data;
5139 reg_list_hdr_blk_index++;
5140
5141 data = 0x08000000 | (reg_num * 4);
5142 dst_ptr[reg_list_hdr_blk_index] = data;
5143 reg_list_hdr_blk_index++;
5144
5145 for (k = 0; k < reg_num; k++) {
5146 data = cs_data[i].section[j].extent[k];
5147 dst_ptr[reg_list_blk_index + k] = data;
5148 }
5149 reg_list_mc_addr += reg_num * 4;
5150 reg_list_blk_index += reg_num;
5151 }
5152 }
5153 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
5154
5155 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
5156 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
347e7592
AD		 5157
5158 return 0;
5159}
5160
f8f84ac5 5161static void si_rlc_reset(struct radeon_device *rdev)
d719cef3 5162{
f8f84ac5 5163 u32 tmp = RREG32(GRBM_SOFT_RESET);
d719cef3 5164
f8f84ac5
AD		 5165 tmp |= SOFT_RESET_RLC;
5166 WREG32(GRBM_SOFT_RESET, tmp);
5167 udelay(50);
5168 tmp &= ~SOFT_RESET_RLC;
5169 WREG32(GRBM_SOFT_RESET, tmp);
5170 udelay(50);
d719cef3
AD		 5171}
5172
347e7592
AD		 5173static void si_rlc_stop(struct radeon_device *rdev)
5174{
5175 WREG32(RLC_CNTL, 0);
d719cef3
AD		 5176
5177 si_enable_gui_idle_interrupt(rdev, false);
5178
5179 si_wait_for_rlc_serdes(rdev);
347e7592
AD		 5180}
5181
5182static void si_rlc_start(struct radeon_device *rdev)
5183{
5184 WREG32(RLC_CNTL, RLC_ENABLE);
d719cef3
AD		 5185
5186 si_enable_gui_idle_interrupt(rdev, true);
5187
5188 udelay(50);
5189}
5190
5191static bool si_lbpw_supported(struct radeon_device *rdev)
5192{
5193 u32 tmp;
5194
5195 /* Enable LBPW only for DDR3 */
5196 tmp = RREG32(MC_SEQ_MISC0);
5197 if ((tmp & 0xF0000000) == 0xB0000000)
5198 return true;
5199 return false;
5200}
5201
5202static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5203{
5204 u32 tmp;
5205
5206 tmp = RREG32(RLC_LB_CNTL);
5207 if (enable)
5208 tmp |= LOAD_BALANCE_ENABLE;
5209 else
5210 tmp &= ~LOAD_BALANCE_ENABLE;
5211 WREG32(RLC_LB_CNTL, tmp);
5212
5213 if (!enable) {
5214 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5215 WREG32(SPI_LB_CU_MASK, 0x00ff);
5216 }
347e7592
AD		 5217}
5218
5219static int si_rlc_resume(struct radeon_device *rdev)
5220{
5221 u32 i;
5222 const __be32 *fw_data;
5223
5224 if (!rdev->rlc_fw)
5225 return -EINVAL;
5226
5227 si_rlc_stop(rdev);
5228
f8f84ac5
AD		 5229 si_rlc_reset(rdev);
5230
5231 si_init_pg(rdev);
5232
5233 si_init_cg(rdev);
5234
347e7592
AD		 5235 WREG32(RLC_RL_BASE, 0);
5236 WREG32(RLC_RL_SIZE, 0);
5237 WREG32(RLC_LB_CNTL, 0);
5238 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5239 WREG32(RLC_LB_CNTR_INIT, 0);
d719cef3 5240 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
347e7592 5241
347e7592
AD		 5242 WREG32(RLC_MC_CNTL, 0);
5243 WREG32(RLC_UCODE_CNTL, 0);
5244
5245 fw_data = (const __be32 *)rdev->rlc_fw->data;
5246 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5247 WREG32(RLC_UCODE_ADDR, i);
5248 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5249 }
5250 WREG32(RLC_UCODE_ADDR, 0);
5251
d719cef3
AD		 5252 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5253
347e7592
AD		 5254 si_rlc_start(rdev);
5255
5256 return 0;
5257}
5258
25a857fb
AD		 5259static void si_enable_interrupts(struct radeon_device *rdev)
5260{
5261 u32 ih_cntl = RREG32(IH_CNTL);
5262 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5263
5264 ih_cntl |= ENABLE_INTR;
5265 ih_rb_cntl |= IH_RB_ENABLE;
5266 WREG32(IH_CNTL, ih_cntl);
5267 WREG32(IH_RB_CNTL, ih_rb_cntl);
5268 rdev->ih.enabled = true;
5269}
5270
5271static void si_disable_interrupts(struct radeon_device *rdev)
5272{
5273 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5274 u32 ih_cntl = RREG32(IH_CNTL);
5275
5276 ih_rb_cntl &= ~IH_RB_ENABLE;
5277 ih_cntl &= ~ENABLE_INTR;
5278 WREG32(IH_RB_CNTL, ih_rb_cntl);
5279 WREG32(IH_CNTL, ih_cntl);
5280 /* set rptr, wptr to 0 */
5281 WREG32(IH_RB_RPTR, 0);
5282 WREG32(IH_RB_WPTR, 0);
5283 rdev->ih.enabled = false;
25a857fb
AD		 5284 rdev->ih.rptr = 0;
5285}
5286
5287static void si_disable_interrupt_state(struct radeon_device *rdev)
5288{
5289 u32 tmp;
5290
5291 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5292 WREG32(CP_INT_CNTL_RING1, 0);
5293 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 5294 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5295 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5296 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5297 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 5298 WREG32(GRBM_INT_CNTL, 0);
5153550a
AD		 5299 if (rdev->num_crtc >= 2) {
5300 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5301 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5302 }
25a857fb
AD		 5303 if (rdev->num_crtc >= 4) {
5304 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5305 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5306 }
5307 if (rdev->num_crtc >= 6) {
5308 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5309 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5310 }
5311
5153550a
AD		 5312 if (rdev->num_crtc >= 2) {
5313 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5314 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5315 }
25a857fb
AD		 5316 if (rdev->num_crtc >= 4) {
5317 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5318 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5319 }
5320 if (rdev->num_crtc >= 6) {
5321 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5322 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5323 }
5324
5153550a
AD		 5325 if (!ASIC_IS_NODCE(rdev)) {
5326 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
5327
5328 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5329 WREG32(DC_HPD1_INT_CONTROL, tmp);
5330 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5331 WREG32(DC_HPD2_INT_CONTROL, tmp);
5332 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5333 WREG32(DC_HPD3_INT_CONTROL, tmp);
5334 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5335 WREG32(DC_HPD4_INT_CONTROL, tmp);
5336 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5337 WREG32(DC_HPD5_INT_CONTROL, tmp);
5338 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5339 WREG32(DC_HPD6_INT_CONTROL, tmp);
5340 }
25a857fb
AD		 5341}
5342
5343static int si_irq_init(struct radeon_device *rdev)
5344{
5345 int ret = 0;
5346 int rb_bufsz;
5347 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5348
5349 /* allocate ring */
5350 ret = r600_ih_ring_alloc(rdev);
5351 if (ret)
5352 return ret;
5353
5354 /* disable irqs */
5355 si_disable_interrupts(rdev);
5356
5357 /* init rlc */
5358 ret = si_rlc_resume(rdev);
5359 if (ret) {
5360 r600_ih_ring_fini(rdev);
5361 return ret;
5362 }
5363
5364 /* setup interrupt control */
5365 /* set dummy read address to ring address */
5366 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5367 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5368 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5369 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5370 */
5371 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5372 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5373 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5374 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5375
5376 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5377 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
5378
5379 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5380 IH_WPTR_OVERFLOW_CLEAR |
5381 (rb_bufsz << 1));
5382
5383 if (rdev->wb.enabled)
5384 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5385
5386 /* set the writeback address whether it's enabled or not */
5387 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5388 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5389
5390 WREG32(IH_RB_CNTL, ih_rb_cntl);
5391
5392 /* set rptr, wptr to 0 */
5393 WREG32(IH_RB_RPTR, 0);
5394 WREG32(IH_RB_WPTR, 0);
5395
5396 /* Default settings for IH_CNTL (disabled at first) */
5397 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5398 /* RPTR_REARM only works if msi's are enabled */
5399 if (rdev->msi_enabled)
5400 ih_cntl |= RPTR_REARM;
5401 WREG32(IH_CNTL, ih_cntl);
5402
5403 /* force the active interrupt state to all disabled */
5404 si_disable_interrupt_state(rdev);
5405
2099810f
DA		 5406 pci_set_master(rdev->pdev);
5407
25a857fb
AD		 5408 /* enable irqs */
5409 si_enable_interrupts(rdev);
5410
5411 return ret;
5412}
5413
5414int si_irq_set(struct radeon_device *rdev)
5415{
5416 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
5417 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5418 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5153550a 5419 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
25a857fb
AD		 5420 u32 grbm_int_cntl = 0;
5421 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
8c5fd7ef 5422 u32 dma_cntl, dma_cntl1;
25a857fb
AD		 5423
5424 if (!rdev->irq.installed) {
5425 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5426 return -EINVAL;
5427 }
5428 /* don't enable anything if the ih is disabled */
5429 if (!rdev->ih.enabled) {
5430 si_disable_interrupts(rdev);
5431 /* force the active interrupt state to all disabled */
5432 si_disable_interrupt_state(rdev);
5433 return 0;
5434 }
5435
5153550a
AD		 5436 if (!ASIC_IS_NODCE(rdev)) {
5437 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5438 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5439 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5440 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5441 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5442 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5443 }
25a857fb 5444
8c5fd7ef
AD		 5445 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5446 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5447
25a857fb 5448 /* enable CP interrupts on all rings */
736fc37f 5449 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 5450 DRM_DEBUG("si_irq_set: sw int gfx\n");
5451 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5452 }
736fc37f 5453 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 5454 DRM_DEBUG("si_irq_set: sw int cp1\n");
5455 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5456 }
736fc37f 5457 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 5458 DRM_DEBUG("si_irq_set: sw int cp2\n");
5459 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5460 }
8c5fd7ef
AD		 5461 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5462 DRM_DEBUG("si_irq_set: sw int dma\n");
5463 dma_cntl |= TRAP_ENABLE;
5464 }
5465
5466 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5467 DRM_DEBUG("si_irq_set: sw int dma1\n");
5468 dma_cntl1 |= TRAP_ENABLE;
5469 }
25a857fb 5470 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 5471 atomic_read(&rdev->irq.pflip[0])) {
25a857fb
AD		 5472 DRM_DEBUG("si_irq_set: vblank 0\n");
5473 crtc1 |= VBLANK_INT_MASK;
5474 }
5475 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 5476 atomic_read(&rdev->irq.pflip[1])) {
25a857fb
AD		 5477 DRM_DEBUG("si_irq_set: vblank 1\n");
5478 crtc2 |= VBLANK_INT_MASK;
5479 }
5480 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 5481 atomic_read(&rdev->irq.pflip[2])) {
25a857fb
AD		 5482 DRM_DEBUG("si_irq_set: vblank 2\n");
5483 crtc3 |= VBLANK_INT_MASK;
5484 }
5485 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 5486 atomic_read(&rdev->irq.pflip[3])) {
25a857fb
AD		 5487 DRM_DEBUG("si_irq_set: vblank 3\n");
5488 crtc4 |= VBLANK_INT_MASK;
5489 }
5490 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 5491 atomic_read(&rdev->irq.pflip[4])) {
25a857fb
AD		 5492 DRM_DEBUG("si_irq_set: vblank 4\n");
5493 crtc5 |= VBLANK_INT_MASK;
5494 }
5495 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 5496 atomic_read(&rdev->irq.pflip[5])) {
25a857fb
AD		 5497 DRM_DEBUG("si_irq_set: vblank 5\n");
5498 crtc6 |= VBLANK_INT_MASK;
5499 }
5500 if (rdev->irq.hpd[0]) {
5501 DRM_DEBUG("si_irq_set: hpd 1\n");
5502 hpd1 |= DC_HPDx_INT_EN;
5503 }
5504 if (rdev->irq.hpd[1]) {
5505 DRM_DEBUG("si_irq_set: hpd 2\n");
5506 hpd2 |= DC_HPDx_INT_EN;
5507 }
5508 if (rdev->irq.hpd[2]) {
5509 DRM_DEBUG("si_irq_set: hpd 3\n");
5510 hpd3 |= DC_HPDx_INT_EN;
5511 }
5512 if (rdev->irq.hpd[3]) {
5513 DRM_DEBUG("si_irq_set: hpd 4\n");
5514 hpd4 |= DC_HPDx_INT_EN;
5515 }
5516 if (rdev->irq.hpd[4]) {
5517 DRM_DEBUG("si_irq_set: hpd 5\n");
5518 hpd5 |= DC_HPDx_INT_EN;
5519 }
5520 if (rdev->irq.hpd[5]) {
5521 DRM_DEBUG("si_irq_set: hpd 6\n");
5522 hpd6 |= DC_HPDx_INT_EN;
5523 }
25a857fb
AD		 5524
5525 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5526 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5527 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5528
8c5fd7ef
AD		 5529 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5530 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5531
25a857fb
AD		 5532 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5533
5153550a
AD		 5534 if (rdev->num_crtc >= 2) {
5535 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5536 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5537 }
25a857fb
AD		 5538 if (rdev->num_crtc >= 4) {
5539 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5540 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5541 }
5542 if (rdev->num_crtc >= 6) {
5543 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5544 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5545 }
5546
5153550a
AD		 5547 if (rdev->num_crtc >= 2) {
5548 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5549 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5550 }
25a857fb
AD		 5551 if (rdev->num_crtc >= 4) {
5552 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5553 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5554 }
5555 if (rdev->num_crtc >= 6) {
5556 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5557 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5558 }
5559
5153550a
AD		 5560 if (!ASIC_IS_NODCE(rdev)) {
5561 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5562 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5563 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5564 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5565 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5566 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5567 }
25a857fb
AD		 5568
5569 return 0;
5570}
5571
5572static inline void si_irq_ack(struct radeon_device *rdev)
5573{
5574 u32 tmp;
5575
5153550a
AD		 5576 if (ASIC_IS_NODCE(rdev))
5577 return;
5578
25a857fb
AD		 5579 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5580 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5581 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5582 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5583 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5584 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5585 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5586 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5587 if (rdev->num_crtc >= 4) {
5588 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5589 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5590 }
5591 if (rdev->num_crtc >= 6) {
5592 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5593 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5594 }
5595
5596 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5597 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5598 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5599 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5600 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5601 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5602 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5603 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5604 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5605 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5606 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5607 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5608
5609 if (rdev->num_crtc >= 4) {
5610 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5611 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5612 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5613 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5614 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5615 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5616 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5617 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5618 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5619 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5620 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5621 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5622 }
5623
5624 if (rdev->num_crtc >= 6) {
5625 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5626 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5627 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5628 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5629 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5630 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5631 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5632 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5633 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5634 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5635 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5636 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5637 }
5638
5639 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5640 tmp = RREG32(DC_HPD1_INT_CONTROL);
5641 tmp |= DC_HPDx_INT_ACK;
5642 WREG32(DC_HPD1_INT_CONTROL, tmp);
5643 }
5644 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5645 tmp = RREG32(DC_HPD2_INT_CONTROL);
5646 tmp |= DC_HPDx_INT_ACK;
5647 WREG32(DC_HPD2_INT_CONTROL, tmp);
5648 }
5649 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5650 tmp = RREG32(DC_HPD3_INT_CONTROL);
5651 tmp |= DC_HPDx_INT_ACK;
5652 WREG32(DC_HPD3_INT_CONTROL, tmp);
5653 }
5654 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5655 tmp = RREG32(DC_HPD4_INT_CONTROL);
5656 tmp |= DC_HPDx_INT_ACK;
5657 WREG32(DC_HPD4_INT_CONTROL, tmp);
5658 }
5659 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5660 tmp = RREG32(DC_HPD5_INT_CONTROL);
5661 tmp |= DC_HPDx_INT_ACK;
5662 WREG32(DC_HPD5_INT_CONTROL, tmp);
5663 }
5664 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5665 tmp = RREG32(DC_HPD5_INT_CONTROL);
5666 tmp |= DC_HPDx_INT_ACK;
5667 WREG32(DC_HPD6_INT_CONTROL, tmp);
5668 }
5669}
5670
5671static void si_irq_disable(struct radeon_device *rdev)
5672{
5673 si_disable_interrupts(rdev);
5674 /* Wait and acknowledge irq */
5675 mdelay(1);
5676 si_irq_ack(rdev);
5677 si_disable_interrupt_state(rdev);
5678}
5679
5680static void si_irq_suspend(struct radeon_device *rdev)
5681{
5682 si_irq_disable(rdev);
5683 si_rlc_stop(rdev);
5684}
5685
9b136d51
AD		 5686static void si_irq_fini(struct radeon_device *rdev)
5687{
5688 si_irq_suspend(rdev);
5689 r600_ih_ring_fini(rdev);
5690}
5691
25a857fb
AD		 5692static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
5693{
5694 u32 wptr, tmp;
5695
5696 if (rdev->wb.enabled)
5697 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
5698 else
5699 wptr = RREG32(IH_RB_WPTR);
5700
5701 if (wptr & RB_OVERFLOW) {
5702 /* When a ring buffer overflow happen start parsing interrupt
5703 * from the last not overwritten vector (wptr + 16). Hopefully
5704 * this should allow us to catchup.
5705 */
5706 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
5707 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
5708 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
5709 tmp = RREG32(IH_RB_CNTL);
5710 tmp |= IH_WPTR_OVERFLOW_CLEAR;
5711 WREG32(IH_RB_CNTL, tmp);
5712 }
5713 return (wptr & rdev->ih.ptr_mask);
5714}
5715
5716/* SI IV Ring
5717 * Each IV ring entry is 128 bits:
5718 * [7:0] - interrupt source id
5719 * [31:8] - reserved
5720 * [59:32] - interrupt source data
5721 * [63:60] - reserved
5722 * [71:64] - RINGID
5723 * [79:72] - VMID
5724 * [127:80] - reserved
5725 */
5726int si_irq_process(struct radeon_device *rdev)
5727{
5728 u32 wptr;
5729 u32 rptr;
5730 u32 src_id, src_data, ring_id;
5731 u32 ring_index;
25a857fb
AD		 5732 bool queue_hotplug = false;
5733
5734 if (!rdev->ih.enabled || rdev->shutdown)
5735 return IRQ_NONE;
5736
5737 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 5738
5739restart_ih:
5740 /* is somebody else already processing irqs? */
5741 if (atomic_xchg(&rdev->ih.lock, 1))
5742 return IRQ_NONE;
5743
25a857fb
AD		 5744 rptr = rdev->ih.rptr;
5745 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
5746
25a857fb
AD		 5747 /* Order reading of wptr vs. reading of IH ring data */
5748 rmb();
5749
5750 /* display interrupts */
5751 si_irq_ack(rdev);
5752
25a857fb
AD		 5753 while (rptr != wptr) {
5754 /* wptr/rptr are in bytes! */
5755 ring_index = rptr / 4;
5756 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
5757 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
5758 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
5759
5760 switch (src_id) {
5761 case 1: /* D1 vblank/vline */
5762 switch (src_data) {
5763 case 0: /* D1 vblank */
5764 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
5765 if (rdev->irq.crtc_vblank_int[0]) {
5766 drm_handle_vblank(rdev->ddev, 0);
5767 rdev->pm.vblank_sync = true;
5768 wake_up(&rdev->irq.vblank_queue);
5769 }
736fc37f 5770 if (atomic_read(&rdev->irq.pflip[0]))
25a857fb
AD		 5771 radeon_crtc_handle_flip(rdev, 0);
5772 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
5773 DRM_DEBUG("IH: D1 vblank\n");
5774 }
5775 break;
5776 case 1: /* D1 vline */
5777 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
5778 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
5779 DRM_DEBUG("IH: D1 vline\n");
5780 }
5781 break;
5782 default:
5783 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5784 break;
5785 }
5786 break;
5787 case 2: /* D2 vblank/vline */
5788 switch (src_data) {
5789 case 0: /* D2 vblank */
5790 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
5791 if (rdev->irq.crtc_vblank_int[1]) {
5792 drm_handle_vblank(rdev->ddev, 1);
5793 rdev->pm.vblank_sync = true;
5794 wake_up(&rdev->irq.vblank_queue);
5795 }
736fc37f 5796 if (atomic_read(&rdev->irq.pflip[1]))
25a857fb
AD		 5797 radeon_crtc_handle_flip(rdev, 1);
5798 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
5799 DRM_DEBUG("IH: D2 vblank\n");
5800 }
5801 break;
5802 case 1: /* D2 vline */
5803 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
5804 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
5805 DRM_DEBUG("IH: D2 vline\n");
5806 }
5807 break;
5808 default:
5809 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5810 break;
5811 }
5812 break;
5813 case 3: /* D3 vblank/vline */
5814 switch (src_data) {
5815 case 0: /* D3 vblank */
5816 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
5817 if (rdev->irq.crtc_vblank_int[2]) {
5818 drm_handle_vblank(rdev->ddev, 2);
5819 rdev->pm.vblank_sync = true;
5820 wake_up(&rdev->irq.vblank_queue);
5821 }
736fc37f 5822 if (atomic_read(&rdev->irq.pflip[2]))
25a857fb
AD		 5823 radeon_crtc_handle_flip(rdev, 2);
5824 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5825 DRM_DEBUG("IH: D3 vblank\n");
5826 }
5827 break;
5828 case 1: /* D3 vline */
5829 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
5830 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5831 DRM_DEBUG("IH: D3 vline\n");
5832 }
5833 break;
5834 default:
5835 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5836 break;
5837 }
5838 break;
5839 case 4: /* D4 vblank/vline */
5840 switch (src_data) {
5841 case 0: /* D4 vblank */
5842 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
5843 if (rdev->irq.crtc_vblank_int[3]) {
5844 drm_handle_vblank(rdev->ddev, 3);
5845 rdev->pm.vblank_sync = true;
5846 wake_up(&rdev->irq.vblank_queue);
5847 }
736fc37f 5848 if (atomic_read(&rdev->irq.pflip[3]))
25a857fb
AD		 5849 radeon_crtc_handle_flip(rdev, 3);
5850 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5851 DRM_DEBUG("IH: D4 vblank\n");
5852 }
5853 break;
5854 case 1: /* D4 vline */
5855 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
5856 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5857 DRM_DEBUG("IH: D4 vline\n");
5858 }
5859 break;
5860 default:
5861 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5862 break;
5863 }
5864 break;
5865 case 5: /* D5 vblank/vline */
5866 switch (src_data) {
5867 case 0: /* D5 vblank */
5868 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
5869 if (rdev->irq.crtc_vblank_int[4]) {
5870 drm_handle_vblank(rdev->ddev, 4);
5871 rdev->pm.vblank_sync = true;
5872 wake_up(&rdev->irq.vblank_queue);
5873 }
736fc37f 5874 if (atomic_read(&rdev->irq.pflip[4]))
25a857fb
AD		 5875 radeon_crtc_handle_flip(rdev, 4);
5876 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5877 DRM_DEBUG("IH: D5 vblank\n");
5878 }
5879 break;
5880 case 1: /* D5 vline */
5881 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
5882 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5883 DRM_DEBUG("IH: D5 vline\n");
5884 }
5885 break;
5886 default:
5887 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5888 break;
5889 }
5890 break;
5891 case 6: /* D6 vblank/vline */
5892 switch (src_data) {
5893 case 0: /* D6 vblank */
5894 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
5895 if (rdev->irq.crtc_vblank_int[5]) {
5896 drm_handle_vblank(rdev->ddev, 5);
5897 rdev->pm.vblank_sync = true;
5898 wake_up(&rdev->irq.vblank_queue);
5899 }
736fc37f 5900 if (atomic_read(&rdev->irq.pflip[5]))
25a857fb
AD		 5901 radeon_crtc_handle_flip(rdev, 5);
5902 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5903 DRM_DEBUG("IH: D6 vblank\n");
5904 }
5905 break;
5906 case 1: /* D6 vline */
5907 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
5908 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5909 DRM_DEBUG("IH: D6 vline\n");
5910 }
5911 break;
5912 default:
5913 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5914 break;
5915 }
5916 break;
5917 case 42: /* HPD hotplug */
5918 switch (src_data) {
5919 case 0:
5920 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5921 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5922 queue_hotplug = true;
5923 DRM_DEBUG("IH: HPD1\n");
5924 }
5925 break;
5926 case 1:
5927 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5928 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5929 queue_hotplug = true;
5930 DRM_DEBUG("IH: HPD2\n");
5931 }
5932 break;
5933 case 2:
5934 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5935 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5936 queue_hotplug = true;
5937 DRM_DEBUG("IH: HPD3\n");
5938 }
5939 break;
5940 case 3:
5941 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5942 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5943 queue_hotplug = true;
5944 DRM_DEBUG("IH: HPD4\n");
5945 }
5946 break;
5947 case 4:
5948 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5949 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5950 queue_hotplug = true;
5951 DRM_DEBUG("IH: HPD5\n");
5952 }
5953 break;
5954 case 5:
5955 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5956 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5957 queue_hotplug = true;
5958 DRM_DEBUG("IH: HPD6\n");
5959 }
5960 break;
5961 default:
5962 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5963 break;
5964 }
5965 break;
ae133a11
CK		 5966 case 146:
5967 case 147:
5968 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5969 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5970 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5971 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5972 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
5973 /* reset addr and status */
5974 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5975 break;
25a857fb
AD		 5976 case 176: /* RINGID0 CP_INT */
5977 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5978 break;
5979 case 177: /* RINGID1 CP_INT */
5980 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5981 break;
5982 case 178: /* RINGID2 CP_INT */
5983 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5984 break;
5985 case 181: /* CP EOP event */
5986 DRM_DEBUG("IH: CP EOP\n");
5987 switch (ring_id) {
5988 case 0:
5989 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5990 break;
5991 case 1:
5992 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5993 break;
5994 case 2:
5995 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5996 break;
5997 }
5998 break;
8c5fd7ef
AD		 5999 case 224: /* DMA trap event */
6000 DRM_DEBUG("IH: DMA trap\n");
6001 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6002 break;
25a857fb
AD		 6003 case 233: /* GUI IDLE */
6004 DRM_DEBUG("IH: GUI idle\n");
25a857fb 6005 break;
8c5fd7ef
AD		 6006 case 244: /* DMA trap event */
6007 DRM_DEBUG("IH: DMA1 trap\n");
6008 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6009 break;
25a857fb
AD		 6010 default:
6011 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6012 break;
6013 }
6014
6015 /* wptr/rptr are in bytes! */
6016 rptr += 16;
6017 rptr &= rdev->ih.ptr_mask;
6018 }
25a857fb
AD		 6019 if (queue_hotplug)
6020 schedule_work(&rdev->hotplug_work);
6021 rdev->ih.rptr = rptr;
6022 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 6023 atomic_set(&rdev->ih.lock, 0);
6024
6025 /* make sure wptr hasn't changed while processing */
6026 wptr = si_get_ih_wptr(rdev);
6027 if (wptr != rptr)
6028 goto restart_ih;
6029
25a857fb
AD		 6030 return IRQ_HANDLED;
6031}
6032
8c5fd7ef
AD		 6033/**
6034 * si_copy_dma - copy pages using the DMA engine
6035 *
6036 * @rdev: radeon_device pointer
6037 * @src_offset: src GPU address
6038 * @dst_offset: dst GPU address
6039 * @num_gpu_pages: number of GPU pages to xfer
6040 * @fence: radeon fence object
6041 *
6042 * Copy GPU paging using the DMA engine (SI).
6043 * Used by the radeon ttm implementation to move pages if
6044 * registered as the asic copy callback.
6045 */
6046int si_copy_dma(struct radeon_device *rdev,
6047 uint64_t src_offset, uint64_t dst_offset,
6048 unsigned num_gpu_pages,
6049 struct radeon_fence **fence)
6050{
6051 struct radeon_semaphore *sem = NULL;
6052 int ring_index = rdev->asic->copy.dma_ring_index;
6053 struct radeon_ring *ring = &rdev->ring[ring_index];
6054 u32 size_in_bytes, cur_size_in_bytes;
6055 int i, num_loops;
6056 int r = 0;
6057
6058 r = radeon_semaphore_create(rdev, &sem);
6059 if (r) {
6060 DRM_ERROR("radeon: moving bo (%d).\n", r);
6061 return r;
6062 }
6063
6064 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
6065 num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
6066 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
6067 if (r) {
6068 DRM_ERROR("radeon: moving bo (%d).\n", r);
6069 radeon_semaphore_free(rdev, &sem, NULL);
6070 return r;
6071 }
6072
6073 if (radeon_fence_need_sync(*fence, ring->idx)) {
6074 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
6075 ring->idx);
6076 radeon_fence_note_sync(*fence, ring->idx);
6077 } else {
6078 radeon_semaphore_free(rdev, &sem, NULL);
6079 }
6080
6081 for (i = 0; i < num_loops; i++) {
6082 cur_size_in_bytes = size_in_bytes;
6083 if (cur_size_in_bytes > 0xFFFFF)
6084 cur_size_in_bytes = 0xFFFFF;
6085 size_in_bytes -= cur_size_in_bytes;
6086 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
6087 radeon_ring_write(ring, dst_offset & 0xffffffff);
6088 radeon_ring_write(ring, src_offset & 0xffffffff);
6089 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
6090 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
6091 src_offset += cur_size_in_bytes;
6092 dst_offset += cur_size_in_bytes;
6093 }
6094
6095 r = radeon_fence_emit(rdev, fence, ring->idx);
6096 if (r) {
6097 radeon_ring_unlock_undo(rdev, ring);
6098 return r;
6099 }
6100
6101 radeon_ring_unlock_commit(rdev, ring);
6102 radeon_semaphore_free(rdev, &sem, *fence);
6103
6104 return r;
6105}
6106
9b136d51
AD		 6107/*
6108 * startup/shutdown callbacks
6109 */
6110static int si_startup(struct radeon_device *rdev)
6111{
6112 struct radeon_ring *ring;
6113 int r;
6114
b9d305df
AD		 6115 /* enable pcie gen2/3 link */
6116 si_pcie_gen3_enable(rdev);
e0bcf165
AD		 6117 /* enable aspm */
6118 si_program_aspm(rdev);
b9d305df 6119
9b136d51
AD		 6120 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6121 !rdev->rlc_fw || !rdev->mc_fw) {
6122 r = si_init_microcode(rdev);
6123 if (r) {
6124 DRM_ERROR("Failed to load firmware!\n");
6125 return r;
6126 }
6127 }
6128
6129 r = si_mc_load_microcode(rdev);
6130 if (r) {
6131 DRM_ERROR("Failed to load MC firmware!\n");
6132 return r;
6133 }
6134
6135 r = r600_vram_scratch_init(rdev);
6136 if (r)
6137 return r;
6138
6139 si_mc_program(rdev);
6140 r = si_pcie_gart_enable(rdev);
6141 if (r)
6142 return r;
6143 si_gpu_init(rdev);
6144
9b136d51
AD		 6145 /* allocate rlc buffers */
6146 r = si_rlc_init(rdev);
6147 if (r) {
6148 DRM_ERROR("Failed to init rlc BOs!\n");
6149 return r;
6150 }
6151
6152 /* allocate wb buffer */
6153 r = radeon_wb_init(rdev);
6154 if (r)
6155 return r;
6156
6157 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6158 if (r) {
6159 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6160 return r;
6161 }
6162
6163 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6164 if (r) {
6165 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6166 return r;
6167 }
6168
6169 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6170 if (r) {
6171 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6172 return r;
6173 }
6174
8c5fd7ef
AD		 6175 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6176 if (r) {
6177 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6178 return r;
6179 }
6180
6181 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6182 if (r) {
6183 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6184 return r;
6185 }
6186
1df0d523
AD		 6187 if (rdev->has_uvd) {
6188 r = rv770_uvd_resume(rdev);
6189 if (!r) {
6190 r = radeon_fence_driver_start_ring(rdev,
6191 R600_RING_TYPE_UVD_INDEX);
6192 if (r)
6193 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6194 }
f2ba57b5 6195 if (r)
1df0d523 6196 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
f2ba57b5 6197 }
f2ba57b5 6198
9b136d51 6199 /* Enable IRQ */
e49f3959
AH		 6200 if (!rdev->irq.installed) {
6201 r = radeon_irq_kms_init(rdev);
6202 if (r)
6203 return r;
6204 }
6205
9b136d51
AD		 6206 r = si_irq_init(rdev);
6207 if (r) {
6208 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6209 radeon_irq_kms_fini(rdev);
6210 return r;
6211 }
6212 si_irq_set(rdev);
6213
6214 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6215 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6216 CP_RB0_RPTR, CP_RB0_WPTR,
6217 0, 0xfffff, RADEON_CP_PACKET2);
6218 if (r)
6219 return r;
6220
6221 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6222 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6223 CP_RB1_RPTR, CP_RB1_WPTR,
6224 0, 0xfffff, RADEON_CP_PACKET2);
6225 if (r)
6226 return r;
6227
6228 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6229 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6230 CP_RB2_RPTR, CP_RB2_WPTR,
6231 0, 0xfffff, RADEON_CP_PACKET2);
6232 if (r)
6233 return r;
6234
8c5fd7ef
AD		 6235 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6236 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6237 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
6238 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
6239 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6240 if (r)
6241 return r;
6242
6243 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6244 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6245 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
6246 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
6247 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6248 if (r)
6249 return r;
6250
9b136d51
AD		 6251 r = si_cp_load_microcode(rdev);
6252 if (r)
6253 return r;
6254 r = si_cp_resume(rdev);
6255 if (r)
6256 return r;
6257
8c5fd7ef
AD		 6258 r = cayman_dma_resume(rdev);
6259 if (r)
6260 return r;
6261
1df0d523
AD		 6262 if (rdev->has_uvd) {
6263 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6264 if (ring->ring_size) {
6265 r = radeon_ring_init(rdev, ring, ring->ring_size,
6266 R600_WB_UVD_RPTR_OFFSET,
6267 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
6268 0, 0xfffff, RADEON_CP_PACKET2);
6269 if (!r)
6270 r = r600_uvd_init(rdev);
6271 if (r)
6272 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6273 }
f2ba57b5
CK		 6274 }
6275
2898c348
CK		 6276 r = radeon_ib_pool_init(rdev);
6277 if (r) {
6278 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 6279 return r;
2898c348 6280 }
9b136d51 6281
c6105f24
CK		 6282 r = radeon_vm_manager_init(rdev);
6283 if (r) {
6284 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 6285 return r;
c6105f24 6286 }
9b136d51
AD		 6287
6288 return 0;
6289}
6290
6291int si_resume(struct radeon_device *rdev)
6292{
6293 int r;
6294
6295 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6296 * posting will perform necessary task to bring back GPU into good
6297 * shape.
6298 */
6299 /* post card */
6300 atom_asic_init(rdev->mode_info.atom_context);
6301
205996c0
AD		 6302 /* init golden registers */
6303 si_init_golden_registers(rdev);
6304
9b136d51
AD		 6305 rdev->accel_working = true;
6306 r = si_startup(rdev);
6307 if (r) {
6308 DRM_ERROR("si startup failed on resume\n");
6309 rdev->accel_working = false;
6310 return r;
6311 }
6312
6313 return r;
6314
6315}
6316
6317int si_suspend(struct radeon_device *rdev)
6318{
fa3daf9a 6319 radeon_vm_manager_fini(rdev);
9b136d51 6320 si_cp_enable(rdev, false);
8c5fd7ef 6321 cayman_dma_stop(rdev);
1df0d523
AD		 6322 if (rdev->has_uvd) {
6323 r600_uvd_rbc_stop(rdev);
6324 radeon_uvd_suspend(rdev);
6325 }
9b136d51
AD		 6326 si_irq_suspend(rdev);
6327 radeon_wb_disable(rdev);
6328 si_pcie_gart_disable(rdev);
6329 return 0;
6330}
6331
6332/* Plan is to move initialization in that function and use
6333 * helper function so that radeon_device_init pretty much
6334 * do nothing more than calling asic specific function. This
6335 * should also allow to remove a bunch of callback function
6336 * like vram_info.
6337 */
6338int si_init(struct radeon_device *rdev)
6339{
6340 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6341 int r;
6342
9b136d51
AD		 6343 /* Read BIOS */
6344 if (!radeon_get_bios(rdev)) {
6345 if (ASIC_IS_AVIVO(rdev))
6346 return -EINVAL;
6347 }
6348 /* Must be an ATOMBIOS */
6349 if (!rdev->is_atom_bios) {
6350 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6351 return -EINVAL;
6352 }
6353 r = radeon_atombios_init(rdev);
6354 if (r)
6355 return r;
6356
6357 /* Post card if necessary */
6358 if (!radeon_card_posted(rdev)) {
6359 if (!rdev->bios) {
6360 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6361 return -EINVAL;
6362 }
6363 DRM_INFO("GPU not posted. posting now...\n");
6364 atom_asic_init(rdev->mode_info.atom_context);
6365 }
205996c0
AD		 6366 /* init golden registers */
6367 si_init_golden_registers(rdev);
9b136d51
AD		 6368 /* Initialize scratch registers */
6369 si_scratch_init(rdev);
6370 /* Initialize surface registers */
6371 radeon_surface_init(rdev);
6372 /* Initialize clocks */
6373 radeon_get_clock_info(rdev->ddev);
6374
6375 /* Fence driver */
6376 r = radeon_fence_driver_init(rdev);
6377 if (r)
6378 return r;
6379
6380 /* initialize memory controller */
6381 r = si_mc_init(rdev);
6382 if (r)
6383 return r;
6384 /* Memory manager */
6385 r = radeon_bo_init(rdev);
6386 if (r)
6387 return r;
6388
9b136d51
AD		 6389 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6390 ring->ring_obj = NULL;
6391 r600_ring_init(rdev, ring, 1024 * 1024);
6392
6393 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6394 ring->ring_obj = NULL;
6395 r600_ring_init(rdev, ring, 1024 * 1024);
6396
6397 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6398 ring->ring_obj = NULL;
6399 r600_ring_init(rdev, ring, 1024 * 1024);
6400
8c5fd7ef
AD		 6401 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6402 ring->ring_obj = NULL;
6403 r600_ring_init(rdev, ring, 64 * 1024);
6404
6405 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6406 ring->ring_obj = NULL;
6407 r600_ring_init(rdev, ring, 64 * 1024);
6408
1df0d523
AD		 6409 if (rdev->has_uvd) {
6410 r = radeon_uvd_init(rdev);
6411 if (!r) {
6412 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6413 ring->ring_obj = NULL;
6414 r600_ring_init(rdev, ring, 4096);
6415 }
f2ba57b5
CK		 6416 }
6417
9b136d51
AD		 6418 rdev->ih.ring_obj = NULL;
6419 r600_ih_ring_init(rdev, 64 * 1024);
6420
6421 r = r600_pcie_gart_init(rdev);
6422 if (r)
6423 return r;
6424
9b136d51 6425 rdev->accel_working = true;
9b136d51
AD		 6426 r = si_startup(rdev);
6427 if (r) {
6428 dev_err(rdev->dev, "disabling GPU acceleration\n");
6429 si_cp_fini(rdev);
8c5fd7ef 6430 cayman_dma_fini(rdev);
9b136d51
AD		 6431 si_irq_fini(rdev);
6432 si_rlc_fini(rdev);
6433 radeon_wb_fini(rdev);
2898c348 6434 radeon_ib_pool_fini(rdev);
9b136d51
AD		 6435 radeon_vm_manager_fini(rdev);
6436 radeon_irq_kms_fini(rdev);
6437 si_pcie_gart_fini(rdev);
6438 rdev->accel_working = false;
6439 }
6440
6441 /* Don't start up if the MC ucode is missing.
6442 * The default clocks and voltages before the MC ucode
6443 * is loaded are not suffient for advanced operations.
6444 */
6445 if (!rdev->mc_fw) {
6446 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6447 return -EINVAL;
6448 }
6449
6450 return 0;
6451}
6452
6453void si_fini(struct radeon_device *rdev)
6454{
9b136d51 6455 si_cp_fini(rdev);
8c5fd7ef 6456 cayman_dma_fini(rdev);
9b136d51
AD		 6457 si_irq_fini(rdev);
6458 si_rlc_fini(rdev);
f8f84ac5
AD		 6459 si_fini_cg(rdev);
6460 si_fini_pg(rdev);
9b136d51
AD		 6461 radeon_wb_fini(rdev);
6462 radeon_vm_manager_fini(rdev);
2898c348 6463 radeon_ib_pool_fini(rdev);
9b136d51 6464 radeon_irq_kms_fini(rdev);
1df0d523
AD		 6465 if (rdev->has_uvd)
6466 radeon_uvd_fini(rdev);
9b136d51
AD		 6467 si_pcie_gart_fini(rdev);
6468 r600_vram_scratch_fini(rdev);
6469 radeon_gem_fini(rdev);
9b136d51
AD		 6470 radeon_fence_driver_fini(rdev);
6471 radeon_bo_fini(rdev);
6472 radeon_atombios_fini(rdev);
6473 kfree(rdev->bios);
6474 rdev->bios = NULL;
6475}
6476
6759a0a7 6477/**
d0418894 6478 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 6479 *
6480 * @rdev: radeon_device pointer
6481 *
6482 * Fetches a GPU clock counter snapshot (SI).
6483 * Returns the 64 bit clock counter snapshot.
6484 */
d0418894 6485uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 6486{
6487 uint64_t clock;
6488
6489 mutex_lock(&rdev->gpu_clock_mutex);
6490 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6491 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6492 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6493 mutex_unlock(&rdev->gpu_clock_mutex);
6494 return clock;
6495}
2539eb02 6496
2539eb02
CK		 6497int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6498{
facd112d 6499 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 6500 int r;
6501
4ed10835
CK		 6502 /* bypass vclk and dclk with bclk */
6503 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6504 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6505 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6506
6507 /* put PLL in bypass mode */
6508 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6509
6510 if (!vclk || !dclk) {
6511 /* keep the Bypass mode, put PLL to sleep */
6512 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6513 return 0;
6514 }
6515
facd112d
CK		 6516 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6517 16384, 0x03FFFFFF, 0, 128, 5,
6518 &fb_div, &vclk_div, &dclk_div);
6519 if (r)
6520 return r;
2539eb02
CK		 6521
6522 /* set RESET_ANTI_MUX to 0 */
6523 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6524
6525 /* set VCO_MODE to 1 */
6526 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6527
6528 /* toggle UPLL_SLEEP to 1 then back to 0 */
6529 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6530 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6531
6532 /* deassert UPLL_RESET */
6533 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6534
6535 mdelay(1);
6536
facd112d 6537 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6538 if (r)
6539 return r;
6540
6541 /* assert UPLL_RESET again */
6542 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6543
6544 /* disable spread spectrum. */
6545 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6546
6547 /* set feedback divider */
facd112d 6548 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 6549
6550 /* set ref divider to 0 */
6551 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6552
facd112d 6553 if (fb_div < 307200)
2539eb02
CK		 6554 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6555 else
6556 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6557
6558 /* set PDIV_A and PDIV_B */
6559 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 6560 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 6561 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6562
6563 /* give the PLL some time to settle */
6564 mdelay(15);
6565
6566 /* deassert PLL_RESET */
6567 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6568
6569 mdelay(15);
6570
6571 /* switch from bypass mode to normal mode */
6572 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6573
facd112d 6574 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6575 if (r)
6576 return r;
6577
6578 /* switch VCLK and DCLK selection */
6579 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6580 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6581 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6582
6583 mdelay(100);
6584
6585 return 0;
6586}
b9d305df
AD		 6587
6588static void si_pcie_gen3_enable(struct radeon_device *rdev)
6589{
6590 struct pci_dev *root = rdev->pdev->bus->self;
6591 int bridge_pos, gpu_pos;
6592 u32 speed_cntl, mask, current_data_rate;
6593 int ret, i;
6594 u16 tmp16;
6595
6596 if (radeon_pcie_gen2 == 0)
6597 return;
6598
6599 if (rdev->flags & RADEON_IS_IGP)
6600 return;
6601
6602 if (!(rdev->flags & RADEON_IS_PCIE))
6603 return;
6604
6605 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6606 if (ret != 0)
6607 return;
6608
6609 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6610 return;
6611
6612 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6613 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6614 LC_CURRENT_DATA_RATE_SHIFT;
6615 if (mask & DRM_PCIE_SPEED_80) {
6616 if (current_data_rate == 2) {
6617 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6618 return;
6619 }
6620 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6621 } else if (mask & DRM_PCIE_SPEED_50) {
6622 if (current_data_rate == 1) {
6623 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6624 return;
6625 }
6626 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6627 }
6628
6629 bridge_pos = pci_pcie_cap(root);
6630 if (!bridge_pos)
6631 return;
6632
6633 gpu_pos = pci_pcie_cap(rdev->pdev);
6634 if (!gpu_pos)
6635 return;
6636
6637 if (mask & DRM_PCIE_SPEED_80) {
6638 /* re-try equalization if gen3 is not already enabled */
6639 if (current_data_rate != 2) {
6640 u16 bridge_cfg, gpu_cfg;
6641 u16 bridge_cfg2, gpu_cfg2;
6642 u32 max_lw, current_lw, tmp;
6643
6644 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6645 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6646
6647 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6648 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6649
6650 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6651 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6652
6653 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6654 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6655 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6656
6657 if (current_lw < max_lw) {
6658 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6659 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6660 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6661 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6662 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6663 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6664 }
6665 }
6666
6667 for (i = 0; i < 10; i++) {
6668 /* check status */
6669 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6670 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6671 break;
6672
6673 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6674 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6675
6676 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6677 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6678
6679 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6680 tmp |= LC_SET_QUIESCE;
6681 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6682
6683 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6684 tmp |= LC_REDO_EQ;
6685 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6686
6687 mdelay(100);
6688
6689 /* linkctl */
6690 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6691 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6692 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6693 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6694
6695 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
6696 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6697 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
6698 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6699
6700 /* linkctl2 */
6701 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
6702 tmp16 &= ~((1 << 4) | (7 << 9));
6703 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
6704 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
6705
6706 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6707 tmp16 &= ~((1 << 4) | (7 << 9));
6708 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
6709 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6710
6711 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6712 tmp &= ~LC_SET_QUIESCE;
6713 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6714 }
6715 }
6716 }
6717
6718 /* set the link speed */
6719 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
6720 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
6721 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6722
6723 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6724 tmp16 &= ~0xf;
6725 if (mask & DRM_PCIE_SPEED_80)
6726 tmp16 |= 3; /* gen3 */
6727 else if (mask & DRM_PCIE_SPEED_50)
6728 tmp16 |= 2; /* gen2 */
6729 else
6730 tmp16 |= 1; /* gen1 */
6731 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6732
6733 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6734 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
6735 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6736
6737 for (i = 0; i < rdev->usec_timeout; i++) {
6738 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6739 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
6740 break;
6741 udelay(1);
6742 }
6743}
6744
e0bcf165
AD		 6745static void si_program_aspm(struct radeon_device *rdev)
6746{
6747 u32 data, orig;
6748 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
6749 bool disable_clkreq = false;
6750
6751 if (!(rdev->flags & RADEON_IS_PCIE))
6752 return;
6753
6754 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6755 data &= ~LC_XMIT_N_FTS_MASK;
6756 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
6757 if (orig != data)
6758 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
6759
6760 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
6761 data |= LC_GO_TO_RECOVERY;
6762 if (orig != data)
6763 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
6764
6765 orig = data = RREG32_PCIE(PCIE_P_CNTL);
6766 data |= P_IGNORE_EDB_ERR;
6767 if (orig != data)
6768 WREG32_PCIE(PCIE_P_CNTL, data);
6769
6770 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6771 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
6772 data |= LC_PMI_TO_L1_DIS;
6773 if (!disable_l0s)
6774 data |= LC_L0S_INACTIVITY(7);
6775
6776 if (!disable_l1) {
6777 data |= LC_L1_INACTIVITY(7);
6778 data &= ~LC_PMI_TO_L1_DIS;
6779 if (orig != data)
6780 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6781
6782 if (!disable_plloff_in_l1) {
6783 bool clk_req_support;
6784
6785 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6786 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6787 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6788 if (orig != data)
6789 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6790
6791 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6792 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6793 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6794 if (orig != data)
6795 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6796
6797 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6798 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6799 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6800 if (orig != data)
6801 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6802
6803 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6804 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6805 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6806 if (orig != data)
6807 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6808
6809 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
6810 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6811 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6812 if (orig != data)
6813 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6814
6815 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6816 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6817 if (orig != data)
6818 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6819
6820 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
6821 data &= ~PLL_RAMP_UP_TIME_2_MASK;
6822 if (orig != data)
6823 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
6824
6825 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
6826 data &= ~PLL_RAMP_UP_TIME_3_MASK;
6827 if (orig != data)
6828 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
6829
6830 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6831 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6832 if (orig != data)
6833 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6834
6835 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6836 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6837 if (orig != data)
6838 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6839
6840 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
6841 data &= ~PLL_RAMP_UP_TIME_2_MASK;
6842 if (orig != data)
6843 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
6844
6845 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
6846 data &= ~PLL_RAMP_UP_TIME_3_MASK;
6847 if (orig != data)
6848 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
6849 }
6850 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6851 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
6852 data |= LC_DYN_LANES_PWR_STATE(3);
6853 if (orig != data)
6854 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
6855
6856 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
6857 data &= ~LS2_EXIT_TIME_MASK;
6858 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
6859 data |= LS2_EXIT_TIME(5);
6860 if (orig != data)
6861 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
6862
6863 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
6864 data &= ~LS2_EXIT_TIME_MASK;
6865 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
6866 data |= LS2_EXIT_TIME(5);
6867 if (orig != data)
6868 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
6869
6870 if (!disable_clkreq) {
6871 struct pci_dev *root = rdev->pdev->bus->self;
6872 u32 lnkcap;
6873
6874 clk_req_support = false;
6875 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
6876 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
6877 clk_req_support = true;
6878 } else {
6879 clk_req_support = false;
6880 }
6881
6882 if (clk_req_support) {
6883 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
6884 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
6885 if (orig != data)
6886 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
6887
6888 orig = data = RREG32(THM_CLK_CNTL);
6889 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
6890 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
6891 if (orig != data)
6892 WREG32(THM_CLK_CNTL, data);
6893
6894 orig = data = RREG32(MISC_CLK_CNTL);
6895 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
6896 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
6897 if (orig != data)
6898 WREG32(MISC_CLK_CNTL, data);
6899
6900 orig = data = RREG32(CG_CLKPIN_CNTL);
6901 data &= ~BCLK_AS_XCLK;
6902 if (orig != data)
6903 WREG32(CG_CLKPIN_CNTL, data);
6904
6905 orig = data = RREG32(CG_CLKPIN_CNTL_2);
6906 data &= ~FORCE_BIF_REFCLK_EN;
6907 if (orig != data)
6908 WREG32(CG_CLKPIN_CNTL_2, data);
6909
6910 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
6911 data &= ~MPLL_CLKOUT_SEL_MASK;
6912 data |= MPLL_CLKOUT_SEL(4);
6913 if (orig != data)
6914 WREG32(MPLL_BYPASSCLK_SEL, data);
6915
6916 orig = data = RREG32(SPLL_CNTL_MODE);
6917 data &= ~SPLL_REFCLK_SEL_MASK;
6918 if (orig != data)
6919 WREG32(SPLL_CNTL_MODE, data);
6920 }
6921 }
6922 } else {
6923 if (orig != data)
6924 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6925 }
6926
6927 orig = data = RREG32_PCIE(PCIE_CNTL2);
6928 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
6929 if (orig != data)
6930 WREG32_PCIE(PCIE_CNTL2, data);
6931
6932 if (!disable_l0s) {
6933 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6934 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
6935 data = RREG32_PCIE(PCIE_LC_STATUS1);
6936 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
6937 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6938 data &= ~LC_L0S_INACTIVITY_MASK;
6939 if (orig != data)
6940 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6941 }
6942 }
6943 }
6944}
Linux 6.x block layer and io_uring tree(s)