projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon: improve PLL params if we don't match exactly v2
[linux-2.6-block.git] / drivers / gpu / drm / radeon / si.c
CommitLineData
43b3cd99
AD		 1/*
2 * Copyright 2011 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
0f0de06c 24#include <linux/firmware.h>
0f0de06c
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
760285e7 27#include <drm/drmP.h>
43b3cd99
AD		 28#include "radeon.h"
29#include "radeon_asic.h"
760285e7 30#include <drm/radeon_drm.h>
43b3cd99
AD		 31#include "sid.h"
32#include "atom.h"
48c0c902 33#include "si_blit_shaders.h"
bd8cd539 34#include "clearstate_si.h"
a0ceada6 35#include "radeon_ucode.h"
43b3cd99 36
0f0de06c
AD		 37
38MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
1ebe9280 42MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
0f0de06c 43MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
a9e61410 44MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
0f0de06c
AD		 45MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
46MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
47MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
48MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
1ebe9280 49MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
0f0de06c 50MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
a9e61410 51MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
0f0de06c
AD		 52MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
53MODULE_FIRMWARE("radeon/VERDE_me.bin");
54MODULE_FIRMWARE("radeon/VERDE_ce.bin");
55MODULE_FIRMWARE("radeon/VERDE_mc.bin");
1ebe9280 56MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
0f0de06c 57MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
a9e61410 58MODULE_FIRMWARE("radeon/VERDE_smc.bin");
bcc7f5d2
AD		 59MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
60MODULE_FIRMWARE("radeon/OLAND_me.bin");
61MODULE_FIRMWARE("radeon/OLAND_ce.bin");
62MODULE_FIRMWARE("radeon/OLAND_mc.bin");
1ebe9280 63MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
bcc7f5d2 64MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
a9e61410 65MODULE_FIRMWARE("radeon/OLAND_smc.bin");
c04c00b4
AD		 66MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
67MODULE_FIRMWARE("radeon/HAINAN_me.bin");
68MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
69MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
1ebe9280 70MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
c04c00b4 71MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
a9e61410 72MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
0f0de06c 73
b9d305df 74static void si_pcie_gen3_enable(struct radeon_device *rdev);
e0bcf165 75static void si_program_aspm(struct radeon_device *rdev);
1fd11777
AD		 76extern void sumo_rlc_fini(struct radeon_device *rdev);
77extern int sumo_rlc_init(struct radeon_device *rdev);
25a857fb
AD		 78extern int r600_ih_ring_alloc(struct radeon_device *rdev);
79extern void r600_ih_ring_fini(struct radeon_device *rdev);
0a96d72b 80extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
c476dde2
AD		 81extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
82extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
ca7db22b 83extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
1c534671 84extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
014bb209 85extern bool evergreen_is_display_hung(struct radeon_device *rdev);
811e4d58
AD		 86static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
87 bool enable);
4a5c8ea5
AD		 88static void si_init_pg(struct radeon_device *rdev);
89static void si_init_cg(struct radeon_device *rdev);
a6f4ae8d
AD		 90static void si_fini_pg(struct radeon_device *rdev);
91static void si_fini_cg(struct radeon_device *rdev);
92static void si_rlc_stop(struct radeon_device *rdev);
0a96d72b 93
6d8cf000
AD		 94static const u32 verde_rlc_save_restore_register_list[] =
95{
96 (0x8000 << 16) | (0x98f4 >> 2),
97 0x00000000,
98 (0x8040 << 16) | (0x98f4 >> 2),
99 0x00000000,
100 (0x8000 << 16) | (0xe80 >> 2),
101 0x00000000,
102 (0x8040 << 16) | (0xe80 >> 2),
103 0x00000000,
104 (0x8000 << 16) | (0x89bc >> 2),
105 0x00000000,
106 (0x8040 << 16) | (0x89bc >> 2),
107 0x00000000,
108 (0x8000 << 16) | (0x8c1c >> 2),
109 0x00000000,
110 (0x8040 << 16) | (0x8c1c >> 2),
111 0x00000000,
112 (0x9c00 << 16) | (0x98f0 >> 2),
113 0x00000000,
114 (0x9c00 << 16) | (0xe7c >> 2),
115 0x00000000,
116 (0x8000 << 16) | (0x9148 >> 2),
117 0x00000000,
118 (0x8040 << 16) | (0x9148 >> 2),
119 0x00000000,
120 (0x9c00 << 16) | (0x9150 >> 2),
121 0x00000000,
122 (0x9c00 << 16) | (0x897c >> 2),
123 0x00000000,
124 (0x9c00 << 16) | (0x8d8c >> 2),
125 0x00000000,
126 (0x9c00 << 16) | (0xac54 >> 2),
127 0X00000000,
128 0x3,
129 (0x9c00 << 16) | (0x98f8 >> 2),
130 0x00000000,
131 (0x9c00 << 16) | (0x9910 >> 2),
132 0x00000000,
133 (0x9c00 << 16) | (0x9914 >> 2),
134 0x00000000,
135 (0x9c00 << 16) | (0x9918 >> 2),
136 0x00000000,
137 (0x9c00 << 16) | (0x991c >> 2),
138 0x00000000,
139 (0x9c00 << 16) | (0x9920 >> 2),
140 0x00000000,
141 (0x9c00 << 16) | (0x9924 >> 2),
142 0x00000000,
143 (0x9c00 << 16) | (0x9928 >> 2),
144 0x00000000,
145 (0x9c00 << 16) | (0x992c >> 2),
146 0x00000000,
147 (0x9c00 << 16) | (0x9930 >> 2),
148 0x00000000,
149 (0x9c00 << 16) | (0x9934 >> 2),
150 0x00000000,
151 (0x9c00 << 16) | (0x9938 >> 2),
152 0x00000000,
153 (0x9c00 << 16) | (0x993c >> 2),
154 0x00000000,
155 (0x9c00 << 16) | (0x9940 >> 2),
156 0x00000000,
157 (0x9c00 << 16) | (0x9944 >> 2),
158 0x00000000,
159 (0x9c00 << 16) | (0x9948 >> 2),
160 0x00000000,
161 (0x9c00 << 16) | (0x994c >> 2),
162 0x00000000,
163 (0x9c00 << 16) | (0x9950 >> 2),
164 0x00000000,
165 (0x9c00 << 16) | (0x9954 >> 2),
166 0x00000000,
167 (0x9c00 << 16) | (0x9958 >> 2),
168 0x00000000,
169 (0x9c00 << 16) | (0x995c >> 2),
170 0x00000000,
171 (0x9c00 << 16) | (0x9960 >> 2),
172 0x00000000,
173 (0x9c00 << 16) | (0x9964 >> 2),
174 0x00000000,
175 (0x9c00 << 16) | (0x9968 >> 2),
176 0x00000000,
177 (0x9c00 << 16) | (0x996c >> 2),
178 0x00000000,
179 (0x9c00 << 16) | (0x9970 >> 2),
180 0x00000000,
181 (0x9c00 << 16) | (0x9974 >> 2),
182 0x00000000,
183 (0x9c00 << 16) | (0x9978 >> 2),
184 0x00000000,
185 (0x9c00 << 16) | (0x997c >> 2),
186 0x00000000,
187 (0x9c00 << 16) | (0x9980 >> 2),
188 0x00000000,
189 (0x9c00 << 16) | (0x9984 >> 2),
190 0x00000000,
191 (0x9c00 << 16) | (0x9988 >> 2),
192 0x00000000,
193 (0x9c00 << 16) | (0x998c >> 2),
194 0x00000000,
195 (0x9c00 << 16) | (0x8c00 >> 2),
196 0x00000000,
197 (0x9c00 << 16) | (0x8c14 >> 2),
198 0x00000000,
199 (0x9c00 << 16) | (0x8c04 >> 2),
200 0x00000000,
201 (0x9c00 << 16) | (0x8c08 >> 2),
202 0x00000000,
203 (0x8000 << 16) | (0x9b7c >> 2),
204 0x00000000,
205 (0x8040 << 16) | (0x9b7c >> 2),
206 0x00000000,
207 (0x8000 << 16) | (0xe84 >> 2),
208 0x00000000,
209 (0x8040 << 16) | (0xe84 >> 2),
210 0x00000000,
211 (0x8000 << 16) | (0x89c0 >> 2),
212 0x00000000,
213 (0x8040 << 16) | (0x89c0 >> 2),
214 0x00000000,
215 (0x8000 << 16) | (0x914c >> 2),
216 0x00000000,
217 (0x8040 << 16) | (0x914c >> 2),
218 0x00000000,
219 (0x8000 << 16) | (0x8c20 >> 2),
220 0x00000000,
221 (0x8040 << 16) | (0x8c20 >> 2),
222 0x00000000,
223 (0x8000 << 16) | (0x9354 >> 2),
224 0x00000000,
225 (0x8040 << 16) | (0x9354 >> 2),
226 0x00000000,
227 (0x9c00 << 16) | (0x9060 >> 2),
228 0x00000000,
229 (0x9c00 << 16) | (0x9364 >> 2),
230 0x00000000,
231 (0x9c00 << 16) | (0x9100 >> 2),
232 0x00000000,
233 (0x9c00 << 16) | (0x913c >> 2),
234 0x00000000,
235 (0x8000 << 16) | (0x90e0 >> 2),
236 0x00000000,
237 (0x8000 << 16) | (0x90e4 >> 2),
238 0x00000000,
239 (0x8000 << 16) | (0x90e8 >> 2),
240 0x00000000,
241 (0x8040 << 16) | (0x90e0 >> 2),
242 0x00000000,
243 (0x8040 << 16) | (0x90e4 >> 2),
244 0x00000000,
245 (0x8040 << 16) | (0x90e8 >> 2),
246 0x00000000,
247 (0x9c00 << 16) | (0x8bcc >> 2),
248 0x00000000,
249 (0x9c00 << 16) | (0x8b24 >> 2),
250 0x00000000,
251 (0x9c00 << 16) | (0x88c4 >> 2),
252 0x00000000,
253 (0x9c00 << 16) | (0x8e50 >> 2),
254 0x00000000,
255 (0x9c00 << 16) | (0x8c0c >> 2),
256 0x00000000,
257 (0x9c00 << 16) | (0x8e58 >> 2),
258 0x00000000,
259 (0x9c00 << 16) | (0x8e5c >> 2),
260 0x00000000,
261 (0x9c00 << 16) | (0x9508 >> 2),
262 0x00000000,
263 (0x9c00 << 16) | (0x950c >> 2),
264 0x00000000,
265 (0x9c00 << 16) | (0x9494 >> 2),
266 0x00000000,
267 (0x9c00 << 16) | (0xac0c >> 2),
268 0x00000000,
269 (0x9c00 << 16) | (0xac10 >> 2),
270 0x00000000,
271 (0x9c00 << 16) | (0xac14 >> 2),
272 0x00000000,
273 (0x9c00 << 16) | (0xae00 >> 2),
274 0x00000000,
275 (0x9c00 << 16) | (0xac08 >> 2),
276 0x00000000,
277 (0x9c00 << 16) | (0x88d4 >> 2),
278 0x00000000,
279 (0x9c00 << 16) | (0x88c8 >> 2),
280 0x00000000,
281 (0x9c00 << 16) | (0x88cc >> 2),
282 0x00000000,
283 (0x9c00 << 16) | (0x89b0 >> 2),
284 0x00000000,
285 (0x9c00 << 16) | (0x8b10 >> 2),
286 0x00000000,
287 (0x9c00 << 16) | (0x8a14 >> 2),
288 0x00000000,
289 (0x9c00 << 16) | (0x9830 >> 2),
290 0x00000000,
291 (0x9c00 << 16) | (0x9834 >> 2),
292 0x00000000,
293 (0x9c00 << 16) | (0x9838 >> 2),
294 0x00000000,
295 (0x9c00 << 16) | (0x9a10 >> 2),
296 0x00000000,
297 (0x8000 << 16) | (0x9870 >> 2),
298 0x00000000,
299 (0x8000 << 16) | (0x9874 >> 2),
300 0x00000000,
301 (0x8001 << 16) | (0x9870 >> 2),
302 0x00000000,
303 (0x8001 << 16) | (0x9874 >> 2),
304 0x00000000,
305 (0x8040 << 16) | (0x9870 >> 2),
306 0x00000000,
307 (0x8040 << 16) | (0x9874 >> 2),
308 0x00000000,
309 (0x8041 << 16) | (0x9870 >> 2),
310 0x00000000,
311 (0x8041 << 16) | (0x9874 >> 2),
312 0x00000000,
313 0x00000000
314};
315
205996c0
AD		 316static const u32 tahiti_golden_rlc_registers[] =
317{
318 0xc424, 0xffffffff, 0x00601005,
319 0xc47c, 0xffffffff, 0x10104040,
320 0xc488, 0xffffffff, 0x0100000a,
321 0xc314, 0xffffffff, 0x00000800,
322 0xc30c, 0xffffffff, 0x800000f4,
323 0xf4a8, 0xffffffff, 0x00000000
324};
325
326static const u32 tahiti_golden_registers[] =
327{
328 0x9a10, 0x00010000, 0x00018208,
329 0x9830, 0xffffffff, 0x00000000,
330 0x9834, 0xf00fffff, 0x00000400,
331 0x9838, 0x0002021c, 0x00020200,
332 0xc78, 0x00000080, 0x00000000,
333 0xd030, 0x000300c0, 0x00800040,
334 0xd830, 0x000300c0, 0x00800040,
335 0x5bb0, 0x000000f0, 0x00000070,
336 0x5bc0, 0x00200000, 0x50100000,
337 0x7030, 0x31000311, 0x00000011,
338 0x277c, 0x00000003, 0x000007ff,
339 0x240c, 0x000007ff, 0x00000000,
340 0x8a14, 0xf000001f, 0x00000007,
341 0x8b24, 0xffffffff, 0x00ffffff,
342 0x8b10, 0x0000ff0f, 0x00000000,
343 0x28a4c, 0x07ffffff, 0x4e000000,
344 0x28350, 0x3f3f3fff, 0x2a00126a,
345 0x30, 0x000000ff, 0x0040,
346 0x34, 0x00000040, 0x00004040,
347 0x9100, 0x07ffffff, 0x03000000,
348 0x8e88, 0x01ff1f3f, 0x00000000,
349 0x8e84, 0x01ff1f3f, 0x00000000,
350 0x9060, 0x0000007f, 0x00000020,
351 0x9508, 0x00010000, 0x00010000,
352 0xac14, 0x00000200, 0x000002fb,
353 0xac10, 0xffffffff, 0x0000543b,
354 0xac0c, 0xffffffff, 0xa9210876,
355 0x88d0, 0xffffffff, 0x000fff40,
356 0x88d4, 0x0000001f, 0x00000010,
357 0x1410, 0x20000000, 0x20fffed8,
358 0x15c0, 0x000c0fc0, 0x000c0400
359};
360
361static const u32 tahiti_golden_registers2[] =
362{
363 0xc64, 0x00000001, 0x00000001
364};
365
366static const u32 pitcairn_golden_rlc_registers[] =
367{
368 0xc424, 0xffffffff, 0x00601004,
369 0xc47c, 0xffffffff, 0x10102020,
370 0xc488, 0xffffffff, 0x01000020,
371 0xc314, 0xffffffff, 0x00000800,
372 0xc30c, 0xffffffff, 0x800000a4
373};
374
375static const u32 pitcairn_golden_registers[] =
376{
377 0x9a10, 0x00010000, 0x00018208,
378 0x9830, 0xffffffff, 0x00000000,
379 0x9834, 0xf00fffff, 0x00000400,
380 0x9838, 0x0002021c, 0x00020200,
381 0xc78, 0x00000080, 0x00000000,
382 0xd030, 0x000300c0, 0x00800040,
383 0xd830, 0x000300c0, 0x00800040,
384 0x5bb0, 0x000000f0, 0x00000070,
385 0x5bc0, 0x00200000, 0x50100000,
386 0x7030, 0x31000311, 0x00000011,
387 0x2ae4, 0x00073ffe, 0x000022a2,
388 0x240c, 0x000007ff, 0x00000000,
389 0x8a14, 0xf000001f, 0x00000007,
390 0x8b24, 0xffffffff, 0x00ffffff,
391 0x8b10, 0x0000ff0f, 0x00000000,
392 0x28a4c, 0x07ffffff, 0x4e000000,
393 0x28350, 0x3f3f3fff, 0x2a00126a,
394 0x30, 0x000000ff, 0x0040,
395 0x34, 0x00000040, 0x00004040,
396 0x9100, 0x07ffffff, 0x03000000,
397 0x9060, 0x0000007f, 0x00000020,
398 0x9508, 0x00010000, 0x00010000,
399 0xac14, 0x000003ff, 0x000000f7,
400 0xac10, 0xffffffff, 0x00000000,
401 0xac0c, 0xffffffff, 0x32761054,
402 0x88d4, 0x0000001f, 0x00000010,
403 0x15c0, 0x000c0fc0, 0x000c0400
404};
405
406static const u32 verde_golden_rlc_registers[] =
407{
408 0xc424, 0xffffffff, 0x033f1005,
409 0xc47c, 0xffffffff, 0x10808020,
410 0xc488, 0xffffffff, 0x00800008,
411 0xc314, 0xffffffff, 0x00001000,
412 0xc30c, 0xffffffff, 0x80010014
413};
414
415static const u32 verde_golden_registers[] =
416{
417 0x9a10, 0x00010000, 0x00018208,
418 0x9830, 0xffffffff, 0x00000000,
419 0x9834, 0xf00fffff, 0x00000400,
420 0x9838, 0x0002021c, 0x00020200,
421 0xc78, 0x00000080, 0x00000000,
422 0xd030, 0x000300c0, 0x00800040,
423 0xd030, 0x000300c0, 0x00800040,
424 0xd830, 0x000300c0, 0x00800040,
425 0xd830, 0x000300c0, 0x00800040,
426 0x5bb0, 0x000000f0, 0x00000070,
427 0x5bc0, 0x00200000, 0x50100000,
428 0x7030, 0x31000311, 0x00000011,
429 0x2ae4, 0x00073ffe, 0x000022a2,
430 0x2ae4, 0x00073ffe, 0x000022a2,
431 0x2ae4, 0x00073ffe, 0x000022a2,
432 0x240c, 0x000007ff, 0x00000000,
433 0x240c, 0x000007ff, 0x00000000,
434 0x240c, 0x000007ff, 0x00000000,
435 0x8a14, 0xf000001f, 0x00000007,
436 0x8a14, 0xf000001f, 0x00000007,
437 0x8a14, 0xf000001f, 0x00000007,
438 0x8b24, 0xffffffff, 0x00ffffff,
439 0x8b10, 0x0000ff0f, 0x00000000,
440 0x28a4c, 0x07ffffff, 0x4e000000,
441 0x28350, 0x3f3f3fff, 0x0000124a,
442 0x28350, 0x3f3f3fff, 0x0000124a,
443 0x28350, 0x3f3f3fff, 0x0000124a,
444 0x30, 0x000000ff, 0x0040,
445 0x34, 0x00000040, 0x00004040,
446 0x9100, 0x07ffffff, 0x03000000,
447 0x9100, 0x07ffffff, 0x03000000,
448 0x8e88, 0x01ff1f3f, 0x00000000,
449 0x8e88, 0x01ff1f3f, 0x00000000,
450 0x8e88, 0x01ff1f3f, 0x00000000,
451 0x8e84, 0x01ff1f3f, 0x00000000,
452 0x8e84, 0x01ff1f3f, 0x00000000,
453 0x8e84, 0x01ff1f3f, 0x00000000,
454 0x9060, 0x0000007f, 0x00000020,
455 0x9508, 0x00010000, 0x00010000,
456 0xac14, 0x000003ff, 0x00000003,
457 0xac14, 0x000003ff, 0x00000003,
458 0xac14, 0x000003ff, 0x00000003,
459 0xac10, 0xffffffff, 0x00000000,
460 0xac10, 0xffffffff, 0x00000000,
461 0xac10, 0xffffffff, 0x00000000,
462 0xac0c, 0xffffffff, 0x00001032,
463 0xac0c, 0xffffffff, 0x00001032,
464 0xac0c, 0xffffffff, 0x00001032,
465 0x88d4, 0x0000001f, 0x00000010,
466 0x88d4, 0x0000001f, 0x00000010,
467 0x88d4, 0x0000001f, 0x00000010,
468 0x15c0, 0x000c0fc0, 0x000c0400
469};
470
471static const u32 oland_golden_rlc_registers[] =
472{
473 0xc424, 0xffffffff, 0x00601005,
474 0xc47c, 0xffffffff, 0x10104040,
475 0xc488, 0xffffffff, 0x0100000a,
476 0xc314, 0xffffffff, 0x00000800,
477 0xc30c, 0xffffffff, 0x800000f4
478};
479
480static const u32 oland_golden_registers[] =
481{
482 0x9a10, 0x00010000, 0x00018208,
483 0x9830, 0xffffffff, 0x00000000,
484 0x9834, 0xf00fffff, 0x00000400,
485 0x9838, 0x0002021c, 0x00020200,
486 0xc78, 0x00000080, 0x00000000,
487 0xd030, 0x000300c0, 0x00800040,
488 0xd830, 0x000300c0, 0x00800040,
489 0x5bb0, 0x000000f0, 0x00000070,
490 0x5bc0, 0x00200000, 0x50100000,
491 0x7030, 0x31000311, 0x00000011,
492 0x2ae4, 0x00073ffe, 0x000022a2,
493 0x240c, 0x000007ff, 0x00000000,
494 0x8a14, 0xf000001f, 0x00000007,
495 0x8b24, 0xffffffff, 0x00ffffff,
496 0x8b10, 0x0000ff0f, 0x00000000,
497 0x28a4c, 0x07ffffff, 0x4e000000,
498 0x28350, 0x3f3f3fff, 0x00000082,
499 0x30, 0x000000ff, 0x0040,
500 0x34, 0x00000040, 0x00004040,
501 0x9100, 0x07ffffff, 0x03000000,
502 0x9060, 0x0000007f, 0x00000020,
503 0x9508, 0x00010000, 0x00010000,
504 0xac14, 0x000003ff, 0x000000f3,
505 0xac10, 0xffffffff, 0x00000000,
506 0xac0c, 0xffffffff, 0x00003210,
507 0x88d4, 0x0000001f, 0x00000010,
508 0x15c0, 0x000c0fc0, 0x000c0400
509};
510
fffbdda4
AD		 511static const u32 hainan_golden_registers[] =
512{
513 0x9a10, 0x00010000, 0x00018208,
514 0x9830, 0xffffffff, 0x00000000,
515 0x9834, 0xf00fffff, 0x00000400,
516 0x9838, 0x0002021c, 0x00020200,
517 0xd0c0, 0xff000fff, 0x00000100,
518 0xd030, 0x000300c0, 0x00800040,
519 0xd8c0, 0xff000fff, 0x00000100,
520 0xd830, 0x000300c0, 0x00800040,
521 0x2ae4, 0x00073ffe, 0x000022a2,
522 0x240c, 0x000007ff, 0x00000000,
523 0x8a14, 0xf000001f, 0x00000007,
524 0x8b24, 0xffffffff, 0x00ffffff,
525 0x8b10, 0x0000ff0f, 0x00000000,
526 0x28a4c, 0x07ffffff, 0x4e000000,
527 0x28350, 0x3f3f3fff, 0x00000000,
528 0x30, 0x000000ff, 0x0040,
529 0x34, 0x00000040, 0x00004040,
530 0x9100, 0x03e00000, 0x03600000,
531 0x9060, 0x0000007f, 0x00000020,
532 0x9508, 0x00010000, 0x00010000,
533 0xac14, 0x000003ff, 0x000000f1,
534 0xac10, 0xffffffff, 0x00000000,
535 0xac0c, 0xffffffff, 0x00003210,
536 0x88d4, 0x0000001f, 0x00000010,
537 0x15c0, 0x000c0fc0, 0x000c0400
538};
539
540static const u32 hainan_golden_registers2[] =
541{
542 0x98f8, 0xffffffff, 0x02010001
543};
544
205996c0
AD		 545static const u32 tahiti_mgcg_cgcg_init[] =
546{
547 0xc400, 0xffffffff, 0xfffffffc,
548 0x802c, 0xffffffff, 0xe0000000,
549 0x9a60, 0xffffffff, 0x00000100,
550 0x92a4, 0xffffffff, 0x00000100,
551 0xc164, 0xffffffff, 0x00000100,
552 0x9774, 0xffffffff, 0x00000100,
553 0x8984, 0xffffffff, 0x06000100,
554 0x8a18, 0xffffffff, 0x00000100,
555 0x92a0, 0xffffffff, 0x00000100,
556 0xc380, 0xffffffff, 0x00000100,
557 0x8b28, 0xffffffff, 0x00000100,
558 0x9144, 0xffffffff, 0x00000100,
559 0x8d88, 0xffffffff, 0x00000100,
560 0x8d8c, 0xffffffff, 0x00000100,
561 0x9030, 0xffffffff, 0x00000100,
562 0x9034, 0xffffffff, 0x00000100,
563 0x9038, 0xffffffff, 0x00000100,
564 0x903c, 0xffffffff, 0x00000100,
565 0xad80, 0xffffffff, 0x00000100,
566 0xac54, 0xffffffff, 0x00000100,
567 0x897c, 0xffffffff, 0x06000100,
568 0x9868, 0xffffffff, 0x00000100,
569 0x9510, 0xffffffff, 0x00000100,
570 0xaf04, 0xffffffff, 0x00000100,
571 0xae04, 0xffffffff, 0x00000100,
572 0x949c, 0xffffffff, 0x00000100,
573 0x802c, 0xffffffff, 0xe0000000,
574 0x9160, 0xffffffff, 0x00010000,
575 0x9164, 0xffffffff, 0x00030002,
576 0x9168, 0xffffffff, 0x00040007,
577 0x916c, 0xffffffff, 0x00060005,
578 0x9170, 0xffffffff, 0x00090008,
579 0x9174, 0xffffffff, 0x00020001,
580 0x9178, 0xffffffff, 0x00040003,
581 0x917c, 0xffffffff, 0x00000007,
582 0x9180, 0xffffffff, 0x00060005,
583 0x9184, 0xffffffff, 0x00090008,
584 0x9188, 0xffffffff, 0x00030002,
585 0x918c, 0xffffffff, 0x00050004,
586 0x9190, 0xffffffff, 0x00000008,
587 0x9194, 0xffffffff, 0x00070006,
588 0x9198, 0xffffffff, 0x000a0009,
589 0x919c, 0xffffffff, 0x00040003,
590 0x91a0, 0xffffffff, 0x00060005,
591 0x91a4, 0xffffffff, 0x00000009,
592 0x91a8, 0xffffffff, 0x00080007,
593 0x91ac, 0xffffffff, 0x000b000a,
594 0x91b0, 0xffffffff, 0x00050004,
595 0x91b4, 0xffffffff, 0x00070006,
596 0x91b8, 0xffffffff, 0x0008000b,
597 0x91bc, 0xffffffff, 0x000a0009,
598 0x91c0, 0xffffffff, 0x000d000c,
599 0x91c4, 0xffffffff, 0x00060005,
600 0x91c8, 0xffffffff, 0x00080007,
601 0x91cc, 0xffffffff, 0x0000000b,
602 0x91d0, 0xffffffff, 0x000a0009,
603 0x91d4, 0xffffffff, 0x000d000c,
604 0x91d8, 0xffffffff, 0x00070006,
605 0x91dc, 0xffffffff, 0x00090008,
606 0x91e0, 0xffffffff, 0x0000000c,
607 0x91e4, 0xffffffff, 0x000b000a,
608 0x91e8, 0xffffffff, 0x000e000d,
609 0x91ec, 0xffffffff, 0x00080007,
610 0x91f0, 0xffffffff, 0x000a0009,
611 0x91f4, 0xffffffff, 0x0000000d,
612 0x91f8, 0xffffffff, 0x000c000b,
613 0x91fc, 0xffffffff, 0x000f000e,
614 0x9200, 0xffffffff, 0x00090008,
615 0x9204, 0xffffffff, 0x000b000a,
616 0x9208, 0xffffffff, 0x000c000f,
617 0x920c, 0xffffffff, 0x000e000d,
618 0x9210, 0xffffffff, 0x00110010,
619 0x9214, 0xffffffff, 0x000a0009,
620 0x9218, 0xffffffff, 0x000c000b,
621 0x921c, 0xffffffff, 0x0000000f,
622 0x9220, 0xffffffff, 0x000e000d,
623 0x9224, 0xffffffff, 0x00110010,
624 0x9228, 0xffffffff, 0x000b000a,
625 0x922c, 0xffffffff, 0x000d000c,
626 0x9230, 0xffffffff, 0x00000010,
627 0x9234, 0xffffffff, 0x000f000e,
628 0x9238, 0xffffffff, 0x00120011,
629 0x923c, 0xffffffff, 0x000c000b,
630 0x9240, 0xffffffff, 0x000e000d,
631 0x9244, 0xffffffff, 0x00000011,
632 0x9248, 0xffffffff, 0x0010000f,
633 0x924c, 0xffffffff, 0x00130012,
634 0x9250, 0xffffffff, 0x000d000c,
635 0x9254, 0xffffffff, 0x000f000e,
636 0x9258, 0xffffffff, 0x00100013,
637 0x925c, 0xffffffff, 0x00120011,
638 0x9260, 0xffffffff, 0x00150014,
639 0x9264, 0xffffffff, 0x000e000d,
640 0x9268, 0xffffffff, 0x0010000f,
641 0x926c, 0xffffffff, 0x00000013,
642 0x9270, 0xffffffff, 0x00120011,
643 0x9274, 0xffffffff, 0x00150014,
644 0x9278, 0xffffffff, 0x000f000e,
645 0x927c, 0xffffffff, 0x00110010,
646 0x9280, 0xffffffff, 0x00000014,
647 0x9284, 0xffffffff, 0x00130012,
648 0x9288, 0xffffffff, 0x00160015,
649 0x928c, 0xffffffff, 0x0010000f,
650 0x9290, 0xffffffff, 0x00120011,
651 0x9294, 0xffffffff, 0x00000015,
652 0x9298, 0xffffffff, 0x00140013,
653 0x929c, 0xffffffff, 0x00170016,
654 0x9150, 0xffffffff, 0x96940200,
655 0x8708, 0xffffffff, 0x00900100,
656 0xc478, 0xffffffff, 0x00000080,
657 0xc404, 0xffffffff, 0x0020003f,
658 0x30, 0xffffffff, 0x0000001c,
659 0x34, 0x000f0000, 0x000f0000,
660 0x160c, 0xffffffff, 0x00000100,
661 0x1024, 0xffffffff, 0x00000100,
662 0x102c, 0x00000101, 0x00000000,
663 0x20a8, 0xffffffff, 0x00000104,
664 0x264c, 0x000c0000, 0x000c0000,
665 0x2648, 0x000c0000, 0x000c0000,
666 0x55e4, 0xff000fff, 0x00000100,
667 0x55e8, 0x00000001, 0x00000001,
668 0x2f50, 0x00000001, 0x00000001,
669 0x30cc, 0xc0000fff, 0x00000104,
670 0xc1e4, 0x00000001, 0x00000001,
671 0xd0c0, 0xfffffff0, 0x00000100,
672 0xd8c0, 0xfffffff0, 0x00000100
673};
674
675static const u32 pitcairn_mgcg_cgcg_init[] =
676{
677 0xc400, 0xffffffff, 0xfffffffc,
678 0x802c, 0xffffffff, 0xe0000000,
679 0x9a60, 0xffffffff, 0x00000100,
680 0x92a4, 0xffffffff, 0x00000100,
681 0xc164, 0xffffffff, 0x00000100,
682 0x9774, 0xffffffff, 0x00000100,
683 0x8984, 0xffffffff, 0x06000100,
684 0x8a18, 0xffffffff, 0x00000100,
685 0x92a0, 0xffffffff, 0x00000100,
686 0xc380, 0xffffffff, 0x00000100,
687 0x8b28, 0xffffffff, 0x00000100,
688 0x9144, 0xffffffff, 0x00000100,
689 0x8d88, 0xffffffff, 0x00000100,
690 0x8d8c, 0xffffffff, 0x00000100,
691 0x9030, 0xffffffff, 0x00000100,
692 0x9034, 0xffffffff, 0x00000100,
693 0x9038, 0xffffffff, 0x00000100,
694 0x903c, 0xffffffff, 0x00000100,
695 0xad80, 0xffffffff, 0x00000100,
696 0xac54, 0xffffffff, 0x00000100,
697 0x897c, 0xffffffff, 0x06000100,
698 0x9868, 0xffffffff, 0x00000100,
699 0x9510, 0xffffffff, 0x00000100,
700 0xaf04, 0xffffffff, 0x00000100,
701 0xae04, 0xffffffff, 0x00000100,
702 0x949c, 0xffffffff, 0x00000100,
703 0x802c, 0xffffffff, 0xe0000000,
704 0x9160, 0xffffffff, 0x00010000,
705 0x9164, 0xffffffff, 0x00030002,
706 0x9168, 0xffffffff, 0x00040007,
707 0x916c, 0xffffffff, 0x00060005,
708 0x9170, 0xffffffff, 0x00090008,
709 0x9174, 0xffffffff, 0x00020001,
710 0x9178, 0xffffffff, 0x00040003,
711 0x917c, 0xffffffff, 0x00000007,
712 0x9180, 0xffffffff, 0x00060005,
713 0x9184, 0xffffffff, 0x00090008,
714 0x9188, 0xffffffff, 0x00030002,
715 0x918c, 0xffffffff, 0x00050004,
716 0x9190, 0xffffffff, 0x00000008,
717 0x9194, 0xffffffff, 0x00070006,
718 0x9198, 0xffffffff, 0x000a0009,
719 0x919c, 0xffffffff, 0x00040003,
720 0x91a0, 0xffffffff, 0x00060005,
721 0x91a4, 0xffffffff, 0x00000009,
722 0x91a8, 0xffffffff, 0x00080007,
723 0x91ac, 0xffffffff, 0x000b000a,
724 0x91b0, 0xffffffff, 0x00050004,
725 0x91b4, 0xffffffff, 0x00070006,
726 0x91b8, 0xffffffff, 0x0008000b,
727 0x91bc, 0xffffffff, 0x000a0009,
728 0x91c0, 0xffffffff, 0x000d000c,
729 0x9200, 0xffffffff, 0x00090008,
730 0x9204, 0xffffffff, 0x000b000a,
731 0x9208, 0xffffffff, 0x000c000f,
732 0x920c, 0xffffffff, 0x000e000d,
733 0x9210, 0xffffffff, 0x00110010,
734 0x9214, 0xffffffff, 0x000a0009,
735 0x9218, 0xffffffff, 0x000c000b,
736 0x921c, 0xffffffff, 0x0000000f,
737 0x9220, 0xffffffff, 0x000e000d,
738 0x9224, 0xffffffff, 0x00110010,
739 0x9228, 0xffffffff, 0x000b000a,
740 0x922c, 0xffffffff, 0x000d000c,
741 0x9230, 0xffffffff, 0x00000010,
742 0x9234, 0xffffffff, 0x000f000e,
743 0x9238, 0xffffffff, 0x00120011,
744 0x923c, 0xffffffff, 0x000c000b,
745 0x9240, 0xffffffff, 0x000e000d,
746 0x9244, 0xffffffff, 0x00000011,
747 0x9248, 0xffffffff, 0x0010000f,
748 0x924c, 0xffffffff, 0x00130012,
749 0x9250, 0xffffffff, 0x000d000c,
750 0x9254, 0xffffffff, 0x000f000e,
751 0x9258, 0xffffffff, 0x00100013,
752 0x925c, 0xffffffff, 0x00120011,
753 0x9260, 0xffffffff, 0x00150014,
754 0x9150, 0xffffffff, 0x96940200,
755 0x8708, 0xffffffff, 0x00900100,
756 0xc478, 0xffffffff, 0x00000080,
757 0xc404, 0xffffffff, 0x0020003f,
758 0x30, 0xffffffff, 0x0000001c,
759 0x34, 0x000f0000, 0x000f0000,
760 0x160c, 0xffffffff, 0x00000100,
761 0x1024, 0xffffffff, 0x00000100,
762 0x102c, 0x00000101, 0x00000000,
763 0x20a8, 0xffffffff, 0x00000104,
764 0x55e4, 0xff000fff, 0x00000100,
765 0x55e8, 0x00000001, 0x00000001,
766 0x2f50, 0x00000001, 0x00000001,
767 0x30cc, 0xc0000fff, 0x00000104,
768 0xc1e4, 0x00000001, 0x00000001,
769 0xd0c0, 0xfffffff0, 0x00000100,
770 0xd8c0, 0xfffffff0, 0x00000100
771};
772
773static const u32 verde_mgcg_cgcg_init[] =
774{
775 0xc400, 0xffffffff, 0xfffffffc,
776 0x802c, 0xffffffff, 0xe0000000,
777 0x9a60, 0xffffffff, 0x00000100,
778 0x92a4, 0xffffffff, 0x00000100,
779 0xc164, 0xffffffff, 0x00000100,
780 0x9774, 0xffffffff, 0x00000100,
781 0x8984, 0xffffffff, 0x06000100,
782 0x8a18, 0xffffffff, 0x00000100,
783 0x92a0, 0xffffffff, 0x00000100,
784 0xc380, 0xffffffff, 0x00000100,
785 0x8b28, 0xffffffff, 0x00000100,
786 0x9144, 0xffffffff, 0x00000100,
787 0x8d88, 0xffffffff, 0x00000100,
788 0x8d8c, 0xffffffff, 0x00000100,
789 0x9030, 0xffffffff, 0x00000100,
790 0x9034, 0xffffffff, 0x00000100,
791 0x9038, 0xffffffff, 0x00000100,
792 0x903c, 0xffffffff, 0x00000100,
793 0xad80, 0xffffffff, 0x00000100,
794 0xac54, 0xffffffff, 0x00000100,
795 0x897c, 0xffffffff, 0x06000100,
796 0x9868, 0xffffffff, 0x00000100,
797 0x9510, 0xffffffff, 0x00000100,
798 0xaf04, 0xffffffff, 0x00000100,
799 0xae04, 0xffffffff, 0x00000100,
800 0x949c, 0xffffffff, 0x00000100,
801 0x802c, 0xffffffff, 0xe0000000,
802 0x9160, 0xffffffff, 0x00010000,
803 0x9164, 0xffffffff, 0x00030002,
804 0x9168, 0xffffffff, 0x00040007,
805 0x916c, 0xffffffff, 0x00060005,
806 0x9170, 0xffffffff, 0x00090008,
807 0x9174, 0xffffffff, 0x00020001,
808 0x9178, 0xffffffff, 0x00040003,
809 0x917c, 0xffffffff, 0x00000007,
810 0x9180, 0xffffffff, 0x00060005,
811 0x9184, 0xffffffff, 0x00090008,
812 0x9188, 0xffffffff, 0x00030002,
813 0x918c, 0xffffffff, 0x00050004,
814 0x9190, 0xffffffff, 0x00000008,
815 0x9194, 0xffffffff, 0x00070006,
816 0x9198, 0xffffffff, 0x000a0009,
817 0x919c, 0xffffffff, 0x00040003,
818 0x91a0, 0xffffffff, 0x00060005,
819 0x91a4, 0xffffffff, 0x00000009,
820 0x91a8, 0xffffffff, 0x00080007,
821 0x91ac, 0xffffffff, 0x000b000a,
822 0x91b0, 0xffffffff, 0x00050004,
823 0x91b4, 0xffffffff, 0x00070006,
824 0x91b8, 0xffffffff, 0x0008000b,
825 0x91bc, 0xffffffff, 0x000a0009,
826 0x91c0, 0xffffffff, 0x000d000c,
827 0x9200, 0xffffffff, 0x00090008,
828 0x9204, 0xffffffff, 0x000b000a,
829 0x9208, 0xffffffff, 0x000c000f,
830 0x920c, 0xffffffff, 0x000e000d,
831 0x9210, 0xffffffff, 0x00110010,
832 0x9214, 0xffffffff, 0x000a0009,
833 0x9218, 0xffffffff, 0x000c000b,
834 0x921c, 0xffffffff, 0x0000000f,
835 0x9220, 0xffffffff, 0x000e000d,
836 0x9224, 0xffffffff, 0x00110010,
837 0x9228, 0xffffffff, 0x000b000a,
838 0x922c, 0xffffffff, 0x000d000c,
839 0x9230, 0xffffffff, 0x00000010,
840 0x9234, 0xffffffff, 0x000f000e,
841 0x9238, 0xffffffff, 0x00120011,
842 0x923c, 0xffffffff, 0x000c000b,
843 0x9240, 0xffffffff, 0x000e000d,
844 0x9244, 0xffffffff, 0x00000011,
845 0x9248, 0xffffffff, 0x0010000f,
846 0x924c, 0xffffffff, 0x00130012,
847 0x9250, 0xffffffff, 0x000d000c,
848 0x9254, 0xffffffff, 0x000f000e,
849 0x9258, 0xffffffff, 0x00100013,
850 0x925c, 0xffffffff, 0x00120011,
851 0x9260, 0xffffffff, 0x00150014,
852 0x9150, 0xffffffff, 0x96940200,
853 0x8708, 0xffffffff, 0x00900100,
854 0xc478, 0xffffffff, 0x00000080,
855 0xc404, 0xffffffff, 0x0020003f,
856 0x30, 0xffffffff, 0x0000001c,
857 0x34, 0x000f0000, 0x000f0000,
858 0x160c, 0xffffffff, 0x00000100,
859 0x1024, 0xffffffff, 0x00000100,
860 0x102c, 0x00000101, 0x00000000,
861 0x20a8, 0xffffffff, 0x00000104,
862 0x264c, 0x000c0000, 0x000c0000,
863 0x2648, 0x000c0000, 0x000c0000,
864 0x55e4, 0xff000fff, 0x00000100,
865 0x55e8, 0x00000001, 0x00000001,
866 0x2f50, 0x00000001, 0x00000001,
867 0x30cc, 0xc0000fff, 0x00000104,
868 0xc1e4, 0x00000001, 0x00000001,
869 0xd0c0, 0xfffffff0, 0x00000100,
870 0xd8c0, 0xfffffff0, 0x00000100
871};
872
873static const u32 oland_mgcg_cgcg_init[] =
874{
875 0xc400, 0xffffffff, 0xfffffffc,
876 0x802c, 0xffffffff, 0xe0000000,
877 0x9a60, 0xffffffff, 0x00000100,
878 0x92a4, 0xffffffff, 0x00000100,
879 0xc164, 0xffffffff, 0x00000100,
880 0x9774, 0xffffffff, 0x00000100,
881 0x8984, 0xffffffff, 0x06000100,
882 0x8a18, 0xffffffff, 0x00000100,
883 0x92a0, 0xffffffff, 0x00000100,
884 0xc380, 0xffffffff, 0x00000100,
885 0x8b28, 0xffffffff, 0x00000100,
886 0x9144, 0xffffffff, 0x00000100,
887 0x8d88, 0xffffffff, 0x00000100,
888 0x8d8c, 0xffffffff, 0x00000100,
889 0x9030, 0xffffffff, 0x00000100,
890 0x9034, 0xffffffff, 0x00000100,
891 0x9038, 0xffffffff, 0x00000100,
892 0x903c, 0xffffffff, 0x00000100,
893 0xad80, 0xffffffff, 0x00000100,
894 0xac54, 0xffffffff, 0x00000100,
895 0x897c, 0xffffffff, 0x06000100,
896 0x9868, 0xffffffff, 0x00000100,
897 0x9510, 0xffffffff, 0x00000100,
898 0xaf04, 0xffffffff, 0x00000100,
899 0xae04, 0xffffffff, 0x00000100,
900 0x949c, 0xffffffff, 0x00000100,
901 0x802c, 0xffffffff, 0xe0000000,
902 0x9160, 0xffffffff, 0x00010000,
903 0x9164, 0xffffffff, 0x00030002,
904 0x9168, 0xffffffff, 0x00040007,
905 0x916c, 0xffffffff, 0x00060005,
906 0x9170, 0xffffffff, 0x00090008,
907 0x9174, 0xffffffff, 0x00020001,
908 0x9178, 0xffffffff, 0x00040003,
909 0x917c, 0xffffffff, 0x00000007,
910 0x9180, 0xffffffff, 0x00060005,
911 0x9184, 0xffffffff, 0x00090008,
912 0x9188, 0xffffffff, 0x00030002,
913 0x918c, 0xffffffff, 0x00050004,
914 0x9190, 0xffffffff, 0x00000008,
915 0x9194, 0xffffffff, 0x00070006,
916 0x9198, 0xffffffff, 0x000a0009,
917 0x919c, 0xffffffff, 0x00040003,
918 0x91a0, 0xffffffff, 0x00060005,
919 0x91a4, 0xffffffff, 0x00000009,
920 0x91a8, 0xffffffff, 0x00080007,
921 0x91ac, 0xffffffff, 0x000b000a,
922 0x91b0, 0xffffffff, 0x00050004,
923 0x91b4, 0xffffffff, 0x00070006,
924 0x91b8, 0xffffffff, 0x0008000b,
925 0x91bc, 0xffffffff, 0x000a0009,
926 0x91c0, 0xffffffff, 0x000d000c,
927 0x91c4, 0xffffffff, 0x00060005,
928 0x91c8, 0xffffffff, 0x00080007,
929 0x91cc, 0xffffffff, 0x0000000b,
930 0x91d0, 0xffffffff, 0x000a0009,
931 0x91d4, 0xffffffff, 0x000d000c,
932 0x9150, 0xffffffff, 0x96940200,
933 0x8708, 0xffffffff, 0x00900100,
934 0xc478, 0xffffffff, 0x00000080,
935 0xc404, 0xffffffff, 0x0020003f,
936 0x30, 0xffffffff, 0x0000001c,
937 0x34, 0x000f0000, 0x000f0000,
938 0x160c, 0xffffffff, 0x00000100,
939 0x1024, 0xffffffff, 0x00000100,
940 0x102c, 0x00000101, 0x00000000,
941 0x20a8, 0xffffffff, 0x00000104,
942 0x264c, 0x000c0000, 0x000c0000,
943 0x2648, 0x000c0000, 0x000c0000,
944 0x55e4, 0xff000fff, 0x00000100,
945 0x55e8, 0x00000001, 0x00000001,
946 0x2f50, 0x00000001, 0x00000001,
947 0x30cc, 0xc0000fff, 0x00000104,
948 0xc1e4, 0x00000001, 0x00000001,
949 0xd0c0, 0xfffffff0, 0x00000100,
950 0xd8c0, 0xfffffff0, 0x00000100
951};
952
fffbdda4
AD		 953static const u32 hainan_mgcg_cgcg_init[] =
954{
955 0xc400, 0xffffffff, 0xfffffffc,
956 0x802c, 0xffffffff, 0xe0000000,
957 0x9a60, 0xffffffff, 0x00000100,
958 0x92a4, 0xffffffff, 0x00000100,
959 0xc164, 0xffffffff, 0x00000100,
960 0x9774, 0xffffffff, 0x00000100,
961 0x8984, 0xffffffff, 0x06000100,
962 0x8a18, 0xffffffff, 0x00000100,
963 0x92a0, 0xffffffff, 0x00000100,
964 0xc380, 0xffffffff, 0x00000100,
965 0x8b28, 0xffffffff, 0x00000100,
966 0x9144, 0xffffffff, 0x00000100,
967 0x8d88, 0xffffffff, 0x00000100,
968 0x8d8c, 0xffffffff, 0x00000100,
969 0x9030, 0xffffffff, 0x00000100,
970 0x9034, 0xffffffff, 0x00000100,
971 0x9038, 0xffffffff, 0x00000100,
972 0x903c, 0xffffffff, 0x00000100,
973 0xad80, 0xffffffff, 0x00000100,
974 0xac54, 0xffffffff, 0x00000100,
975 0x897c, 0xffffffff, 0x06000100,
976 0x9868, 0xffffffff, 0x00000100,
977 0x9510, 0xffffffff, 0x00000100,
978 0xaf04, 0xffffffff, 0x00000100,
979 0xae04, 0xffffffff, 0x00000100,
980 0x949c, 0xffffffff, 0x00000100,
981 0x802c, 0xffffffff, 0xe0000000,
982 0x9160, 0xffffffff, 0x00010000,
983 0x9164, 0xffffffff, 0x00030002,
984 0x9168, 0xffffffff, 0x00040007,
985 0x916c, 0xffffffff, 0x00060005,
986 0x9170, 0xffffffff, 0x00090008,
987 0x9174, 0xffffffff, 0x00020001,
988 0x9178, 0xffffffff, 0x00040003,
989 0x917c, 0xffffffff, 0x00000007,
990 0x9180, 0xffffffff, 0x00060005,
991 0x9184, 0xffffffff, 0x00090008,
992 0x9188, 0xffffffff, 0x00030002,
993 0x918c, 0xffffffff, 0x00050004,
994 0x9190, 0xffffffff, 0x00000008,
995 0x9194, 0xffffffff, 0x00070006,
996 0x9198, 0xffffffff, 0x000a0009,
997 0x919c, 0xffffffff, 0x00040003,
998 0x91a0, 0xffffffff, 0x00060005,
999 0x91a4, 0xffffffff, 0x00000009,
1000 0x91a8, 0xffffffff, 0x00080007,
1001 0x91ac, 0xffffffff, 0x000b000a,
1002 0x91b0, 0xffffffff, 0x00050004,
1003 0x91b4, 0xffffffff, 0x00070006,
1004 0x91b8, 0xffffffff, 0x0008000b,
1005 0x91bc, 0xffffffff, 0x000a0009,
1006 0x91c0, 0xffffffff, 0x000d000c,
1007 0x91c4, 0xffffffff, 0x00060005,
1008 0x91c8, 0xffffffff, 0x00080007,
1009 0x91cc, 0xffffffff, 0x0000000b,
1010 0x91d0, 0xffffffff, 0x000a0009,
1011 0x91d4, 0xffffffff, 0x000d000c,
1012 0x9150, 0xffffffff, 0x96940200,
1013 0x8708, 0xffffffff, 0x00900100,
1014 0xc478, 0xffffffff, 0x00000080,
1015 0xc404, 0xffffffff, 0x0020003f,
1016 0x30, 0xffffffff, 0x0000001c,
1017 0x34, 0x000f0000, 0x000f0000,
1018 0x160c, 0xffffffff, 0x00000100,
1019 0x1024, 0xffffffff, 0x00000100,
1020 0x20a8, 0xffffffff, 0x00000104,
1021 0x264c, 0x000c0000, 0x000c0000,
1022 0x2648, 0x000c0000, 0x000c0000,
1023 0x2f50, 0x00000001, 0x00000001,
1024 0x30cc, 0xc0000fff, 0x00000104,
1025 0xc1e4, 0x00000001, 0x00000001,
1026 0xd0c0, 0xfffffff0, 0x00000100,
1027 0xd8c0, 0xfffffff0, 0x00000100
1028};
1029
205996c0
AD		 1030static u32 verde_pg_init[] =
1031{
1032 0x353c, 0xffffffff, 0x40000,
1033 0x3538, 0xffffffff, 0x200010ff,
1034 0x353c, 0xffffffff, 0x0,
1035 0x353c, 0xffffffff, 0x0,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x7007,
1040 0x3538, 0xffffffff, 0x300010ff,
1041 0x353c, 0xffffffff, 0x0,
1042 0x353c, 0xffffffff, 0x0,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x400000,
1047 0x3538, 0xffffffff, 0x100010ff,
1048 0x353c, 0xffffffff, 0x0,
1049 0x353c, 0xffffffff, 0x0,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x120200,
1054 0x3538, 0xffffffff, 0x500010ff,
1055 0x353c, 0xffffffff, 0x0,
1056 0x353c, 0xffffffff, 0x0,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x0,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x1e1e16,
1061 0x3538, 0xffffffff, 0x600010ff,
1062 0x353c, 0xffffffff, 0x0,
1063 0x353c, 0xffffffff, 0x0,
1064 0x353c, 0xffffffff, 0x0,
1065 0x353c, 0xffffffff, 0x0,
1066 0x353c, 0xffffffff, 0x0,
1067 0x353c, 0xffffffff, 0x171f1e,
1068 0x3538, 0xffffffff, 0x700010ff,
1069 0x353c, 0xffffffff, 0x0,
1070 0x353c, 0xffffffff, 0x0,
1071 0x353c, 0xffffffff, 0x0,
1072 0x353c, 0xffffffff, 0x0,
1073 0x353c, 0xffffffff, 0x0,
1074 0x353c, 0xffffffff, 0x0,
1075 0x3538, 0xffffffff, 0x9ff,
1076 0x3500, 0xffffffff, 0x0,
1077 0x3504, 0xffffffff, 0x10000800,
1078 0x3504, 0xffffffff, 0xf,
1079 0x3504, 0xffffffff, 0xf,
1080 0x3500, 0xffffffff, 0x4,
1081 0x3504, 0xffffffff, 0x1000051e,
1082 0x3504, 0xffffffff, 0xffff,
1083 0x3504, 0xffffffff, 0xffff,
1084 0x3500, 0xffffffff, 0x8,
1085 0x3504, 0xffffffff, 0x80500,
1086 0x3500, 0xffffffff, 0x12,
1087 0x3504, 0xffffffff, 0x9050c,
1088 0x3500, 0xffffffff, 0x1d,
1089 0x3504, 0xffffffff, 0xb052c,
1090 0x3500, 0xffffffff, 0x2a,
1091 0x3504, 0xffffffff, 0x1053e,
1092 0x3500, 0xffffffff, 0x2d,
1093 0x3504, 0xffffffff, 0x10546,
1094 0x3500, 0xffffffff, 0x30,
1095 0x3504, 0xffffffff, 0xa054e,
1096 0x3500, 0xffffffff, 0x3c,
1097 0x3504, 0xffffffff, 0x1055f,
1098 0x3500, 0xffffffff, 0x3f,
1099 0x3504, 0xffffffff, 0x10567,
1100 0x3500, 0xffffffff, 0x42,
1101 0x3504, 0xffffffff, 0x1056f,
1102 0x3500, 0xffffffff, 0x45,
1103 0x3504, 0xffffffff, 0x10572,
1104 0x3500, 0xffffffff, 0x48,
1105 0x3504, 0xffffffff, 0x20575,
1106 0x3500, 0xffffffff, 0x4c,
1107 0x3504, 0xffffffff, 0x190801,
1108 0x3500, 0xffffffff, 0x67,
1109 0x3504, 0xffffffff, 0x1082a,
1110 0x3500, 0xffffffff, 0x6a,
1111 0x3504, 0xffffffff, 0x1b082d,
1112 0x3500, 0xffffffff, 0x87,
1113 0x3504, 0xffffffff, 0x310851,
1114 0x3500, 0xffffffff, 0xba,
1115 0x3504, 0xffffffff, 0x891,
1116 0x3500, 0xffffffff, 0xbc,
1117 0x3504, 0xffffffff, 0x893,
1118 0x3500, 0xffffffff, 0xbe,
1119 0x3504, 0xffffffff, 0x20895,
1120 0x3500, 0xffffffff, 0xc2,
1121 0x3504, 0xffffffff, 0x20899,
1122 0x3500, 0xffffffff, 0xc6,
1123 0x3504, 0xffffffff, 0x2089d,
1124 0x3500, 0xffffffff, 0xca,
1125 0x3504, 0xffffffff, 0x8a1,
1126 0x3500, 0xffffffff, 0xcc,
1127 0x3504, 0xffffffff, 0x8a3,
1128 0x3500, 0xffffffff, 0xce,
1129 0x3504, 0xffffffff, 0x308a5,
1130 0x3500, 0xffffffff, 0xd3,
1131 0x3504, 0xffffffff, 0x6d08cd,
1132 0x3500, 0xffffffff, 0x142,
1133 0x3504, 0xffffffff, 0x2000095a,
1134 0x3504, 0xffffffff, 0x1,
1135 0x3500, 0xffffffff, 0x144,
1136 0x3504, 0xffffffff, 0x301f095b,
1137 0x3500, 0xffffffff, 0x165,
1138 0x3504, 0xffffffff, 0xc094d,
1139 0x3500, 0xffffffff, 0x173,
1140 0x3504, 0xffffffff, 0xf096d,
1141 0x3500, 0xffffffff, 0x184,
1142 0x3504, 0xffffffff, 0x15097f,
1143 0x3500, 0xffffffff, 0x19b,
1144 0x3504, 0xffffffff, 0xc0998,
1145 0x3500, 0xffffffff, 0x1a9,
1146 0x3504, 0xffffffff, 0x409a7,
1147 0x3500, 0xffffffff, 0x1af,
1148 0x3504, 0xffffffff, 0xcdc,
1149 0x3500, 0xffffffff, 0x1b1,
1150 0x3504, 0xffffffff, 0x800,
1151 0x3508, 0xffffffff, 0x6c9b2000,
1152 0x3510, 0xfc00, 0x2000,
1153 0x3544, 0xffffffff, 0xfc0,
1154 0x28d4, 0x00000100, 0x100
1155};
1156
1157static void si_init_golden_registers(struct radeon_device *rdev)
1158{
1159 switch (rdev->family) {
1160 case CHIP_TAHITI:
1161 radeon_program_register_sequence(rdev,
1162 tahiti_golden_registers,
1163 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1164 radeon_program_register_sequence(rdev,
1165 tahiti_golden_rlc_registers,
1166 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1167 radeon_program_register_sequence(rdev,
1168 tahiti_mgcg_cgcg_init,
1169 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1170 radeon_program_register_sequence(rdev,
1171 tahiti_golden_registers2,
1172 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1173 break;
1174 case CHIP_PITCAIRN:
1175 radeon_program_register_sequence(rdev,
1176 pitcairn_golden_registers,
1177 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1178 radeon_program_register_sequence(rdev,
1179 pitcairn_golden_rlc_registers,
1180 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1181 radeon_program_register_sequence(rdev,
1182 pitcairn_mgcg_cgcg_init,
1183 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1184 break;
1185 case CHIP_VERDE:
1186 radeon_program_register_sequence(rdev,
1187 verde_golden_registers,
1188 (const u32)ARRAY_SIZE(verde_golden_registers));
1189 radeon_program_register_sequence(rdev,
1190 verde_golden_rlc_registers,
1191 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1192 radeon_program_register_sequence(rdev,
1193 verde_mgcg_cgcg_init,
1194 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1195 radeon_program_register_sequence(rdev,
1196 verde_pg_init,
1197 (const u32)ARRAY_SIZE(verde_pg_init));
1198 break;
1199 case CHIP_OLAND:
1200 radeon_program_register_sequence(rdev,
1201 oland_golden_registers,
1202 (const u32)ARRAY_SIZE(oland_golden_registers));
1203 radeon_program_register_sequence(rdev,
1204 oland_golden_rlc_registers,
1205 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1206 radeon_program_register_sequence(rdev,
1207 oland_mgcg_cgcg_init,
1208 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1209 break;
fffbdda4
AD		 1210 case CHIP_HAINAN:
1211 radeon_program_register_sequence(rdev,
1212 hainan_golden_registers,
1213 (const u32)ARRAY_SIZE(hainan_golden_registers));
1214 radeon_program_register_sequence(rdev,
1215 hainan_golden_registers2,
1216 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1217 radeon_program_register_sequence(rdev,
1218 hainan_mgcg_cgcg_init,
1219 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1220 break;
205996c0
AD		 1221 default:
1222 break;
1223 }
1224}
1225
454d2e2a
AD		 1226#define PCIE_BUS_CLK 10000
1227#define TCLK (PCIE_BUS_CLK / 10)
1228
1229/**
1230 * si_get_xclk - get the xclk
1231 *
1232 * @rdev: radeon_device pointer
1233 *
1234 * Returns the reference clock used by the gfx engine
1235 * (SI).
1236 */
1237u32 si_get_xclk(struct radeon_device *rdev)
1238{
1239 u32 reference_clock = rdev->clock.spll.reference_freq;
1240 u32 tmp;
1241
1242 tmp = RREG32(CG_CLKPIN_CNTL_2);
1243 if (tmp & MUX_TCLK_TO_XCLK)
1244 return TCLK;
1245
1246 tmp = RREG32(CG_CLKPIN_CNTL);
1247 if (tmp & XTALIN_DIVIDE)
1248 return reference_clock / 4;
1249
1250 return reference_clock;
1251}
1252
1bd47d2e
AD		 1253/* get temperature in millidegrees */
1254int si_get_temp(struct radeon_device *rdev)
1255{
1256 u32 temp;
1257 int actual_temp = 0;
1258
1259 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1260 CTF_TEMP_SHIFT;
1261
1262 if (temp & 0x200)
1263 actual_temp = 255;
1264 else
1265 actual_temp = temp & 0x1ff;
1266
1267 actual_temp = (actual_temp * 1000);
1268
1269 return actual_temp;
1270}
1271
8b074dd6
AD		 1272#define TAHITI_IO_MC_REGS_SIZE 36
1273
1274static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1275 {0x0000006f, 0x03044000},
1276 {0x00000070, 0x0480c018},
1277 {0x00000071, 0x00000040},
1278 {0x00000072, 0x01000000},
1279 {0x00000074, 0x000000ff},
1280 {0x00000075, 0x00143400},
1281 {0x00000076, 0x08ec0800},
1282 {0x00000077, 0x040000cc},
1283 {0x00000079, 0x00000000},
1284 {0x0000007a, 0x21000409},
1285 {0x0000007c, 0x00000000},
1286 {0x0000007d, 0xe8000000},
1287 {0x0000007e, 0x044408a8},
1288 {0x0000007f, 0x00000003},
1289 {0x00000080, 0x00000000},
1290 {0x00000081, 0x01000000},
1291 {0x00000082, 0x02000000},
1292 {0x00000083, 0x00000000},
1293 {0x00000084, 0xe3f3e4f4},
1294 {0x00000085, 0x00052024},
1295 {0x00000087, 0x00000000},
1296 {0x00000088, 0x66036603},
1297 {0x00000089, 0x01000000},
1298 {0x0000008b, 0x1c0a0000},
1299 {0x0000008c, 0xff010000},
1300 {0x0000008e, 0xffffefff},
1301 {0x0000008f, 0xfff3efff},
1302 {0x00000090, 0xfff3efbf},
1303 {0x00000094, 0x00101101},
1304 {0x00000095, 0x00000fff},
1305 {0x00000096, 0x00116fff},
1306 {0x00000097, 0x60010000},
1307 {0x00000098, 0x10010000},
1308 {0x00000099, 0x00006000},
1309 {0x0000009a, 0x00001000},
1310 {0x0000009f, 0x00a77400}
1311};
1312
1313static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1314 {0x0000006f, 0x03044000},
1315 {0x00000070, 0x0480c018},
1316 {0x00000071, 0x00000040},
1317 {0x00000072, 0x01000000},
1318 {0x00000074, 0x000000ff},
1319 {0x00000075, 0x00143400},
1320 {0x00000076, 0x08ec0800},
1321 {0x00000077, 0x040000cc},
1322 {0x00000079, 0x00000000},
1323 {0x0000007a, 0x21000409},
1324 {0x0000007c, 0x00000000},
1325 {0x0000007d, 0xe8000000},
1326 {0x0000007e, 0x044408a8},
1327 {0x0000007f, 0x00000003},
1328 {0x00000080, 0x00000000},
1329 {0x00000081, 0x01000000},
1330 {0x00000082, 0x02000000},
1331 {0x00000083, 0x00000000},
1332 {0x00000084, 0xe3f3e4f4},
1333 {0x00000085, 0x00052024},
1334 {0x00000087, 0x00000000},
1335 {0x00000088, 0x66036603},
1336 {0x00000089, 0x01000000},
1337 {0x0000008b, 0x1c0a0000},
1338 {0x0000008c, 0xff010000},
1339 {0x0000008e, 0xffffefff},
1340 {0x0000008f, 0xfff3efff},
1341 {0x00000090, 0xfff3efbf},
1342 {0x00000094, 0x00101101},
1343 {0x00000095, 0x00000fff},
1344 {0x00000096, 0x00116fff},
1345 {0x00000097, 0x60010000},
1346 {0x00000098, 0x10010000},
1347 {0x00000099, 0x00006000},
1348 {0x0000009a, 0x00001000},
1349 {0x0000009f, 0x00a47400}
1350};
1351
1352static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1353 {0x0000006f, 0x03044000},
1354 {0x00000070, 0x0480c018},
1355 {0x00000071, 0x00000040},
1356 {0x00000072, 0x01000000},
1357 {0x00000074, 0x000000ff},
1358 {0x00000075, 0x00143400},
1359 {0x00000076, 0x08ec0800},
1360 {0x00000077, 0x040000cc},
1361 {0x00000079, 0x00000000},
1362 {0x0000007a, 0x21000409},
1363 {0x0000007c, 0x00000000},
1364 {0x0000007d, 0xe8000000},
1365 {0x0000007e, 0x044408a8},
1366 {0x0000007f, 0x00000003},
1367 {0x00000080, 0x00000000},
1368 {0x00000081, 0x01000000},
1369 {0x00000082, 0x02000000},
1370 {0x00000083, 0x00000000},
1371 {0x00000084, 0xe3f3e4f4},
1372 {0x00000085, 0x00052024},
1373 {0x00000087, 0x00000000},
1374 {0x00000088, 0x66036603},
1375 {0x00000089, 0x01000000},
1376 {0x0000008b, 0x1c0a0000},
1377 {0x0000008c, 0xff010000},
1378 {0x0000008e, 0xffffefff},
1379 {0x0000008f, 0xfff3efff},
1380 {0x00000090, 0xfff3efbf},
1381 {0x00000094, 0x00101101},
1382 {0x00000095, 0x00000fff},
1383 {0x00000096, 0x00116fff},
1384 {0x00000097, 0x60010000},
1385 {0x00000098, 0x10010000},
1386 {0x00000099, 0x00006000},
1387 {0x0000009a, 0x00001000},
1388 {0x0000009f, 0x00a37400}
1389};
1390
bcc7f5d2
AD		 1391static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1392 {0x0000006f, 0x03044000},
1393 {0x00000070, 0x0480c018},
1394 {0x00000071, 0x00000040},
1395 {0x00000072, 0x01000000},
1396 {0x00000074, 0x000000ff},
1397 {0x00000075, 0x00143400},
1398 {0x00000076, 0x08ec0800},
1399 {0x00000077, 0x040000cc},
1400 {0x00000079, 0x00000000},
1401 {0x0000007a, 0x21000409},
1402 {0x0000007c, 0x00000000},
1403 {0x0000007d, 0xe8000000},
1404 {0x0000007e, 0x044408a8},
1405 {0x0000007f, 0x00000003},
1406 {0x00000080, 0x00000000},
1407 {0x00000081, 0x01000000},
1408 {0x00000082, 0x02000000},
1409 {0x00000083, 0x00000000},
1410 {0x00000084, 0xe3f3e4f4},
1411 {0x00000085, 0x00052024},
1412 {0x00000087, 0x00000000},
1413 {0x00000088, 0x66036603},
1414 {0x00000089, 0x01000000},
1415 {0x0000008b, 0x1c0a0000},
1416 {0x0000008c, 0xff010000},
1417 {0x0000008e, 0xffffefff},
1418 {0x0000008f, 0xfff3efff},
1419 {0x00000090, 0xfff3efbf},
1420 {0x00000094, 0x00101101},
1421 {0x00000095, 0x00000fff},
1422 {0x00000096, 0x00116fff},
1423 {0x00000097, 0x60010000},
1424 {0x00000098, 0x10010000},
1425 {0x00000099, 0x00006000},
1426 {0x0000009a, 0x00001000},
1427 {0x0000009f, 0x00a17730}
1428};
1429
c04c00b4
AD		 1430static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1431 {0x0000006f, 0x03044000},
1432 {0x00000070, 0x0480c018},
1433 {0x00000071, 0x00000040},
1434 {0x00000072, 0x01000000},
1435 {0x00000074, 0x000000ff},
1436 {0x00000075, 0x00143400},
1437 {0x00000076, 0x08ec0800},
1438 {0x00000077, 0x040000cc},
1439 {0x00000079, 0x00000000},
1440 {0x0000007a, 0x21000409},
1441 {0x0000007c, 0x00000000},
1442 {0x0000007d, 0xe8000000},
1443 {0x0000007e, 0x044408a8},
1444 {0x0000007f, 0x00000003},
1445 {0x00000080, 0x00000000},
1446 {0x00000081, 0x01000000},
1447 {0x00000082, 0x02000000},
1448 {0x00000083, 0x00000000},
1449 {0x00000084, 0xe3f3e4f4},
1450 {0x00000085, 0x00052024},
1451 {0x00000087, 0x00000000},
1452 {0x00000088, 0x66036603},
1453 {0x00000089, 0x01000000},
1454 {0x0000008b, 0x1c0a0000},
1455 {0x0000008c, 0xff010000},
1456 {0x0000008e, 0xffffefff},
1457 {0x0000008f, 0xfff3efff},
1458 {0x00000090, 0xfff3efbf},
1459 {0x00000094, 0x00101101},
1460 {0x00000095, 0x00000fff},
1461 {0x00000096, 0x00116fff},
1462 {0x00000097, 0x60010000},
1463 {0x00000098, 0x10010000},
1464 {0x00000099, 0x00006000},
1465 {0x0000009a, 0x00001000},
1466 {0x0000009f, 0x00a07730}
1467};
1468
8b074dd6 1469/* ucode loading */
6c7bccea 1470int si_mc_load_microcode(struct radeon_device *rdev)
8b074dd6
AD		 1471{
1472 const __be32 *fw_data;
1473 u32 running, blackout = 0;
1474 u32 *io_mc_regs;
1ebe9280 1475 int i, regs_size, ucode_size = rdev->mc_fw->size / 4;
8b074dd6
AD		 1476
1477 if (!rdev->mc_fw)
1478 return -EINVAL;
1479
1480 switch (rdev->family) {
1481 case CHIP_TAHITI:
1482 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
8b074dd6
AD		 1483 regs_size = TAHITI_IO_MC_REGS_SIZE;
1484 break;
1485 case CHIP_PITCAIRN:
1486 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
8b074dd6
AD		 1487 regs_size = TAHITI_IO_MC_REGS_SIZE;
1488 break;
1489 case CHIP_VERDE:
1490 default:
1491 io_mc_regs = (u32 *)&verde_io_mc_regs;
8b074dd6
AD		 1492 regs_size = TAHITI_IO_MC_REGS_SIZE;
1493 break;
bcc7f5d2
AD		 1494 case CHIP_OLAND:
1495 io_mc_regs = (u32 *)&oland_io_mc_regs;
bcc7f5d2
AD		 1496 regs_size = TAHITI_IO_MC_REGS_SIZE;
1497 break;
c04c00b4
AD		 1498 case CHIP_HAINAN:
1499 io_mc_regs = (u32 *)&hainan_io_mc_regs;
c04c00b4
AD		 1500 regs_size = TAHITI_IO_MC_REGS_SIZE;
1501 break;
8b074dd6
AD		 1502 }
1503
1504 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1505
1506 if (running == 0) {
1507 if (running) {
1508 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1509 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1510 }
1511
1512 /* reset the engine and set to writable */
1513 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1514 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1515
1516 /* load mc io regs */
1517 for (i = 0; i < regs_size; i++) {
1518 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1519 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1520 }
1521 /* load the MC ucode */
1522 fw_data = (const __be32 *)rdev->mc_fw->data;
1523 for (i = 0; i < ucode_size; i++)
1524 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1525
1526 /* put the engine back into the active state */
1527 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1528 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1529 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1530
1531 /* wait for training to complete */
1532 for (i = 0; i < rdev->usec_timeout; i++) {
1533 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1534 break;
1535 udelay(1);
1536 }
1537 for (i = 0; i < rdev->usec_timeout; i++) {
1538 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1539 break;
1540 udelay(1);
1541 }
1542
1543 if (running)
1544 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1545 }
1546
1547 return 0;
1548}
1549
0f0de06c
AD		 1550static int si_init_microcode(struct radeon_device *rdev)
1551{
0f0de06c
AD		 1552 const char *chip_name;
1553 const char *rlc_chip_name;
1554 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1ebe9280 1555 size_t smc_req_size, mc2_req_size;
0f0de06c
AD		 1556 char fw_name[30];
1557 int err;
1558
1559 DRM_DEBUG("\n");
1560
0f0de06c
AD		 1561 switch (rdev->family) {
1562 case CHIP_TAHITI:
1563 chip_name = "TAHITI";
1564 rlc_chip_name = "TAHITI";
1565 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1566 me_req_size = SI_PM4_UCODE_SIZE * 4;
1567 ce_req_size = SI_CE_UCODE_SIZE * 4;
1568 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1569 mc_req_size = SI_MC_UCODE_SIZE * 4;
1ebe9280 1570 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
a9e61410 1571 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1572 break;
1573 case CHIP_PITCAIRN:
1574 chip_name = "PITCAIRN";
1575 rlc_chip_name = "PITCAIRN";
1576 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1577 me_req_size = SI_PM4_UCODE_SIZE * 4;
1578 ce_req_size = SI_CE_UCODE_SIZE * 4;
1579 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1580 mc_req_size = SI_MC_UCODE_SIZE * 4;
1ebe9280 1581 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
a9e61410 1582 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
0f0de06c
AD		 1583 break;
1584 case CHIP_VERDE:
1585 chip_name = "VERDE";
1586 rlc_chip_name = "VERDE";
1587 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1588 me_req_size = SI_PM4_UCODE_SIZE * 4;
1589 ce_req_size = SI_CE_UCODE_SIZE * 4;
1590 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1591 mc_req_size = SI_MC_UCODE_SIZE * 4;
1ebe9280 1592 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
a9e61410 1593 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
0f0de06c 1594 break;
bcc7f5d2
AD		 1595 case CHIP_OLAND:
1596 chip_name = "OLAND";
1597 rlc_chip_name = "OLAND";
1598 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1599 me_req_size = SI_PM4_UCODE_SIZE * 4;
1600 ce_req_size = SI_CE_UCODE_SIZE * 4;
1601 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1ebe9280 1602 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1603 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
bcc7f5d2 1604 break;
c04c00b4
AD		 1605 case CHIP_HAINAN:
1606 chip_name = "HAINAN";
1607 rlc_chip_name = "HAINAN";
1608 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1609 me_req_size = SI_PM4_UCODE_SIZE * 4;
1610 ce_req_size = SI_CE_UCODE_SIZE * 4;
1611 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1ebe9280 1612 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
a9e61410 1613 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
c04c00b4 1614 break;
0f0de06c
AD		 1615 default: BUG();
1616 }
1617
1618 DRM_INFO("Loading %s Microcode\n", chip_name);
1619
1620 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
0a168933 1621 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
0f0de06c
AD		 1622 if (err)
1623 goto out;
1624 if (rdev->pfp_fw->size != pfp_req_size) {
1625 printk(KERN_ERR
1626 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1627 rdev->pfp_fw->size, fw_name);
1628 err = -EINVAL;
1629 goto out;
1630 }
1631
1632 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
0a168933 1633 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
0f0de06c
AD		 1634 if (err)
1635 goto out;
1636 if (rdev->me_fw->size != me_req_size) {
1637 printk(KERN_ERR
1638 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1639 rdev->me_fw->size, fw_name);
1640 err = -EINVAL;
1641 }
1642
1643 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
0a168933 1644 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
0f0de06c
AD		 1645 if (err)
1646 goto out;
1647 if (rdev->ce_fw->size != ce_req_size) {
1648 printk(KERN_ERR
1649 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1650 rdev->ce_fw->size, fw_name);
1651 err = -EINVAL;
1652 }
1653
1654 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
0a168933 1655 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
0f0de06c
AD		 1656 if (err)
1657 goto out;
1658 if (rdev->rlc_fw->size != rlc_req_size) {
1659 printk(KERN_ERR
1660 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1661 rdev->rlc_fw->size, fw_name);
1662 err = -EINVAL;
1663 }
1664
1ebe9280 1665 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
0a168933 1666 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1ebe9280
AD		 1667 if (err) {
1668 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1669 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1670 if (err)
1671 goto out;
1672 }
1673 if ((rdev->mc_fw->size != mc_req_size) &&
1674 (rdev->mc_fw->size != mc2_req_size)) {
0f0de06c
AD		 1675 printk(KERN_ERR
1676 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1677 rdev->mc_fw->size, fw_name);
1678 err = -EINVAL;
1679 }
1ebe9280 1680 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
0f0de06c 1681
a9e61410 1682 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
0a168933 1683 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
8a53fa23
AD		 1684 if (err) {
1685 printk(KERN_ERR
1686 "smc: error loading firmware \"%s\"\n",
1687 fw_name);
1688 release_firmware(rdev->smc_fw);
1689 rdev->smc_fw = NULL;
d8367112 1690 err = 0;
8a53fa23 1691 } else if (rdev->smc_fw->size != smc_req_size) {
a9e61410
AD		 1692 printk(KERN_ERR
1693 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1694 rdev->smc_fw->size, fw_name);
1695 err = -EINVAL;
1696 }
1697
0f0de06c 1698out:
0f0de06c
AD		 1699 if (err) {
1700 if (err != -EINVAL)
1701 printk(KERN_ERR
1702 "si_cp: Failed to load firmware \"%s\"\n",
1703 fw_name);
1704 release_firmware(rdev->pfp_fw);
1705 rdev->pfp_fw = NULL;
1706 release_firmware(rdev->me_fw);
1707 rdev->me_fw = NULL;
1708 release_firmware(rdev->ce_fw);
1709 rdev->ce_fw = NULL;
1710 release_firmware(rdev->rlc_fw);
1711 rdev->rlc_fw = NULL;
1712 release_firmware(rdev->mc_fw);
1713 rdev->mc_fw = NULL;
a9e61410
AD		 1714 release_firmware(rdev->smc_fw);
1715 rdev->smc_fw = NULL;
0f0de06c
AD		 1716 }
1717 return err;
1718}
1719
43b3cd99
AD		 1720/* watermark setup */
1721static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1722 struct radeon_crtc *radeon_crtc,
1723 struct drm_display_mode *mode,
1724 struct drm_display_mode *other_mode)
1725{
290d2457
AD		 1726 u32 tmp, buffer_alloc, i;
1727 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
43b3cd99
AD		 1728 /*
1729 * Line Buffer Setup
1730 * There are 3 line buffers, each one shared by 2 display controllers.
1731 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1732 * the display controllers. The paritioning is done via one of four
1733 * preset allocations specified in bits 21:20:
1734 * 0 - half lb
1735 * 2 - whole lb, other crtc must be disabled
1736 */
1737 /* this can get tricky if we have two large displays on a paired group
1738 * of crtcs. Ideally for multiple large displays we'd assign them to
1739 * non-linked crtcs for maximum line buffer allocation.
1740 */
1741 if (radeon_crtc->base.enabled && mode) {
290d2457 1742 if (other_mode) {
43b3cd99 1743 tmp = 0; /* 1/2 */
290d2457
AD		 1744 buffer_alloc = 1;
1745 } else {
43b3cd99 1746 tmp = 2; /* whole */
290d2457
AD		 1747 buffer_alloc = 2;
1748 }
1749 } else {
43b3cd99 1750 tmp = 0;
290d2457
AD		 1751 buffer_alloc = 0;
1752 }
43b3cd99
AD		 1753
1754 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1755 DC_LB_MEMORY_CONFIG(tmp));
1756
290d2457
AD		 1757 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1758 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1759 for (i = 0; i < rdev->usec_timeout; i++) {
1760 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1761 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1762 break;
1763 udelay(1);
1764 }
1765
43b3cd99
AD		 1766 if (radeon_crtc->base.enabled && mode) {
1767 switch (tmp) {
1768 case 0:
1769 default:
1770 return 4096 * 2;
1771 case 2:
1772 return 8192 * 2;
1773 }
1774 }
1775
1776 /* controller not enabled, so no lb used */
1777 return 0;
1778}
1779
ca7db22b 1780static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
43b3cd99
AD		 1781{
1782 u32 tmp = RREG32(MC_SHARED_CHMAP);
1783
1784 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1785 case 0:
1786 default:
1787 return 1;
1788 case 1:
1789 return 2;
1790 case 2:
1791 return 4;
1792 case 3:
1793 return 8;
1794 case 4:
1795 return 3;
1796 case 5:
1797 return 6;
1798 case 6:
1799 return 10;
1800 case 7:
1801 return 12;
1802 case 8:
1803 return 16;
1804 }
1805}
1806
1807struct dce6_wm_params {
1808 u32 dram_channels; /* number of dram channels */
1809 u32 yclk; /* bandwidth per dram data pin in kHz */
1810 u32 sclk; /* engine clock in kHz */
1811 u32 disp_clk; /* display clock in kHz */
1812 u32 src_width; /* viewport width */
1813 u32 active_time; /* active display time in ns */
1814 u32 blank_time; /* blank time in ns */
1815 bool interlaced; /* mode is interlaced */
1816 fixed20_12 vsc; /* vertical scale ratio */
1817 u32 num_heads; /* number of active crtcs */
1818 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1819 u32 lb_size; /* line buffer allocated to pipe */
1820 u32 vtaps; /* vertical scaler taps */
1821};
1822
1823static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1824{
1825 /* Calculate raw DRAM Bandwidth */
1826 fixed20_12 dram_efficiency; /* 0.7 */
1827 fixed20_12 yclk, dram_channels, bandwidth;
1828 fixed20_12 a;
1829
1830 a.full = dfixed_const(1000);
1831 yclk.full = dfixed_const(wm->yclk);
1832 yclk.full = dfixed_div(yclk, a);
1833 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1834 a.full = dfixed_const(10);
1835 dram_efficiency.full = dfixed_const(7);
1836 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1837 bandwidth.full = dfixed_mul(dram_channels, yclk);
1838 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1839
1840 return dfixed_trunc(bandwidth);
1841}
1842
1843static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1844{
1845 /* Calculate DRAM Bandwidth and the part allocated to display. */
1846 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1847 fixed20_12 yclk, dram_channels, bandwidth;
1848 fixed20_12 a;
1849
1850 a.full = dfixed_const(1000);
1851 yclk.full = dfixed_const(wm->yclk);
1852 yclk.full = dfixed_div(yclk, a);
1853 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1854 a.full = dfixed_const(10);
1855 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1856 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1857 bandwidth.full = dfixed_mul(dram_channels, yclk);
1858 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1859
1860 return dfixed_trunc(bandwidth);
1861}
1862
1863static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1864{
1865 /* Calculate the display Data return Bandwidth */
1866 fixed20_12 return_efficiency; /* 0.8 */
1867 fixed20_12 sclk, bandwidth;
1868 fixed20_12 a;
1869
1870 a.full = dfixed_const(1000);
1871 sclk.full = dfixed_const(wm->sclk);
1872 sclk.full = dfixed_div(sclk, a);
1873 a.full = dfixed_const(10);
1874 return_efficiency.full = dfixed_const(8);
1875 return_efficiency.full = dfixed_div(return_efficiency, a);
1876 a.full = dfixed_const(32);
1877 bandwidth.full = dfixed_mul(a, sclk);
1878 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1879
1880 return dfixed_trunc(bandwidth);
1881}
1882
1883static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1884{
1885 return 32;
1886}
1887
1888static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1889{
1890 /* Calculate the DMIF Request Bandwidth */
1891 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1892 fixed20_12 disp_clk, sclk, bandwidth;
1893 fixed20_12 a, b1, b2;
1894 u32 min_bandwidth;
1895
1896 a.full = dfixed_const(1000);
1897 disp_clk.full = dfixed_const(wm->disp_clk);
1898 disp_clk.full = dfixed_div(disp_clk, a);
1899 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1900 b1.full = dfixed_mul(a, disp_clk);
1901
1902 a.full = dfixed_const(1000);
1903 sclk.full = dfixed_const(wm->sclk);
1904 sclk.full = dfixed_div(sclk, a);
1905 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1906 b2.full = dfixed_mul(a, sclk);
1907
1908 a.full = dfixed_const(10);
1909 disp_clk_request_efficiency.full = dfixed_const(8);
1910 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1911
1912 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1913
1914 a.full = dfixed_const(min_bandwidth);
1915 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1916
1917 return dfixed_trunc(bandwidth);
1918}
1919
1920static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1921{
1922 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1923 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1924 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1925 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1926
1927 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1928}
1929
1930static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1931{
1932 /* Calculate the display mode Average Bandwidth
1933 * DisplayMode should contain the source and destination dimensions,
1934 * timing, etc.
1935 */
1936 fixed20_12 bpp;
1937 fixed20_12 line_time;
1938 fixed20_12 src_width;
1939 fixed20_12 bandwidth;
1940 fixed20_12 a;
1941
1942 a.full = dfixed_const(1000);
1943 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1944 line_time.full = dfixed_div(line_time, a);
1945 bpp.full = dfixed_const(wm->bytes_per_pixel);
1946 src_width.full = dfixed_const(wm->src_width);
1947 bandwidth.full = dfixed_mul(src_width, bpp);
1948 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1949 bandwidth.full = dfixed_div(bandwidth, line_time);
1950
1951 return dfixed_trunc(bandwidth);
1952}
1953
1954static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1955{
1956 /* First calcualte the latency in ns */
1957 u32 mc_latency = 2000; /* 2000 ns. */
1958 u32 available_bandwidth = dce6_available_bandwidth(wm);
1959 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1960 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1961 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1962 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1963 (wm->num_heads * cursor_line_pair_return_time);
1964 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1965 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1966 u32 tmp, dmif_size = 12288;
1967 fixed20_12 a, b, c;
1968
1969 if (wm->num_heads == 0)
1970 return 0;
1971
1972 a.full = dfixed_const(2);
1973 b.full = dfixed_const(1);
1974 if ((wm->vsc.full > a.full) ||
1975 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1976 (wm->vtaps >= 5) ||
1977 ((wm->vsc.full >= a.full) && wm->interlaced))
1978 max_src_lines_per_dst_line = 4;
1979 else
1980 max_src_lines_per_dst_line = 2;
1981
1982 a.full = dfixed_const(available_bandwidth);
1983 b.full = dfixed_const(wm->num_heads);
1984 a.full = dfixed_div(a, b);
1985
1986 b.full = dfixed_const(mc_latency + 512);
1987 c.full = dfixed_const(wm->disp_clk);
1988 b.full = dfixed_div(b, c);
1989
1990 c.full = dfixed_const(dmif_size);
1991 b.full = dfixed_div(c, b);
1992
1993 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1994
1995 b.full = dfixed_const(1000);
1996 c.full = dfixed_const(wm->disp_clk);
1997 b.full = dfixed_div(c, b);
1998 c.full = dfixed_const(wm->bytes_per_pixel);
1999 b.full = dfixed_mul(b, c);
2000
2001 lb_fill_bw = min(tmp, dfixed_trunc(b));
2002
2003 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2004 b.full = dfixed_const(1000);
2005 c.full = dfixed_const(lb_fill_bw);
2006 b.full = dfixed_div(c, b);
2007 a.full = dfixed_div(a, b);
2008 line_fill_time = dfixed_trunc(a);
2009
2010 if (line_fill_time < wm->active_time)
2011 return latency;
2012 else
2013 return latency + (line_fill_time - wm->active_time);
2014
2015}
2016
2017static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2018{
2019 if (dce6_average_bandwidth(wm) <=
2020 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2021 return true;
2022 else
2023 return false;
2024};
2025
2026static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2027{
2028 if (dce6_average_bandwidth(wm) <=
2029 (dce6_available_bandwidth(wm) / wm->num_heads))
2030 return true;
2031 else
2032 return false;
2033};
2034
2035static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2036{
2037 u32 lb_partitions = wm->lb_size / wm->src_width;
2038 u32 line_time = wm->active_time + wm->blank_time;
2039 u32 latency_tolerant_lines;
2040 u32 latency_hiding;
2041 fixed20_12 a;
2042
2043 a.full = dfixed_const(1);
2044 if (wm->vsc.full > a.full)
2045 latency_tolerant_lines = 1;
2046 else {
2047 if (lb_partitions <= (wm->vtaps + 1))
2048 latency_tolerant_lines = 1;
2049 else
2050 latency_tolerant_lines = 2;
2051 }
2052
2053 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2054
2055 if (dce6_latency_watermark(wm) <= latency_hiding)
2056 return true;
2057 else
2058 return false;
2059}
2060
2061static void dce6_program_watermarks(struct radeon_device *rdev,
2062 struct radeon_crtc *radeon_crtc,
2063 u32 lb_size, u32 num_heads)
2064{
2065 struct drm_display_mode *mode = &radeon_crtc->base.mode;
c696e53f
AD		 2066 struct dce6_wm_params wm_low, wm_high;
2067 u32 dram_channels;
43b3cd99
AD		 2068 u32 pixel_period;
2069 u32 line_time = 0;
2070 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2071 u32 priority_a_mark = 0, priority_b_mark = 0;
2072 u32 priority_a_cnt = PRIORITY_OFF;
2073 u32 priority_b_cnt = PRIORITY_OFF;
2074 u32 tmp, arb_control3;
2075 fixed20_12 a, b, c;
2076
2077 if (radeon_crtc->base.enabled && num_heads && mode) {
2078 pixel_period = 1000000 / (u32)mode->clock;
2079 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2080 priority_a_cnt = 0;
2081 priority_b_cnt = 0;
2082
ca7db22b 2083 if (rdev->family == CHIP_ARUBA)
c696e53f 2084 dram_channels = evergreen_get_number_of_dram_channels(rdev);
ca7db22b 2085 else
c696e53f
AD		 2086 dram_channels = si_get_number_of_dram_channels(rdev);
2087
2088 /* watermark for high clocks */
2089 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2090 wm_high.yclk =
2091 radeon_dpm_get_mclk(rdev, false) * 10;
2092 wm_high.sclk =
2093 radeon_dpm_get_sclk(rdev, false) * 10;
2094 } else {
2095 wm_high.yclk = rdev->pm.current_mclk * 10;
2096 wm_high.sclk = rdev->pm.current_sclk * 10;
2097 }
2098
2099 wm_high.disp_clk = mode->clock;
2100 wm_high.src_width = mode->crtc_hdisplay;
2101 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2102 wm_high.blank_time = line_time - wm_high.active_time;
2103 wm_high.interlaced = false;
2104 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2105 wm_high.interlaced = true;
2106 wm_high.vsc = radeon_crtc->vsc;
2107 wm_high.vtaps = 1;
2108 if (radeon_crtc->rmx_type != RMX_OFF)
2109 wm_high.vtaps = 2;
2110 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2111 wm_high.lb_size = lb_size;
2112 wm_high.dram_channels = dram_channels;
2113 wm_high.num_heads = num_heads;
2114
2115 /* watermark for low clocks */
2116 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2117 wm_low.yclk =
2118 radeon_dpm_get_mclk(rdev, true) * 10;
2119 wm_low.sclk =
2120 radeon_dpm_get_sclk(rdev, true) * 10;
2121 } else {
2122 wm_low.yclk = rdev->pm.current_mclk * 10;
2123 wm_low.sclk = rdev->pm.current_sclk * 10;
2124 }
2125
2126 wm_low.disp_clk = mode->clock;
2127 wm_low.src_width = mode->crtc_hdisplay;
2128 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2129 wm_low.blank_time = line_time - wm_low.active_time;
2130 wm_low.interlaced = false;
2131 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2132 wm_low.interlaced = true;
2133 wm_low.vsc = radeon_crtc->vsc;
2134 wm_low.vtaps = 1;
2135 if (radeon_crtc->rmx_type != RMX_OFF)
2136 wm_low.vtaps = 2;
2137 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2138 wm_low.lb_size = lb_size;
2139 wm_low.dram_channels = dram_channels;
2140 wm_low.num_heads = num_heads;
43b3cd99
AD		 2141
2142 /* set for high clocks */
c696e53f 2143 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
43b3cd99 2144 /* set for low clocks */
c696e53f 2145 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
43b3cd99
AD		 2146
2147 /* possibly force display priority to high */
2148 /* should really do this at mode validation time... */
c696e53f
AD		 2149 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2150 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2151 !dce6_check_latency_hiding(&wm_high) ||
2152 (rdev->disp_priority == 2)) {
2153 DRM_DEBUG_KMS("force priority to high\n");
2154 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2155 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2156 }
2157 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2158 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2159 !dce6_check_latency_hiding(&wm_low) ||
43b3cd99
AD		 2160 (rdev->disp_priority == 2)) {
2161 DRM_DEBUG_KMS("force priority to high\n");
2162 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2163 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2164 }
2165
2166 a.full = dfixed_const(1000);
2167 b.full = dfixed_const(mode->clock);
2168 b.full = dfixed_div(b, a);
2169 c.full = dfixed_const(latency_watermark_a);
2170 c.full = dfixed_mul(c, b);
2171 c.full = dfixed_mul(c, radeon_crtc->hsc);
2172 c.full = dfixed_div(c, a);
2173 a.full = dfixed_const(16);
2174 c.full = dfixed_div(c, a);
2175 priority_a_mark = dfixed_trunc(c);
2176 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2177
2178 a.full = dfixed_const(1000);
2179 b.full = dfixed_const(mode->clock);
2180 b.full = dfixed_div(b, a);
2181 c.full = dfixed_const(latency_watermark_b);
2182 c.full = dfixed_mul(c, b);
2183 c.full = dfixed_mul(c, radeon_crtc->hsc);
2184 c.full = dfixed_div(c, a);
2185 a.full = dfixed_const(16);
2186 c.full = dfixed_div(c, a);
2187 priority_b_mark = dfixed_trunc(c);
2188 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2189 }
2190
2191 /* select wm A */
2192 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2193 tmp = arb_control3;
2194 tmp &= ~LATENCY_WATERMARK_MASK(3);
2195 tmp |= LATENCY_WATERMARK_MASK(1);
2196 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2197 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2198 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2199 LATENCY_HIGH_WATERMARK(line_time)));
2200 /* select wm B */
2201 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2202 tmp &= ~LATENCY_WATERMARK_MASK(3);
2203 tmp |= LATENCY_WATERMARK_MASK(2);
2204 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2205 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2206 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2207 LATENCY_HIGH_WATERMARK(line_time)));
2208 /* restore original selection */
2209 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2210
2211 /* write the priority marks */
2212 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2213 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2214
7178d2a6
AD		 2215 /* save values for DPM */
2216 radeon_crtc->line_time = line_time;
2217 radeon_crtc->wm_high = latency_watermark_a;
2218 radeon_crtc->wm_low = latency_watermark_b;
43b3cd99
AD		 2219}
2220
2221void dce6_bandwidth_update(struct radeon_device *rdev)
2222{
2223 struct drm_display_mode *mode0 = NULL;
2224 struct drm_display_mode *mode1 = NULL;
2225 u32 num_heads = 0, lb_size;
2226 int i;
2227
2228 radeon_update_display_priority(rdev);
2229
2230 for (i = 0; i < rdev->num_crtc; i++) {
2231 if (rdev->mode_info.crtcs[i]->base.enabled)
2232 num_heads++;
2233 }
2234 for (i = 0; i < rdev->num_crtc; i += 2) {
2235 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2236 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2237 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2238 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2239 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2240 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2241 }
2242}
2243
0a96d72b
AD		 2244/*
2245 * Core functions
2246 */
0a96d72b
AD		 2247static void si_tiling_mode_table_init(struct radeon_device *rdev)
2248{
2249 const u32 num_tile_mode_states = 32;
2250 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2251
2252 switch (rdev->config.si.mem_row_size_in_kb) {
2253 case 1:
2254 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2255 break;
2256 case 2:
2257 default:
2258 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2259 break;
2260 case 4:
2261 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2262 break;
2263 }
2264
2265 if ((rdev->family == CHIP_TAHITI) ||
2266 (rdev->family == CHIP_PITCAIRN)) {
2267 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2268 switch (reg_offset) {
2269 case 0: /* non-AA compressed depth or any compressed stencil */
2270 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2271 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2272 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2273 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2274 NUM_BANKS(ADDR_SURF_16_BANK) |
2275 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2276 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2277 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2278 break;
2279 case 1: /* 2xAA/4xAA compressed depth only */
2280 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2281 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2282 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2283 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2284 NUM_BANKS(ADDR_SURF_16_BANK) |
2285 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2286 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2287 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2288 break;
2289 case 2: /* 8xAA compressed depth only */
2290 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2291 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2292 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2293 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2294 NUM_BANKS(ADDR_SURF_16_BANK) |
2295 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2296 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2297 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2298 break;
2299 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2300 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2301 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2302 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2303 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2304 NUM_BANKS(ADDR_SURF_16_BANK) |
2305 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2306 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2307 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2308 break;
2309 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2310 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2311 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2312 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2313 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2314 NUM_BANKS(ADDR_SURF_16_BANK) |
2315 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2316 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2317 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2318 break;
2319 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2320 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2321 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2322 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2323 TILE_SPLIT(split_equal_to_row_size) |
2324 NUM_BANKS(ADDR_SURF_16_BANK) |
2325 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2326 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2327 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2328 break;
2329 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2330 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2331 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2332 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2333 TILE_SPLIT(split_equal_to_row_size) |
2334 NUM_BANKS(ADDR_SURF_16_BANK) |
2335 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2336 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2337 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2338 break;
2339 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2340 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2341 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2342 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2343 TILE_SPLIT(split_equal_to_row_size) |
2344 NUM_BANKS(ADDR_SURF_16_BANK) |
2345 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2346 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2347 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2348 break;
2349 case 8: /* 1D and 1D Array Surfaces */
2350 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2351 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2352 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2353 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2354 NUM_BANKS(ADDR_SURF_16_BANK) |
2355 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2356 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2357 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2358 break;
2359 case 9: /* Displayable maps. */
2360 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2361 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2362 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2363 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2364 NUM_BANKS(ADDR_SURF_16_BANK) |
2365 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2366 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2367 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2368 break;
2369 case 10: /* Display 8bpp. */
2370 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2371 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2372 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2373 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2374 NUM_BANKS(ADDR_SURF_16_BANK) |
2375 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2376 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2377 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2378 break;
2379 case 11: /* Display 16bpp. */
2380 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2381 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2382 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2383 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2384 NUM_BANKS(ADDR_SURF_16_BANK) |
2385 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2387 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2388 break;
2389 case 12: /* Display 32bpp. */
2390 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2391 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2392 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2393 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2394 NUM_BANKS(ADDR_SURF_16_BANK) |
2395 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2396 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2397 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2398 break;
2399 case 13: /* Thin. */
2400 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2401 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2402 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2403 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2404 NUM_BANKS(ADDR_SURF_16_BANK) |
2405 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2406 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2407 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2408 break;
2409 case 14: /* Thin 8 bpp. */
2410 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2411 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2412 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2413 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2414 NUM_BANKS(ADDR_SURF_16_BANK) |
2415 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2416 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2417 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2418 break;
2419 case 15: /* Thin 16 bpp. */
2420 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2421 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2422 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2423 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2424 NUM_BANKS(ADDR_SURF_16_BANK) |
2425 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2426 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2427 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2428 break;
2429 case 16: /* Thin 32 bpp. */
2430 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2431 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2432 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2433 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2434 NUM_BANKS(ADDR_SURF_16_BANK) |
2435 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2436 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2437 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2438 break;
2439 case 17: /* Thin 64 bpp. */
2440 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2441 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2442 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2443 TILE_SPLIT(split_equal_to_row_size) |
2444 NUM_BANKS(ADDR_SURF_16_BANK) |
2445 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2446 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2447 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2448 break;
2449 case 21: /* 8 bpp PRT. */
2450 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2451 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2452 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2453 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2454 NUM_BANKS(ADDR_SURF_16_BANK) |
2455 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2456 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2457 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2458 break;
2459 case 22: /* 16 bpp PRT */
2460 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2461 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2462 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2463 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2464 NUM_BANKS(ADDR_SURF_16_BANK) |
2465 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2466 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2467 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2468 break;
2469 case 23: /* 32 bpp PRT */
2470 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2471 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2472 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2473 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2474 NUM_BANKS(ADDR_SURF_16_BANK) |
2475 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2476 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2477 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2478 break;
2479 case 24: /* 64 bpp PRT */
2480 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2481 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2482 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2483 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2484 NUM_BANKS(ADDR_SURF_16_BANK) |
2485 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2488 break;
2489 case 25: /* 128 bpp PRT */
2490 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2491 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2492 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2493 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2494 NUM_BANKS(ADDR_SURF_8_BANK) |
2495 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2496 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2497 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2498 break;
2499 default:
2500 gb_tile_moden = 0;
2501 break;
2502 }
64d7b8be 2503 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2504 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2505 }
d0ae7fcc 2506 } else if ((rdev->family == CHIP_VERDE) ||
8b02859d
AD		 2507 (rdev->family == CHIP_OLAND) ||
2508 (rdev->family == CHIP_HAINAN)) {
0a96d72b
AD		 2509 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2510 switch (reg_offset) {
2511 case 0: /* non-AA compressed depth or any compressed stencil */
2512 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2513 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2514 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2515 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2516 NUM_BANKS(ADDR_SURF_16_BANK) |
2517 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2518 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2519 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2520 break;
2521 case 1: /* 2xAA/4xAA compressed depth only */
2522 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2523 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2524 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2525 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2526 NUM_BANKS(ADDR_SURF_16_BANK) |
2527 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2528 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2529 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2530 break;
2531 case 2: /* 8xAA compressed depth only */
2532 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2533 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2534 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2535 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2536 NUM_BANKS(ADDR_SURF_16_BANK) |
2537 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2538 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2539 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2540 break;
2541 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2542 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2543 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2544 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2545 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2546 NUM_BANKS(ADDR_SURF_16_BANK) |
2547 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2548 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2549 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2550 break;
2551 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2552 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2553 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2554 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2555 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2556 NUM_BANKS(ADDR_SURF_16_BANK) |
2557 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2558 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2559 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2560 break;
2561 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2562 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2563 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2564 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2565 TILE_SPLIT(split_equal_to_row_size) |
2566 NUM_BANKS(ADDR_SURF_16_BANK) |
2567 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2568 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2569 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2570 break;
2571 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2572 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2573 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2574 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2575 TILE_SPLIT(split_equal_to_row_size) |
2576 NUM_BANKS(ADDR_SURF_16_BANK) |
2577 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2578 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2579 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2580 break;
2581 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2582 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2583 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2584 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2585 TILE_SPLIT(split_equal_to_row_size) |
2586 NUM_BANKS(ADDR_SURF_16_BANK) |
2587 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2588 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2589 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2590 break;
2591 case 8: /* 1D and 1D Array Surfaces */
2592 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2593 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2594 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2595 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2596 NUM_BANKS(ADDR_SURF_16_BANK) |
2597 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2598 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2599 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2600 break;
2601 case 9: /* Displayable maps. */
2602 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2603 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2604 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2605 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2606 NUM_BANKS(ADDR_SURF_16_BANK) |
2607 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2608 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2609 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2610 break;
2611 case 10: /* Display 8bpp. */
2612 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2613 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2614 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2615 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2616 NUM_BANKS(ADDR_SURF_16_BANK) |
2617 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2618 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2619 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2620 break;
2621 case 11: /* Display 16bpp. */
2622 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2623 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2624 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2625 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2626 NUM_BANKS(ADDR_SURF_16_BANK) |
2627 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2629 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2630 break;
2631 case 12: /* Display 32bpp. */
2632 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2633 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2634 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2635 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2636 NUM_BANKS(ADDR_SURF_16_BANK) |
2637 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2638 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2639 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2640 break;
2641 case 13: /* Thin. */
2642 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2643 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2644 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2645 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2646 NUM_BANKS(ADDR_SURF_16_BANK) |
2647 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2648 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2649 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2650 break;
2651 case 14: /* Thin 8 bpp. */
2652 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2653 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2654 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2655 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2656 NUM_BANKS(ADDR_SURF_16_BANK) |
2657 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2658 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2659 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2660 break;
2661 case 15: /* Thin 16 bpp. */
2662 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2663 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2664 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2665 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2666 NUM_BANKS(ADDR_SURF_16_BANK) |
2667 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2668 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2669 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2670 break;
2671 case 16: /* Thin 32 bpp. */
2672 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2673 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2674 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2675 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2676 NUM_BANKS(ADDR_SURF_16_BANK) |
2677 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2678 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2679 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2680 break;
2681 case 17: /* Thin 64 bpp. */
2682 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2683 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2684 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2685 TILE_SPLIT(split_equal_to_row_size) |
2686 NUM_BANKS(ADDR_SURF_16_BANK) |
2687 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2688 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2689 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2690 break;
2691 case 21: /* 8 bpp PRT. */
2692 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2693 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2694 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2695 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2696 NUM_BANKS(ADDR_SURF_16_BANK) |
2697 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2698 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2699 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2700 break;
2701 case 22: /* 16 bpp PRT */
2702 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2703 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2704 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2705 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2706 NUM_BANKS(ADDR_SURF_16_BANK) |
2707 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2708 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2709 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2710 break;
2711 case 23: /* 32 bpp PRT */
2712 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2713 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2714 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2715 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2716 NUM_BANKS(ADDR_SURF_16_BANK) |
2717 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2718 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2719 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2720 break;
2721 case 24: /* 64 bpp PRT */
2722 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2723 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2724 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2725 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2726 NUM_BANKS(ADDR_SURF_16_BANK) |
2727 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2728 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2729 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2730 break;
2731 case 25: /* 128 bpp PRT */
2732 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2733 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2734 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2735 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2736 NUM_BANKS(ADDR_SURF_8_BANK) |
2737 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2738 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2739 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2740 break;
2741 default:
2742 gb_tile_moden = 0;
2743 break;
2744 }
64d7b8be 2745 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
0a96d72b
AD		 2746 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2747 }
2748 } else
2749 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2750}
2751
1a8ca750
AD		 2752static void si_select_se_sh(struct radeon_device *rdev,
2753 u32 se_num, u32 sh_num)
2754{
2755 u32 data = INSTANCE_BROADCAST_WRITES;
2756
2757 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
79b52d6a 2758 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
1a8ca750
AD		 2759 else if (se_num == 0xffffffff)
2760 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2761 else if (sh_num == 0xffffffff)
2762 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2763 else
2764 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2765 WREG32(GRBM_GFX_INDEX, data);
2766}
2767
2768static u32 si_create_bitmask(u32 bit_width)
2769{
2770 u32 i, mask = 0;
2771
2772 for (i = 0; i < bit_width; i++) {
2773 mask <<= 1;
2774 mask |= 1;
2775 }
2776 return mask;
2777}
2778
2779static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2780{
2781 u32 data, mask;
2782
2783 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2784 if (data & 1)
2785 data &= INACTIVE_CUS_MASK;
2786 else
2787 data = 0;
2788 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2789
2790 data >>= INACTIVE_CUS_SHIFT;
2791
2792 mask = si_create_bitmask(cu_per_sh);
2793
2794 return ~data & mask;
2795}
2796
2797static void si_setup_spi(struct radeon_device *rdev,
2798 u32 se_num, u32 sh_per_se,
2799 u32 cu_per_sh)
2800{
2801 int i, j, k;
2802 u32 data, mask, active_cu;
2803
2804 for (i = 0; i < se_num; i++) {
2805 for (j = 0; j < sh_per_se; j++) {
2806 si_select_se_sh(rdev, i, j);
2807 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2808 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2809
2810 mask = 1;
2811 for (k = 0; k < 16; k++) {
2812 mask <<= k;
2813 if (active_cu & mask) {
2814 data &= ~mask;
2815 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2816 break;
2817 }
2818 }
2819 }
2820 }
2821 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2822}
2823
2824static u32 si_get_rb_disabled(struct radeon_device *rdev,
9fadb352 2825 u32 max_rb_num_per_se,
1a8ca750
AD		 2826 u32 sh_per_se)
2827{
2828 u32 data, mask;
2829
2830 data = RREG32(CC_RB_BACKEND_DISABLE);
2831 if (data & 1)
2832 data &= BACKEND_DISABLE_MASK;
2833 else
2834 data = 0;
2835 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2836
2837 data >>= BACKEND_DISABLE_SHIFT;
2838
9fadb352 2839 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
1a8ca750
AD		 2840
2841 return data & mask;
2842}
2843
2844static void si_setup_rb(struct radeon_device *rdev,
2845 u32 se_num, u32 sh_per_se,
9fadb352 2846 u32 max_rb_num_per_se)
1a8ca750
AD		 2847{
2848 int i, j;
2849 u32 data, mask;
2850 u32 disabled_rbs = 0;
2851 u32 enabled_rbs = 0;
2852
2853 for (i = 0; i < se_num; i++) {
2854 for (j = 0; j < sh_per_se; j++) {
2855 si_select_se_sh(rdev, i, j);
9fadb352 2856 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
1a8ca750
AD		 2857 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2858 }
2859 }
2860 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2861
2862 mask = 1;
9fadb352 2863 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
1a8ca750
AD		 2864 if (!(disabled_rbs & mask))
2865 enabled_rbs |= mask;
2866 mask <<= 1;
2867 }
2868
439a1cff
MO		 2869 rdev->config.si.backend_enable_mask = enabled_rbs;
2870
1a8ca750
AD		 2871 for (i = 0; i < se_num; i++) {
2872 si_select_se_sh(rdev, i, 0xffffffff);
2873 data = 0;
2874 for (j = 0; j < sh_per_se; j++) {
2875 switch (enabled_rbs & 3) {
2876 case 1:
2877 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2878 break;
2879 case 2:
2880 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2881 break;
2882 case 3:
2883 default:
2884 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2885 break;
2886 }
2887 enabled_rbs >>= 2;
2888 }
2889 WREG32(PA_SC_RASTER_CONFIG, data);
2890 }
2891 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2892}
2893
0a96d72b
AD		 2894static void si_gpu_init(struct radeon_device *rdev)
2895{
0a96d72b
AD		 2896 u32 gb_addr_config = 0;
2897 u32 mc_shared_chmap, mc_arb_ramcfg;
0a96d72b 2898 u32 sx_debug_1;
0a96d72b
AD		 2899 u32 hdp_host_path_cntl;
2900 u32 tmp;
2901 int i, j;
2902
2903 switch (rdev->family) {
2904 case CHIP_TAHITI:
2905 rdev->config.si.max_shader_engines = 2;
0a96d72b 2906 rdev->config.si.max_tile_pipes = 12;
1a8ca750
AD		 2907 rdev->config.si.max_cu_per_sh = 8;
2908 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2909 rdev->config.si.max_backends_per_se = 4;
2910 rdev->config.si.max_texture_channel_caches = 12;
2911 rdev->config.si.max_gprs = 256;
2912 rdev->config.si.max_gs_threads = 32;
2913 rdev->config.si.max_hw_contexts = 8;
2914
2915 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2916 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2917 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2918 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2919 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2920 break;
2921 case CHIP_PITCAIRN:
2922 rdev->config.si.max_shader_engines = 2;
0a96d72b 2923 rdev->config.si.max_tile_pipes = 8;
1a8ca750
AD		 2924 rdev->config.si.max_cu_per_sh = 5;
2925 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2926 rdev->config.si.max_backends_per_se = 4;
2927 rdev->config.si.max_texture_channel_caches = 8;
2928 rdev->config.si.max_gprs = 256;
2929 rdev->config.si.max_gs_threads = 32;
2930 rdev->config.si.max_hw_contexts = 8;
2931
2932 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2933 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2934 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2935 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2936 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
0a96d72b
AD		 2937 break;
2938 case CHIP_VERDE:
2939 default:
2940 rdev->config.si.max_shader_engines = 1;
0a96d72b 2941 rdev->config.si.max_tile_pipes = 4;
468ef1a5 2942 rdev->config.si.max_cu_per_sh = 5;
1a8ca750 2943 rdev->config.si.max_sh_per_se = 2;
0a96d72b
AD		 2944 rdev->config.si.max_backends_per_se = 4;
2945 rdev->config.si.max_texture_channel_caches = 4;
2946 rdev->config.si.max_gprs = 256;
2947 rdev->config.si.max_gs_threads = 32;
2948 rdev->config.si.max_hw_contexts = 8;
2949
d0ae7fcc
AD		 2950 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2951 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2952 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2953 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2954 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2955 break;
2956 case CHIP_OLAND:
2957 rdev->config.si.max_shader_engines = 1;
2958 rdev->config.si.max_tile_pipes = 4;
2959 rdev->config.si.max_cu_per_sh = 6;
2960 rdev->config.si.max_sh_per_se = 1;
2961 rdev->config.si.max_backends_per_se = 2;
2962 rdev->config.si.max_texture_channel_caches = 4;
2963 rdev->config.si.max_gprs = 256;
2964 rdev->config.si.max_gs_threads = 16;
2965 rdev->config.si.max_hw_contexts = 8;
2966
0a96d72b
AD		 2967 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2968 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2969 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2970 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
1a8ca750 2971 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
0a96d72b 2972 break;
8b02859d
AD		 2973 case CHIP_HAINAN:
2974 rdev->config.si.max_shader_engines = 1;
2975 rdev->config.si.max_tile_pipes = 4;
2976 rdev->config.si.max_cu_per_sh = 5;
2977 rdev->config.si.max_sh_per_se = 1;
2978 rdev->config.si.max_backends_per_se = 1;
2979 rdev->config.si.max_texture_channel_caches = 2;
2980 rdev->config.si.max_gprs = 256;
2981 rdev->config.si.max_gs_threads = 16;
2982 rdev->config.si.max_hw_contexts = 8;
2983
2984 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2985 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2986 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2987 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2988 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2989 break;
0a96d72b
AD		 2990 }
2991
2992 /* Initialize HDP */
2993 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2994 WREG32((0x2c14 + j), 0x00000000);
2995 WREG32((0x2c18 + j), 0x00000000);
2996 WREG32((0x2c1c + j), 0x00000000);
2997 WREG32((0x2c20 + j), 0x00000000);
2998 WREG32((0x2c24 + j), 0x00000000);
2999 }
3000
3001 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3002
3003 evergreen_fix_pci_max_read_req_size(rdev);
3004
3005 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3006
3007 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3008 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3009
0a96d72b 3010 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
0a96d72b
AD		 3011 rdev->config.si.mem_max_burst_length_bytes = 256;
3012 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3013 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3014 if (rdev->config.si.mem_row_size_in_kb > 4)
3015 rdev->config.si.mem_row_size_in_kb = 4;
3016 /* XXX use MC settings? */
3017 rdev->config.si.shader_engine_tile_size = 32;
3018 rdev->config.si.num_gpus = 1;
3019 rdev->config.si.multi_gpu_tile_size = 64;
3020
1a8ca750
AD		 3021 /* fix up row size */
3022 gb_addr_config &= ~ROW_SIZE_MASK;
0a96d72b
AD		 3023 switch (rdev->config.si.mem_row_size_in_kb) {
3024 case 1:
3025 default:
3026 gb_addr_config |= ROW_SIZE(0);
3027 break;
3028 case 2:
3029 gb_addr_config |= ROW_SIZE(1);
3030 break;
3031 case 4:
3032 gb_addr_config |= ROW_SIZE(2);
3033 break;
3034 }
3035
0a96d72b
AD		 3036 /* setup tiling info dword. gb_addr_config is not adequate since it does
3037 * not have bank info, so create a custom tiling dword.
3038 * bits 3:0 num_pipes
3039 * bits 7:4 num_banks
3040 * bits 11:8 group_size
3041 * bits 15:12 row_size
3042 */
3043 rdev->config.si.tile_config = 0;
3044 switch (rdev->config.si.num_tile_pipes) {
3045 case 1:
3046 rdev->config.si.tile_config |= (0 << 0);
3047 break;
3048 case 2:
3049 rdev->config.si.tile_config |= (1 << 0);
3050 break;
3051 case 4:
3052 rdev->config.si.tile_config |= (2 << 0);
3053 break;
3054 case 8:
3055 default:
3056 /* XXX what about 12? */
3057 rdev->config.si.tile_config |= (3 << 0);
3058 break;
dca571a6
CK		 3059 }
3060 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3061 case 0: /* four banks */
1a8ca750 3062 rdev->config.si.tile_config |= 0 << 4;
dca571a6
CK		 3063 break;
3064 case 1: /* eight banks */
3065 rdev->config.si.tile_config |= 1 << 4;
3066 break;
3067 case 2: /* sixteen banks */
3068 default:
3069 rdev->config.si.tile_config |= 2 << 4;
3070 break;
3071 }
0a96d72b
AD		 3072 rdev->config.si.tile_config |=
3073 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3074 rdev->config.si.tile_config |=
3075 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3076
0a96d72b
AD		 3077 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3078 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
7c1c7c18 3079 WREG32(DMIF_ADDR_CALC, gb_addr_config);
0a96d72b 3080 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
8c5fd7ef
AD		 3081 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3082 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1df0d523
AD		 3083 if (rdev->has_uvd) {
3084 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3085 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3086 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3087 }
0a96d72b 3088
1a8ca750 3089 si_tiling_mode_table_init(rdev);
0a96d72b 3090
1a8ca750
AD		 3091 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3092 rdev->config.si.max_sh_per_se,
3093 rdev->config.si.max_backends_per_se);
0a96d72b 3094
1a8ca750
AD		 3095 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3096 rdev->config.si.max_sh_per_se,
3097 rdev->config.si.max_cu_per_sh);
0a96d72b 3098
0a96d72b
AD		 3099
3100 /* set HW defaults for 3D engine */
3101 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3102 ROQ_IB2_START(0x2b)));
3103 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3104
3105 sx_debug_1 = RREG32(SX_DEBUG_1);
3106 WREG32(SX_DEBUG_1, sx_debug_1);
3107
3108 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3109
3110 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3111 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3112 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3113 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3114
3115 WREG32(VGT_NUM_INSTANCES, 1);
3116
3117 WREG32(CP_PERFMON_CNTL, 0);
3118
3119 WREG32(SQ_CONFIG, 0);
3120
3121 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3122 FORCE_EOV_MAX_REZ_CNT(255)));
3123
3124 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3125 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3126
3127 WREG32(VGT_GS_VERTEX_REUSE, 16);
3128 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3129
3130 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3131 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3132 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3133 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3134 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3135 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3136 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3137 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3138
3139 tmp = RREG32(HDP_MISC_CNTL);
3140 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3141 WREG32(HDP_MISC_CNTL, tmp);
3142
3143 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3144 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3145
3146 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3147
3148 udelay(50);
3149}
c476dde2 3150
2ece2e8b
AD		 3151/*
3152 * GPU scratch registers helpers function.
3153 */
3154static void si_scratch_init(struct radeon_device *rdev)
3155{
3156 int i;
3157
3158 rdev->scratch.num_reg = 7;
3159 rdev->scratch.reg_base = SCRATCH_REG0;
3160 for (i = 0; i < rdev->scratch.num_reg; i++) {
3161 rdev->scratch.free[i] = true;
3162 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3163 }
3164}
3165
3166void si_fence_ring_emit(struct radeon_device *rdev,
3167 struct radeon_fence *fence)
3168{
3169 struct radeon_ring *ring = &rdev->ring[fence->ring];
3170 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3171
3172 /* flush read cache over gart */
3173 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3174 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3175 radeon_ring_write(ring, 0);
3176 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3177 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3178 PACKET3_TC_ACTION_ENA |
3179 PACKET3_SH_KCACHE_ACTION_ENA |
3180 PACKET3_SH_ICACHE_ACTION_ENA);
3181 radeon_ring_write(ring, 0xFFFFFFFF);
3182 radeon_ring_write(ring, 0);
3183 radeon_ring_write(ring, 10); /* poll interval */
3184 /* EVENT_WRITE_EOP - flush caches, send int */
3185 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3186 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3187 radeon_ring_write(ring, addr & 0xffffffff);
3188 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3189 radeon_ring_write(ring, fence->seq);
3190 radeon_ring_write(ring, 0);
3191}
3192
3193/*
3194 * IB stuff
3195 */
3196void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3197{
876dc9f3 3198 struct radeon_ring *ring = &rdev->ring[ib->ring];
2ece2e8b
AD		 3199 u32 header;
3200
a85a7da4
AD		 3201 if (ib->is_const_ib) {
3202 /* set switch buffer packet before const IB */
3203 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3204 radeon_ring_write(ring, 0);
45df6803 3205
2ece2e8b 3206 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
a85a7da4 3207 } else {
89d35807 3208 u32 next_rptr;
a85a7da4 3209 if (ring->rptr_save_reg) {
89d35807 3210 next_rptr = ring->wptr + 3 + 4 + 8;
a85a7da4
AD		 3211 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3212 radeon_ring_write(ring, ((ring->rptr_save_reg -
3213 PACKET3_SET_CONFIG_REG_START) >> 2));
3214 radeon_ring_write(ring, next_rptr);
89d35807
AD		 3215 } else if (rdev->wb.enabled) {
3216 next_rptr = ring->wptr + 5 + 4 + 8;
3217 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3218 radeon_ring_write(ring, (1 << 8));
3219 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3220 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3221 radeon_ring_write(ring, next_rptr);
a85a7da4
AD		 3222 }
3223
2ece2e8b 3224 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
a85a7da4 3225 }
2ece2e8b
AD		 3226
3227 radeon_ring_write(ring, header);
3228 radeon_ring_write(ring,
3229#ifdef __BIG_ENDIAN
3230 (2 << 0) |
3231#endif
3232 (ib->gpu_addr & 0xFFFFFFFC));
3233 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4bf3dd92
CK		 3234 radeon_ring_write(ring, ib->length_dw |
3235 (ib->vm ? (ib->vm->id << 24) : 0));
2ece2e8b 3236
a85a7da4
AD		 3237 if (!ib->is_const_ib) {
3238 /* flush read cache over gart for this vmid */
3239 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3240 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
4bf3dd92 3241 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
a85a7da4
AD		 3242 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3243 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3244 PACKET3_TC_ACTION_ENA |
3245 PACKET3_SH_KCACHE_ACTION_ENA |
3246 PACKET3_SH_ICACHE_ACTION_ENA);
3247 radeon_ring_write(ring, 0xFFFFFFFF);
3248 radeon_ring_write(ring, 0);
3249 radeon_ring_write(ring, 10); /* poll interval */
3250 }
2ece2e8b
AD		 3251}
3252
48c0c902
AD		 3253/*
3254 * CP.
3255 */
3256static void si_cp_enable(struct radeon_device *rdev, bool enable)
3257{
3258 if (enable)
3259 WREG32(CP_ME_CNTL, 0);
3260 else {
50efa51a
AD		 3261 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3262 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
48c0c902
AD		 3263 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3264 WREG32(SCRATCH_UMSK, 0);
8c5fd7ef
AD		 3265 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3266 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3267 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
48c0c902
AD		 3268 }
3269 udelay(50);
3270}
3271
3272static int si_cp_load_microcode(struct radeon_device *rdev)
3273{
3274 const __be32 *fw_data;
3275 int i;
3276
3277 if (!rdev->me_fw || !rdev->pfp_fw)
3278 return -EINVAL;
3279
3280 si_cp_enable(rdev, false);
3281
3282 /* PFP */
3283 fw_data = (const __be32 *)rdev->pfp_fw->data;
3284 WREG32(CP_PFP_UCODE_ADDR, 0);
3285 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3286 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3287 WREG32(CP_PFP_UCODE_ADDR, 0);
3288
3289 /* CE */
3290 fw_data = (const __be32 *)rdev->ce_fw->data;
3291 WREG32(CP_CE_UCODE_ADDR, 0);
3292 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3293 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3294 WREG32(CP_CE_UCODE_ADDR, 0);
3295
3296 /* ME */
3297 fw_data = (const __be32 *)rdev->me_fw->data;
3298 WREG32(CP_ME_RAM_WADDR, 0);
3299 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3300 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3301 WREG32(CP_ME_RAM_WADDR, 0);
3302
3303 WREG32(CP_PFP_UCODE_ADDR, 0);
3304 WREG32(CP_CE_UCODE_ADDR, 0);
3305 WREG32(CP_ME_RAM_WADDR, 0);
3306 WREG32(CP_ME_RAM_RADDR, 0);
3307 return 0;
3308}
3309
3310static int si_cp_start(struct radeon_device *rdev)
3311{
3312 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3313 int r, i;
3314
3315 r = radeon_ring_lock(rdev, ring, 7 + 4);
3316 if (r) {
3317 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3318 return r;
3319 }
3320 /* init the CP */
3321 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3322 radeon_ring_write(ring, 0x1);
3323 radeon_ring_write(ring, 0x0);
3324 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3325 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3326 radeon_ring_write(ring, 0);
3327 radeon_ring_write(ring, 0);
3328
3329 /* init the CE partitions */
3330 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3331 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3332 radeon_ring_write(ring, 0xc000);
3333 radeon_ring_write(ring, 0xe000);
3334 radeon_ring_unlock_commit(rdev, ring);
3335
3336 si_cp_enable(rdev, true);
3337
3338 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3339 if (r) {
3340 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3341 return r;
3342 }
3343
3344 /* setup clear context state */
3345 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3346 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3347
3348 for (i = 0; i < si_default_size; i++)
3349 radeon_ring_write(ring, si_default_state[i]);
3350
3351 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3352 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3353
3354 /* set clear context state */
3355 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3356 radeon_ring_write(ring, 0);
3357
3358 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3359 radeon_ring_write(ring, 0x00000316);
3360 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3361 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3362
3363 radeon_ring_unlock_commit(rdev, ring);
3364
3365 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3366 ring = &rdev->ring[i];
3367 r = radeon_ring_lock(rdev, ring, 2);
3368
3369 /* clear the compute context state */
3370 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3371 radeon_ring_write(ring, 0);
3372
3373 radeon_ring_unlock_commit(rdev, ring);
3374 }
3375
3376 return 0;
3377}
3378
3379static void si_cp_fini(struct radeon_device *rdev)
3380{
45df6803 3381 struct radeon_ring *ring;
48c0c902 3382 si_cp_enable(rdev, false);
45df6803
CK		 3383
3384 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3385 radeon_ring_fini(rdev, ring);
3386 radeon_scratch_free(rdev, ring->rptr_save_reg);
3387
3388 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3389 radeon_ring_fini(rdev, ring);
3390 radeon_scratch_free(rdev, ring->rptr_save_reg);
3391
3392 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3393 radeon_ring_fini(rdev, ring);
3394 radeon_scratch_free(rdev, ring->rptr_save_reg);
48c0c902
AD		 3395}
3396
3397static int si_cp_resume(struct radeon_device *rdev)
3398{
3399 struct radeon_ring *ring;
3400 u32 tmp;
3401 u32 rb_bufsz;
3402 int r;
3403
811e4d58
AD		 3404 si_enable_gui_idle_interrupt(rdev, false);
3405
48c0c902
AD		 3406 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3407 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3408
3409 /* Set the write pointer delay */
3410 WREG32(CP_RB_WPTR_DELAY, 0);
3411
3412 WREG32(CP_DEBUG, 0);
3413 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3414
3415 /* ring 0 - compute and gfx */
3416 /* Set ring buffer size */
3417 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 3418 rb_bufsz = order_base_2(ring->ring_size / 8);
3419 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3420#ifdef __BIG_ENDIAN
3421 tmp |= BUF_SWAP_32BIT;
3422#endif
3423 WREG32(CP_RB0_CNTL, tmp);
3424
3425 /* Initialize the ring buffer's read and write pointers */
3426 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3427 ring->wptr = 0;
3428 WREG32(CP_RB0_WPTR, ring->wptr);
3429
48fc7f7e 3430 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3431 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3432 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3433
3434 if (rdev->wb.enabled)
3435 WREG32(SCRATCH_UMSK, 0xff);
3436 else {
3437 tmp |= RB_NO_UPDATE;
3438 WREG32(SCRATCH_UMSK, 0);
3439 }
3440
3441 mdelay(1);
3442 WREG32(CP_RB0_CNTL, tmp);
3443
3444 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3445
48c0c902
AD		 3446 /* ring1 - compute only */
3447 /* Set ring buffer size */
3448 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
b72a8925
DV		 3449 rb_bufsz = order_base_2(ring->ring_size / 8);
3450 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3451#ifdef __BIG_ENDIAN
3452 tmp |= BUF_SWAP_32BIT;
3453#endif
3454 WREG32(CP_RB1_CNTL, tmp);
3455
3456 /* Initialize the ring buffer's read and write pointers */
3457 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3458 ring->wptr = 0;
3459 WREG32(CP_RB1_WPTR, ring->wptr);
3460
48fc7f7e 3461 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3462 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3463 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3464
3465 mdelay(1);
3466 WREG32(CP_RB1_CNTL, tmp);
3467
3468 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3469
48c0c902
AD		 3470 /* ring2 - compute only */
3471 /* Set ring buffer size */
3472 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
b72a8925
DV		 3473 rb_bufsz = order_base_2(ring->ring_size / 8);
3474 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
48c0c902
AD		 3475#ifdef __BIG_ENDIAN
3476 tmp |= BUF_SWAP_32BIT;
3477#endif
3478 WREG32(CP_RB2_CNTL, tmp);
3479
3480 /* Initialize the ring buffer's read and write pointers */
3481 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3482 ring->wptr = 0;
3483 WREG32(CP_RB2_WPTR, ring->wptr);
3484
48fc7f7e 3485 /* set the wb address whether it's enabled or not */
48c0c902
AD		 3486 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3487 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3488
3489 mdelay(1);
3490 WREG32(CP_RB2_CNTL, tmp);
3491
3492 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3493
48c0c902
AD		 3494 /* start the rings */
3495 si_cp_start(rdev);
3496 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3497 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3498 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3499 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3500 if (r) {
3501 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3502 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3503 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3504 return r;
3505 }
3506 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3507 if (r) {
3508 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3509 }
3510 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3511 if (r) {
3512 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3513 }
3514
811e4d58
AD		 3515 si_enable_gui_idle_interrupt(rdev, true);
3516
50efa51a
AD		 3517 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3518 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3519
48c0c902
AD		 3520 return 0;
3521}
3522
2483b4ea 3523u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
06bc6df0 3524{
014bb209 3525 u32 reset_mask = 0;
1c534671 3526 u32 tmp;
06bc6df0 3527
014bb209
AD		 3528 /* GRBM_STATUS */
3529 tmp = RREG32(GRBM_STATUS);
3530 if (tmp & (PA_BUSY | SC_BUSY |
3531 BCI_BUSY | SX_BUSY |
3532 TA_BUSY | VGT_BUSY |
3533 DB_BUSY | CB_BUSY |
3534 GDS_BUSY | SPI_BUSY |
3535 IA_BUSY | IA_BUSY_NO_DMA))
3536 reset_mask |= RADEON_RESET_GFX;
3537
3538 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3539 CP_BUSY | CP_COHERENCY_BUSY))
3540 reset_mask |= RADEON_RESET_CP;
3541
3542 if (tmp & GRBM_EE_BUSY)
3543 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3544
3545 /* GRBM_STATUS2 */
3546 tmp = RREG32(GRBM_STATUS2);
3547 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3548 reset_mask |= RADEON_RESET_RLC;
3549
3550 /* DMA_STATUS_REG 0 */
3551 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3552 if (!(tmp & DMA_IDLE))
3553 reset_mask |= RADEON_RESET_DMA;
3554
3555 /* DMA_STATUS_REG 1 */
3556 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3557 if (!(tmp & DMA_IDLE))
3558 reset_mask |= RADEON_RESET_DMA1;
3559
3560 /* SRBM_STATUS2 */
3561 tmp = RREG32(SRBM_STATUS2);
3562 if (tmp & DMA_BUSY)
3563 reset_mask |= RADEON_RESET_DMA;
3564
3565 if (tmp & DMA1_BUSY)
3566 reset_mask |= RADEON_RESET_DMA1;
3567
3568 /* SRBM_STATUS */
3569 tmp = RREG32(SRBM_STATUS);
3570
3571 if (tmp & IH_BUSY)
3572 reset_mask |= RADEON_RESET_IH;
3573
3574 if (tmp & SEM_BUSY)
3575 reset_mask |= RADEON_RESET_SEM;
3576
3577 if (tmp & GRBM_RQ_PENDING)
3578 reset_mask |= RADEON_RESET_GRBM;
3579
3580 if (tmp & VMC_BUSY)
3581 reset_mask |= RADEON_RESET_VMC;
19fc42ed 3582
014bb209
AD		 3583 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3584 MCC_BUSY | MCD_BUSY))
3585 reset_mask |= RADEON_RESET_MC;
3586
3587 if (evergreen_is_display_hung(rdev))
3588 reset_mask |= RADEON_RESET_DISPLAY;
3589
3590 /* VM_L2_STATUS */
3591 tmp = RREG32(VM_L2_STATUS);
3592 if (tmp & L2_BUSY)
3593 reset_mask |= RADEON_RESET_VMC;
3594
d808fc88
AD		 3595 /* Skip MC reset as it's mostly likely not hung, just busy */
3596 if (reset_mask & RADEON_RESET_MC) {
3597 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3598 reset_mask &= ~RADEON_RESET_MC;
3599 }
3600
014bb209
AD		 3601 return reset_mask;
3602}
3603
3604static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3605{
3606 struct evergreen_mc_save save;
3607 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3608 u32 tmp;
19fc42ed 3609
06bc6df0 3610 if (reset_mask == 0)
014bb209 3611 return;
06bc6df0
AD		 3612
3613 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3614
1c534671 3615 evergreen_print_gpu_status_regs(rdev);
06bc6df0
AD		 3616 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3617 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3618 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3619 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3620
a6f4ae8d
AD		 3621 /* disable PG/CG */
3622 si_fini_pg(rdev);
3623 si_fini_cg(rdev);
3624
3625 /* stop the rlc */
3626 si_rlc_stop(rdev);
3627
1c534671
AD		 3628 /* Disable CP parsing/prefetching */
3629 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3630
3631 if (reset_mask & RADEON_RESET_DMA) {
3632 /* dma0 */
3633 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3634 tmp &= ~DMA_RB_ENABLE;
3635 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
014bb209
AD		 3636 }
3637 if (reset_mask & RADEON_RESET_DMA1) {
1c534671
AD		 3638 /* dma1 */
3639 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3640 tmp &= ~DMA_RB_ENABLE;
3641 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3642 }
3643
f770d78a
AD		 3644 udelay(50);
3645
3646 evergreen_mc_stop(rdev, &save);
3647 if (evergreen_mc_wait_for_idle(rdev)) {
3648 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3649 }
3650
1c534671
AD		 3651 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3652 grbm_soft_reset = SOFT_RESET_CB |
3653 SOFT_RESET_DB |
3654 SOFT_RESET_GDS |
3655 SOFT_RESET_PA |
3656 SOFT_RESET_SC |
3657 SOFT_RESET_BCI |
3658 SOFT_RESET_SPI |
3659 SOFT_RESET_SX |
3660 SOFT_RESET_TC |
3661 SOFT_RESET_TA |
3662 SOFT_RESET_VGT |
3663 SOFT_RESET_IA;
3664 }
3665
3666 if (reset_mask & RADEON_RESET_CP) {
3667 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3668
3669 srbm_soft_reset |= SOFT_RESET_GRBM;
3670 }
06bc6df0
AD		 3671
3672 if (reset_mask & RADEON_RESET_DMA)
014bb209
AD		 3673 srbm_soft_reset |= SOFT_RESET_DMA;
3674
3675 if (reset_mask & RADEON_RESET_DMA1)
3676 srbm_soft_reset |= SOFT_RESET_DMA1;
3677
3678 if (reset_mask & RADEON_RESET_DISPLAY)
3679 srbm_soft_reset |= SOFT_RESET_DC;
3680
3681 if (reset_mask & RADEON_RESET_RLC)
3682 grbm_soft_reset |= SOFT_RESET_RLC;
3683
3684 if (reset_mask & RADEON_RESET_SEM)
3685 srbm_soft_reset |= SOFT_RESET_SEM;
3686
3687 if (reset_mask & RADEON_RESET_IH)
3688 srbm_soft_reset |= SOFT_RESET_IH;
3689
3690 if (reset_mask & RADEON_RESET_GRBM)
3691 srbm_soft_reset |= SOFT_RESET_GRBM;
3692
3693 if (reset_mask & RADEON_RESET_VMC)
3694 srbm_soft_reset |= SOFT_RESET_VMC;
3695
3696 if (reset_mask & RADEON_RESET_MC)
3697 srbm_soft_reset |= SOFT_RESET_MC;
1c534671
AD		 3698
3699 if (grbm_soft_reset) {
3700 tmp = RREG32(GRBM_SOFT_RESET);
3701 tmp |= grbm_soft_reset;
3702 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3703 WREG32(GRBM_SOFT_RESET, tmp);
3704 tmp = RREG32(GRBM_SOFT_RESET);
3705
3706 udelay(50);
3707
3708 tmp &= ~grbm_soft_reset;
3709 WREG32(GRBM_SOFT_RESET, tmp);
3710 tmp = RREG32(GRBM_SOFT_RESET);
3711 }
3712
3713 if (srbm_soft_reset) {
3714 tmp = RREG32(SRBM_SOFT_RESET);
3715 tmp |= srbm_soft_reset;
3716 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3717 WREG32(SRBM_SOFT_RESET, tmp);
3718 tmp = RREG32(SRBM_SOFT_RESET);
3719
3720 udelay(50);
3721
3722 tmp &= ~srbm_soft_reset;
3723 WREG32(SRBM_SOFT_RESET, tmp);
3724 tmp = RREG32(SRBM_SOFT_RESET);
3725 }
06bc6df0
AD		 3726
3727 /* Wait a little for things to settle down */
3728 udelay(50);
3729
c476dde2 3730 evergreen_mc_resume(rdev, &save);
1c534671
AD		 3731 udelay(50);
3732
1c534671 3733 evergreen_print_gpu_status_regs(rdev);
c476dde2
AD		 3734}
3735
4a5c8ea5
AD		 3736static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3737{
3738 u32 tmp, i;
3739
3740 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3741 tmp |= SPLL_BYPASS_EN;
3742 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3743
3744 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3745 tmp |= SPLL_CTLREQ_CHG;
3746 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3747
3748 for (i = 0; i < rdev->usec_timeout; i++) {
3749 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3750 break;
3751 udelay(1);
3752 }
3753
3754 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3755 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3756 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3757
3758 tmp = RREG32(MPLL_CNTL_MODE);
3759 tmp &= ~MPLL_MCLK_SEL;
3760 WREG32(MPLL_CNTL_MODE, tmp);
3761}
3762
3763static void si_spll_powerdown(struct radeon_device *rdev)
3764{
3765 u32 tmp;
3766
3767 tmp = RREG32(SPLL_CNTL_MODE);
3768 tmp |= SPLL_SW_DIR_CONTROL;
3769 WREG32(SPLL_CNTL_MODE, tmp);
3770
3771 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3772 tmp |= SPLL_RESET;
3773 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3774
3775 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3776 tmp |= SPLL_SLEEP;
3777 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3778
3779 tmp = RREG32(SPLL_CNTL_MODE);
3780 tmp &= ~SPLL_SW_DIR_CONTROL;
3781 WREG32(SPLL_CNTL_MODE, tmp);
3782}
3783
3784static void si_gpu_pci_config_reset(struct radeon_device *rdev)
3785{
3786 struct evergreen_mc_save save;
3787 u32 tmp, i;
3788
3789 dev_info(rdev->dev, "GPU pci config reset\n");
3790
3791 /* disable dpm? */
3792
3793 /* disable cg/pg */
3794 si_fini_pg(rdev);
3795 si_fini_cg(rdev);
3796
3797 /* Disable CP parsing/prefetching */
3798 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3799 /* dma0 */
3800 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3801 tmp &= ~DMA_RB_ENABLE;
3802 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3803 /* dma1 */
3804 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3805 tmp &= ~DMA_RB_ENABLE;
3806 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3807 /* XXX other engines? */
3808
3809 /* halt the rlc, disable cp internal ints */
3810 si_rlc_stop(rdev);
3811
3812 udelay(50);
3813
3814 /* disable mem access */
3815 evergreen_mc_stop(rdev, &save);
3816 if (evergreen_mc_wait_for_idle(rdev)) {
3817 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3818 }
3819
3820 /* set mclk/sclk to bypass */
3821 si_set_clk_bypass_mode(rdev);
3822 /* powerdown spll */
3823 si_spll_powerdown(rdev);
3824 /* disable BM */
3825 pci_clear_master(rdev->pdev);
3826 /* reset */
3827 radeon_pci_config_reset(rdev);
3828 /* wait for asic to come out of reset */
3829 for (i = 0; i < rdev->usec_timeout; i++) {
3830 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3831 break;
3832 udelay(1);
3833 }
3834}
3835
c476dde2
AD		 3836int si_asic_reset(struct radeon_device *rdev)
3837{
014bb209
AD		 3838 u32 reset_mask;
3839
3840 reset_mask = si_gpu_check_soft_reset(rdev);
3841
3842 if (reset_mask)
3843 r600_set_bios_scratch_engine_hung(rdev, true);
3844
4a5c8ea5 3845 /* try soft reset */
014bb209
AD		 3846 si_gpu_soft_reset(rdev, reset_mask);
3847
3848 reset_mask = si_gpu_check_soft_reset(rdev);
3849
4a5c8ea5
AD		 3850 /* try pci config reset */
3851 if (reset_mask && radeon_hard_reset)
3852 si_gpu_pci_config_reset(rdev);
3853
3854 reset_mask = si_gpu_check_soft_reset(rdev);
3855
014bb209
AD		 3856 if (!reset_mask)
3857 r600_set_bios_scratch_engine_hung(rdev, false);
3858
3859 return 0;
c476dde2
AD		 3860}
3861
123bc183
AD		 3862/**
3863 * si_gfx_is_lockup - Check if the GFX engine is locked up
3864 *
3865 * @rdev: radeon_device pointer
3866 * @ring: radeon_ring structure holding ring information
3867 *
3868 * Check if the GFX engine is locked up.
3869 * Returns true if the engine appears to be locked up, false if not.
3870 */
3871bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3872{
3873 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3874
3875 if (!(reset_mask & (RADEON_RESET_GFX |
3876 RADEON_RESET_COMPUTE |
3877 RADEON_RESET_CP))) {
ff212f25 3878 radeon_ring_lockup_update(rdev, ring);
123bc183
AD		 3879 return false;
3880 }
123bc183
AD		 3881 return radeon_ring_test_lockup(rdev, ring);
3882}
3883
d2800ee5
AD		 3884/* MC */
3885static void si_mc_program(struct radeon_device *rdev)
3886{
3887 struct evergreen_mc_save save;
3888 u32 tmp;
3889 int i, j;
3890
3891 /* Initialize HDP */
3892 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3893 WREG32((0x2c14 + j), 0x00000000);
3894 WREG32((0x2c18 + j), 0x00000000);
3895 WREG32((0x2c1c + j), 0x00000000);
3896 WREG32((0x2c20 + j), 0x00000000);
3897 WREG32((0x2c24 + j), 0x00000000);
3898 }
3899 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3900
3901 evergreen_mc_stop(rdev, &save);
3902 if (radeon_mc_wait_for_idle(rdev)) {
3903 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3904 }
5153550a
AD		 3905 if (!ASIC_IS_NODCE(rdev))
3906 /* Lockout access through VGA aperture*/
3907 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
d2800ee5
AD		 3908 /* Update configuration */
3909 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3910 rdev->mc.vram_start >> 12);
3911 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3912 rdev->mc.vram_end >> 12);
3913 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3914 rdev->vram_scratch.gpu_addr >> 12);
3915 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3916 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3917 WREG32(MC_VM_FB_LOCATION, tmp);
3918 /* XXX double check these! */
3919 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3920 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3921 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3922 WREG32(MC_VM_AGP_BASE, 0);
3923 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3924 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3925 if (radeon_mc_wait_for_idle(rdev)) {
3926 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3927 }
3928 evergreen_mc_resume(rdev, &save);
5153550a
AD		 3929 if (!ASIC_IS_NODCE(rdev)) {
3930 /* we need to own VRAM, so turn off the VGA renderer here
3931 * to stop it overwriting our objects */
3932 rv515_vga_render_disable(rdev);
3933 }
d2800ee5
AD		 3934}
3935
1c49165d
AD		 3936void si_vram_gtt_location(struct radeon_device *rdev,
3937 struct radeon_mc *mc)
d2800ee5
AD		 3938{
3939 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3940 /* leave room for at least 1024M GTT */
3941 dev_warn(rdev->dev, "limiting VRAM\n");
3942 mc->real_vram_size = 0xFFC0000000ULL;
3943 mc->mc_vram_size = 0xFFC0000000ULL;
3944 }
9ed8b1f9 3945 radeon_vram_location(rdev, &rdev->mc, 0);
d2800ee5 3946 rdev->mc.gtt_base_align = 0;
9ed8b1f9 3947 radeon_gtt_location(rdev, mc);
d2800ee5
AD		 3948}
3949
3950static int si_mc_init(struct radeon_device *rdev)
3951{
3952 u32 tmp;
3953 int chansize, numchan;
3954
3955 /* Get VRAM informations */
3956 rdev->mc.vram_is_ddr = true;
3957 tmp = RREG32(MC_ARB_RAMCFG);
3958 if (tmp & CHANSIZE_OVERRIDE) {
3959 chansize = 16;
3960 } else if (tmp & CHANSIZE_MASK) {
3961 chansize = 64;
3962 } else {
3963 chansize = 32;
3964 }
3965 tmp = RREG32(MC_SHARED_CHMAP);
3966 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3967 case 0:
3968 default:
3969 numchan = 1;
3970 break;
3971 case 1:
3972 numchan = 2;
3973 break;
3974 case 2:
3975 numchan = 4;
3976 break;
3977 case 3:
3978 numchan = 8;
3979 break;
3980 case 4:
3981 numchan = 3;
3982 break;
3983 case 5:
3984 numchan = 6;
3985 break;
3986 case 6:
3987 numchan = 10;
3988 break;
3989 case 7:
3990 numchan = 12;
3991 break;
3992 case 8:
3993 numchan = 16;
3994 break;
3995 }
3996 rdev->mc.vram_width = numchan * chansize;
3997 /* Could aper size report 0 ? */
3998 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3999 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4000 /* size in MB on si */
0ca223b0
AD		 4001 tmp = RREG32(CONFIG_MEMSIZE);
4002 /* some boards may have garbage in the upper 16 bits */
4003 if (tmp & 0xffff0000) {
4004 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4005 if (tmp & 0xffff)
4006 tmp &= 0xffff;
4007 }
4008 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4009 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
d2800ee5
AD		 4010 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4011 si_vram_gtt_location(rdev, &rdev->mc);
4012 radeon_update_bandwidth_info(rdev);
4013
4014 return 0;
4015}
4016
4017/*
4018 * GART
4019 */
4020void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4021{
4022 /* flush hdp cache */
4023 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4024
4025 /* bits 0-15 are the VM contexts0-15 */
4026 WREG32(VM_INVALIDATE_REQUEST, 1);
4027}
4028
1109ca09 4029static int si_pcie_gart_enable(struct radeon_device *rdev)
d2800ee5
AD		 4030{
4031 int r, i;
4032
4033 if (rdev->gart.robj == NULL) {
4034 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4035 return -EINVAL;
4036 }
4037 r = radeon_gart_table_vram_pin(rdev);
4038 if (r)
4039 return r;
4040 radeon_gart_restore(rdev);
4041 /* Setup TLB control */
4042 WREG32(MC_VM_MX_L1_TLB_CNTL,
4043 (0xA << 7) |
4044 ENABLE_L1_TLB |
4045 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4046 ENABLE_ADVANCED_DRIVER_MODEL |
4047 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4048 /* Setup L2 cache */
4049 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4050 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4051 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4052 EFFECTIVE_L2_QUEUE_SIZE(7) |
4053 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4054 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4055 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4056 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4057 /* setup context0 */
4058 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4059 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4060 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4061 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4062 (u32)(rdev->dummy_page.addr >> 12));
4063 WREG32(VM_CONTEXT0_CNTL2, 0);
4064 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4065 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4066
4067 WREG32(0x15D4, 0);
4068 WREG32(0x15D8, 0);
4069 WREG32(0x15DC, 0);
4070
4071 /* empty context1-15 */
d2800ee5
AD		 4072 /* set vm size, must be a multiple of 4 */
4073 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
c21b328e 4074 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
23d4f1f2
AD		 4075 /* Assign the pt base to something valid for now; the pts used for
4076 * the VMs are determined by the application and setup and assigned
4077 * on the fly in the vm part of radeon_gart.c
4078 */
d2800ee5
AD		 4079 for (i = 1; i < 16; i++) {
4080 if (i < 8)
4081 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4082 rdev->gart.table_addr >> 12);
4083 else
4084 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4085 rdev->gart.table_addr >> 12);
4086 }
4087
4088 /* enable context1-15 */
4089 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4090 (u32)(rdev->dummy_page.addr >> 12));
ae133a11 4091 WREG32(VM_CONTEXT1_CNTL2, 4);
fa87e62d 4092 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
ae133a11
CK		 4093 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4094 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4095 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4096 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4097 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4098 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4099 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4100 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4101 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4102 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4103 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4104 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
d2800ee5
AD		 4105
4106 si_pcie_gart_tlb_flush(rdev);
4107 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4108 (unsigned)(rdev->mc.gtt_size >> 20),
4109 (unsigned long long)rdev->gart.table_addr);
4110 rdev->gart.ready = true;
4111 return 0;
4112}
4113
1109ca09 4114static void si_pcie_gart_disable(struct radeon_device *rdev)
d2800ee5
AD		 4115{
4116 /* Disable all tables */
4117 WREG32(VM_CONTEXT0_CNTL, 0);
4118 WREG32(VM_CONTEXT1_CNTL, 0);
4119 /* Setup TLB control */
4120 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4121 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4122 /* Setup L2 cache */
4123 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4124 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4125 EFFECTIVE_L2_QUEUE_SIZE(7) |
4126 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4127 WREG32(VM_L2_CNTL2, 0);
4128 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4129 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4130 radeon_gart_table_vram_unpin(rdev);
4131}
4132
1109ca09 4133static void si_pcie_gart_fini(struct radeon_device *rdev)
d2800ee5
AD		 4134{
4135 si_pcie_gart_disable(rdev);
4136 radeon_gart_table_vram_free(rdev);
4137 radeon_gart_fini(rdev);
4138}
4139
498dd8b3
AD		 4140/* vm parser */
4141static bool si_vm_reg_valid(u32 reg)
4142{
4143 /* context regs are fine */
4144 if (reg >= 0x28000)
4145 return true;
4146
4147 /* check config regs */
4148 switch (reg) {
4149 case GRBM_GFX_INDEX:
f418b88a 4150 case CP_STRMOUT_CNTL:
498dd8b3
AD		 4151 case VGT_VTX_VECT_EJECT_REG:
4152 case VGT_CACHE_INVALIDATION:
4153 case VGT_ESGS_RING_SIZE:
4154 case VGT_GSVS_RING_SIZE:
4155 case VGT_GS_VERTEX_REUSE:
4156 case VGT_PRIMITIVE_TYPE:
4157 case VGT_INDEX_TYPE:
4158 case VGT_NUM_INDICES:
4159 case VGT_NUM_INSTANCES:
4160 case VGT_TF_RING_SIZE:
4161 case VGT_HS_OFFCHIP_PARAM:
4162 case VGT_TF_MEMORY_BASE:
4163 case PA_CL_ENHANCE:
4164 case PA_SU_LINE_STIPPLE_VALUE:
4165 case PA_SC_LINE_STIPPLE_STATE:
4166 case PA_SC_ENHANCE:
4167 case SQC_CACHES:
4168 case SPI_STATIC_THREAD_MGMT_1:
4169 case SPI_STATIC_THREAD_MGMT_2:
4170 case SPI_STATIC_THREAD_MGMT_3:
4171 case SPI_PS_MAX_WAVE_ID:
4172 case SPI_CONFIG_CNTL:
4173 case SPI_CONFIG_CNTL_1:
4174 case TA_CNTL_AUX:
4175 return true;
4176 default:
4177 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4178 return false;
4179 }
4180}
4181
4182static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4183 u32 *ib, struct radeon_cs_packet *pkt)
4184{
4185 switch (pkt->opcode) {
4186 case PACKET3_NOP:
4187 case PACKET3_SET_BASE:
4188 case PACKET3_SET_CE_DE_COUNTERS:
4189 case PACKET3_LOAD_CONST_RAM:
4190 case PACKET3_WRITE_CONST_RAM:
4191 case PACKET3_WRITE_CONST_RAM_OFFSET:
4192 case PACKET3_DUMP_CONST_RAM:
4193 case PACKET3_INCREMENT_CE_COUNTER:
4194 case PACKET3_WAIT_ON_DE_COUNTER:
4195 case PACKET3_CE_WRITE:
4196 break;
4197 default:
4198 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4199 return -EINVAL;
4200 }
4201 return 0;
4202}
4203
e5b9e750
TS		 4204static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4205{
4206 u32 start_reg, reg, i;
4207 u32 command = ib[idx + 4];
4208 u32 info = ib[idx + 1];
4209 u32 idx_value = ib[idx];
4210 if (command & PACKET3_CP_DMA_CMD_SAS) {
4211 /* src address space is register */
4212 if (((info & 0x60000000) >> 29) == 0) {
4213 start_reg = idx_value << 2;
4214 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4215 reg = start_reg;
4216 if (!si_vm_reg_valid(reg)) {
4217 DRM_ERROR("CP DMA Bad SRC register\n");
4218 return -EINVAL;
4219 }
4220 } else {
4221 for (i = 0; i < (command & 0x1fffff); i++) {
4222 reg = start_reg + (4 * i);
4223 if (!si_vm_reg_valid(reg)) {
4224 DRM_ERROR("CP DMA Bad SRC register\n");
4225 return -EINVAL;
4226 }
4227 }
4228 }
4229 }
4230 }
4231 if (command & PACKET3_CP_DMA_CMD_DAS) {
4232 /* dst address space is register */
4233 if (((info & 0x00300000) >> 20) == 0) {
4234 start_reg = ib[idx + 2];
4235 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4236 reg = start_reg;
4237 if (!si_vm_reg_valid(reg)) {
4238 DRM_ERROR("CP DMA Bad DST register\n");
4239 return -EINVAL;
4240 }
4241 } else {
4242 for (i = 0; i < (command & 0x1fffff); i++) {
4243 reg = start_reg + (4 * i);
4244 if (!si_vm_reg_valid(reg)) {
4245 DRM_ERROR("CP DMA Bad DST register\n");
4246 return -EINVAL;
4247 }
4248 }
4249 }
4250 }
4251 }
4252 return 0;
4253}
4254
498dd8b3
AD		 4255static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4256 u32 *ib, struct radeon_cs_packet *pkt)
4257{
e5b9e750 4258 int r;
498dd8b3
AD		 4259 u32 idx = pkt->idx + 1;
4260 u32 idx_value = ib[idx];
4261 u32 start_reg, end_reg, reg, i;
4262
4263 switch (pkt->opcode) {
4264 case PACKET3_NOP:
4265 case PACKET3_SET_BASE:
4266 case PACKET3_CLEAR_STATE:
4267 case PACKET3_INDEX_BUFFER_SIZE:
4268 case PACKET3_DISPATCH_DIRECT:
4269 case PACKET3_DISPATCH_INDIRECT:
4270 case PACKET3_ALLOC_GDS:
4271 case PACKET3_WRITE_GDS_RAM:
4272 case PACKET3_ATOMIC_GDS:
4273 case PACKET3_ATOMIC:
4274 case PACKET3_OCCLUSION_QUERY:
4275 case PACKET3_SET_PREDICATION:
4276 case PACKET3_COND_EXEC:
4277 case PACKET3_PRED_EXEC:
4278 case PACKET3_DRAW_INDIRECT:
4279 case PACKET3_DRAW_INDEX_INDIRECT:
4280 case PACKET3_INDEX_BASE:
4281 case PACKET3_DRAW_INDEX_2:
4282 case PACKET3_CONTEXT_CONTROL:
4283 case PACKET3_INDEX_TYPE:
4284 case PACKET3_DRAW_INDIRECT_MULTI:
4285 case PACKET3_DRAW_INDEX_AUTO:
4286 case PACKET3_DRAW_INDEX_IMMD:
4287 case PACKET3_NUM_INSTANCES:
4288 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4289 case PACKET3_STRMOUT_BUFFER_UPDATE:
4290 case PACKET3_DRAW_INDEX_OFFSET_2:
4291 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4292 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4293 case PACKET3_MPEG_INDEX:
4294 case PACKET3_WAIT_REG_MEM:
4295 case PACKET3_MEM_WRITE:
4296 case PACKET3_PFP_SYNC_ME:
4297 case PACKET3_SURFACE_SYNC:
4298 case PACKET3_EVENT_WRITE:
4299 case PACKET3_EVENT_WRITE_EOP:
4300 case PACKET3_EVENT_WRITE_EOS:
4301 case PACKET3_SET_CONTEXT_REG:
4302 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4303 case PACKET3_SET_SH_REG:
4304 case PACKET3_SET_SH_REG_OFFSET:
4305 case PACKET3_INCREMENT_DE_COUNTER:
4306 case PACKET3_WAIT_ON_CE_COUNTER:
4307 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4308 case PACKET3_ME_WRITE:
4309 break;
4310 case PACKET3_COPY_DATA:
4311 if ((idx_value & 0xf00) == 0) {
4312 reg = ib[idx + 3] * 4;
4313 if (!si_vm_reg_valid(reg))
4314 return -EINVAL;
4315 }
4316 break;
4317 case PACKET3_WRITE_DATA:
4318 if ((idx_value & 0xf00) == 0) {
4319 start_reg = ib[idx + 1] * 4;
4320 if (idx_value & 0x10000) {
4321 if (!si_vm_reg_valid(start_reg))
4322 return -EINVAL;
4323 } else {
4324 for (i = 0; i < (pkt->count - 2); i++) {
4325 reg = start_reg + (4 * i);
4326 if (!si_vm_reg_valid(reg))
4327 return -EINVAL;
4328 }
4329 }
4330 }
4331 break;
4332 case PACKET3_COND_WRITE:
4333 if (idx_value & 0x100) {
4334 reg = ib[idx + 5] * 4;
4335 if (!si_vm_reg_valid(reg))
4336 return -EINVAL;
4337 }
4338 break;
4339 case PACKET3_COPY_DW:
4340 if (idx_value & 0x2) {
4341 reg = ib[idx + 3] * 4;
4342 if (!si_vm_reg_valid(reg))
4343 return -EINVAL;
4344 }
4345 break;
4346 case PACKET3_SET_CONFIG_REG:
4347 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4348 end_reg = 4 * pkt->count + start_reg - 4;
4349 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4350 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4351 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4352 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4353 return -EINVAL;
4354 }
4355 for (i = 0; i < pkt->count; i++) {
4356 reg = start_reg + (4 * i);
4357 if (!si_vm_reg_valid(reg))
4358 return -EINVAL;
4359 }
4360 break;
5aa709be 4361 case PACKET3_CP_DMA:
e5b9e750
TS		 4362 r = si_vm_packet3_cp_dma_check(ib, idx);
4363 if (r)
4364 return r;
5aa709be 4365 break;
498dd8b3
AD		 4366 default:
4367 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4368 return -EINVAL;
4369 }
4370 return 0;
4371}
4372
4373static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4374 u32 *ib, struct radeon_cs_packet *pkt)
4375{
e5b9e750 4376 int r;
498dd8b3
AD		 4377 u32 idx = pkt->idx + 1;
4378 u32 idx_value = ib[idx];
4379 u32 start_reg, reg, i;
4380
4381 switch (pkt->opcode) {
4382 case PACKET3_NOP:
4383 case PACKET3_SET_BASE:
4384 case PACKET3_CLEAR_STATE:
4385 case PACKET3_DISPATCH_DIRECT:
4386 case PACKET3_DISPATCH_INDIRECT:
4387 case PACKET3_ALLOC_GDS:
4388 case PACKET3_WRITE_GDS_RAM:
4389 case PACKET3_ATOMIC_GDS:
4390 case PACKET3_ATOMIC:
4391 case PACKET3_OCCLUSION_QUERY:
4392 case PACKET3_SET_PREDICATION:
4393 case PACKET3_COND_EXEC:
4394 case PACKET3_PRED_EXEC:
4395 case PACKET3_CONTEXT_CONTROL:
4396 case PACKET3_STRMOUT_BUFFER_UPDATE:
4397 case PACKET3_WAIT_REG_MEM:
4398 case PACKET3_MEM_WRITE:
4399 case PACKET3_PFP_SYNC_ME:
4400 case PACKET3_SURFACE_SYNC:
4401 case PACKET3_EVENT_WRITE:
4402 case PACKET3_EVENT_WRITE_EOP:
4403 case PACKET3_EVENT_WRITE_EOS:
4404 case PACKET3_SET_CONTEXT_REG:
4405 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4406 case PACKET3_SET_SH_REG:
4407 case PACKET3_SET_SH_REG_OFFSET:
4408 case PACKET3_INCREMENT_DE_COUNTER:
4409 case PACKET3_WAIT_ON_CE_COUNTER:
4410 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4411 case PACKET3_ME_WRITE:
4412 break;
4413 case PACKET3_COPY_DATA:
4414 if ((idx_value & 0xf00) == 0) {
4415 reg = ib[idx + 3] * 4;
4416 if (!si_vm_reg_valid(reg))
4417 return -EINVAL;
4418 }
4419 break;
4420 case PACKET3_WRITE_DATA:
4421 if ((idx_value & 0xf00) == 0) {
4422 start_reg = ib[idx + 1] * 4;
4423 if (idx_value & 0x10000) {
4424 if (!si_vm_reg_valid(start_reg))
4425 return -EINVAL;
4426 } else {
4427 for (i = 0; i < (pkt->count - 2); i++) {
4428 reg = start_reg + (4 * i);
4429 if (!si_vm_reg_valid(reg))
4430 return -EINVAL;
4431 }
4432 }
4433 }
4434 break;
4435 case PACKET3_COND_WRITE:
4436 if (idx_value & 0x100) {
4437 reg = ib[idx + 5] * 4;
4438 if (!si_vm_reg_valid(reg))
4439 return -EINVAL;
4440 }
4441 break;
4442 case PACKET3_COPY_DW:
4443 if (idx_value & 0x2) {
4444 reg = ib[idx + 3] * 4;
4445 if (!si_vm_reg_valid(reg))
4446 return -EINVAL;
4447 }
4448 break;
e5b9e750
TS		 4449 case PACKET3_CP_DMA:
4450 r = si_vm_packet3_cp_dma_check(ib, idx);
4451 if (r)
4452 return r;
4453 break;
498dd8b3
AD		 4454 default:
4455 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4456 return -EINVAL;
4457 }
4458 return 0;
4459}
4460
4461int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4462{
4463 int ret = 0;
4464 u32 idx = 0;
4465 struct radeon_cs_packet pkt;
4466
4467 do {
4468 pkt.idx = idx;
4e872ae2
IH		 4469 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4470 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
498dd8b3
AD		 4471 pkt.one_reg_wr = 0;
4472 switch (pkt.type) {
4e872ae2 4473 case RADEON_PACKET_TYPE0:
498dd8b3
AD		 4474 dev_err(rdev->dev, "Packet0 not allowed!\n");
4475 ret = -EINVAL;
4476 break;
4e872ae2 4477 case RADEON_PACKET_TYPE2:
498dd8b3
AD		 4478 idx += 1;
4479 break;
4e872ae2
IH		 4480 case RADEON_PACKET_TYPE3:
4481 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
498dd8b3
AD		 4482 if (ib->is_const_ib)
4483 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4484 else {
876dc9f3 4485 switch (ib->ring) {
498dd8b3
AD		 4486 case RADEON_RING_TYPE_GFX_INDEX:
4487 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4488 break;
4489 case CAYMAN_RING_TYPE_CP1_INDEX:
4490 case CAYMAN_RING_TYPE_CP2_INDEX:
4491 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4492 break;
4493 default:
876dc9f3 4494 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
498dd8b3
AD		 4495 ret = -EINVAL;
4496 break;
4497 }
4498 }
4499 idx += pkt.count + 2;
4500 break;
4501 default:
4502 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4503 ret = -EINVAL;
4504 break;
4505 }
4506 if (ret)
4507 break;
4508 } while (idx < ib->length_dw);
4509
4510 return ret;
4511}
4512
d2800ee5
AD		 4513/*
4514 * vm
4515 */
4516int si_vm_init(struct radeon_device *rdev)
4517{
4518 /* number of VMs */
4519 rdev->vm_manager.nvm = 16;
4520 /* base offset of vram pages */
4521 rdev->vm_manager.vram_base_offset = 0;
4522
4523 return 0;
4524}
4525
4526void si_vm_fini(struct radeon_device *rdev)
4527{
4528}
4529
fbf6dc7a
AD		 4530/**
4531 * si_vm_decode_fault - print human readable fault info
4532 *
4533 * @rdev: radeon_device pointer
4534 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4535 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4536 *
4537 * Print human readable fault information (SI).
4538 */
4539static void si_vm_decode_fault(struct radeon_device *rdev,
4540 u32 status, u32 addr)
4541{
4542 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4543 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4544 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4545 char *block;
4546
4547 if (rdev->family == CHIP_TAHITI) {
4548 switch (mc_id) {
4549 case 160:
4550 case 144:
4551 case 96:
4552 case 80:
4553 case 224:
4554 case 208:
4555 case 32:
4556 case 16:
4557 block = "CB";
4558 break;
4559 case 161:
4560 case 145:
4561 case 97:
4562 case 81:
4563 case 225:
4564 case 209:
4565 case 33:
4566 case 17:
4567 block = "CB_FMASK";
4568 break;
4569 case 162:
4570 case 146:
4571 case 98:
4572 case 82:
4573 case 226:
4574 case 210:
4575 case 34:
4576 case 18:
4577 block = "CB_CMASK";
4578 break;
4579 case 163:
4580 case 147:
4581 case 99:
4582 case 83:
4583 case 227:
4584 case 211:
4585 case 35:
4586 case 19:
4587 block = "CB_IMMED";
4588 break;
4589 case 164:
4590 case 148:
4591 case 100:
4592 case 84:
4593 case 228:
4594 case 212:
4595 case 36:
4596 case 20:
4597 block = "DB";
4598 break;
4599 case 165:
4600 case 149:
4601 case 101:
4602 case 85:
4603 case 229:
4604 case 213:
4605 case 37:
4606 case 21:
4607 block = "DB_HTILE";
4608 break;
4609 case 167:
4610 case 151:
4611 case 103:
4612 case 87:
4613 case 231:
4614 case 215:
4615 case 39:
4616 case 23:
4617 block = "DB_STEN";
4618 break;
4619 case 72:
4620 case 68:
4621 case 64:
4622 case 8:
4623 case 4:
4624 case 0:
4625 case 136:
4626 case 132:
4627 case 128:
4628 case 200:
4629 case 196:
4630 case 192:
4631 block = "TC";
4632 break;
4633 case 112:
4634 case 48:
4635 block = "CP";
4636 break;
4637 case 49:
4638 case 177:
4639 case 50:
4640 case 178:
4641 block = "SH";
4642 break;
4643 case 53:
4644 case 190:
4645 block = "VGT";
4646 break;
4647 case 117:
4648 block = "IH";
4649 break;
4650 case 51:
4651 case 115:
4652 block = "RLC";
4653 break;
4654 case 119:
4655 case 183:
4656 block = "DMA0";
4657 break;
4658 case 61:
4659 block = "DMA1";
4660 break;
4661 case 248:
4662 case 120:
4663 block = "HDP";
4664 break;
4665 default:
4666 block = "unknown";
4667 break;
4668 }
4669 } else {
4670 switch (mc_id) {
4671 case 32:
4672 case 16:
4673 case 96:
4674 case 80:
4675 case 160:
4676 case 144:
4677 case 224:
4678 case 208:
4679 block = "CB";
4680 break;
4681 case 33:
4682 case 17:
4683 case 97:
4684 case 81:
4685 case 161:
4686 case 145:
4687 case 225:
4688 case 209:
4689 block = "CB_FMASK";
4690 break;
4691 case 34:
4692 case 18:
4693 case 98:
4694 case 82:
4695 case 162:
4696 case 146:
4697 case 226:
4698 case 210:
4699 block = "CB_CMASK";
4700 break;
4701 case 35:
4702 case 19:
4703 case 99:
4704 case 83:
4705 case 163:
4706 case 147:
4707 case 227:
4708 case 211:
4709 block = "CB_IMMED";
4710 break;
4711 case 36:
4712 case 20:
4713 case 100:
4714 case 84:
4715 case 164:
4716 case 148:
4717 case 228:
4718 case 212:
4719 block = "DB";
4720 break;
4721 case 37:
4722 case 21:
4723 case 101:
4724 case 85:
4725 case 165:
4726 case 149:
4727 case 229:
4728 case 213:
4729 block = "DB_HTILE";
4730 break;
4731 case 39:
4732 case 23:
4733 case 103:
4734 case 87:
4735 case 167:
4736 case 151:
4737 case 231:
4738 case 215:
4739 block = "DB_STEN";
4740 break;
4741 case 72:
4742 case 68:
4743 case 8:
4744 case 4:
4745 case 136:
4746 case 132:
4747 case 200:
4748 case 196:
4749 block = "TC";
4750 break;
4751 case 112:
4752 case 48:
4753 block = "CP";
4754 break;
4755 case 49:
4756 case 177:
4757 case 50:
4758 case 178:
4759 block = "SH";
4760 break;
4761 case 53:
4762 block = "VGT";
4763 break;
4764 case 117:
4765 block = "IH";
4766 break;
4767 case 51:
4768 case 115:
4769 block = "RLC";
4770 break;
4771 case 119:
4772 case 183:
4773 block = "DMA0";
4774 break;
4775 case 61:
4776 block = "DMA1";
4777 break;
4778 case 248:
4779 case 120:
4780 block = "HDP";
4781 break;
4782 default:
4783 block = "unknown";
4784 break;
4785 }
4786 }
4787
4788 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
4789 protections, vmid, addr,
4790 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
4791 block, mc_id);
4792}
4793
498522b4 4794void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
d2800ee5 4795{
498522b4 4796 struct radeon_ring *ring = &rdev->ring[ridx];
d2800ee5 4797
ee60e29f 4798 if (vm == NULL)
d2800ee5
AD		 4799 return;
4800
76c44f2c
AD		 4801 /* write new base address */
4802 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4803 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4804 WRITE_DATA_DST_SEL(0)));
4805
ee60e29f 4806 if (vm->id < 8) {
76c44f2c
AD		 4807 radeon_ring_write(ring,
4808 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
ee60e29f 4809 } else {
76c44f2c
AD		 4810 radeon_ring_write(ring,
4811 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
ee60e29f 4812 }
76c44f2c 4813 radeon_ring_write(ring, 0);
fa87e62d 4814 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
ee60e29f 4815
d2800ee5 4816 /* flush hdp cache */
76c44f2c
AD		 4817 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4818 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4819 WRITE_DATA_DST_SEL(0)));
4820 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4821 radeon_ring_write(ring, 0);
ee60e29f
CK		 4822 radeon_ring_write(ring, 0x1);
4823
d2800ee5 4824 /* bits 0-15 are the VM contexts0-15 */
76c44f2c
AD		 4825 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4826 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4827 WRITE_DATA_DST_SEL(0)));
4828 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4829 radeon_ring_write(ring, 0);
498522b4 4830 radeon_ring_write(ring, 1 << vm->id);
58f8cf56
CK		 4831
4832 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4833 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4834 radeon_ring_write(ring, 0x0);
d2800ee5
AD		 4835}
4836
f8f84ac5
AD		 4837/*
4838 * Power and clock gating
4839 */
4840static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4841{
4842 int i;
4843
4844 for (i = 0; i < rdev->usec_timeout; i++) {
4845 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4846 break;
4847 udelay(1);
4848 }
4849
4850 for (i = 0; i < rdev->usec_timeout; i++) {
4851 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4852 break;
4853 udelay(1);
4854 }
4855}
4856
4857static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4858 bool enable)
4859{
4860 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4861 u32 mask;
4862 int i;
4863
4864 if (enable)
4865 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4866 else
4867 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4868 WREG32(CP_INT_CNTL_RING0, tmp);
4869
4870 if (!enable) {
4871 /* read a gfx register */
4872 tmp = RREG32(DB_DEPTH_INFO);
4873
4874 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4875 for (i = 0; i < rdev->usec_timeout; i++) {
4876 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4877 break;
4878 udelay(1);
4879 }
4880 }
4881}
4882
4883static void si_set_uvd_dcm(struct radeon_device *rdev,
4884 bool sw_mode)
4885{
4886 u32 tmp, tmp2;
4887
4888 tmp = RREG32(UVD_CGC_CTRL);
4889 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4890 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4891
4892 if (sw_mode) {
4893 tmp &= ~0x7ffff800;
4894 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4895 } else {
4896 tmp |= 0x7ffff800;
4897 tmp2 = 0;
4898 }
4899
4900 WREG32(UVD_CGC_CTRL, tmp);
4901 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4902}
4903
22c775ce 4904void si_init_uvd_internal_cg(struct radeon_device *rdev)
f8f84ac5
AD		 4905{
4906 bool hw_mode = true;
4907
4908 if (hw_mode) {
4909 si_set_uvd_dcm(rdev, false);
4910 } else {
4911 u32 tmp = RREG32(UVD_CGC_CTRL);
4912 tmp &= ~DCM;
4913 WREG32(UVD_CGC_CTRL, tmp);
4914 }
4915}
4916
4917static u32 si_halt_rlc(struct radeon_device *rdev)
4918{
4919 u32 data, orig;
4920
4921 orig = data = RREG32(RLC_CNTL);
4922
4923 if (data & RLC_ENABLE) {
4924 data &= ~RLC_ENABLE;
4925 WREG32(RLC_CNTL, data);
4926
4927 si_wait_for_rlc_serdes(rdev);
4928 }
4929
4930 return orig;
4931}
4932
4933static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4934{
4935 u32 tmp;
4936
4937 tmp = RREG32(RLC_CNTL);
4938 if (tmp != rlc)
4939 WREG32(RLC_CNTL, rlc);
4940}
4941
4942static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4943{
4944 u32 data, orig;
4945
4946 orig = data = RREG32(DMA_PG);
e16866ec 4947 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
f8f84ac5
AD		 4948 data |= PG_CNTL_ENABLE;
4949 else
4950 data &= ~PG_CNTL_ENABLE;
4951 if (orig != data)
4952 WREG32(DMA_PG, data);
4953}
4954
4955static void si_init_dma_pg(struct radeon_device *rdev)
4956{
4957 u32 tmp;
4958
4959 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4960 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4961
4962 for (tmp = 0; tmp < 5; tmp++)
4963 WREG32(DMA_PGFSM_WRITE, 0);
4964}
4965
4966static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4967 bool enable)
4968{
4969 u32 tmp;
4970
2b19d17f 4971 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
f8f84ac5
AD		 4972 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4973 WREG32(RLC_TTOP_D, tmp);
4974
4975 tmp = RREG32(RLC_PG_CNTL);
4976 tmp |= GFX_PG_ENABLE;
4977 WREG32(RLC_PG_CNTL, tmp);
4978
4979 tmp = RREG32(RLC_AUTO_PG_CTRL);
4980 tmp |= AUTO_PG_EN;
4981 WREG32(RLC_AUTO_PG_CTRL, tmp);
4982 } else {
4983 tmp = RREG32(RLC_AUTO_PG_CTRL);
4984 tmp &= ~AUTO_PG_EN;
4985 WREG32(RLC_AUTO_PG_CTRL, tmp);
4986
4987 tmp = RREG32(DB_RENDER_CONTROL);
4988 }
4989}
4990
4991static void si_init_gfx_cgpg(struct radeon_device *rdev)
4992{
4993 u32 tmp;
4994
4995 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4996
4997 tmp = RREG32(RLC_PG_CNTL);
4998 tmp |= GFX_PG_SRC;
4999 WREG32(RLC_PG_CNTL, tmp);
5000
5001 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5002
5003 tmp = RREG32(RLC_AUTO_PG_CTRL);
5004
5005 tmp &= ~GRBM_REG_SGIT_MASK;
5006 tmp |= GRBM_REG_SGIT(0x700);
5007 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5008 WREG32(RLC_AUTO_PG_CTRL, tmp);
5009}
5010
ba19031a 5011static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
f8f84ac5
AD		 5012{
5013 u32 mask = 0, tmp, tmp1;
5014 int i;
5015
5016 si_select_se_sh(rdev, se, sh);
5017 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5018 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5019 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5020
5021 tmp &= 0xffff0000;
5022
5023 tmp |= tmp1;
5024 tmp >>= 16;
5025
5026 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5027 mask <<= 1;
5028 mask |= 1;
5029 }
5030
5031 return (~tmp) & mask;
5032}
5033
5034static void si_init_ao_cu_mask(struct radeon_device *rdev)
5035{
5036 u32 i, j, k, active_cu_number = 0;
5037 u32 mask, counter, cu_bitmap;
5038 u32 tmp = 0;
5039
5040 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5041 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5042 mask = 1;
5043 cu_bitmap = 0;
5044 counter = 0;
5045 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
ba19031a 5046 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
f8f84ac5
AD		 5047 if (counter < 2)
5048 cu_bitmap |= mask;
5049 counter++;
5050 }
5051 mask <<= 1;
5052 }
5053
5054 active_cu_number += counter;
5055 tmp |= (cu_bitmap << (i * 16 + j * 8));
5056 }
5057 }
5058
5059 WREG32(RLC_PG_AO_CU_MASK, tmp);
5060
5061 tmp = RREG32(RLC_MAX_PG_CU);
5062 tmp &= ~MAX_PU_CU_MASK;
5063 tmp |= MAX_PU_CU(active_cu_number);
5064 WREG32(RLC_MAX_PG_CU, tmp);
5065}
5066
5067static void si_enable_cgcg(struct radeon_device *rdev,
5068 bool enable)
5069{
5070 u32 data, orig, tmp;
5071
5072 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5073
e16866ec 5074 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5594a558 5075 si_enable_gui_idle_interrupt(rdev, true);
f8f84ac5 5076
f8f84ac5
AD		 5077 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5078
5079 tmp = si_halt_rlc(rdev);
5080
5081 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5082 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5083 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5084
5085 si_wait_for_rlc_serdes(rdev);
5086
5087 si_update_rlc(rdev, tmp);
5088
5089 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5090
5091 data |= CGCG_EN | CGLS_EN;
5092 } else {
5594a558
AD		 5093 si_enable_gui_idle_interrupt(rdev, false);
5094
f8f84ac5
AD		 5095 RREG32(CB_CGTT_SCLK_CTRL);
5096 RREG32(CB_CGTT_SCLK_CTRL);
5097 RREG32(CB_CGTT_SCLK_CTRL);
5098 RREG32(CB_CGTT_SCLK_CTRL);
5099
5100 data &= ~(CGCG_EN | CGLS_EN);
5101 }
5102
5103 if (orig != data)
5104 WREG32(RLC_CGCG_CGLS_CTRL, data);
5105}
5106
5107static void si_enable_mgcg(struct radeon_device *rdev,
5108 bool enable)
5109{
5110 u32 data, orig, tmp = 0;
5111
e16866ec 5112 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
f8f84ac5
AD		 5113 orig = data = RREG32(CGTS_SM_CTRL_REG);
5114 data = 0x96940200;
5115 if (orig != data)
5116 WREG32(CGTS_SM_CTRL_REG, data);
5117
e16866ec
AD		 5118 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5119 orig = data = RREG32(CP_MEM_SLP_CNTL);
5120 data |= CP_MEM_LS_EN;
5121 if (orig != data)
5122 WREG32(CP_MEM_SLP_CNTL, data);
5123 }
f8f84ac5
AD		 5124
5125 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5126 data &= 0xffffffc0;
5127 if (orig != data)
5128 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5129
5130 tmp = si_halt_rlc(rdev);
5131
5132 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5133 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5134 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5135
5136 si_update_rlc(rdev, tmp);
5137 } else {
5138 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5139 data |= 0x00000003;
5140 if (orig != data)
5141 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5142
5143 data = RREG32(CP_MEM_SLP_CNTL);
5144 if (data & CP_MEM_LS_EN) {
5145 data &= ~CP_MEM_LS_EN;
5146 WREG32(CP_MEM_SLP_CNTL, data);
5147 }
5148 orig = data = RREG32(CGTS_SM_CTRL_REG);
5149 data |= LS_OVERRIDE | OVERRIDE;
5150 if (orig != data)
5151 WREG32(CGTS_SM_CTRL_REG, data);
5152
5153 tmp = si_halt_rlc(rdev);
5154
5155 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5156 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5157 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5158
5159 si_update_rlc(rdev, tmp);
5160 }
5161}
5162
5163static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5164 bool enable)
5165{
5166 u32 orig, data, tmp;
5167
e16866ec 5168 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
f8f84ac5
AD		 5169 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5170 tmp |= 0x3fff;
5171 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5172
5173 orig = data = RREG32(UVD_CGC_CTRL);
5174 data |= DCM;
5175 if (orig != data)
5176 WREG32(UVD_CGC_CTRL, data);
5177
5178 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5179 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5180 } else {
5181 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5182 tmp &= ~0x3fff;
5183 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5184
5185 orig = data = RREG32(UVD_CGC_CTRL);
5186 data &= ~DCM;
5187 if (orig != data)
5188 WREG32(UVD_CGC_CTRL, data);
5189
5190 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5191 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5192 }
5193}
5194
5195static const u32 mc_cg_registers[] =
5196{
5197 MC_HUB_MISC_HUB_CG,
5198 MC_HUB_MISC_SIP_CG,
5199 MC_HUB_MISC_VM_CG,
5200 MC_XPB_CLK_GAT,
5201 ATC_MISC_CG,
5202 MC_CITF_MISC_WR_CG,
5203 MC_CITF_MISC_RD_CG,
5204 MC_CITF_MISC_VM_CG,
5205 VM_L2_CG,
5206};
5207
5208static void si_enable_mc_ls(struct radeon_device *rdev,
5209 bool enable)
5210{
5211 int i;
5212 u32 orig, data;
5213
5214 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5215 orig = data = RREG32(mc_cg_registers[i]);
e16866ec 5216 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
f8f84ac5
AD		 5217 data |= MC_LS_ENABLE;
5218 else
5219 data &= ~MC_LS_ENABLE;
5220 if (data != orig)
5221 WREG32(mc_cg_registers[i], data);
5222 }
5223}
5224
e16866ec
AD		 5225static void si_enable_mc_mgcg(struct radeon_device *rdev,
5226 bool enable)
f8f84ac5 5227{
e16866ec
AD		 5228 int i;
5229 u32 orig, data;
5230
5231 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5232 orig = data = RREG32(mc_cg_registers[i]);
5233 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5234 data |= MC_CG_ENABLE;
5235 else
5236 data &= ~MC_CG_ENABLE;
5237 if (data != orig)
5238 WREG32(mc_cg_registers[i], data);
f8f84ac5
AD		 5239 }
5240}
5241
e16866ec
AD		 5242static void si_enable_dma_mgcg(struct radeon_device *rdev,
5243 bool enable)
f8f84ac5 5244{
e16866ec
AD		 5245 u32 orig, data, offset;
5246 int i;
f8f84ac5 5247
e16866ec
AD		 5248 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5249 for (i = 0; i < 2; i++) {
5250 if (i == 0)
5251 offset = DMA0_REGISTER_OFFSET;
5252 else
5253 offset = DMA1_REGISTER_OFFSET;
5254 orig = data = RREG32(DMA_POWER_CNTL + offset);
5255 data &= ~MEM_POWER_OVERRIDE;
5256 if (data != orig)
5257 WREG32(DMA_POWER_CNTL + offset, data);
5258 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5259 }
5260 } else {
5261 for (i = 0; i < 2; i++) {
5262 if (i == 0)
5263 offset = DMA0_REGISTER_OFFSET;
5264 else
5265 offset = DMA1_REGISTER_OFFSET;
5266 orig = data = RREG32(DMA_POWER_CNTL + offset);
5267 data |= MEM_POWER_OVERRIDE;
5268 if (data != orig)
5269 WREG32(DMA_POWER_CNTL + offset, data);
5270
5271 orig = data = RREG32(DMA_CLK_CTRL + offset);
5272 data = 0xff000000;
5273 if (data != orig)
5274 WREG32(DMA_CLK_CTRL + offset, data);
5275 }
5276 }
f8f84ac5
AD		 5277}
5278
e16866ec
AD		 5279static void si_enable_bif_mgls(struct radeon_device *rdev,
5280 bool enable)
f8f84ac5 5281{
e16866ec 5282 u32 orig, data;
f8f84ac5 5283
e16866ec 5284 orig = data = RREG32_PCIE(PCIE_CNTL2);
f8f84ac5 5285
e16866ec
AD		 5286 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5287 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5288 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5289 else
5290 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5291 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5292
5293 if (orig != data)
5294 WREG32_PCIE(PCIE_CNTL2, data);
f8f84ac5
AD		 5295}
5296
e16866ec
AD		 5297static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5298 bool enable)
f8f84ac5 5299{
e16866ec 5300 u32 orig, data;
f8f84ac5 5301
e16866ec 5302 orig = data = RREG32(HDP_HOST_PATH_CNTL);
f8f84ac5 5303
e16866ec
AD		 5304 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5305 data &= ~CLOCK_GATING_DIS;
5306 else
5307 data |= CLOCK_GATING_DIS;
5308
5309 if (orig != data)
5310 WREG32(HDP_HOST_PATH_CNTL, data);
f8f84ac5
AD		 5311}
5312
e16866ec
AD		 5313static void si_enable_hdp_ls(struct radeon_device *rdev,
5314 bool enable)
347e7592 5315{
e16866ec 5316 u32 orig, data;
347e7592 5317
e16866ec 5318 orig = data = RREG32(HDP_MEM_POWER_LS);
347e7592 5319
e16866ec
AD		 5320 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5321 data |= HDP_LS_ENABLE;
5322 else
5323 data &= ~HDP_LS_ENABLE;
347e7592 5324
e16866ec
AD		 5325 if (orig != data)
5326 WREG32(HDP_MEM_POWER_LS, data);
347e7592
AD		 5327}
5328
68e3a092
AD		 5329static void si_update_cg(struct radeon_device *rdev,
5330 u32 block, bool enable)
347e7592 5331{
e16866ec 5332 if (block & RADEON_CG_BLOCK_GFX) {
811e4d58 5333 si_enable_gui_idle_interrupt(rdev, false);
e16866ec
AD		 5334 /* order matters! */
5335 if (enable) {
5336 si_enable_mgcg(rdev, true);
5337 si_enable_cgcg(rdev, true);
5338 } else {
5339 si_enable_cgcg(rdev, false);
5340 si_enable_mgcg(rdev, false);
347e7592 5341 }
811e4d58 5342 si_enable_gui_idle_interrupt(rdev, true);
347e7592
AD		 5343 }
5344
e16866ec
AD		 5345 if (block & RADEON_CG_BLOCK_MC) {
5346 si_enable_mc_mgcg(rdev, enable);
5347 si_enable_mc_ls(rdev, enable);
347e7592 5348 }
e16866ec
AD		 5349
5350 if (block & RADEON_CG_BLOCK_SDMA) {
5351 si_enable_dma_mgcg(rdev, enable);
347e7592
AD		 5352 }
5353
e16866ec
AD		 5354 if (block & RADEON_CG_BLOCK_BIF) {
5355 si_enable_bif_mgls(rdev, enable);
6d8cf000 5356 }
6d8cf000 5357
e16866ec
AD		 5358 if (block & RADEON_CG_BLOCK_UVD) {
5359 if (rdev->has_uvd) {
5360 si_enable_uvd_mgcg(rdev, enable);
bd8cd539
AD		 5361 }
5362 }
bd8cd539 5363
e16866ec
AD		 5364 if (block & RADEON_CG_BLOCK_HDP) {
5365 si_enable_hdp_mgcg(rdev, enable);
5366 si_enable_hdp_ls(rdev, enable);
347e7592 5367 }
e16866ec 5368}
f8f84ac5
AD		 5369
5370static void si_init_cg(struct radeon_device *rdev)
5371{
e16866ec
AD		 5372 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5373 RADEON_CG_BLOCK_MC |
5374 RADEON_CG_BLOCK_SDMA |
5375 RADEON_CG_BLOCK_BIF |
5376 RADEON_CG_BLOCK_HDP), true);
b2d70917 5377 if (rdev->has_uvd) {
e16866ec 5378 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
f8f84ac5 5379 si_init_uvd_internal_cg(rdev);
347e7592 5380 }
f8f84ac5 5381}
bd8cd539 5382
f8f84ac5
AD		 5383static void si_fini_cg(struct radeon_device *rdev)
5384{
0116e1ef 5385 if (rdev->has_uvd) {
e16866ec 5386 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
347e7592 5387 }
e16866ec
AD		 5388 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5389 RADEON_CG_BLOCK_MC |
5390 RADEON_CG_BLOCK_SDMA |
5391 RADEON_CG_BLOCK_BIF |
5392 RADEON_CG_BLOCK_HDP), false);
5393}
5394
59a82d0e
AD		 5395u32 si_get_csb_size(struct radeon_device *rdev)
5396{
5397 u32 count = 0;
5398 const struct cs_section_def *sect = NULL;
5399 const struct cs_extent_def *ext = NULL;
5400
5401 if (rdev->rlc.cs_data == NULL)
5402 return 0;
5403
5404 /* begin clear state */
5405 count += 2;
5406 /* context control state */
5407 count += 3;
5408
5409 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5410 for (ext = sect->section; ext->extent != NULL; ++ext) {
5411 if (sect->id == SECT_CONTEXT)
5412 count += 2 + ext->reg_count;
5413 else
5414 return 0;
5415 }
bd8cd539 5416 }
59a82d0e
AD		 5417 /* pa_sc_raster_config */
5418 count += 3;
5419 /* end clear state */
5420 count += 2;
5421 /* clear state */
5422 count += 2;
5423
5424 return count;
5425}
5426
5427void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5428{
5429 u32 count = 0, i;
5430 const struct cs_section_def *sect = NULL;
5431 const struct cs_extent_def *ext = NULL;
5432
5433 if (rdev->rlc.cs_data == NULL)
5434 return;
5435 if (buffer == NULL)
5436 return;
5437
6ba81e53
AD		 5438 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5439 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
59a82d0e 5440
6ba81e53
AD		 5441 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5442 buffer[count++] = cpu_to_le32(0x80000000);
5443 buffer[count++] = cpu_to_le32(0x80000000);
59a82d0e
AD		 5444
5445 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5446 for (ext = sect->section; ext->extent != NULL; ++ext) {
5447 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 5448 buffer[count++] =
5449 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5450 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
59a82d0e 5451 for (i = 0; i < ext->reg_count; i++)
6ba81e53 5452 buffer[count++] = cpu_to_le32(ext->extent[i]);
59a82d0e
AD		 5453 } else {
5454 return;
bd8cd539 5455 }
bd8cd539
AD		 5456 }
5457 }
bd8cd539 5458
6ba81e53
AD		 5459 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5460 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
59a82d0e
AD		 5461 switch (rdev->family) {
5462 case CHIP_TAHITI:
5463 case CHIP_PITCAIRN:
6ba81e53 5464 buffer[count++] = cpu_to_le32(0x2a00126a);
59a82d0e
AD		 5465 break;
5466 case CHIP_VERDE:
6ba81e53 5467 buffer[count++] = cpu_to_le32(0x0000124a);
59a82d0e
AD		 5468 break;
5469 case CHIP_OLAND:
6ba81e53 5470 buffer[count++] = cpu_to_le32(0x00000082);
59a82d0e
AD		 5471 break;
5472 case CHIP_HAINAN:
6ba81e53 5473 buffer[count++] = cpu_to_le32(0x00000000);
59a82d0e
AD		 5474 break;
5475 default:
6ba81e53 5476 buffer[count++] = cpu_to_le32(0x00000000);
59a82d0e
AD		 5477 break;
5478 }
5479
6ba81e53
AD		 5480 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5481 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
347e7592 5482
6ba81e53
AD		 5483 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5484 buffer[count++] = cpu_to_le32(0);
59a82d0e
AD		 5485}
5486
f8f84ac5
AD		 5487static void si_init_pg(struct radeon_device *rdev)
5488{
0116e1ef
AD		 5489 if (rdev->pg_flags) {
5490 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5491 si_init_dma_pg(rdev);
0116e1ef 5492 }
f8f84ac5 5493 si_init_ao_cu_mask(rdev);
2b19d17f 5494 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
0116e1ef 5495 si_init_gfx_cgpg(rdev);
aa34dba8
AD		 5496 } else {
5497 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5498 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
0116e1ef 5499 }
ca6ebb39
AD		 5500 si_enable_dma_pg(rdev, true);
5501 si_enable_gfx_cgpg(rdev, true);
f8f84ac5
AD		 5502 } else {
5503 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5504 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5505 }
5506}
5507
5508static void si_fini_pg(struct radeon_device *rdev)
5509{
0116e1ef 5510 if (rdev->pg_flags) {
ca6ebb39
AD		 5511 si_enable_dma_pg(rdev, false);
5512 si_enable_gfx_cgpg(rdev, false);
f8f84ac5 5513 }
347e7592
AD		 5514}
5515
347e7592
AD		 5516/*
5517 * RLC
5518 */
866d83de 5519void si_rlc_reset(struct radeon_device *rdev)
d719cef3 5520{
f8f84ac5 5521 u32 tmp = RREG32(GRBM_SOFT_RESET);
d719cef3 5522
f8f84ac5
AD		 5523 tmp |= SOFT_RESET_RLC;
5524 WREG32(GRBM_SOFT_RESET, tmp);
5525 udelay(50);
5526 tmp &= ~SOFT_RESET_RLC;
5527 WREG32(GRBM_SOFT_RESET, tmp);
5528 udelay(50);
d719cef3
AD		 5529}
5530
347e7592
AD		 5531static void si_rlc_stop(struct radeon_device *rdev)
5532{
5533 WREG32(RLC_CNTL, 0);
d719cef3
AD		 5534
5535 si_enable_gui_idle_interrupt(rdev, false);
5536
5537 si_wait_for_rlc_serdes(rdev);
347e7592
AD		 5538}
5539
5540static void si_rlc_start(struct radeon_device *rdev)
5541{
5542 WREG32(RLC_CNTL, RLC_ENABLE);
d719cef3
AD		 5543
5544 si_enable_gui_idle_interrupt(rdev, true);
5545
5546 udelay(50);
5547}
5548
5549static bool si_lbpw_supported(struct radeon_device *rdev)
5550{
5551 u32 tmp;
5552
5553 /* Enable LBPW only for DDR3 */
5554 tmp = RREG32(MC_SEQ_MISC0);
5555 if ((tmp & 0xF0000000) == 0xB0000000)
5556 return true;
5557 return false;
5558}
5559
5560static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5561{
5562 u32 tmp;
5563
5564 tmp = RREG32(RLC_LB_CNTL);
5565 if (enable)
5566 tmp |= LOAD_BALANCE_ENABLE;
5567 else
5568 tmp &= ~LOAD_BALANCE_ENABLE;
5569 WREG32(RLC_LB_CNTL, tmp);
5570
5571 if (!enable) {
5572 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5573 WREG32(SPI_LB_CU_MASK, 0x00ff);
5574 }
347e7592
AD		 5575}
5576
5577static int si_rlc_resume(struct radeon_device *rdev)
5578{
5579 u32 i;
5580 const __be32 *fw_data;
5581
5582 if (!rdev->rlc_fw)
5583 return -EINVAL;
5584
5585 si_rlc_stop(rdev);
5586
f8f84ac5
AD		 5587 si_rlc_reset(rdev);
5588
5589 si_init_pg(rdev);
5590
5591 si_init_cg(rdev);
5592
347e7592
AD		 5593 WREG32(RLC_RL_BASE, 0);
5594 WREG32(RLC_RL_SIZE, 0);
5595 WREG32(RLC_LB_CNTL, 0);
5596 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5597 WREG32(RLC_LB_CNTR_INIT, 0);
d719cef3 5598 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
347e7592 5599
347e7592
AD		 5600 WREG32(RLC_MC_CNTL, 0);
5601 WREG32(RLC_UCODE_CNTL, 0);
5602
5603 fw_data = (const __be32 *)rdev->rlc_fw->data;
5604 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5605 WREG32(RLC_UCODE_ADDR, i);
5606 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5607 }
5608 WREG32(RLC_UCODE_ADDR, 0);
5609
d719cef3
AD		 5610 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5611
347e7592
AD		 5612 si_rlc_start(rdev);
5613
5614 return 0;
5615}
5616
25a857fb
AD		 5617static void si_enable_interrupts(struct radeon_device *rdev)
5618{
5619 u32 ih_cntl = RREG32(IH_CNTL);
5620 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5621
5622 ih_cntl |= ENABLE_INTR;
5623 ih_rb_cntl |= IH_RB_ENABLE;
5624 WREG32(IH_CNTL, ih_cntl);
5625 WREG32(IH_RB_CNTL, ih_rb_cntl);
5626 rdev->ih.enabled = true;
5627}
5628
5629static void si_disable_interrupts(struct radeon_device *rdev)
5630{
5631 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5632 u32 ih_cntl = RREG32(IH_CNTL);
5633
5634 ih_rb_cntl &= ~IH_RB_ENABLE;
5635 ih_cntl &= ~ENABLE_INTR;
5636 WREG32(IH_RB_CNTL, ih_rb_cntl);
5637 WREG32(IH_CNTL, ih_cntl);
5638 /* set rptr, wptr to 0 */
5639 WREG32(IH_RB_RPTR, 0);
5640 WREG32(IH_RB_WPTR, 0);
5641 rdev->ih.enabled = false;
25a857fb
AD		 5642 rdev->ih.rptr = 0;
5643}
5644
5645static void si_disable_interrupt_state(struct radeon_device *rdev)
5646{
5647 u32 tmp;
5648
811e4d58
AD		 5649 tmp = RREG32(CP_INT_CNTL_RING0) &
5650 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5651 WREG32(CP_INT_CNTL_RING0, tmp);
25a857fb
AD		 5652 WREG32(CP_INT_CNTL_RING1, 0);
5653 WREG32(CP_INT_CNTL_RING2, 0);
8c5fd7ef
AD		 5654 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5655 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5656 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5657 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
25a857fb 5658 WREG32(GRBM_INT_CNTL, 0);
5153550a
AD		 5659 if (rdev->num_crtc >= 2) {
5660 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5661 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5662 }
25a857fb
AD		 5663 if (rdev->num_crtc >= 4) {
5664 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5665 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5666 }
5667 if (rdev->num_crtc >= 6) {
5668 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5669 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5670 }
5671
5153550a
AD		 5672 if (rdev->num_crtc >= 2) {
5673 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5674 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5675 }
25a857fb
AD		 5676 if (rdev->num_crtc >= 4) {
5677 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5678 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5679 }
5680 if (rdev->num_crtc >= 6) {
5681 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5682 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5683 }
5684
5153550a 5685 if (!ASIC_IS_NODCE(rdev)) {
e9a321c6 5686 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5153550a
AD		 5687
5688 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5689 WREG32(DC_HPD1_INT_CONTROL, tmp);
5690 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5691 WREG32(DC_HPD2_INT_CONTROL, tmp);
5692 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5693 WREG32(DC_HPD3_INT_CONTROL, tmp);
5694 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5695 WREG32(DC_HPD4_INT_CONTROL, tmp);
5696 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5697 WREG32(DC_HPD5_INT_CONTROL, tmp);
5698 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5699 WREG32(DC_HPD6_INT_CONTROL, tmp);
5700 }
25a857fb
AD		 5701}
5702
5703static int si_irq_init(struct radeon_device *rdev)
5704{
5705 int ret = 0;
5706 int rb_bufsz;
5707 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5708
5709 /* allocate ring */
5710 ret = r600_ih_ring_alloc(rdev);
5711 if (ret)
5712 return ret;
5713
5714 /* disable irqs */
5715 si_disable_interrupts(rdev);
5716
5717 /* init rlc */
5718 ret = si_rlc_resume(rdev);
5719 if (ret) {
5720 r600_ih_ring_fini(rdev);
5721 return ret;
5722 }
5723
5724 /* setup interrupt control */
5725 /* set dummy read address to ring address */
5726 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5727 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5728 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5729 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5730 */
5731 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5732 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5733 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5734 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5735
5736 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 5737 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
25a857fb
AD		 5738
5739 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5740 IH_WPTR_OVERFLOW_CLEAR |
5741 (rb_bufsz << 1));
5742
5743 if (rdev->wb.enabled)
5744 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5745
5746 /* set the writeback address whether it's enabled or not */
5747 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5748 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5749
5750 WREG32(IH_RB_CNTL, ih_rb_cntl);
5751
5752 /* set rptr, wptr to 0 */
5753 WREG32(IH_RB_RPTR, 0);
5754 WREG32(IH_RB_WPTR, 0);
5755
5756 /* Default settings for IH_CNTL (disabled at first) */
5757 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5758 /* RPTR_REARM only works if msi's are enabled */
5759 if (rdev->msi_enabled)
5760 ih_cntl |= RPTR_REARM;
5761 WREG32(IH_CNTL, ih_cntl);
5762
5763 /* force the active interrupt state to all disabled */
5764 si_disable_interrupt_state(rdev);
5765
2099810f
DA		 5766 pci_set_master(rdev->pdev);
5767
25a857fb
AD		 5768 /* enable irqs */
5769 si_enable_interrupts(rdev);
5770
5771 return ret;
5772}
5773
5774int si_irq_set(struct radeon_device *rdev)
5775{
811e4d58 5776 u32 cp_int_cntl;
25a857fb
AD		 5777 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5778 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5153550a 5779 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
25a857fb
AD		 5780 u32 grbm_int_cntl = 0;
5781 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
8c5fd7ef 5782 u32 dma_cntl, dma_cntl1;
a9e61410 5783 u32 thermal_int = 0;
25a857fb
AD		 5784
5785 if (!rdev->irq.installed) {
5786 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5787 return -EINVAL;
5788 }
5789 /* don't enable anything if the ih is disabled */
5790 if (!rdev->ih.enabled) {
5791 si_disable_interrupts(rdev);
5792 /* force the active interrupt state to all disabled */
5793 si_disable_interrupt_state(rdev);
5794 return 0;
5795 }
5796
811e4d58
AD		 5797 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
5798 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5799
5153550a
AD		 5800 if (!ASIC_IS_NODCE(rdev)) {
5801 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5802 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5803 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5804 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5805 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5806 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5807 }
25a857fb 5808
8c5fd7ef
AD		 5809 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5810 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5811
a9e61410
AD		 5812 thermal_int = RREG32(CG_THERMAL_INT) &
5813 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5814
25a857fb 5815 /* enable CP interrupts on all rings */
736fc37f 5816 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
25a857fb
AD		 5817 DRM_DEBUG("si_irq_set: sw int gfx\n");
5818 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5819 }
736fc37f 5820 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
25a857fb
AD		 5821 DRM_DEBUG("si_irq_set: sw int cp1\n");
5822 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5823 }
736fc37f 5824 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
25a857fb
AD		 5825 DRM_DEBUG("si_irq_set: sw int cp2\n");
5826 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5827 }
8c5fd7ef
AD		 5828 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5829 DRM_DEBUG("si_irq_set: sw int dma\n");
5830 dma_cntl |= TRAP_ENABLE;
5831 }
5832
5833 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5834 DRM_DEBUG("si_irq_set: sw int dma1\n");
5835 dma_cntl1 |= TRAP_ENABLE;
5836 }
25a857fb 5837 if (rdev->irq.crtc_vblank_int[0] ||
736fc37f 5838 atomic_read(&rdev->irq.pflip[0])) {
25a857fb
AD		 5839 DRM_DEBUG("si_irq_set: vblank 0\n");
5840 crtc1 |= VBLANK_INT_MASK;
5841 }
5842 if (rdev->irq.crtc_vblank_int[1] ||
736fc37f 5843 atomic_read(&rdev->irq.pflip[1])) {
25a857fb
AD		 5844 DRM_DEBUG("si_irq_set: vblank 1\n");
5845 crtc2 |= VBLANK_INT_MASK;
5846 }
5847 if (rdev->irq.crtc_vblank_int[2] ||
736fc37f 5848 atomic_read(&rdev->irq.pflip[2])) {
25a857fb
AD		 5849 DRM_DEBUG("si_irq_set: vblank 2\n");
5850 crtc3 |= VBLANK_INT_MASK;
5851 }
5852 if (rdev->irq.crtc_vblank_int[3] ||
736fc37f 5853 atomic_read(&rdev->irq.pflip[3])) {
25a857fb
AD		 5854 DRM_DEBUG("si_irq_set: vblank 3\n");
5855 crtc4 |= VBLANK_INT_MASK;
5856 }
5857 if (rdev->irq.crtc_vblank_int[4] ||
736fc37f 5858 atomic_read(&rdev->irq.pflip[4])) {
25a857fb
AD		 5859 DRM_DEBUG("si_irq_set: vblank 4\n");
5860 crtc5 |= VBLANK_INT_MASK;
5861 }
5862 if (rdev->irq.crtc_vblank_int[5] ||
736fc37f 5863 atomic_read(&rdev->irq.pflip[5])) {
25a857fb
AD		 5864 DRM_DEBUG("si_irq_set: vblank 5\n");
5865 crtc6 |= VBLANK_INT_MASK;
5866 }
5867 if (rdev->irq.hpd[0]) {
5868 DRM_DEBUG("si_irq_set: hpd 1\n");
5869 hpd1 |= DC_HPDx_INT_EN;
5870 }
5871 if (rdev->irq.hpd[1]) {
5872 DRM_DEBUG("si_irq_set: hpd 2\n");
5873 hpd2 |= DC_HPDx_INT_EN;
5874 }
5875 if (rdev->irq.hpd[2]) {
5876 DRM_DEBUG("si_irq_set: hpd 3\n");
5877 hpd3 |= DC_HPDx_INT_EN;
5878 }
5879 if (rdev->irq.hpd[3]) {
5880 DRM_DEBUG("si_irq_set: hpd 4\n");
5881 hpd4 |= DC_HPDx_INT_EN;
5882 }
5883 if (rdev->irq.hpd[4]) {
5884 DRM_DEBUG("si_irq_set: hpd 5\n");
5885 hpd5 |= DC_HPDx_INT_EN;
5886 }
5887 if (rdev->irq.hpd[5]) {
5888 DRM_DEBUG("si_irq_set: hpd 6\n");
5889 hpd6 |= DC_HPDx_INT_EN;
5890 }
25a857fb
AD		 5891
5892 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5893 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5894 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5895
8c5fd7ef
AD		 5896 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5897 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5898
25a857fb
AD		 5899 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5900
a9e61410
AD		 5901 if (rdev->irq.dpm_thermal) {
5902 DRM_DEBUG("dpm thermal\n");
5903 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5904 }
5905
5153550a
AD		 5906 if (rdev->num_crtc >= 2) {
5907 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5908 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5909 }
25a857fb
AD		 5910 if (rdev->num_crtc >= 4) {
5911 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5912 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5913 }
5914 if (rdev->num_crtc >= 6) {
5915 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5916 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5917 }
5918
5153550a
AD		 5919 if (rdev->num_crtc >= 2) {
5920 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5921 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5922 }
25a857fb
AD		 5923 if (rdev->num_crtc >= 4) {
5924 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5925 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5926 }
5927 if (rdev->num_crtc >= 6) {
5928 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5929 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5930 }
5931
5153550a
AD		 5932 if (!ASIC_IS_NODCE(rdev)) {
5933 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5934 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5935 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5936 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5937 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5938 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5939 }
25a857fb 5940
a9e61410
AD		 5941 WREG32(CG_THERMAL_INT, thermal_int);
5942
25a857fb
AD		 5943 return 0;
5944}
5945
5946static inline void si_irq_ack(struct radeon_device *rdev)
5947{
5948 u32 tmp;
5949
5153550a
AD		 5950 if (ASIC_IS_NODCE(rdev))
5951 return;
5952
25a857fb
AD		 5953 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5954 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5955 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5956 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5957 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5958 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5959 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5960 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5961 if (rdev->num_crtc >= 4) {
5962 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5963 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5964 }
5965 if (rdev->num_crtc >= 6) {
5966 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5967 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5968 }
5969
5970 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5971 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5972 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5973 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5974 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5975 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5976 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5977 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5978 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5979 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5980 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5981 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5982
5983 if (rdev->num_crtc >= 4) {
5984 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5985 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5986 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5987 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5988 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5989 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5990 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5991 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5992 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5993 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5994 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5995 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5996 }
5997
5998 if (rdev->num_crtc >= 6) {
5999 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
6000 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6001 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
6002 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6003 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6004 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6005 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6006 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6007 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6008 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6009 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6010 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6011 }
6012
6013 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6014 tmp = RREG32(DC_HPD1_INT_CONTROL);
6015 tmp |= DC_HPDx_INT_ACK;
6016 WREG32(DC_HPD1_INT_CONTROL, tmp);
6017 }
6018 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6019 tmp = RREG32(DC_HPD2_INT_CONTROL);
6020 tmp |= DC_HPDx_INT_ACK;
6021 WREG32(DC_HPD2_INT_CONTROL, tmp);
6022 }
6023 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6024 tmp = RREG32(DC_HPD3_INT_CONTROL);
6025 tmp |= DC_HPDx_INT_ACK;
6026 WREG32(DC_HPD3_INT_CONTROL, tmp);
6027 }
6028 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6029 tmp = RREG32(DC_HPD4_INT_CONTROL);
6030 tmp |= DC_HPDx_INT_ACK;
6031 WREG32(DC_HPD4_INT_CONTROL, tmp);
6032 }
6033 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6034 tmp = RREG32(DC_HPD5_INT_CONTROL);
6035 tmp |= DC_HPDx_INT_ACK;
6036 WREG32(DC_HPD5_INT_CONTROL, tmp);
6037 }
6038 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6039 tmp = RREG32(DC_HPD5_INT_CONTROL);
6040 tmp |= DC_HPDx_INT_ACK;
6041 WREG32(DC_HPD6_INT_CONTROL, tmp);
6042 }
6043}
6044
6045static void si_irq_disable(struct radeon_device *rdev)
6046{
6047 si_disable_interrupts(rdev);
6048 /* Wait and acknowledge irq */
6049 mdelay(1);
6050 si_irq_ack(rdev);
6051 si_disable_interrupt_state(rdev);
6052}
6053
6054static void si_irq_suspend(struct radeon_device *rdev)
6055{
6056 si_irq_disable(rdev);
6057 si_rlc_stop(rdev);
6058}
6059
9b136d51
AD		 6060static void si_irq_fini(struct radeon_device *rdev)
6061{
6062 si_irq_suspend(rdev);
6063 r600_ih_ring_fini(rdev);
6064}
6065
25a857fb
AD		 6066static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6067{
6068 u32 wptr, tmp;
6069
6070 if (rdev->wb.enabled)
6071 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6072 else
6073 wptr = RREG32(IH_RB_WPTR);
6074
6075 if (wptr & RB_OVERFLOW) {
6076 /* When a ring buffer overflow happen start parsing interrupt
6077 * from the last not overwritten vector (wptr + 16). Hopefully
6078 * this should allow us to catchup.
6079 */
6080 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6081 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6082 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6083 tmp = RREG32(IH_RB_CNTL);
6084 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6085 WREG32(IH_RB_CNTL, tmp);
6086 }
6087 return (wptr & rdev->ih.ptr_mask);
6088}
6089
6090/* SI IV Ring
6091 * Each IV ring entry is 128 bits:
6092 * [7:0] - interrupt source id
6093 * [31:8] - reserved
6094 * [59:32] - interrupt source data
6095 * [63:60] - reserved
6096 * [71:64] - RINGID
6097 * [79:72] - VMID
6098 * [127:80] - reserved
6099 */
6100int si_irq_process(struct radeon_device *rdev)
6101{
6102 u32 wptr;
6103 u32 rptr;
6104 u32 src_id, src_data, ring_id;
6105 u32 ring_index;
25a857fb 6106 bool queue_hotplug = false;
a9e61410 6107 bool queue_thermal = false;
fbf6dc7a 6108 u32 status, addr;
25a857fb
AD		 6109
6110 if (!rdev->ih.enabled || rdev->shutdown)
6111 return IRQ_NONE;
6112
6113 wptr = si_get_ih_wptr(rdev);
c20dc369
CK		 6114
6115restart_ih:
6116 /* is somebody else already processing irqs? */
6117 if (atomic_xchg(&rdev->ih.lock, 1))
6118 return IRQ_NONE;
6119
25a857fb
AD		 6120 rptr = rdev->ih.rptr;
6121 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6122
25a857fb
AD		 6123 /* Order reading of wptr vs. reading of IH ring data */
6124 rmb();
6125
6126 /* display interrupts */
6127 si_irq_ack(rdev);
6128
25a857fb
AD		 6129 while (rptr != wptr) {
6130 /* wptr/rptr are in bytes! */
6131 ring_index = rptr / 4;
6132 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6133 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6134 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6135
6136 switch (src_id) {
6137 case 1: /* D1 vblank/vline */
6138 switch (src_data) {
6139 case 0: /* D1 vblank */
6140 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6141 if (rdev->irq.crtc_vblank_int[0]) {
6142 drm_handle_vblank(rdev->ddev, 0);
6143 rdev->pm.vblank_sync = true;
6144 wake_up(&rdev->irq.vblank_queue);
6145 }
736fc37f 6146 if (atomic_read(&rdev->irq.pflip[0]))
25a857fb
AD		 6147 radeon_crtc_handle_flip(rdev, 0);
6148 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6149 DRM_DEBUG("IH: D1 vblank\n");
6150 }
6151 break;
6152 case 1: /* D1 vline */
6153 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6154 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6155 DRM_DEBUG("IH: D1 vline\n");
6156 }
6157 break;
6158 default:
6159 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6160 break;
6161 }
6162 break;
6163 case 2: /* D2 vblank/vline */
6164 switch (src_data) {
6165 case 0: /* D2 vblank */
6166 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6167 if (rdev->irq.crtc_vblank_int[1]) {
6168 drm_handle_vblank(rdev->ddev, 1);
6169 rdev->pm.vblank_sync = true;
6170 wake_up(&rdev->irq.vblank_queue);
6171 }
736fc37f 6172 if (atomic_read(&rdev->irq.pflip[1]))
25a857fb
AD		 6173 radeon_crtc_handle_flip(rdev, 1);
6174 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6175 DRM_DEBUG("IH: D2 vblank\n");
6176 }
6177 break;
6178 case 1: /* D2 vline */
6179 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6180 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6181 DRM_DEBUG("IH: D2 vline\n");
6182 }
6183 break;
6184 default:
6185 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6186 break;
6187 }
6188 break;
6189 case 3: /* D3 vblank/vline */
6190 switch (src_data) {
6191 case 0: /* D3 vblank */
6192 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6193 if (rdev->irq.crtc_vblank_int[2]) {
6194 drm_handle_vblank(rdev->ddev, 2);
6195 rdev->pm.vblank_sync = true;
6196 wake_up(&rdev->irq.vblank_queue);
6197 }
736fc37f 6198 if (atomic_read(&rdev->irq.pflip[2]))
25a857fb
AD		 6199 radeon_crtc_handle_flip(rdev, 2);
6200 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6201 DRM_DEBUG("IH: D3 vblank\n");
6202 }
6203 break;
6204 case 1: /* D3 vline */
6205 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6206 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6207 DRM_DEBUG("IH: D3 vline\n");
6208 }
6209 break;
6210 default:
6211 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6212 break;
6213 }
6214 break;
6215 case 4: /* D4 vblank/vline */
6216 switch (src_data) {
6217 case 0: /* D4 vblank */
6218 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6219 if (rdev->irq.crtc_vblank_int[3]) {
6220 drm_handle_vblank(rdev->ddev, 3);
6221 rdev->pm.vblank_sync = true;
6222 wake_up(&rdev->irq.vblank_queue);
6223 }
736fc37f 6224 if (atomic_read(&rdev->irq.pflip[3]))
25a857fb
AD		 6225 radeon_crtc_handle_flip(rdev, 3);
6226 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6227 DRM_DEBUG("IH: D4 vblank\n");
6228 }
6229 break;
6230 case 1: /* D4 vline */
6231 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6232 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6233 DRM_DEBUG("IH: D4 vline\n");
6234 }
6235 break;
6236 default:
6237 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6238 break;
6239 }
6240 break;
6241 case 5: /* D5 vblank/vline */
6242 switch (src_data) {
6243 case 0: /* D5 vblank */
6244 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6245 if (rdev->irq.crtc_vblank_int[4]) {
6246 drm_handle_vblank(rdev->ddev, 4);
6247 rdev->pm.vblank_sync = true;
6248 wake_up(&rdev->irq.vblank_queue);
6249 }
736fc37f 6250 if (atomic_read(&rdev->irq.pflip[4]))
25a857fb
AD		 6251 radeon_crtc_handle_flip(rdev, 4);
6252 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6253 DRM_DEBUG("IH: D5 vblank\n");
6254 }
6255 break;
6256 case 1: /* D5 vline */
6257 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6258 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6259 DRM_DEBUG("IH: D5 vline\n");
6260 }
6261 break;
6262 default:
6263 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6264 break;
6265 }
6266 break;
6267 case 6: /* D6 vblank/vline */
6268 switch (src_data) {
6269 case 0: /* D6 vblank */
6270 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6271 if (rdev->irq.crtc_vblank_int[5]) {
6272 drm_handle_vblank(rdev->ddev, 5);
6273 rdev->pm.vblank_sync = true;
6274 wake_up(&rdev->irq.vblank_queue);
6275 }
736fc37f 6276 if (atomic_read(&rdev->irq.pflip[5]))
25a857fb
AD		 6277 radeon_crtc_handle_flip(rdev, 5);
6278 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6279 DRM_DEBUG("IH: D6 vblank\n");
6280 }
6281 break;
6282 case 1: /* D6 vline */
6283 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6284 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6285 DRM_DEBUG("IH: D6 vline\n");
6286 }
6287 break;
6288 default:
6289 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6290 break;
6291 }
6292 break;
6293 case 42: /* HPD hotplug */
6294 switch (src_data) {
6295 case 0:
6296 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6297 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6298 queue_hotplug = true;
6299 DRM_DEBUG("IH: HPD1\n");
6300 }
6301 break;
6302 case 1:
6303 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6304 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6305 queue_hotplug = true;
6306 DRM_DEBUG("IH: HPD2\n");
6307 }
6308 break;
6309 case 2:
6310 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6311 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6312 queue_hotplug = true;
6313 DRM_DEBUG("IH: HPD3\n");
6314 }
6315 break;
6316 case 3:
6317 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6318 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6319 queue_hotplug = true;
6320 DRM_DEBUG("IH: HPD4\n");
6321 }
6322 break;
6323 case 4:
6324 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6325 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6326 queue_hotplug = true;
6327 DRM_DEBUG("IH: HPD5\n");
6328 }
6329 break;
6330 case 5:
6331 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6332 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6333 queue_hotplug = true;
6334 DRM_DEBUG("IH: HPD6\n");
6335 }
6336 break;
6337 default:
6338 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6339 break;
6340 }
6341 break;
b927e1c2
CK		 6342 case 124: /* UVD */
6343 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6344 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6345 break;
ae133a11
CK		 6346 case 146:
6347 case 147:
fbf6dc7a
AD		 6348 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6349 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
ae133a11
CK		 6350 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6351 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
fbf6dc7a 6352 addr);
ae133a11 6353 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
fbf6dc7a
AD		 6354 status);
6355 si_vm_decode_fault(rdev, status, addr);
ae133a11
CK		 6356 /* reset addr and status */
6357 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6358 break;
25a857fb
AD		 6359 case 176: /* RINGID0 CP_INT */
6360 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6361 break;
6362 case 177: /* RINGID1 CP_INT */
6363 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6364 break;
6365 case 178: /* RINGID2 CP_INT */
6366 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6367 break;
6368 case 181: /* CP EOP event */
6369 DRM_DEBUG("IH: CP EOP\n");
6370 switch (ring_id) {
6371 case 0:
6372 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6373 break;
6374 case 1:
6375 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6376 break;
6377 case 2:
6378 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6379 break;
6380 }
6381 break;
8c5fd7ef
AD		 6382 case 224: /* DMA trap event */
6383 DRM_DEBUG("IH: DMA trap\n");
6384 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6385 break;
a9e61410
AD		 6386 case 230: /* thermal low to high */
6387 DRM_DEBUG("IH: thermal low to high\n");
6388 rdev->pm.dpm.thermal.high_to_low = false;
6389 queue_thermal = true;
6390 break;
6391 case 231: /* thermal high to low */
6392 DRM_DEBUG("IH: thermal high to low\n");
6393 rdev->pm.dpm.thermal.high_to_low = true;
6394 queue_thermal = true;
6395 break;
25a857fb
AD		 6396 case 233: /* GUI IDLE */
6397 DRM_DEBUG("IH: GUI idle\n");
25a857fb 6398 break;
8c5fd7ef
AD		 6399 case 244: /* DMA trap event */
6400 DRM_DEBUG("IH: DMA1 trap\n");
6401 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6402 break;
25a857fb
AD		 6403 default:
6404 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6405 break;
6406 }
6407
6408 /* wptr/rptr are in bytes! */
6409 rptr += 16;
6410 rptr &= rdev->ih.ptr_mask;
6411 }
25a857fb
AD		 6412 if (queue_hotplug)
6413 schedule_work(&rdev->hotplug_work);
a9e61410
AD		 6414 if (queue_thermal && rdev->pm.dpm_enabled)
6415 schedule_work(&rdev->pm.dpm.thermal.work);
25a857fb
AD		 6416 rdev->ih.rptr = rptr;
6417 WREG32(IH_RB_RPTR, rdev->ih.rptr);
c20dc369
CK		 6418 atomic_set(&rdev->ih.lock, 0);
6419
6420 /* make sure wptr hasn't changed while processing */
6421 wptr = si_get_ih_wptr(rdev);
6422 if (wptr != rptr)
6423 goto restart_ih;
6424
25a857fb
AD		 6425 return IRQ_HANDLED;
6426}
6427
9b136d51
AD		 6428/*
6429 * startup/shutdown callbacks
6430 */
6431static int si_startup(struct radeon_device *rdev)
6432{
6433 struct radeon_ring *ring;
6434 int r;
6435
b9d305df
AD		 6436 /* enable pcie gen2/3 link */
6437 si_pcie_gen3_enable(rdev);
e0bcf165
AD		 6438 /* enable aspm */
6439 si_program_aspm(rdev);
b9d305df 6440
e5903d39
AD		 6441 /* scratch needs to be initialized before MC */
6442 r = r600_vram_scratch_init(rdev);
6443 if (r)
6444 return r;
6445
6fab3feb
AD		 6446 si_mc_program(rdev);
6447
6c7bccea
AD		 6448 if (!rdev->pm.dpm_enabled) {
6449 r = si_mc_load_microcode(rdev);
6450 if (r) {
6451 DRM_ERROR("Failed to load MC firmware!\n");
6452 return r;
6453 }
9b136d51
AD		 6454 }
6455
9b136d51
AD		 6456 r = si_pcie_gart_enable(rdev);
6457 if (r)
6458 return r;
6459 si_gpu_init(rdev);
6460
9b136d51 6461 /* allocate rlc buffers */
1fd11777
AD		 6462 if (rdev->family == CHIP_VERDE) {
6463 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6464 rdev->rlc.reg_list_size =
6465 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6466 }
6467 rdev->rlc.cs_data = si_cs_data;
6468 r = sumo_rlc_init(rdev);
9b136d51
AD		 6469 if (r) {
6470 DRM_ERROR("Failed to init rlc BOs!\n");
6471 return r;
6472 }
6473
6474 /* allocate wb buffer */
6475 r = radeon_wb_init(rdev);
6476 if (r)
6477 return r;
6478
6479 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6480 if (r) {
6481 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6482 return r;
6483 }
6484
6485 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6486 if (r) {
6487 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6488 return r;
6489 }
6490
6491 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6492 if (r) {
6493 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6494 return r;
6495 }
6496
8c5fd7ef
AD		 6497 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6498 if (r) {
6499 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6500 return r;
6501 }
6502
6503 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6504 if (r) {
6505 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6506 return r;
6507 }
6508
1df0d523 6509 if (rdev->has_uvd) {
e409b128 6510 r = uvd_v2_2_resume(rdev);
1df0d523
AD		 6511 if (!r) {
6512 r = radeon_fence_driver_start_ring(rdev,
6513 R600_RING_TYPE_UVD_INDEX);
6514 if (r)
6515 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6516 }
f2ba57b5 6517 if (r)
1df0d523 6518 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
f2ba57b5 6519 }
f2ba57b5 6520
9b136d51 6521 /* Enable IRQ */
e49f3959
AH		 6522 if (!rdev->irq.installed) {
6523 r = radeon_irq_kms_init(rdev);
6524 if (r)
6525 return r;
6526 }
6527
9b136d51
AD		 6528 r = si_irq_init(rdev);
6529 if (r) {
6530 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6531 radeon_irq_kms_fini(rdev);
6532 return r;
6533 }
6534 si_irq_set(rdev);
6535
6536 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6537 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2e1e6dad 6538 RADEON_CP_PACKET2);
9b136d51
AD		 6539 if (r)
6540 return r;
6541
6542 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6543 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
2e1e6dad 6544 RADEON_CP_PACKET2);
9b136d51
AD		 6545 if (r)
6546 return r;
6547
6548 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6549 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
2e1e6dad 6550 RADEON_CP_PACKET2);
9b136d51
AD		 6551 if (r)
6552 return r;
6553
8c5fd7ef
AD		 6554 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6555 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 6556 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6557 if (r)
6558 return r;
6559
6560 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6561 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 6562 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
8c5fd7ef
AD		 6563 if (r)
6564 return r;
6565
9b136d51
AD		 6566 r = si_cp_load_microcode(rdev);
6567 if (r)
6568 return r;
6569 r = si_cp_resume(rdev);
6570 if (r)
6571 return r;
6572
8c5fd7ef
AD		 6573 r = cayman_dma_resume(rdev);
6574 if (r)
6575 return r;
6576
1df0d523
AD		 6577 if (rdev->has_uvd) {
6578 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6579 if (ring->ring_size) {
02c9f7fa 6580 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 6581 RADEON_CP_PACKET2);
1df0d523 6582 if (!r)
e409b128 6583 r = uvd_v1_0_init(rdev);
1df0d523
AD		 6584 if (r)
6585 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6586 }
f2ba57b5
CK		 6587 }
6588
2898c348
CK		 6589 r = radeon_ib_pool_init(rdev);
6590 if (r) {
6591 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
9b136d51 6592 return r;
2898c348 6593 }
9b136d51 6594
c6105f24
CK		 6595 r = radeon_vm_manager_init(rdev);
6596 if (r) {
6597 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
9b136d51 6598 return r;
c6105f24 6599 }
9b136d51 6600
b530602f
AD		 6601 r = dce6_audio_init(rdev);
6602 if (r)
6603 return r;
6604
9b136d51
AD		 6605 return 0;
6606}
6607
6608int si_resume(struct radeon_device *rdev)
6609{
6610 int r;
6611
6612 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6613 * posting will perform necessary task to bring back GPU into good
6614 * shape.
6615 */
6616 /* post card */
6617 atom_asic_init(rdev->mode_info.atom_context);
6618
205996c0
AD		 6619 /* init golden registers */
6620 si_init_golden_registers(rdev);
6621
bc6a6295
AD		 6622 if (rdev->pm.pm_method == PM_METHOD_DPM)
6623 radeon_pm_resume(rdev);
6c7bccea 6624
9b136d51
AD		 6625 rdev->accel_working = true;
6626 r = si_startup(rdev);
6627 if (r) {
6628 DRM_ERROR("si startup failed on resume\n");
6629 rdev->accel_working = false;
6630 return r;
6631 }
6632
6633 return r;
6634
6635}
6636
6637int si_suspend(struct radeon_device *rdev)
6638{
6c7bccea 6639 radeon_pm_suspend(rdev);
b530602f 6640 dce6_audio_fini(rdev);
fa3daf9a 6641 radeon_vm_manager_fini(rdev);
9b136d51 6642 si_cp_enable(rdev, false);
8c5fd7ef 6643 cayman_dma_stop(rdev);
1df0d523 6644 if (rdev->has_uvd) {
e409b128 6645 uvd_v1_0_fini(rdev);
1df0d523
AD		 6646 radeon_uvd_suspend(rdev);
6647 }
e16866ec
AD		 6648 si_fini_pg(rdev);
6649 si_fini_cg(rdev);
9b136d51
AD		 6650 si_irq_suspend(rdev);
6651 radeon_wb_disable(rdev);
6652 si_pcie_gart_disable(rdev);
6653 return 0;
6654}
6655
6656/* Plan is to move initialization in that function and use
6657 * helper function so that radeon_device_init pretty much
6658 * do nothing more than calling asic specific function. This
6659 * should also allow to remove a bunch of callback function
6660 * like vram_info.
6661 */
6662int si_init(struct radeon_device *rdev)
6663{
6664 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6665 int r;
6666
9b136d51
AD		 6667 /* Read BIOS */
6668 if (!radeon_get_bios(rdev)) {
6669 if (ASIC_IS_AVIVO(rdev))
6670 return -EINVAL;
6671 }
6672 /* Must be an ATOMBIOS */
6673 if (!rdev->is_atom_bios) {
6674 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6675 return -EINVAL;
6676 }
6677 r = radeon_atombios_init(rdev);
6678 if (r)
6679 return r;
6680
6681 /* Post card if necessary */
6682 if (!radeon_card_posted(rdev)) {
6683 if (!rdev->bios) {
6684 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6685 return -EINVAL;
6686 }
6687 DRM_INFO("GPU not posted. posting now...\n");
6688 atom_asic_init(rdev->mode_info.atom_context);
6689 }
205996c0
AD		 6690 /* init golden registers */
6691 si_init_golden_registers(rdev);
9b136d51
AD		 6692 /* Initialize scratch registers */
6693 si_scratch_init(rdev);
6694 /* Initialize surface registers */
6695 radeon_surface_init(rdev);
6696 /* Initialize clocks */
6697 radeon_get_clock_info(rdev->ddev);
6698
6699 /* Fence driver */
6700 r = radeon_fence_driver_init(rdev);
6701 if (r)
6702 return r;
6703
6704 /* initialize memory controller */
6705 r = si_mc_init(rdev);
6706 if (r)
6707 return r;
6708 /* Memory manager */
6709 r = radeon_bo_init(rdev);
6710 if (r)
6711 return r;
6712
01ac8794
AD		 6713 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6714 !rdev->rlc_fw || !rdev->mc_fw) {
6715 r = si_init_microcode(rdev);
6716 if (r) {
6717 DRM_ERROR("Failed to load firmware!\n");
6718 return r;
6719 }
6720 }
6721
6c7bccea
AD		 6722 /* Initialize power management */
6723 radeon_pm_init(rdev);
6724
9b136d51
AD		 6725 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6726 ring->ring_obj = NULL;
6727 r600_ring_init(rdev, ring, 1024 * 1024);
6728
6729 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6730 ring->ring_obj = NULL;
6731 r600_ring_init(rdev, ring, 1024 * 1024);
6732
6733 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6734 ring->ring_obj = NULL;
6735 r600_ring_init(rdev, ring, 1024 * 1024);
6736
8c5fd7ef
AD		 6737 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6738 ring->ring_obj = NULL;
6739 r600_ring_init(rdev, ring, 64 * 1024);
6740
6741 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6742 ring->ring_obj = NULL;
6743 r600_ring_init(rdev, ring, 64 * 1024);
6744
1df0d523
AD		 6745 if (rdev->has_uvd) {
6746 r = radeon_uvd_init(rdev);
6747 if (!r) {
6748 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6749 ring->ring_obj = NULL;
6750 r600_ring_init(rdev, ring, 4096);
6751 }
f2ba57b5
CK		 6752 }
6753
9b136d51
AD		 6754 rdev->ih.ring_obj = NULL;
6755 r600_ih_ring_init(rdev, 64 * 1024);
6756
6757 r = r600_pcie_gart_init(rdev);
6758 if (r)
6759 return r;
6760
9b136d51 6761 rdev->accel_working = true;
9b136d51
AD		 6762 r = si_startup(rdev);
6763 if (r) {
6764 dev_err(rdev->dev, "disabling GPU acceleration\n");
6765 si_cp_fini(rdev);
8c5fd7ef 6766 cayman_dma_fini(rdev);
9b136d51 6767 si_irq_fini(rdev);
1fd11777 6768 sumo_rlc_fini(rdev);
9b136d51 6769 radeon_wb_fini(rdev);
2898c348 6770 radeon_ib_pool_fini(rdev);
9b136d51
AD		 6771 radeon_vm_manager_fini(rdev);
6772 radeon_irq_kms_fini(rdev);
6773 si_pcie_gart_fini(rdev);
6774 rdev->accel_working = false;
6775 }
6776
6777 /* Don't start up if the MC ucode is missing.
6778 * The default clocks and voltages before the MC ucode
6779 * is loaded are not suffient for advanced operations.
6780 */
6781 if (!rdev->mc_fw) {
6782 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6783 return -EINVAL;
6784 }
6785
6786 return 0;
6787}
6788
6789void si_fini(struct radeon_device *rdev)
6790{
6c7bccea 6791 radeon_pm_fini(rdev);
9b136d51 6792 si_cp_fini(rdev);
8c5fd7ef 6793 cayman_dma_fini(rdev);
f8f84ac5 6794 si_fini_pg(rdev);
e16866ec 6795 si_fini_cg(rdev);
9b136d51 6796 si_irq_fini(rdev);
1fd11777 6797 sumo_rlc_fini(rdev);
9b136d51
AD		 6798 radeon_wb_fini(rdev);
6799 radeon_vm_manager_fini(rdev);
2898c348 6800 radeon_ib_pool_fini(rdev);
9b136d51 6801 radeon_irq_kms_fini(rdev);
2858c00d 6802 if (rdev->has_uvd) {
e409b128 6803 uvd_v1_0_fini(rdev);
1df0d523 6804 radeon_uvd_fini(rdev);
2858c00d 6805 }
9b136d51
AD		 6806 si_pcie_gart_fini(rdev);
6807 r600_vram_scratch_fini(rdev);
6808 radeon_gem_fini(rdev);
9b136d51
AD		 6809 radeon_fence_driver_fini(rdev);
6810 radeon_bo_fini(rdev);
6811 radeon_atombios_fini(rdev);
6812 kfree(rdev->bios);
6813 rdev->bios = NULL;
6814}
6815
6759a0a7 6816/**
d0418894 6817 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6759a0a7
MO		 6818 *
6819 * @rdev: radeon_device pointer
6820 *
6821 * Fetches a GPU clock counter snapshot (SI).
6822 * Returns the 64 bit clock counter snapshot.
6823 */
d0418894 6824uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6759a0a7
MO		 6825{
6826 uint64_t clock;
6827
6828 mutex_lock(&rdev->gpu_clock_mutex);
6829 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6830 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6831 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6832 mutex_unlock(&rdev->gpu_clock_mutex);
6833 return clock;
6834}
2539eb02 6835
2539eb02
CK		 6836int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6837{
facd112d 6838 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
2539eb02
CK		 6839 int r;
6840
4ed10835
CK		 6841 /* bypass vclk and dclk with bclk */
6842 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6843 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6844 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6845
6846 /* put PLL in bypass mode */
6847 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6848
6849 if (!vclk || !dclk) {
6850 /* keep the Bypass mode, put PLL to sleep */
6851 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6852 return 0;
6853 }
6854
facd112d
CK		 6855 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6856 16384, 0x03FFFFFF, 0, 128, 5,
6857 &fb_div, &vclk_div, &dclk_div);
6858 if (r)
6859 return r;
2539eb02
CK		 6860
6861 /* set RESET_ANTI_MUX to 0 */
6862 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6863
6864 /* set VCO_MODE to 1 */
6865 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6866
6867 /* toggle UPLL_SLEEP to 1 then back to 0 */
6868 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6869 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6870
6871 /* deassert UPLL_RESET */
6872 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6873
6874 mdelay(1);
6875
facd112d 6876 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6877 if (r)
6878 return r;
6879
6880 /* assert UPLL_RESET again */
6881 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6882
6883 /* disable spread spectrum. */
6884 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6885
6886 /* set feedback divider */
facd112d 6887 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
2539eb02
CK		 6888
6889 /* set ref divider to 0 */
6890 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6891
facd112d 6892 if (fb_div < 307200)
2539eb02
CK		 6893 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6894 else
6895 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6896
6897 /* set PDIV_A and PDIV_B */
6898 WREG32_P(CG_UPLL_FUNC_CNTL_2,
facd112d 6899 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
2539eb02
CK		 6900 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6901
6902 /* give the PLL some time to settle */
6903 mdelay(15);
6904
6905 /* deassert PLL_RESET */
6906 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6907
6908 mdelay(15);
6909
6910 /* switch from bypass mode to normal mode */
6911 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6912
facd112d 6913 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
2539eb02
CK		 6914 if (r)
6915 return r;
6916
6917 /* switch VCLK and DCLK selection */
6918 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6919 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6920 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6921
6922 mdelay(100);
6923
6924 return 0;
6925}
b9d305df
AD		 6926
6927static void si_pcie_gen3_enable(struct radeon_device *rdev)
6928{
6929 struct pci_dev *root = rdev->pdev->bus->self;
6930 int bridge_pos, gpu_pos;
6931 u32 speed_cntl, mask, current_data_rate;
6932 int ret, i;
6933 u16 tmp16;
6934
6935 if (radeon_pcie_gen2 == 0)
6936 return;
6937
6938 if (rdev->flags & RADEON_IS_IGP)
6939 return;
6940
6941 if (!(rdev->flags & RADEON_IS_PCIE))
6942 return;
6943
6944 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6945 if (ret != 0)
6946 return;
6947
6948 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6949 return;
6950
6951 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6952 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6953 LC_CURRENT_DATA_RATE_SHIFT;
6954 if (mask & DRM_PCIE_SPEED_80) {
6955 if (current_data_rate == 2) {
6956 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6957 return;
6958 }
6959 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6960 } else if (mask & DRM_PCIE_SPEED_50) {
6961 if (current_data_rate == 1) {
6962 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6963 return;
6964 }
6965 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6966 }
6967
6968 bridge_pos = pci_pcie_cap(root);
6969 if (!bridge_pos)
6970 return;
6971
6972 gpu_pos = pci_pcie_cap(rdev->pdev);
6973 if (!gpu_pos)
6974 return;
6975
6976 if (mask & DRM_PCIE_SPEED_80) {
6977 /* re-try equalization if gen3 is not already enabled */
6978 if (current_data_rate != 2) {
6979 u16 bridge_cfg, gpu_cfg;
6980 u16 bridge_cfg2, gpu_cfg2;
6981 u32 max_lw, current_lw, tmp;
6982
6983 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6984 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6985
6986 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6987 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6988
6989 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6990 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6991
6992 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6993 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6994 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6995
6996 if (current_lw < max_lw) {
6997 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6998 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6999 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7000 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7001 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7002 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7003 }
7004 }
7005
7006 for (i = 0; i < 10; i++) {
7007 /* check status */
7008 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
7009 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7010 break;
7011
7012 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7013 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7014
7015 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7016 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7017
7018 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7019 tmp |= LC_SET_QUIESCE;
7020 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7021
7022 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7023 tmp |= LC_REDO_EQ;
7024 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7025
7026 mdelay(100);
7027
7028 /* linkctl */
7029 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7030 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7031 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7032 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7033
7034 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7035 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7036 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7037 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7038
7039 /* linkctl2 */
7040 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7041 tmp16 &= ~((1 << 4) | (7 << 9));
7042 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7043 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7044
7045 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7046 tmp16 &= ~((1 << 4) | (7 << 9));
7047 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7048 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7049
7050 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7051 tmp &= ~LC_SET_QUIESCE;
7052 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7053 }
7054 }
7055 }
7056
7057 /* set the link speed */
7058 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7059 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7060 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7061
7062 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7063 tmp16 &= ~0xf;
7064 if (mask & DRM_PCIE_SPEED_80)
7065 tmp16 |= 3; /* gen3 */
7066 else if (mask & DRM_PCIE_SPEED_50)
7067 tmp16 |= 2; /* gen2 */
7068 else
7069 tmp16 |= 1; /* gen1 */
7070 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7071
7072 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7073 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7074 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7075
7076 for (i = 0; i < rdev->usec_timeout; i++) {
7077 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7078 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7079 break;
7080 udelay(1);
7081 }
7082}
7083
e0bcf165
AD		 7084static void si_program_aspm(struct radeon_device *rdev)
7085{
7086 u32 data, orig;
7087 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7088 bool disable_clkreq = false;
7089
1294d4a3
AD		 7090 if (radeon_aspm == 0)
7091 return;
7092
e0bcf165
AD		 7093 if (!(rdev->flags & RADEON_IS_PCIE))
7094 return;
7095
7096 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7097 data &= ~LC_XMIT_N_FTS_MASK;
7098 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7099 if (orig != data)
7100 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7101
7102 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7103 data |= LC_GO_TO_RECOVERY;
7104 if (orig != data)
7105 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7106
7107 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7108 data |= P_IGNORE_EDB_ERR;
7109 if (orig != data)
7110 WREG32_PCIE(PCIE_P_CNTL, data);
7111
7112 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7113 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7114 data |= LC_PMI_TO_L1_DIS;
7115 if (!disable_l0s)
7116 data |= LC_L0S_INACTIVITY(7);
7117
7118 if (!disable_l1) {
7119 data |= LC_L1_INACTIVITY(7);
7120 data &= ~LC_PMI_TO_L1_DIS;
7121 if (orig != data)
7122 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7123
7124 if (!disable_plloff_in_l1) {
7125 bool clk_req_support;
7126
7127 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7128 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7129 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7130 if (orig != data)
7131 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7132
7133 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7134 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7135 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7136 if (orig != data)
7137 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7138
7139 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7140 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7141 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7142 if (orig != data)
7143 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7144
7145 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7146 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7147 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7148 if (orig != data)
7149 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7150
7151 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7152 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7153 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7154 if (orig != data)
7155 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7156
7157 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7158 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7159 if (orig != data)
7160 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7161
7162 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7163 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7164 if (orig != data)
7165 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7166
7167 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7168 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7169 if (orig != data)
7170 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7171
7172 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7173 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7174 if (orig != data)
7175 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7176
7177 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7178 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7179 if (orig != data)
7180 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7181
7182 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7183 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7184 if (orig != data)
7185 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7186
7187 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7188 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7189 if (orig != data)
7190 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7191 }
7192 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7193 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7194 data |= LC_DYN_LANES_PWR_STATE(3);
7195 if (orig != data)
7196 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7197
7198 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7199 data &= ~LS2_EXIT_TIME_MASK;
7200 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7201 data |= LS2_EXIT_TIME(5);
7202 if (orig != data)
7203 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7204
7205 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7206 data &= ~LS2_EXIT_TIME_MASK;
7207 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7208 data |= LS2_EXIT_TIME(5);
7209 if (orig != data)
7210 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7211
7212 if (!disable_clkreq) {
7213 struct pci_dev *root = rdev->pdev->bus->self;
7214 u32 lnkcap;
7215
7216 clk_req_support = false;
7217 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7218 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7219 clk_req_support = true;
7220 } else {
7221 clk_req_support = false;
7222 }
7223
7224 if (clk_req_support) {
7225 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7226 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7227 if (orig != data)
7228 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7229
7230 orig = data = RREG32(THM_CLK_CNTL);
7231 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7232 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7233 if (orig != data)
7234 WREG32(THM_CLK_CNTL, data);
7235
7236 orig = data = RREG32(MISC_CLK_CNTL);
7237 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7238 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7239 if (orig != data)
7240 WREG32(MISC_CLK_CNTL, data);
7241
7242 orig = data = RREG32(CG_CLKPIN_CNTL);
7243 data &= ~BCLK_AS_XCLK;
7244 if (orig != data)
7245 WREG32(CG_CLKPIN_CNTL, data);
7246
7247 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7248 data &= ~FORCE_BIF_REFCLK_EN;
7249 if (orig != data)
7250 WREG32(CG_CLKPIN_CNTL_2, data);
7251
7252 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7253 data &= ~MPLL_CLKOUT_SEL_MASK;
7254 data |= MPLL_CLKOUT_SEL(4);
7255 if (orig != data)
7256 WREG32(MPLL_BYPASSCLK_SEL, data);
7257
7258 orig = data = RREG32(SPLL_CNTL_MODE);
7259 data &= ~SPLL_REFCLK_SEL_MASK;
7260 if (orig != data)
7261 WREG32(SPLL_CNTL_MODE, data);
7262 }
7263 }
7264 } else {
7265 if (orig != data)
7266 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7267 }
7268
7269 orig = data = RREG32_PCIE(PCIE_CNTL2);
7270 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7271 if (orig != data)
7272 WREG32_PCIE(PCIE_CNTL2, data);
7273
7274 if (!disable_l0s) {
7275 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7276 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7277 data = RREG32_PCIE(PCIE_LC_STATUS1);
7278 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7279 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7280 data &= ~LC_L0S_INACTIVITY_MASK;
7281 if (orig != data)
7282 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7283 }
7284 }
7285 }
7286}
Linux 6.x block layer and io_uring tree(s)