2 * Copyright 2011 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
27 #include <linux/module.h>
30 #include "radeon_asic.h"
31 #include <drm/radeon_drm.h>
34 #include "si_blit_shaders.h"
35 #include "clearstate_si.h"
36 #include "radeon_ucode.h"
39 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
40 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
41 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
42 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
43 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
44 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
45 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
46 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
47 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
48 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
49 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
50 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
51 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
52 MODULE_FIRMWARE("radeon/VERDE_me.bin");
53 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
54 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
55 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
56 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
57 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
58 MODULE_FIRMWARE("radeon/OLAND_me.bin");
59 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
60 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
61 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
62 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
63 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
64 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
65 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
66 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
67 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
68 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
70 static void si_pcie_gen3_enable(struct radeon_device *rdev);
71 static void si_program_aspm(struct radeon_device *rdev);
72 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
73 extern void r600_ih_ring_fini(struct radeon_device *rdev);
74 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
75 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
76 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
77 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
78 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
79 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
81 static const u32 verde_rlc_save_restore_register_list[] =
83 (0x8000 << 16) | (0x98f4 >> 2),
85 (0x8040 << 16) | (0x98f4 >> 2),
87 (0x8000 << 16) | (0xe80 >> 2),
89 (0x8040 << 16) | (0xe80 >> 2),
91 (0x8000 << 16) | (0x89bc >> 2),
93 (0x8040 << 16) | (0x89bc >> 2),
95 (0x8000 << 16) | (0x8c1c >> 2),
97 (0x8040 << 16) | (0x8c1c >> 2),
99 (0x9c00 << 16) | (0x98f0 >> 2),
101 (0x9c00 << 16) | (0xe7c >> 2),
103 (0x8000 << 16) | (0x9148 >> 2),
105 (0x8040 << 16) | (0x9148 >> 2),
107 (0x9c00 << 16) | (0x9150 >> 2),
109 (0x9c00 << 16) | (0x897c >> 2),
111 (0x9c00 << 16) | (0x8d8c >> 2),
113 (0x9c00 << 16) | (0xac54 >> 2),
116 (0x9c00 << 16) | (0x98f8 >> 2),
118 (0x9c00 << 16) | (0x9910 >> 2),
120 (0x9c00 << 16) | (0x9914 >> 2),
122 (0x9c00 << 16) | (0x9918 >> 2),
124 (0x9c00 << 16) | (0x991c >> 2),
126 (0x9c00 << 16) | (0x9920 >> 2),
128 (0x9c00 << 16) | (0x9924 >> 2),
130 (0x9c00 << 16) | (0x9928 >> 2),
132 (0x9c00 << 16) | (0x992c >> 2),
134 (0x9c00 << 16) | (0x9930 >> 2),
136 (0x9c00 << 16) | (0x9934 >> 2),
138 (0x9c00 << 16) | (0x9938 >> 2),
140 (0x9c00 << 16) | (0x993c >> 2),
142 (0x9c00 << 16) | (0x9940 >> 2),
144 (0x9c00 << 16) | (0x9944 >> 2),
146 (0x9c00 << 16) | (0x9948 >> 2),
148 (0x9c00 << 16) | (0x994c >> 2),
150 (0x9c00 << 16) | (0x9950 >> 2),
152 (0x9c00 << 16) | (0x9954 >> 2),
154 (0x9c00 << 16) | (0x9958 >> 2),
156 (0x9c00 << 16) | (0x995c >> 2),
158 (0x9c00 << 16) | (0x9960 >> 2),
160 (0x9c00 << 16) | (0x9964 >> 2),
162 (0x9c00 << 16) | (0x9968 >> 2),
164 (0x9c00 << 16) | (0x996c >> 2),
166 (0x9c00 << 16) | (0x9970 >> 2),
168 (0x9c00 << 16) | (0x9974 >> 2),
170 (0x9c00 << 16) | (0x9978 >> 2),
172 (0x9c00 << 16) | (0x997c >> 2),
174 (0x9c00 << 16) | (0x9980 >> 2),
176 (0x9c00 << 16) | (0x9984 >> 2),
178 (0x9c00 << 16) | (0x9988 >> 2),
180 (0x9c00 << 16) | (0x998c >> 2),
182 (0x9c00 << 16) | (0x8c00 >> 2),
184 (0x9c00 << 16) | (0x8c14 >> 2),
186 (0x9c00 << 16) | (0x8c04 >> 2),
188 (0x9c00 << 16) | (0x8c08 >> 2),
190 (0x8000 << 16) | (0x9b7c >> 2),
192 (0x8040 << 16) | (0x9b7c >> 2),
194 (0x8000 << 16) | (0xe84 >> 2),
196 (0x8040 << 16) | (0xe84 >> 2),
198 (0x8000 << 16) | (0x89c0 >> 2),
200 (0x8040 << 16) | (0x89c0 >> 2),
202 (0x8000 << 16) | (0x914c >> 2),
204 (0x8040 << 16) | (0x914c >> 2),
206 (0x8000 << 16) | (0x8c20 >> 2),
208 (0x8040 << 16) | (0x8c20 >> 2),
210 (0x8000 << 16) | (0x9354 >> 2),
212 (0x8040 << 16) | (0x9354 >> 2),
214 (0x9c00 << 16) | (0x9060 >> 2),
216 (0x9c00 << 16) | (0x9364 >> 2),
218 (0x9c00 << 16) | (0x9100 >> 2),
220 (0x9c00 << 16) | (0x913c >> 2),
222 (0x8000 << 16) | (0x90e0 >> 2),
224 (0x8000 << 16) | (0x90e4 >> 2),
226 (0x8000 << 16) | (0x90e8 >> 2),
228 (0x8040 << 16) | (0x90e0 >> 2),
230 (0x8040 << 16) | (0x90e4 >> 2),
232 (0x8040 << 16) | (0x90e8 >> 2),
234 (0x9c00 << 16) | (0x8bcc >> 2),
236 (0x9c00 << 16) | (0x8b24 >> 2),
238 (0x9c00 << 16) | (0x88c4 >> 2),
240 (0x9c00 << 16) | (0x8e50 >> 2),
242 (0x9c00 << 16) | (0x8c0c >> 2),
244 (0x9c00 << 16) | (0x8e58 >> 2),
246 (0x9c00 << 16) | (0x8e5c >> 2),
248 (0x9c00 << 16) | (0x9508 >> 2),
250 (0x9c00 << 16) | (0x950c >> 2),
252 (0x9c00 << 16) | (0x9494 >> 2),
254 (0x9c00 << 16) | (0xac0c >> 2),
256 (0x9c00 << 16) | (0xac10 >> 2),
258 (0x9c00 << 16) | (0xac14 >> 2),
260 (0x9c00 << 16) | (0xae00 >> 2),
262 (0x9c00 << 16) | (0xac08 >> 2),
264 (0x9c00 << 16) | (0x88d4 >> 2),
266 (0x9c00 << 16) | (0x88c8 >> 2),
268 (0x9c00 << 16) | (0x88cc >> 2),
270 (0x9c00 << 16) | (0x89b0 >> 2),
272 (0x9c00 << 16) | (0x8b10 >> 2),
274 (0x9c00 << 16) | (0x8a14 >> 2),
276 (0x9c00 << 16) | (0x9830 >> 2),
278 (0x9c00 << 16) | (0x9834 >> 2),
280 (0x9c00 << 16) | (0x9838 >> 2),
282 (0x9c00 << 16) | (0x9a10 >> 2),
284 (0x8000 << 16) | (0x9870 >> 2),
286 (0x8000 << 16) | (0x9874 >> 2),
288 (0x8001 << 16) | (0x9870 >> 2),
290 (0x8001 << 16) | (0x9874 >> 2),
292 (0x8040 << 16) | (0x9870 >> 2),
294 (0x8040 << 16) | (0x9874 >> 2),
296 (0x8041 << 16) | (0x9870 >> 2),
298 (0x8041 << 16) | (0x9874 >> 2),
303 static const u32 tahiti_golden_rlc_registers[] =
305 0xc424, 0xffffffff, 0x00601005,
306 0xc47c, 0xffffffff, 0x10104040,
307 0xc488, 0xffffffff, 0x0100000a,
308 0xc314, 0xffffffff, 0x00000800,
309 0xc30c, 0xffffffff, 0x800000f4,
310 0xf4a8, 0xffffffff, 0x00000000
313 static const u32 tahiti_golden_registers[] =
315 0x9a10, 0x00010000, 0x00018208,
316 0x9830, 0xffffffff, 0x00000000,
317 0x9834, 0xf00fffff, 0x00000400,
318 0x9838, 0x0002021c, 0x00020200,
319 0xc78, 0x00000080, 0x00000000,
320 0xd030, 0x000300c0, 0x00800040,
321 0xd830, 0x000300c0, 0x00800040,
322 0x5bb0, 0x000000f0, 0x00000070,
323 0x5bc0, 0x00200000, 0x50100000,
324 0x7030, 0x31000311, 0x00000011,
325 0x277c, 0x00000003, 0x000007ff,
326 0x240c, 0x000007ff, 0x00000000,
327 0x8a14, 0xf000001f, 0x00000007,
328 0x8b24, 0xffffffff, 0x00ffffff,
329 0x8b10, 0x0000ff0f, 0x00000000,
330 0x28a4c, 0x07ffffff, 0x4e000000,
331 0x28350, 0x3f3f3fff, 0x2a00126a,
332 0x30, 0x000000ff, 0x0040,
333 0x34, 0x00000040, 0x00004040,
334 0x9100, 0x07ffffff, 0x03000000,
335 0x8e88, 0x01ff1f3f, 0x00000000,
336 0x8e84, 0x01ff1f3f, 0x00000000,
337 0x9060, 0x0000007f, 0x00000020,
338 0x9508, 0x00010000, 0x00010000,
339 0xac14, 0x00000200, 0x000002fb,
340 0xac10, 0xffffffff, 0x0000543b,
341 0xac0c, 0xffffffff, 0xa9210876,
342 0x88d0, 0xffffffff, 0x000fff40,
343 0x88d4, 0x0000001f, 0x00000010,
344 0x1410, 0x20000000, 0x20fffed8,
345 0x15c0, 0x000c0fc0, 0x000c0400
348 static const u32 tahiti_golden_registers2[] =
350 0xc64, 0x00000001, 0x00000001
353 static const u32 pitcairn_golden_rlc_registers[] =
355 0xc424, 0xffffffff, 0x00601004,
356 0xc47c, 0xffffffff, 0x10102020,
357 0xc488, 0xffffffff, 0x01000020,
358 0xc314, 0xffffffff, 0x00000800,
359 0xc30c, 0xffffffff, 0x800000a4
362 static const u32 pitcairn_golden_registers[] =
364 0x9a10, 0x00010000, 0x00018208,
365 0x9830, 0xffffffff, 0x00000000,
366 0x9834, 0xf00fffff, 0x00000400,
367 0x9838, 0x0002021c, 0x00020200,
368 0xc78, 0x00000080, 0x00000000,
369 0xd030, 0x000300c0, 0x00800040,
370 0xd830, 0x000300c0, 0x00800040,
371 0x5bb0, 0x000000f0, 0x00000070,
372 0x5bc0, 0x00200000, 0x50100000,
373 0x7030, 0x31000311, 0x00000011,
374 0x2ae4, 0x00073ffe, 0x000022a2,
375 0x240c, 0x000007ff, 0x00000000,
376 0x8a14, 0xf000001f, 0x00000007,
377 0x8b24, 0xffffffff, 0x00ffffff,
378 0x8b10, 0x0000ff0f, 0x00000000,
379 0x28a4c, 0x07ffffff, 0x4e000000,
380 0x28350, 0x3f3f3fff, 0x2a00126a,
381 0x30, 0x000000ff, 0x0040,
382 0x34, 0x00000040, 0x00004040,
383 0x9100, 0x07ffffff, 0x03000000,
384 0x9060, 0x0000007f, 0x00000020,
385 0x9508, 0x00010000, 0x00010000,
386 0xac14, 0x000003ff, 0x000000f7,
387 0xac10, 0xffffffff, 0x00000000,
388 0xac0c, 0xffffffff, 0x32761054,
389 0x88d4, 0x0000001f, 0x00000010,
390 0x15c0, 0x000c0fc0, 0x000c0400
393 static const u32 verde_golden_rlc_registers[] =
395 0xc424, 0xffffffff, 0x033f1005,
396 0xc47c, 0xffffffff, 0x10808020,
397 0xc488, 0xffffffff, 0x00800008,
398 0xc314, 0xffffffff, 0x00001000,
399 0xc30c, 0xffffffff, 0x80010014
402 static const u32 verde_golden_registers[] =
404 0x9a10, 0x00010000, 0x00018208,
405 0x9830, 0xffffffff, 0x00000000,
406 0x9834, 0xf00fffff, 0x00000400,
407 0x9838, 0x0002021c, 0x00020200,
408 0xc78, 0x00000080, 0x00000000,
409 0xd030, 0x000300c0, 0x00800040,
410 0xd030, 0x000300c0, 0x00800040,
411 0xd830, 0x000300c0, 0x00800040,
412 0xd830, 0x000300c0, 0x00800040,
413 0x5bb0, 0x000000f0, 0x00000070,
414 0x5bc0, 0x00200000, 0x50100000,
415 0x7030, 0x31000311, 0x00000011,
416 0x2ae4, 0x00073ffe, 0x000022a2,
417 0x2ae4, 0x00073ffe, 0x000022a2,
418 0x2ae4, 0x00073ffe, 0x000022a2,
419 0x240c, 0x000007ff, 0x00000000,
420 0x240c, 0x000007ff, 0x00000000,
421 0x240c, 0x000007ff, 0x00000000,
422 0x8a14, 0xf000001f, 0x00000007,
423 0x8a14, 0xf000001f, 0x00000007,
424 0x8a14, 0xf000001f, 0x00000007,
425 0x8b24, 0xffffffff, 0x00ffffff,
426 0x8b10, 0x0000ff0f, 0x00000000,
427 0x28a4c, 0x07ffffff, 0x4e000000,
428 0x28350, 0x3f3f3fff, 0x0000124a,
429 0x28350, 0x3f3f3fff, 0x0000124a,
430 0x28350, 0x3f3f3fff, 0x0000124a,
431 0x30, 0x000000ff, 0x0040,
432 0x34, 0x00000040, 0x00004040,
433 0x9100, 0x07ffffff, 0x03000000,
434 0x9100, 0x07ffffff, 0x03000000,
435 0x8e88, 0x01ff1f3f, 0x00000000,
436 0x8e88, 0x01ff1f3f, 0x00000000,
437 0x8e88, 0x01ff1f3f, 0x00000000,
438 0x8e84, 0x01ff1f3f, 0x00000000,
439 0x8e84, 0x01ff1f3f, 0x00000000,
440 0x8e84, 0x01ff1f3f, 0x00000000,
441 0x9060, 0x0000007f, 0x00000020,
442 0x9508, 0x00010000, 0x00010000,
443 0xac14, 0x000003ff, 0x00000003,
444 0xac14, 0x000003ff, 0x00000003,
445 0xac14, 0x000003ff, 0x00000003,
446 0xac10, 0xffffffff, 0x00000000,
447 0xac10, 0xffffffff, 0x00000000,
448 0xac10, 0xffffffff, 0x00000000,
449 0xac0c, 0xffffffff, 0x00001032,
450 0xac0c, 0xffffffff, 0x00001032,
451 0xac0c, 0xffffffff, 0x00001032,
452 0x88d4, 0x0000001f, 0x00000010,
453 0x88d4, 0x0000001f, 0x00000010,
454 0x88d4, 0x0000001f, 0x00000010,
455 0x15c0, 0x000c0fc0, 0x000c0400
458 static const u32 oland_golden_rlc_registers[] =
460 0xc424, 0xffffffff, 0x00601005,
461 0xc47c, 0xffffffff, 0x10104040,
462 0xc488, 0xffffffff, 0x0100000a,
463 0xc314, 0xffffffff, 0x00000800,
464 0xc30c, 0xffffffff, 0x800000f4
467 static const u32 oland_golden_registers[] =
469 0x9a10, 0x00010000, 0x00018208,
470 0x9830, 0xffffffff, 0x00000000,
471 0x9834, 0xf00fffff, 0x00000400,
472 0x9838, 0x0002021c, 0x00020200,
473 0xc78, 0x00000080, 0x00000000,
474 0xd030, 0x000300c0, 0x00800040,
475 0xd830, 0x000300c0, 0x00800040,
476 0x5bb0, 0x000000f0, 0x00000070,
477 0x5bc0, 0x00200000, 0x50100000,
478 0x7030, 0x31000311, 0x00000011,
479 0x2ae4, 0x00073ffe, 0x000022a2,
480 0x240c, 0x000007ff, 0x00000000,
481 0x8a14, 0xf000001f, 0x00000007,
482 0x8b24, 0xffffffff, 0x00ffffff,
483 0x8b10, 0x0000ff0f, 0x00000000,
484 0x28a4c, 0x07ffffff, 0x4e000000,
485 0x28350, 0x3f3f3fff, 0x00000082,
486 0x30, 0x000000ff, 0x0040,
487 0x34, 0x00000040, 0x00004040,
488 0x9100, 0x07ffffff, 0x03000000,
489 0x9060, 0x0000007f, 0x00000020,
490 0x9508, 0x00010000, 0x00010000,
491 0xac14, 0x000003ff, 0x000000f3,
492 0xac10, 0xffffffff, 0x00000000,
493 0xac0c, 0xffffffff, 0x00003210,
494 0x88d4, 0x0000001f, 0x00000010,
495 0x15c0, 0x000c0fc0, 0x000c0400
498 static const u32 hainan_golden_registers[] =
500 0x9a10, 0x00010000, 0x00018208,
501 0x9830, 0xffffffff, 0x00000000,
502 0x9834, 0xf00fffff, 0x00000400,
503 0x9838, 0x0002021c, 0x00020200,
504 0xd0c0, 0xff000fff, 0x00000100,
505 0xd030, 0x000300c0, 0x00800040,
506 0xd8c0, 0xff000fff, 0x00000100,
507 0xd830, 0x000300c0, 0x00800040,
508 0x2ae4, 0x00073ffe, 0x000022a2,
509 0x240c, 0x000007ff, 0x00000000,
510 0x8a14, 0xf000001f, 0x00000007,
511 0x8b24, 0xffffffff, 0x00ffffff,
512 0x8b10, 0x0000ff0f, 0x00000000,
513 0x28a4c, 0x07ffffff, 0x4e000000,
514 0x28350, 0x3f3f3fff, 0x00000000,
515 0x30, 0x000000ff, 0x0040,
516 0x34, 0x00000040, 0x00004040,
517 0x9100, 0x03e00000, 0x03600000,
518 0x9060, 0x0000007f, 0x00000020,
519 0x9508, 0x00010000, 0x00010000,
520 0xac14, 0x000003ff, 0x000000f1,
521 0xac10, 0xffffffff, 0x00000000,
522 0xac0c, 0xffffffff, 0x00003210,
523 0x88d4, 0x0000001f, 0x00000010,
524 0x15c0, 0x000c0fc0, 0x000c0400
527 static const u32 hainan_golden_registers2[] =
529 0x98f8, 0xffffffff, 0x02010001
532 static const u32 tahiti_mgcg_cgcg_init[] =
534 0xc400, 0xffffffff, 0xfffffffc,
535 0x802c, 0xffffffff, 0xe0000000,
536 0x9a60, 0xffffffff, 0x00000100,
537 0x92a4, 0xffffffff, 0x00000100,
538 0xc164, 0xffffffff, 0x00000100,
539 0x9774, 0xffffffff, 0x00000100,
540 0x8984, 0xffffffff, 0x06000100,
541 0x8a18, 0xffffffff, 0x00000100,
542 0x92a0, 0xffffffff, 0x00000100,
543 0xc380, 0xffffffff, 0x00000100,
544 0x8b28, 0xffffffff, 0x00000100,
545 0x9144, 0xffffffff, 0x00000100,
546 0x8d88, 0xffffffff, 0x00000100,
547 0x8d8c, 0xffffffff, 0x00000100,
548 0x9030, 0xffffffff, 0x00000100,
549 0x9034, 0xffffffff, 0x00000100,
550 0x9038, 0xffffffff, 0x00000100,
551 0x903c, 0xffffffff, 0x00000100,
552 0xad80, 0xffffffff, 0x00000100,
553 0xac54, 0xffffffff, 0x00000100,
554 0x897c, 0xffffffff, 0x06000100,
555 0x9868, 0xffffffff, 0x00000100,
556 0x9510, 0xffffffff, 0x00000100,
557 0xaf04, 0xffffffff, 0x00000100,
558 0xae04, 0xffffffff, 0x00000100,
559 0x949c, 0xffffffff, 0x00000100,
560 0x802c, 0xffffffff, 0xe0000000,
561 0x9160, 0xffffffff, 0x00010000,
562 0x9164, 0xffffffff, 0x00030002,
563 0x9168, 0xffffffff, 0x00040007,
564 0x916c, 0xffffffff, 0x00060005,
565 0x9170, 0xffffffff, 0x00090008,
566 0x9174, 0xffffffff, 0x00020001,
567 0x9178, 0xffffffff, 0x00040003,
568 0x917c, 0xffffffff, 0x00000007,
569 0x9180, 0xffffffff, 0x00060005,
570 0x9184, 0xffffffff, 0x00090008,
571 0x9188, 0xffffffff, 0x00030002,
572 0x918c, 0xffffffff, 0x00050004,
573 0x9190, 0xffffffff, 0x00000008,
574 0x9194, 0xffffffff, 0x00070006,
575 0x9198, 0xffffffff, 0x000a0009,
576 0x919c, 0xffffffff, 0x00040003,
577 0x91a0, 0xffffffff, 0x00060005,
578 0x91a4, 0xffffffff, 0x00000009,
579 0x91a8, 0xffffffff, 0x00080007,
580 0x91ac, 0xffffffff, 0x000b000a,
581 0x91b0, 0xffffffff, 0x00050004,
582 0x91b4, 0xffffffff, 0x00070006,
583 0x91b8, 0xffffffff, 0x0008000b,
584 0x91bc, 0xffffffff, 0x000a0009,
585 0x91c0, 0xffffffff, 0x000d000c,
586 0x91c4, 0xffffffff, 0x00060005,
587 0x91c8, 0xffffffff, 0x00080007,
588 0x91cc, 0xffffffff, 0x0000000b,
589 0x91d0, 0xffffffff, 0x000a0009,
590 0x91d4, 0xffffffff, 0x000d000c,
591 0x91d8, 0xffffffff, 0x00070006,
592 0x91dc, 0xffffffff, 0x00090008,
593 0x91e0, 0xffffffff, 0x0000000c,
594 0x91e4, 0xffffffff, 0x000b000a,
595 0x91e8, 0xffffffff, 0x000e000d,
596 0x91ec, 0xffffffff, 0x00080007,
597 0x91f0, 0xffffffff, 0x000a0009,
598 0x91f4, 0xffffffff, 0x0000000d,
599 0x91f8, 0xffffffff, 0x000c000b,
600 0x91fc, 0xffffffff, 0x000f000e,
601 0x9200, 0xffffffff, 0x00090008,
602 0x9204, 0xffffffff, 0x000b000a,
603 0x9208, 0xffffffff, 0x000c000f,
604 0x920c, 0xffffffff, 0x000e000d,
605 0x9210, 0xffffffff, 0x00110010,
606 0x9214, 0xffffffff, 0x000a0009,
607 0x9218, 0xffffffff, 0x000c000b,
608 0x921c, 0xffffffff, 0x0000000f,
609 0x9220, 0xffffffff, 0x000e000d,
610 0x9224, 0xffffffff, 0x00110010,
611 0x9228, 0xffffffff, 0x000b000a,
612 0x922c, 0xffffffff, 0x000d000c,
613 0x9230, 0xffffffff, 0x00000010,
614 0x9234, 0xffffffff, 0x000f000e,
615 0x9238, 0xffffffff, 0x00120011,
616 0x923c, 0xffffffff, 0x000c000b,
617 0x9240, 0xffffffff, 0x000e000d,
618 0x9244, 0xffffffff, 0x00000011,
619 0x9248, 0xffffffff, 0x0010000f,
620 0x924c, 0xffffffff, 0x00130012,
621 0x9250, 0xffffffff, 0x000d000c,
622 0x9254, 0xffffffff, 0x000f000e,
623 0x9258, 0xffffffff, 0x00100013,
624 0x925c, 0xffffffff, 0x00120011,
625 0x9260, 0xffffffff, 0x00150014,
626 0x9264, 0xffffffff, 0x000e000d,
627 0x9268, 0xffffffff, 0x0010000f,
628 0x926c, 0xffffffff, 0x00000013,
629 0x9270, 0xffffffff, 0x00120011,
630 0x9274, 0xffffffff, 0x00150014,
631 0x9278, 0xffffffff, 0x000f000e,
632 0x927c, 0xffffffff, 0x00110010,
633 0x9280, 0xffffffff, 0x00000014,
634 0x9284, 0xffffffff, 0x00130012,
635 0x9288, 0xffffffff, 0x00160015,
636 0x928c, 0xffffffff, 0x0010000f,
637 0x9290, 0xffffffff, 0x00120011,
638 0x9294, 0xffffffff, 0x00000015,
639 0x9298, 0xffffffff, 0x00140013,
640 0x929c, 0xffffffff, 0x00170016,
641 0x9150, 0xffffffff, 0x96940200,
642 0x8708, 0xffffffff, 0x00900100,
643 0xc478, 0xffffffff, 0x00000080,
644 0xc404, 0xffffffff, 0x0020003f,
645 0x30, 0xffffffff, 0x0000001c,
646 0x34, 0x000f0000, 0x000f0000,
647 0x160c, 0xffffffff, 0x00000100,
648 0x1024, 0xffffffff, 0x00000100,
649 0x102c, 0x00000101, 0x00000000,
650 0x20a8, 0xffffffff, 0x00000104,
651 0x264c, 0x000c0000, 0x000c0000,
652 0x2648, 0x000c0000, 0x000c0000,
653 0x55e4, 0xff000fff, 0x00000100,
654 0x55e8, 0x00000001, 0x00000001,
655 0x2f50, 0x00000001, 0x00000001,
656 0x30cc, 0xc0000fff, 0x00000104,
657 0xc1e4, 0x00000001, 0x00000001,
658 0xd0c0, 0xfffffff0, 0x00000100,
659 0xd8c0, 0xfffffff0, 0x00000100
662 static const u32 pitcairn_mgcg_cgcg_init[] =
664 0xc400, 0xffffffff, 0xfffffffc,
665 0x802c, 0xffffffff, 0xe0000000,
666 0x9a60, 0xffffffff, 0x00000100,
667 0x92a4, 0xffffffff, 0x00000100,
668 0xc164, 0xffffffff, 0x00000100,
669 0x9774, 0xffffffff, 0x00000100,
670 0x8984, 0xffffffff, 0x06000100,
671 0x8a18, 0xffffffff, 0x00000100,
672 0x92a0, 0xffffffff, 0x00000100,
673 0xc380, 0xffffffff, 0x00000100,
674 0x8b28, 0xffffffff, 0x00000100,
675 0x9144, 0xffffffff, 0x00000100,
676 0x8d88, 0xffffffff, 0x00000100,
677 0x8d8c, 0xffffffff, 0x00000100,
678 0x9030, 0xffffffff, 0x00000100,
679 0x9034, 0xffffffff, 0x00000100,
680 0x9038, 0xffffffff, 0x00000100,
681 0x903c, 0xffffffff, 0x00000100,
682 0xad80, 0xffffffff, 0x00000100,
683 0xac54, 0xffffffff, 0x00000100,
684 0x897c, 0xffffffff, 0x06000100,
685 0x9868, 0xffffffff, 0x00000100,
686 0x9510, 0xffffffff, 0x00000100,
687 0xaf04, 0xffffffff, 0x00000100,
688 0xae04, 0xffffffff, 0x00000100,
689 0x949c, 0xffffffff, 0x00000100,
690 0x802c, 0xffffffff, 0xe0000000,
691 0x9160, 0xffffffff, 0x00010000,
692 0x9164, 0xffffffff, 0x00030002,
693 0x9168, 0xffffffff, 0x00040007,
694 0x916c, 0xffffffff, 0x00060005,
695 0x9170, 0xffffffff, 0x00090008,
696 0x9174, 0xffffffff, 0x00020001,
697 0x9178, 0xffffffff, 0x00040003,
698 0x917c, 0xffffffff, 0x00000007,
699 0x9180, 0xffffffff, 0x00060005,
700 0x9184, 0xffffffff, 0x00090008,
701 0x9188, 0xffffffff, 0x00030002,
702 0x918c, 0xffffffff, 0x00050004,
703 0x9190, 0xffffffff, 0x00000008,
704 0x9194, 0xffffffff, 0x00070006,
705 0x9198, 0xffffffff, 0x000a0009,
706 0x919c, 0xffffffff, 0x00040003,
707 0x91a0, 0xffffffff, 0x00060005,
708 0x91a4, 0xffffffff, 0x00000009,
709 0x91a8, 0xffffffff, 0x00080007,
710 0x91ac, 0xffffffff, 0x000b000a,
711 0x91b0, 0xffffffff, 0x00050004,
712 0x91b4, 0xffffffff, 0x00070006,
713 0x91b8, 0xffffffff, 0x0008000b,
714 0x91bc, 0xffffffff, 0x000a0009,
715 0x91c0, 0xffffffff, 0x000d000c,
716 0x9200, 0xffffffff, 0x00090008,
717 0x9204, 0xffffffff, 0x000b000a,
718 0x9208, 0xffffffff, 0x000c000f,
719 0x920c, 0xffffffff, 0x000e000d,
720 0x9210, 0xffffffff, 0x00110010,
721 0x9214, 0xffffffff, 0x000a0009,
722 0x9218, 0xffffffff, 0x000c000b,
723 0x921c, 0xffffffff, 0x0000000f,
724 0x9220, 0xffffffff, 0x000e000d,
725 0x9224, 0xffffffff, 0x00110010,
726 0x9228, 0xffffffff, 0x000b000a,
727 0x922c, 0xffffffff, 0x000d000c,
728 0x9230, 0xffffffff, 0x00000010,
729 0x9234, 0xffffffff, 0x000f000e,
730 0x9238, 0xffffffff, 0x00120011,
731 0x923c, 0xffffffff, 0x000c000b,
732 0x9240, 0xffffffff, 0x000e000d,
733 0x9244, 0xffffffff, 0x00000011,
734 0x9248, 0xffffffff, 0x0010000f,
735 0x924c, 0xffffffff, 0x00130012,
736 0x9250, 0xffffffff, 0x000d000c,
737 0x9254, 0xffffffff, 0x000f000e,
738 0x9258, 0xffffffff, 0x00100013,
739 0x925c, 0xffffffff, 0x00120011,
740 0x9260, 0xffffffff, 0x00150014,
741 0x9150, 0xffffffff, 0x96940200,
742 0x8708, 0xffffffff, 0x00900100,
743 0xc478, 0xffffffff, 0x00000080,
744 0xc404, 0xffffffff, 0x0020003f,
745 0x30, 0xffffffff, 0x0000001c,
746 0x34, 0x000f0000, 0x000f0000,
747 0x160c, 0xffffffff, 0x00000100,
748 0x1024, 0xffffffff, 0x00000100,
749 0x102c, 0x00000101, 0x00000000,
750 0x20a8, 0xffffffff, 0x00000104,
751 0x55e4, 0xff000fff, 0x00000100,
752 0x55e8, 0x00000001, 0x00000001,
753 0x2f50, 0x00000001, 0x00000001,
754 0x30cc, 0xc0000fff, 0x00000104,
755 0xc1e4, 0x00000001, 0x00000001,
756 0xd0c0, 0xfffffff0, 0x00000100,
757 0xd8c0, 0xfffffff0, 0x00000100
760 static const u32 verde_mgcg_cgcg_init[] =
762 0xc400, 0xffffffff, 0xfffffffc,
763 0x802c, 0xffffffff, 0xe0000000,
764 0x9a60, 0xffffffff, 0x00000100,
765 0x92a4, 0xffffffff, 0x00000100,
766 0xc164, 0xffffffff, 0x00000100,
767 0x9774, 0xffffffff, 0x00000100,
768 0x8984, 0xffffffff, 0x06000100,
769 0x8a18, 0xffffffff, 0x00000100,
770 0x92a0, 0xffffffff, 0x00000100,
771 0xc380, 0xffffffff, 0x00000100,
772 0x8b28, 0xffffffff, 0x00000100,
773 0x9144, 0xffffffff, 0x00000100,
774 0x8d88, 0xffffffff, 0x00000100,
775 0x8d8c, 0xffffffff, 0x00000100,
776 0x9030, 0xffffffff, 0x00000100,
777 0x9034, 0xffffffff, 0x00000100,
778 0x9038, 0xffffffff, 0x00000100,
779 0x903c, 0xffffffff, 0x00000100,
780 0xad80, 0xffffffff, 0x00000100,
781 0xac54, 0xffffffff, 0x00000100,
782 0x897c, 0xffffffff, 0x06000100,
783 0x9868, 0xffffffff, 0x00000100,
784 0x9510, 0xffffffff, 0x00000100,
785 0xaf04, 0xffffffff, 0x00000100,
786 0xae04, 0xffffffff, 0x00000100,
787 0x949c, 0xffffffff, 0x00000100,
788 0x802c, 0xffffffff, 0xe0000000,
789 0x9160, 0xffffffff, 0x00010000,
790 0x9164, 0xffffffff, 0x00030002,
791 0x9168, 0xffffffff, 0x00040007,
792 0x916c, 0xffffffff, 0x00060005,
793 0x9170, 0xffffffff, 0x00090008,
794 0x9174, 0xffffffff, 0x00020001,
795 0x9178, 0xffffffff, 0x00040003,
796 0x917c, 0xffffffff, 0x00000007,
797 0x9180, 0xffffffff, 0x00060005,
798 0x9184, 0xffffffff, 0x00090008,
799 0x9188, 0xffffffff, 0x00030002,
800 0x918c, 0xffffffff, 0x00050004,
801 0x9190, 0xffffffff, 0x00000008,
802 0x9194, 0xffffffff, 0x00070006,
803 0x9198, 0xffffffff, 0x000a0009,
804 0x919c, 0xffffffff, 0x00040003,
805 0x91a0, 0xffffffff, 0x00060005,
806 0x91a4, 0xffffffff, 0x00000009,
807 0x91a8, 0xffffffff, 0x00080007,
808 0x91ac, 0xffffffff, 0x000b000a,
809 0x91b0, 0xffffffff, 0x00050004,
810 0x91b4, 0xffffffff, 0x00070006,
811 0x91b8, 0xffffffff, 0x0008000b,
812 0x91bc, 0xffffffff, 0x000a0009,
813 0x91c0, 0xffffffff, 0x000d000c,
814 0x9200, 0xffffffff, 0x00090008,
815 0x9204, 0xffffffff, 0x000b000a,
816 0x9208, 0xffffffff, 0x000c000f,
817 0x920c, 0xffffffff, 0x000e000d,
818 0x9210, 0xffffffff, 0x00110010,
819 0x9214, 0xffffffff, 0x000a0009,
820 0x9218, 0xffffffff, 0x000c000b,
821 0x921c, 0xffffffff, 0x0000000f,
822 0x9220, 0xffffffff, 0x000e000d,
823 0x9224, 0xffffffff, 0x00110010,
824 0x9228, 0xffffffff, 0x000b000a,
825 0x922c, 0xffffffff, 0x000d000c,
826 0x9230, 0xffffffff, 0x00000010,
827 0x9234, 0xffffffff, 0x000f000e,
828 0x9238, 0xffffffff, 0x00120011,
829 0x923c, 0xffffffff, 0x000c000b,
830 0x9240, 0xffffffff, 0x000e000d,
831 0x9244, 0xffffffff, 0x00000011,
832 0x9248, 0xffffffff, 0x0010000f,
833 0x924c, 0xffffffff, 0x00130012,
834 0x9250, 0xffffffff, 0x000d000c,
835 0x9254, 0xffffffff, 0x000f000e,
836 0x9258, 0xffffffff, 0x00100013,
837 0x925c, 0xffffffff, 0x00120011,
838 0x9260, 0xffffffff, 0x00150014,
839 0x9150, 0xffffffff, 0x96940200,
840 0x8708, 0xffffffff, 0x00900100,
841 0xc478, 0xffffffff, 0x00000080,
842 0xc404, 0xffffffff, 0x0020003f,
843 0x30, 0xffffffff, 0x0000001c,
844 0x34, 0x000f0000, 0x000f0000,
845 0x160c, 0xffffffff, 0x00000100,
846 0x1024, 0xffffffff, 0x00000100,
847 0x102c, 0x00000101, 0x00000000,
848 0x20a8, 0xffffffff, 0x00000104,
849 0x264c, 0x000c0000, 0x000c0000,
850 0x2648, 0x000c0000, 0x000c0000,
851 0x55e4, 0xff000fff, 0x00000100,
852 0x55e8, 0x00000001, 0x00000001,
853 0x2f50, 0x00000001, 0x00000001,
854 0x30cc, 0xc0000fff, 0x00000104,
855 0xc1e4, 0x00000001, 0x00000001,
856 0xd0c0, 0xfffffff0, 0x00000100,
857 0xd8c0, 0xfffffff0, 0x00000100
860 static const u32 oland_mgcg_cgcg_init[] =
862 0xc400, 0xffffffff, 0xfffffffc,
863 0x802c, 0xffffffff, 0xe0000000,
864 0x9a60, 0xffffffff, 0x00000100,
865 0x92a4, 0xffffffff, 0x00000100,
866 0xc164, 0xffffffff, 0x00000100,
867 0x9774, 0xffffffff, 0x00000100,
868 0x8984, 0xffffffff, 0x06000100,
869 0x8a18, 0xffffffff, 0x00000100,
870 0x92a0, 0xffffffff, 0x00000100,
871 0xc380, 0xffffffff, 0x00000100,
872 0x8b28, 0xffffffff, 0x00000100,
873 0x9144, 0xffffffff, 0x00000100,
874 0x8d88, 0xffffffff, 0x00000100,
875 0x8d8c, 0xffffffff, 0x00000100,
876 0x9030, 0xffffffff, 0x00000100,
877 0x9034, 0xffffffff, 0x00000100,
878 0x9038, 0xffffffff, 0x00000100,
879 0x903c, 0xffffffff, 0x00000100,
880 0xad80, 0xffffffff, 0x00000100,
881 0xac54, 0xffffffff, 0x00000100,
882 0x897c, 0xffffffff, 0x06000100,
883 0x9868, 0xffffffff, 0x00000100,
884 0x9510, 0xffffffff, 0x00000100,
885 0xaf04, 0xffffffff, 0x00000100,
886 0xae04, 0xffffffff, 0x00000100,
887 0x949c, 0xffffffff, 0x00000100,
888 0x802c, 0xffffffff, 0xe0000000,
889 0x9160, 0xffffffff, 0x00010000,
890 0x9164, 0xffffffff, 0x00030002,
891 0x9168, 0xffffffff, 0x00040007,
892 0x916c, 0xffffffff, 0x00060005,
893 0x9170, 0xffffffff, 0x00090008,
894 0x9174, 0xffffffff, 0x00020001,
895 0x9178, 0xffffffff, 0x00040003,
896 0x917c, 0xffffffff, 0x00000007,
897 0x9180, 0xffffffff, 0x00060005,
898 0x9184, 0xffffffff, 0x00090008,
899 0x9188, 0xffffffff, 0x00030002,
900 0x918c, 0xffffffff, 0x00050004,
901 0x9190, 0xffffffff, 0x00000008,
902 0x9194, 0xffffffff, 0x00070006,
903 0x9198, 0xffffffff, 0x000a0009,
904 0x919c, 0xffffffff, 0x00040003,
905 0x91a0, 0xffffffff, 0x00060005,
906 0x91a4, 0xffffffff, 0x00000009,
907 0x91a8, 0xffffffff, 0x00080007,
908 0x91ac, 0xffffffff, 0x000b000a,
909 0x91b0, 0xffffffff, 0x00050004,
910 0x91b4, 0xffffffff, 0x00070006,
911 0x91b8, 0xffffffff, 0x0008000b,
912 0x91bc, 0xffffffff, 0x000a0009,
913 0x91c0, 0xffffffff, 0x000d000c,
914 0x91c4, 0xffffffff, 0x00060005,
915 0x91c8, 0xffffffff, 0x00080007,
916 0x91cc, 0xffffffff, 0x0000000b,
917 0x91d0, 0xffffffff, 0x000a0009,
918 0x91d4, 0xffffffff, 0x000d000c,
919 0x9150, 0xffffffff, 0x96940200,
920 0x8708, 0xffffffff, 0x00900100,
921 0xc478, 0xffffffff, 0x00000080,
922 0xc404, 0xffffffff, 0x0020003f,
923 0x30, 0xffffffff, 0x0000001c,
924 0x34, 0x000f0000, 0x000f0000,
925 0x160c, 0xffffffff, 0x00000100,
926 0x1024, 0xffffffff, 0x00000100,
927 0x102c, 0x00000101, 0x00000000,
928 0x20a8, 0xffffffff, 0x00000104,
929 0x264c, 0x000c0000, 0x000c0000,
930 0x2648, 0x000c0000, 0x000c0000,
931 0x55e4, 0xff000fff, 0x00000100,
932 0x55e8, 0x00000001, 0x00000001,
933 0x2f50, 0x00000001, 0x00000001,
934 0x30cc, 0xc0000fff, 0x00000104,
935 0xc1e4, 0x00000001, 0x00000001,
936 0xd0c0, 0xfffffff0, 0x00000100,
937 0xd8c0, 0xfffffff0, 0x00000100
940 static const u32 hainan_mgcg_cgcg_init[] =
942 0xc400, 0xffffffff, 0xfffffffc,
943 0x802c, 0xffffffff, 0xe0000000,
944 0x9a60, 0xffffffff, 0x00000100,
945 0x92a4, 0xffffffff, 0x00000100,
946 0xc164, 0xffffffff, 0x00000100,
947 0x9774, 0xffffffff, 0x00000100,
948 0x8984, 0xffffffff, 0x06000100,
949 0x8a18, 0xffffffff, 0x00000100,
950 0x92a0, 0xffffffff, 0x00000100,
951 0xc380, 0xffffffff, 0x00000100,
952 0x8b28, 0xffffffff, 0x00000100,
953 0x9144, 0xffffffff, 0x00000100,
954 0x8d88, 0xffffffff, 0x00000100,
955 0x8d8c, 0xffffffff, 0x00000100,
956 0x9030, 0xffffffff, 0x00000100,
957 0x9034, 0xffffffff, 0x00000100,
958 0x9038, 0xffffffff, 0x00000100,
959 0x903c, 0xffffffff, 0x00000100,
960 0xad80, 0xffffffff, 0x00000100,
961 0xac54, 0xffffffff, 0x00000100,
962 0x897c, 0xffffffff, 0x06000100,
963 0x9868, 0xffffffff, 0x00000100,
964 0x9510, 0xffffffff, 0x00000100,
965 0xaf04, 0xffffffff, 0x00000100,
966 0xae04, 0xffffffff, 0x00000100,
967 0x949c, 0xffffffff, 0x00000100,
968 0x802c, 0xffffffff, 0xe0000000,
969 0x9160, 0xffffffff, 0x00010000,
970 0x9164, 0xffffffff, 0x00030002,
971 0x9168, 0xffffffff, 0x00040007,
972 0x916c, 0xffffffff, 0x00060005,
973 0x9170, 0xffffffff, 0x00090008,
974 0x9174, 0xffffffff, 0x00020001,
975 0x9178, 0xffffffff, 0x00040003,
976 0x917c, 0xffffffff, 0x00000007,
977 0x9180, 0xffffffff, 0x00060005,
978 0x9184, 0xffffffff, 0x00090008,
979 0x9188, 0xffffffff, 0x00030002,
980 0x918c, 0xffffffff, 0x00050004,
981 0x9190, 0xffffffff, 0x00000008,
982 0x9194, 0xffffffff, 0x00070006,
983 0x9198, 0xffffffff, 0x000a0009,
984 0x919c, 0xffffffff, 0x00040003,
985 0x91a0, 0xffffffff, 0x00060005,
986 0x91a4, 0xffffffff, 0x00000009,
987 0x91a8, 0xffffffff, 0x00080007,
988 0x91ac, 0xffffffff, 0x000b000a,
989 0x91b0, 0xffffffff, 0x00050004,
990 0x91b4, 0xffffffff, 0x00070006,
991 0x91b8, 0xffffffff, 0x0008000b,
992 0x91bc, 0xffffffff, 0x000a0009,
993 0x91c0, 0xffffffff, 0x000d000c,
994 0x91c4, 0xffffffff, 0x00060005,
995 0x91c8, 0xffffffff, 0x00080007,
996 0x91cc, 0xffffffff, 0x0000000b,
997 0x91d0, 0xffffffff, 0x000a0009,
998 0x91d4, 0xffffffff, 0x000d000c,
999 0x9150, 0xffffffff, 0x96940200,
1000 0x8708, 0xffffffff, 0x00900100,
1001 0xc478, 0xffffffff, 0x00000080,
1002 0xc404, 0xffffffff, 0x0020003f,
1003 0x30, 0xffffffff, 0x0000001c,
1004 0x34, 0x000f0000, 0x000f0000,
1005 0x160c, 0xffffffff, 0x00000100,
1006 0x1024, 0xffffffff, 0x00000100,
1007 0x20a8, 0xffffffff, 0x00000104,
1008 0x264c, 0x000c0000, 0x000c0000,
1009 0x2648, 0x000c0000, 0x000c0000,
1010 0x2f50, 0x00000001, 0x00000001,
1011 0x30cc, 0xc0000fff, 0x00000104,
1012 0xc1e4, 0x00000001, 0x00000001,
1013 0xd0c0, 0xfffffff0, 0x00000100,
1014 0xd8c0, 0xfffffff0, 0x00000100
1017 static u32 verde_pg_init[] =
1019 0x353c, 0xffffffff, 0x40000,
1020 0x3538, 0xffffffff, 0x200010ff,
1021 0x353c, 0xffffffff, 0x0,
1022 0x353c, 0xffffffff, 0x0,
1023 0x353c, 0xffffffff, 0x0,
1024 0x353c, 0xffffffff, 0x0,
1025 0x353c, 0xffffffff, 0x0,
1026 0x353c, 0xffffffff, 0x7007,
1027 0x3538, 0xffffffff, 0x300010ff,
1028 0x353c, 0xffffffff, 0x0,
1029 0x353c, 0xffffffff, 0x0,
1030 0x353c, 0xffffffff, 0x0,
1031 0x353c, 0xffffffff, 0x0,
1032 0x353c, 0xffffffff, 0x0,
1033 0x353c, 0xffffffff, 0x400000,
1034 0x3538, 0xffffffff, 0x100010ff,
1035 0x353c, 0xffffffff, 0x0,
1036 0x353c, 0xffffffff, 0x0,
1037 0x353c, 0xffffffff, 0x0,
1038 0x353c, 0xffffffff, 0x0,
1039 0x353c, 0xffffffff, 0x0,
1040 0x353c, 0xffffffff, 0x120200,
1041 0x3538, 0xffffffff, 0x500010ff,
1042 0x353c, 0xffffffff, 0x0,
1043 0x353c, 0xffffffff, 0x0,
1044 0x353c, 0xffffffff, 0x0,
1045 0x353c, 0xffffffff, 0x0,
1046 0x353c, 0xffffffff, 0x0,
1047 0x353c, 0xffffffff, 0x1e1e16,
1048 0x3538, 0xffffffff, 0x600010ff,
1049 0x353c, 0xffffffff, 0x0,
1050 0x353c, 0xffffffff, 0x0,
1051 0x353c, 0xffffffff, 0x0,
1052 0x353c, 0xffffffff, 0x0,
1053 0x353c, 0xffffffff, 0x0,
1054 0x353c, 0xffffffff, 0x171f1e,
1055 0x3538, 0xffffffff, 0x700010ff,
1056 0x353c, 0xffffffff, 0x0,
1057 0x353c, 0xffffffff, 0x0,
1058 0x353c, 0xffffffff, 0x0,
1059 0x353c, 0xffffffff, 0x0,
1060 0x353c, 0xffffffff, 0x0,
1061 0x353c, 0xffffffff, 0x0,
1062 0x3538, 0xffffffff, 0x9ff,
1063 0x3500, 0xffffffff, 0x0,
1064 0x3504, 0xffffffff, 0x10000800,
1065 0x3504, 0xffffffff, 0xf,
1066 0x3504, 0xffffffff, 0xf,
1067 0x3500, 0xffffffff, 0x4,
1068 0x3504, 0xffffffff, 0x1000051e,
1069 0x3504, 0xffffffff, 0xffff,
1070 0x3504, 0xffffffff, 0xffff,
1071 0x3500, 0xffffffff, 0x8,
1072 0x3504, 0xffffffff, 0x80500,
1073 0x3500, 0xffffffff, 0x12,
1074 0x3504, 0xffffffff, 0x9050c,
1075 0x3500, 0xffffffff, 0x1d,
1076 0x3504, 0xffffffff, 0xb052c,
1077 0x3500, 0xffffffff, 0x2a,
1078 0x3504, 0xffffffff, 0x1053e,
1079 0x3500, 0xffffffff, 0x2d,
1080 0x3504, 0xffffffff, 0x10546,
1081 0x3500, 0xffffffff, 0x30,
1082 0x3504, 0xffffffff, 0xa054e,
1083 0x3500, 0xffffffff, 0x3c,
1084 0x3504, 0xffffffff, 0x1055f,
1085 0x3500, 0xffffffff, 0x3f,
1086 0x3504, 0xffffffff, 0x10567,
1087 0x3500, 0xffffffff, 0x42,
1088 0x3504, 0xffffffff, 0x1056f,
1089 0x3500, 0xffffffff, 0x45,
1090 0x3504, 0xffffffff, 0x10572,
1091 0x3500, 0xffffffff, 0x48,
1092 0x3504, 0xffffffff, 0x20575,
1093 0x3500, 0xffffffff, 0x4c,
1094 0x3504, 0xffffffff, 0x190801,
1095 0x3500, 0xffffffff, 0x67,
1096 0x3504, 0xffffffff, 0x1082a,
1097 0x3500, 0xffffffff, 0x6a,
1098 0x3504, 0xffffffff, 0x1b082d,
1099 0x3500, 0xffffffff, 0x87,
1100 0x3504, 0xffffffff, 0x310851,
1101 0x3500, 0xffffffff, 0xba,
1102 0x3504, 0xffffffff, 0x891,
1103 0x3500, 0xffffffff, 0xbc,
1104 0x3504, 0xffffffff, 0x893,
1105 0x3500, 0xffffffff, 0xbe,
1106 0x3504, 0xffffffff, 0x20895,
1107 0x3500, 0xffffffff, 0xc2,
1108 0x3504, 0xffffffff, 0x20899,
1109 0x3500, 0xffffffff, 0xc6,
1110 0x3504, 0xffffffff, 0x2089d,
1111 0x3500, 0xffffffff, 0xca,
1112 0x3504, 0xffffffff, 0x8a1,
1113 0x3500, 0xffffffff, 0xcc,
1114 0x3504, 0xffffffff, 0x8a3,
1115 0x3500, 0xffffffff, 0xce,
1116 0x3504, 0xffffffff, 0x308a5,
1117 0x3500, 0xffffffff, 0xd3,
1118 0x3504, 0xffffffff, 0x6d08cd,
1119 0x3500, 0xffffffff, 0x142,
1120 0x3504, 0xffffffff, 0x2000095a,
1121 0x3504, 0xffffffff, 0x1,
1122 0x3500, 0xffffffff, 0x144,
1123 0x3504, 0xffffffff, 0x301f095b,
1124 0x3500, 0xffffffff, 0x165,
1125 0x3504, 0xffffffff, 0xc094d,
1126 0x3500, 0xffffffff, 0x173,
1127 0x3504, 0xffffffff, 0xf096d,
1128 0x3500, 0xffffffff, 0x184,
1129 0x3504, 0xffffffff, 0x15097f,
1130 0x3500, 0xffffffff, 0x19b,
1131 0x3504, 0xffffffff, 0xc0998,
1132 0x3500, 0xffffffff, 0x1a9,
1133 0x3504, 0xffffffff, 0x409a7,
1134 0x3500, 0xffffffff, 0x1af,
1135 0x3504, 0xffffffff, 0xcdc,
1136 0x3500, 0xffffffff, 0x1b1,
1137 0x3504, 0xffffffff, 0x800,
1138 0x3508, 0xffffffff, 0x6c9b2000,
1139 0x3510, 0xfc00, 0x2000,
1140 0x3544, 0xffffffff, 0xfc0,
1141 0x28d4, 0x00000100, 0x100
1144 static void si_init_golden_registers(struct radeon_device *rdev)
1146 switch (rdev->family) {
1148 radeon_program_register_sequence(rdev,
1149 tahiti_golden_registers,
1150 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1151 radeon_program_register_sequence(rdev,
1152 tahiti_golden_rlc_registers,
1153 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1154 radeon_program_register_sequence(rdev,
1155 tahiti_mgcg_cgcg_init,
1156 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1157 radeon_program_register_sequence(rdev,
1158 tahiti_golden_registers2,
1159 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1162 radeon_program_register_sequence(rdev,
1163 pitcairn_golden_registers,
1164 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1165 radeon_program_register_sequence(rdev,
1166 pitcairn_golden_rlc_registers,
1167 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1168 radeon_program_register_sequence(rdev,
1169 pitcairn_mgcg_cgcg_init,
1170 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1173 radeon_program_register_sequence(rdev,
1174 verde_golden_registers,
1175 (const u32)ARRAY_SIZE(verde_golden_registers));
1176 radeon_program_register_sequence(rdev,
1177 verde_golden_rlc_registers,
1178 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1179 radeon_program_register_sequence(rdev,
1180 verde_mgcg_cgcg_init,
1181 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1182 radeon_program_register_sequence(rdev,
1184 (const u32)ARRAY_SIZE(verde_pg_init));
1187 radeon_program_register_sequence(rdev,
1188 oland_golden_registers,
1189 (const u32)ARRAY_SIZE(oland_golden_registers));
1190 radeon_program_register_sequence(rdev,
1191 oland_golden_rlc_registers,
1192 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1193 radeon_program_register_sequence(rdev,
1194 oland_mgcg_cgcg_init,
1195 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1198 radeon_program_register_sequence(rdev,
1199 hainan_golden_registers,
1200 (const u32)ARRAY_SIZE(hainan_golden_registers));
1201 radeon_program_register_sequence(rdev,
1202 hainan_golden_registers2,
1203 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1204 radeon_program_register_sequence(rdev,
1205 hainan_mgcg_cgcg_init,
1206 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1213 #define PCIE_BUS_CLK 10000
1214 #define TCLK (PCIE_BUS_CLK / 10)
1217 * si_get_xclk - get the xclk
1219 * @rdev: radeon_device pointer
1221 * Returns the reference clock used by the gfx engine
1224 u32 si_get_xclk(struct radeon_device *rdev)
1226 u32 reference_clock = rdev->clock.spll.reference_freq;
1229 tmp = RREG32(CG_CLKPIN_CNTL_2);
1230 if (tmp & MUX_TCLK_TO_XCLK)
1233 tmp = RREG32(CG_CLKPIN_CNTL);
1234 if (tmp & XTALIN_DIVIDE)
1235 return reference_clock / 4;
1237 return reference_clock;
1240 /* get temperature in millidegrees */
1241 int si_get_temp(struct radeon_device *rdev)
1244 int actual_temp = 0;
1246 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1252 actual_temp = temp & 0x1ff;
1254 actual_temp = (actual_temp * 1000);
1259 #define TAHITI_IO_MC_REGS_SIZE 36
1261 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1262 {0x0000006f, 0x03044000},
1263 {0x00000070, 0x0480c018},
1264 {0x00000071, 0x00000040},
1265 {0x00000072, 0x01000000},
1266 {0x00000074, 0x000000ff},
1267 {0x00000075, 0x00143400},
1268 {0x00000076, 0x08ec0800},
1269 {0x00000077, 0x040000cc},
1270 {0x00000079, 0x00000000},
1271 {0x0000007a, 0x21000409},
1272 {0x0000007c, 0x00000000},
1273 {0x0000007d, 0xe8000000},
1274 {0x0000007e, 0x044408a8},
1275 {0x0000007f, 0x00000003},
1276 {0x00000080, 0x00000000},
1277 {0x00000081, 0x01000000},
1278 {0x00000082, 0x02000000},
1279 {0x00000083, 0x00000000},
1280 {0x00000084, 0xe3f3e4f4},
1281 {0x00000085, 0x00052024},
1282 {0x00000087, 0x00000000},
1283 {0x00000088, 0x66036603},
1284 {0x00000089, 0x01000000},
1285 {0x0000008b, 0x1c0a0000},
1286 {0x0000008c, 0xff010000},
1287 {0x0000008e, 0xffffefff},
1288 {0x0000008f, 0xfff3efff},
1289 {0x00000090, 0xfff3efbf},
1290 {0x00000094, 0x00101101},
1291 {0x00000095, 0x00000fff},
1292 {0x00000096, 0x00116fff},
1293 {0x00000097, 0x60010000},
1294 {0x00000098, 0x10010000},
1295 {0x00000099, 0x00006000},
1296 {0x0000009a, 0x00001000},
1297 {0x0000009f, 0x00a77400}
1300 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1301 {0x0000006f, 0x03044000},
1302 {0x00000070, 0x0480c018},
1303 {0x00000071, 0x00000040},
1304 {0x00000072, 0x01000000},
1305 {0x00000074, 0x000000ff},
1306 {0x00000075, 0x00143400},
1307 {0x00000076, 0x08ec0800},
1308 {0x00000077, 0x040000cc},
1309 {0x00000079, 0x00000000},
1310 {0x0000007a, 0x21000409},
1311 {0x0000007c, 0x00000000},
1312 {0x0000007d, 0xe8000000},
1313 {0x0000007e, 0x044408a8},
1314 {0x0000007f, 0x00000003},
1315 {0x00000080, 0x00000000},
1316 {0x00000081, 0x01000000},
1317 {0x00000082, 0x02000000},
1318 {0x00000083, 0x00000000},
1319 {0x00000084, 0xe3f3e4f4},
1320 {0x00000085, 0x00052024},
1321 {0x00000087, 0x00000000},
1322 {0x00000088, 0x66036603},
1323 {0x00000089, 0x01000000},
1324 {0x0000008b, 0x1c0a0000},
1325 {0x0000008c, 0xff010000},
1326 {0x0000008e, 0xffffefff},
1327 {0x0000008f, 0xfff3efff},
1328 {0x00000090, 0xfff3efbf},
1329 {0x00000094, 0x00101101},
1330 {0x00000095, 0x00000fff},
1331 {0x00000096, 0x00116fff},
1332 {0x00000097, 0x60010000},
1333 {0x00000098, 0x10010000},
1334 {0x00000099, 0x00006000},
1335 {0x0000009a, 0x00001000},
1336 {0x0000009f, 0x00a47400}
1339 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1340 {0x0000006f, 0x03044000},
1341 {0x00000070, 0x0480c018},
1342 {0x00000071, 0x00000040},
1343 {0x00000072, 0x01000000},
1344 {0x00000074, 0x000000ff},
1345 {0x00000075, 0x00143400},
1346 {0x00000076, 0x08ec0800},
1347 {0x00000077, 0x040000cc},
1348 {0x00000079, 0x00000000},
1349 {0x0000007a, 0x21000409},
1350 {0x0000007c, 0x00000000},
1351 {0x0000007d, 0xe8000000},
1352 {0x0000007e, 0x044408a8},
1353 {0x0000007f, 0x00000003},
1354 {0x00000080, 0x00000000},
1355 {0x00000081, 0x01000000},
1356 {0x00000082, 0x02000000},
1357 {0x00000083, 0x00000000},
1358 {0x00000084, 0xe3f3e4f4},
1359 {0x00000085, 0x00052024},
1360 {0x00000087, 0x00000000},
1361 {0x00000088, 0x66036603},
1362 {0x00000089, 0x01000000},
1363 {0x0000008b, 0x1c0a0000},
1364 {0x0000008c, 0xff010000},
1365 {0x0000008e, 0xffffefff},
1366 {0x0000008f, 0xfff3efff},
1367 {0x00000090, 0xfff3efbf},
1368 {0x00000094, 0x00101101},
1369 {0x00000095, 0x00000fff},
1370 {0x00000096, 0x00116fff},
1371 {0x00000097, 0x60010000},
1372 {0x00000098, 0x10010000},
1373 {0x00000099, 0x00006000},
1374 {0x0000009a, 0x00001000},
1375 {0x0000009f, 0x00a37400}
1378 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1379 {0x0000006f, 0x03044000},
1380 {0x00000070, 0x0480c018},
1381 {0x00000071, 0x00000040},
1382 {0x00000072, 0x01000000},
1383 {0x00000074, 0x000000ff},
1384 {0x00000075, 0x00143400},
1385 {0x00000076, 0x08ec0800},
1386 {0x00000077, 0x040000cc},
1387 {0x00000079, 0x00000000},
1388 {0x0000007a, 0x21000409},
1389 {0x0000007c, 0x00000000},
1390 {0x0000007d, 0xe8000000},
1391 {0x0000007e, 0x044408a8},
1392 {0x0000007f, 0x00000003},
1393 {0x00000080, 0x00000000},
1394 {0x00000081, 0x01000000},
1395 {0x00000082, 0x02000000},
1396 {0x00000083, 0x00000000},
1397 {0x00000084, 0xe3f3e4f4},
1398 {0x00000085, 0x00052024},
1399 {0x00000087, 0x00000000},
1400 {0x00000088, 0x66036603},
1401 {0x00000089, 0x01000000},
1402 {0x0000008b, 0x1c0a0000},
1403 {0x0000008c, 0xff010000},
1404 {0x0000008e, 0xffffefff},
1405 {0x0000008f, 0xfff3efff},
1406 {0x00000090, 0xfff3efbf},
1407 {0x00000094, 0x00101101},
1408 {0x00000095, 0x00000fff},
1409 {0x00000096, 0x00116fff},
1410 {0x00000097, 0x60010000},
1411 {0x00000098, 0x10010000},
1412 {0x00000099, 0x00006000},
1413 {0x0000009a, 0x00001000},
1414 {0x0000009f, 0x00a17730}
1417 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1418 {0x0000006f, 0x03044000},
1419 {0x00000070, 0x0480c018},
1420 {0x00000071, 0x00000040},
1421 {0x00000072, 0x01000000},
1422 {0x00000074, 0x000000ff},
1423 {0x00000075, 0x00143400},
1424 {0x00000076, 0x08ec0800},
1425 {0x00000077, 0x040000cc},
1426 {0x00000079, 0x00000000},
1427 {0x0000007a, 0x21000409},
1428 {0x0000007c, 0x00000000},
1429 {0x0000007d, 0xe8000000},
1430 {0x0000007e, 0x044408a8},
1431 {0x0000007f, 0x00000003},
1432 {0x00000080, 0x00000000},
1433 {0x00000081, 0x01000000},
1434 {0x00000082, 0x02000000},
1435 {0x00000083, 0x00000000},
1436 {0x00000084, 0xe3f3e4f4},
1437 {0x00000085, 0x00052024},
1438 {0x00000087, 0x00000000},
1439 {0x00000088, 0x66036603},
1440 {0x00000089, 0x01000000},
1441 {0x0000008b, 0x1c0a0000},
1442 {0x0000008c, 0xff010000},
1443 {0x0000008e, 0xffffefff},
1444 {0x0000008f, 0xfff3efff},
1445 {0x00000090, 0xfff3efbf},
1446 {0x00000094, 0x00101101},
1447 {0x00000095, 0x00000fff},
1448 {0x00000096, 0x00116fff},
1449 {0x00000097, 0x60010000},
1450 {0x00000098, 0x10010000},
1451 {0x00000099, 0x00006000},
1452 {0x0000009a, 0x00001000},
1453 {0x0000009f, 0x00a07730}
1457 static int si_mc_load_microcode(struct radeon_device *rdev)
1459 const __be32 *fw_data;
1460 u32 running, blackout = 0;
1462 int i, ucode_size, regs_size;
1467 switch (rdev->family) {
1469 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1470 ucode_size = SI_MC_UCODE_SIZE;
1471 regs_size = TAHITI_IO_MC_REGS_SIZE;
1474 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1475 ucode_size = SI_MC_UCODE_SIZE;
1476 regs_size = TAHITI_IO_MC_REGS_SIZE;
1480 io_mc_regs = (u32 *)&verde_io_mc_regs;
1481 ucode_size = SI_MC_UCODE_SIZE;
1482 regs_size = TAHITI_IO_MC_REGS_SIZE;
1485 io_mc_regs = (u32 *)&oland_io_mc_regs;
1486 ucode_size = OLAND_MC_UCODE_SIZE;
1487 regs_size = TAHITI_IO_MC_REGS_SIZE;
1490 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1491 ucode_size = OLAND_MC_UCODE_SIZE;
1492 regs_size = TAHITI_IO_MC_REGS_SIZE;
1496 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1500 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1501 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1504 /* reset the engine and set to writable */
1505 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1506 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1508 /* load mc io regs */
1509 for (i = 0; i < regs_size; i++) {
1510 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1511 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1513 /* load the MC ucode */
1514 fw_data = (const __be32 *)rdev->mc_fw->data;
1515 for (i = 0; i < ucode_size; i++)
1516 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1518 /* put the engine back into the active state */
1519 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1520 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1521 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1523 /* wait for training to complete */
1524 for (i = 0; i < rdev->usec_timeout; i++) {
1525 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1529 for (i = 0; i < rdev->usec_timeout; i++) {
1530 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1536 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1542 static int si_init_microcode(struct radeon_device *rdev)
1544 struct platform_device *pdev;
1545 const char *chip_name;
1546 const char *rlc_chip_name;
1547 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1548 size_t smc_req_size;
1554 pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1557 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1561 switch (rdev->family) {
1563 chip_name = "TAHITI";
1564 rlc_chip_name = "TAHITI";
1565 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1566 me_req_size = SI_PM4_UCODE_SIZE * 4;
1567 ce_req_size = SI_CE_UCODE_SIZE * 4;
1568 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1569 mc_req_size = SI_MC_UCODE_SIZE * 4;
1570 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1573 chip_name = "PITCAIRN";
1574 rlc_chip_name = "PITCAIRN";
1575 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1576 me_req_size = SI_PM4_UCODE_SIZE * 4;
1577 ce_req_size = SI_CE_UCODE_SIZE * 4;
1578 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1579 mc_req_size = SI_MC_UCODE_SIZE * 4;
1580 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1583 chip_name = "VERDE";
1584 rlc_chip_name = "VERDE";
1585 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1586 me_req_size = SI_PM4_UCODE_SIZE * 4;
1587 ce_req_size = SI_CE_UCODE_SIZE * 4;
1588 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1589 mc_req_size = SI_MC_UCODE_SIZE * 4;
1590 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1593 chip_name = "OLAND";
1594 rlc_chip_name = "OLAND";
1595 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1596 me_req_size = SI_PM4_UCODE_SIZE * 4;
1597 ce_req_size = SI_CE_UCODE_SIZE * 4;
1598 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1599 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1600 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1603 chip_name = "HAINAN";
1604 rlc_chip_name = "HAINAN";
1605 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1606 me_req_size = SI_PM4_UCODE_SIZE * 4;
1607 ce_req_size = SI_CE_UCODE_SIZE * 4;
1608 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1609 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1610 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1615 DRM_INFO("Loading %s Microcode\n", chip_name);
1617 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1618 err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1621 if (rdev->pfp_fw->size != pfp_req_size) {
1623 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1624 rdev->pfp_fw->size, fw_name);
1629 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1630 err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1633 if (rdev->me_fw->size != me_req_size) {
1635 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1636 rdev->me_fw->size, fw_name);
1640 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1641 err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
1644 if (rdev->ce_fw->size != ce_req_size) {
1646 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1647 rdev->ce_fw->size, fw_name);
1651 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1652 err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1655 if (rdev->rlc_fw->size != rlc_req_size) {
1657 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1658 rdev->rlc_fw->size, fw_name);
1662 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1663 err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
1666 if (rdev->mc_fw->size != mc_req_size) {
1668 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1669 rdev->mc_fw->size, fw_name);
1673 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1674 err = request_firmware(&rdev->smc_fw, fw_name, &pdev->dev);
1677 if (rdev->smc_fw->size != smc_req_size) {
1679 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1680 rdev->smc_fw->size, fw_name);
1685 platform_device_unregister(pdev);
1690 "si_cp: Failed to load firmware \"%s\"\n",
1692 release_firmware(rdev->pfp_fw);
1693 rdev->pfp_fw = NULL;
1694 release_firmware(rdev->me_fw);
1696 release_firmware(rdev->ce_fw);
1698 release_firmware(rdev->rlc_fw);
1699 rdev->rlc_fw = NULL;
1700 release_firmware(rdev->mc_fw);
1702 release_firmware(rdev->smc_fw);
1703 rdev->smc_fw = NULL;
1708 /* watermark setup */
1709 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1710 struct radeon_crtc *radeon_crtc,
1711 struct drm_display_mode *mode,
1712 struct drm_display_mode *other_mode)
1717 * There are 3 line buffers, each one shared by 2 display controllers.
1718 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1719 * the display controllers. The paritioning is done via one of four
1720 * preset allocations specified in bits 21:20:
1722 * 2 - whole lb, other crtc must be disabled
1724 /* this can get tricky if we have two large displays on a paired group
1725 * of crtcs. Ideally for multiple large displays we'd assign them to
1726 * non-linked crtcs for maximum line buffer allocation.
1728 if (radeon_crtc->base.enabled && mode) {
1732 tmp = 2; /* whole */
1736 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1737 DC_LB_MEMORY_CONFIG(tmp));
1739 if (radeon_crtc->base.enabled && mode) {
1749 /* controller not enabled, so no lb used */
1753 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1755 u32 tmp = RREG32(MC_SHARED_CHMAP);
1757 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1780 struct dce6_wm_params {
1781 u32 dram_channels; /* number of dram channels */
1782 u32 yclk; /* bandwidth per dram data pin in kHz */
1783 u32 sclk; /* engine clock in kHz */
1784 u32 disp_clk; /* display clock in kHz */
1785 u32 src_width; /* viewport width */
1786 u32 active_time; /* active display time in ns */
1787 u32 blank_time; /* blank time in ns */
1788 bool interlaced; /* mode is interlaced */
1789 fixed20_12 vsc; /* vertical scale ratio */
1790 u32 num_heads; /* number of active crtcs */
1791 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1792 u32 lb_size; /* line buffer allocated to pipe */
1793 u32 vtaps; /* vertical scaler taps */
1796 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1798 /* Calculate raw DRAM Bandwidth */
1799 fixed20_12 dram_efficiency; /* 0.7 */
1800 fixed20_12 yclk, dram_channels, bandwidth;
1803 a.full = dfixed_const(1000);
1804 yclk.full = dfixed_const(wm->yclk);
1805 yclk.full = dfixed_div(yclk, a);
1806 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1807 a.full = dfixed_const(10);
1808 dram_efficiency.full = dfixed_const(7);
1809 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1810 bandwidth.full = dfixed_mul(dram_channels, yclk);
1811 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1813 return dfixed_trunc(bandwidth);
1816 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1818 /* Calculate DRAM Bandwidth and the part allocated to display. */
1819 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1820 fixed20_12 yclk, dram_channels, bandwidth;
1823 a.full = dfixed_const(1000);
1824 yclk.full = dfixed_const(wm->yclk);
1825 yclk.full = dfixed_div(yclk, a);
1826 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1827 a.full = dfixed_const(10);
1828 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1829 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1830 bandwidth.full = dfixed_mul(dram_channels, yclk);
1831 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1833 return dfixed_trunc(bandwidth);
1836 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1838 /* Calculate the display Data return Bandwidth */
1839 fixed20_12 return_efficiency; /* 0.8 */
1840 fixed20_12 sclk, bandwidth;
1843 a.full = dfixed_const(1000);
1844 sclk.full = dfixed_const(wm->sclk);
1845 sclk.full = dfixed_div(sclk, a);
1846 a.full = dfixed_const(10);
1847 return_efficiency.full = dfixed_const(8);
1848 return_efficiency.full = dfixed_div(return_efficiency, a);
1849 a.full = dfixed_const(32);
1850 bandwidth.full = dfixed_mul(a, sclk);
1851 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1853 return dfixed_trunc(bandwidth);
1856 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1861 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1863 /* Calculate the DMIF Request Bandwidth */
1864 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1865 fixed20_12 disp_clk, sclk, bandwidth;
1866 fixed20_12 a, b1, b2;
1869 a.full = dfixed_const(1000);
1870 disp_clk.full = dfixed_const(wm->disp_clk);
1871 disp_clk.full = dfixed_div(disp_clk, a);
1872 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1873 b1.full = dfixed_mul(a, disp_clk);
1875 a.full = dfixed_const(1000);
1876 sclk.full = dfixed_const(wm->sclk);
1877 sclk.full = dfixed_div(sclk, a);
1878 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1879 b2.full = dfixed_mul(a, sclk);
1881 a.full = dfixed_const(10);
1882 disp_clk_request_efficiency.full = dfixed_const(8);
1883 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1885 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1887 a.full = dfixed_const(min_bandwidth);
1888 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1890 return dfixed_trunc(bandwidth);
1893 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1895 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1896 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1897 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1898 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1900 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1903 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1905 /* Calculate the display mode Average Bandwidth
1906 * DisplayMode should contain the source and destination dimensions,
1910 fixed20_12 line_time;
1911 fixed20_12 src_width;
1912 fixed20_12 bandwidth;
1915 a.full = dfixed_const(1000);
1916 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1917 line_time.full = dfixed_div(line_time, a);
1918 bpp.full = dfixed_const(wm->bytes_per_pixel);
1919 src_width.full = dfixed_const(wm->src_width);
1920 bandwidth.full = dfixed_mul(src_width, bpp);
1921 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1922 bandwidth.full = dfixed_div(bandwidth, line_time);
1924 return dfixed_trunc(bandwidth);
1927 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1929 /* First calcualte the latency in ns */
1930 u32 mc_latency = 2000; /* 2000 ns. */
1931 u32 available_bandwidth = dce6_available_bandwidth(wm);
1932 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1933 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1934 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1935 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1936 (wm->num_heads * cursor_line_pair_return_time);
1937 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1938 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1939 u32 tmp, dmif_size = 12288;
1942 if (wm->num_heads == 0)
1945 a.full = dfixed_const(2);
1946 b.full = dfixed_const(1);
1947 if ((wm->vsc.full > a.full) ||
1948 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1950 ((wm->vsc.full >= a.full) && wm->interlaced))
1951 max_src_lines_per_dst_line = 4;
1953 max_src_lines_per_dst_line = 2;
1955 a.full = dfixed_const(available_bandwidth);
1956 b.full = dfixed_const(wm->num_heads);
1957 a.full = dfixed_div(a, b);
1959 b.full = dfixed_const(mc_latency + 512);
1960 c.full = dfixed_const(wm->disp_clk);
1961 b.full = dfixed_div(b, c);
1963 c.full = dfixed_const(dmif_size);
1964 b.full = dfixed_div(c, b);
1966 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1968 b.full = dfixed_const(1000);
1969 c.full = dfixed_const(wm->disp_clk);
1970 b.full = dfixed_div(c, b);
1971 c.full = dfixed_const(wm->bytes_per_pixel);
1972 b.full = dfixed_mul(b, c);
1974 lb_fill_bw = min(tmp, dfixed_trunc(b));
1976 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1977 b.full = dfixed_const(1000);
1978 c.full = dfixed_const(lb_fill_bw);
1979 b.full = dfixed_div(c, b);
1980 a.full = dfixed_div(a, b);
1981 line_fill_time = dfixed_trunc(a);
1983 if (line_fill_time < wm->active_time)
1986 return latency + (line_fill_time - wm->active_time);
1990 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1992 if (dce6_average_bandwidth(wm) <=
1993 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
1999 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2001 if (dce6_average_bandwidth(wm) <=
2002 (dce6_available_bandwidth(wm) / wm->num_heads))
2008 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2010 u32 lb_partitions = wm->lb_size / wm->src_width;
2011 u32 line_time = wm->active_time + wm->blank_time;
2012 u32 latency_tolerant_lines;
2016 a.full = dfixed_const(1);
2017 if (wm->vsc.full > a.full)
2018 latency_tolerant_lines = 1;
2020 if (lb_partitions <= (wm->vtaps + 1))
2021 latency_tolerant_lines = 1;
2023 latency_tolerant_lines = 2;
2026 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2028 if (dce6_latency_watermark(wm) <= latency_hiding)
2034 static void dce6_program_watermarks(struct radeon_device *rdev,
2035 struct radeon_crtc *radeon_crtc,
2036 u32 lb_size, u32 num_heads)
2038 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2039 struct dce6_wm_params wm_low, wm_high;
2043 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2044 u32 priority_a_mark = 0, priority_b_mark = 0;
2045 u32 priority_a_cnt = PRIORITY_OFF;
2046 u32 priority_b_cnt = PRIORITY_OFF;
2047 u32 tmp, arb_control3;
2050 if (radeon_crtc->base.enabled && num_heads && mode) {
2051 pixel_period = 1000000 / (u32)mode->clock;
2052 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2056 if (rdev->family == CHIP_ARUBA)
2057 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2059 dram_channels = si_get_number_of_dram_channels(rdev);
2061 /* watermark for high clocks */
2062 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2064 radeon_dpm_get_mclk(rdev, false) * 10;
2066 radeon_dpm_get_sclk(rdev, false) * 10;
2068 wm_high.yclk = rdev->pm.current_mclk * 10;
2069 wm_high.sclk = rdev->pm.current_sclk * 10;
2072 wm_high.disp_clk = mode->clock;
2073 wm_high.src_width = mode->crtc_hdisplay;
2074 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2075 wm_high.blank_time = line_time - wm_high.active_time;
2076 wm_high.interlaced = false;
2077 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2078 wm_high.interlaced = true;
2079 wm_high.vsc = radeon_crtc->vsc;
2081 if (radeon_crtc->rmx_type != RMX_OFF)
2083 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2084 wm_high.lb_size = lb_size;
2085 wm_high.dram_channels = dram_channels;
2086 wm_high.num_heads = num_heads;
2088 /* watermark for low clocks */
2089 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2091 radeon_dpm_get_mclk(rdev, true) * 10;
2093 radeon_dpm_get_sclk(rdev, true) * 10;
2095 wm_low.yclk = rdev->pm.current_mclk * 10;
2096 wm_low.sclk = rdev->pm.current_sclk * 10;
2099 wm_low.disp_clk = mode->clock;
2100 wm_low.src_width = mode->crtc_hdisplay;
2101 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2102 wm_low.blank_time = line_time - wm_low.active_time;
2103 wm_low.interlaced = false;
2104 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2105 wm_low.interlaced = true;
2106 wm_low.vsc = radeon_crtc->vsc;
2108 if (radeon_crtc->rmx_type != RMX_OFF)
2110 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2111 wm_low.lb_size = lb_size;
2112 wm_low.dram_channels = dram_channels;
2113 wm_low.num_heads = num_heads;
2115 /* set for high clocks */
2116 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2117 /* set for low clocks */
2118 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2120 /* possibly force display priority to high */
2121 /* should really do this at mode validation time... */
2122 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2123 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2124 !dce6_check_latency_hiding(&wm_high) ||
2125 (rdev->disp_priority == 2)) {
2126 DRM_DEBUG_KMS("force priority to high\n");
2127 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2128 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2130 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2131 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2132 !dce6_check_latency_hiding(&wm_low) ||
2133 (rdev->disp_priority == 2)) {
2134 DRM_DEBUG_KMS("force priority to high\n");
2135 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2136 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2139 a.full = dfixed_const(1000);
2140 b.full = dfixed_const(mode->clock);
2141 b.full = dfixed_div(b, a);
2142 c.full = dfixed_const(latency_watermark_a);
2143 c.full = dfixed_mul(c, b);
2144 c.full = dfixed_mul(c, radeon_crtc->hsc);
2145 c.full = dfixed_div(c, a);
2146 a.full = dfixed_const(16);
2147 c.full = dfixed_div(c, a);
2148 priority_a_mark = dfixed_trunc(c);
2149 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2151 a.full = dfixed_const(1000);
2152 b.full = dfixed_const(mode->clock);
2153 b.full = dfixed_div(b, a);
2154 c.full = dfixed_const(latency_watermark_b);
2155 c.full = dfixed_mul(c, b);
2156 c.full = dfixed_mul(c, radeon_crtc->hsc);
2157 c.full = dfixed_div(c, a);
2158 a.full = dfixed_const(16);
2159 c.full = dfixed_div(c, a);
2160 priority_b_mark = dfixed_trunc(c);
2161 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2165 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2167 tmp &= ~LATENCY_WATERMARK_MASK(3);
2168 tmp |= LATENCY_WATERMARK_MASK(1);
2169 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2170 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2171 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2172 LATENCY_HIGH_WATERMARK(line_time)));
2174 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2175 tmp &= ~LATENCY_WATERMARK_MASK(3);
2176 tmp |= LATENCY_WATERMARK_MASK(2);
2177 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2178 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2179 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2180 LATENCY_HIGH_WATERMARK(line_time)));
2181 /* restore original selection */
2182 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2184 /* write the priority marks */
2185 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2186 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2188 /* save values for DPM */
2189 radeon_crtc->line_time = line_time;
2190 radeon_crtc->wm_high = latency_watermark_a;
2191 radeon_crtc->wm_low = latency_watermark_b;
2194 void dce6_bandwidth_update(struct radeon_device *rdev)
2196 struct drm_display_mode *mode0 = NULL;
2197 struct drm_display_mode *mode1 = NULL;
2198 u32 num_heads = 0, lb_size;
2201 radeon_update_display_priority(rdev);
2203 for (i = 0; i < rdev->num_crtc; i++) {
2204 if (rdev->mode_info.crtcs[i]->base.enabled)
2207 for (i = 0; i < rdev->num_crtc; i += 2) {
2208 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2209 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2210 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2211 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2212 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2213 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2220 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2222 const u32 num_tile_mode_states = 32;
2223 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2225 switch (rdev->config.si.mem_row_size_in_kb) {
2227 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2231 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2234 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2238 if ((rdev->family == CHIP_TAHITI) ||
2239 (rdev->family == CHIP_PITCAIRN)) {
2240 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2241 switch (reg_offset) {
2242 case 0: /* non-AA compressed depth or any compressed stencil */
2243 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2244 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2245 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2246 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2247 NUM_BANKS(ADDR_SURF_16_BANK) |
2248 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2249 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2250 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2252 case 1: /* 2xAA/4xAA compressed depth only */
2253 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2254 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2255 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2256 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2257 NUM_BANKS(ADDR_SURF_16_BANK) |
2258 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2259 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2260 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2262 case 2: /* 8xAA compressed depth only */
2263 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2264 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2265 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2266 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2267 NUM_BANKS(ADDR_SURF_16_BANK) |
2268 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2269 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2270 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2272 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2273 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2274 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2275 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2276 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2277 NUM_BANKS(ADDR_SURF_16_BANK) |
2278 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2279 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2280 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2282 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2283 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2284 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2285 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2286 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2287 NUM_BANKS(ADDR_SURF_16_BANK) |
2288 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2289 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2290 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2292 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2293 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2294 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2295 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2296 TILE_SPLIT(split_equal_to_row_size) |
2297 NUM_BANKS(ADDR_SURF_16_BANK) |
2298 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2299 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2300 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2302 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2303 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2304 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2305 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2306 TILE_SPLIT(split_equal_to_row_size) |
2307 NUM_BANKS(ADDR_SURF_16_BANK) |
2308 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2309 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2310 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2312 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2313 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2314 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2315 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2316 TILE_SPLIT(split_equal_to_row_size) |
2317 NUM_BANKS(ADDR_SURF_16_BANK) |
2318 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2319 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2320 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2322 case 8: /* 1D and 1D Array Surfaces */
2323 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2324 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2325 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2326 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2327 NUM_BANKS(ADDR_SURF_16_BANK) |
2328 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2329 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2330 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2332 case 9: /* Displayable maps. */
2333 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2334 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2335 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2336 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2337 NUM_BANKS(ADDR_SURF_16_BANK) |
2338 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2339 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2340 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2342 case 10: /* Display 8bpp. */
2343 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2344 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2345 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2346 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2347 NUM_BANKS(ADDR_SURF_16_BANK) |
2348 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2349 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2350 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2352 case 11: /* Display 16bpp. */
2353 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2354 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2355 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2356 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2357 NUM_BANKS(ADDR_SURF_16_BANK) |
2358 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2359 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2360 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2362 case 12: /* Display 32bpp. */
2363 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2364 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2365 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2366 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2367 NUM_BANKS(ADDR_SURF_16_BANK) |
2368 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2369 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2370 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2372 case 13: /* Thin. */
2373 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2374 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2375 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2376 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2377 NUM_BANKS(ADDR_SURF_16_BANK) |
2378 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2379 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2380 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2382 case 14: /* Thin 8 bpp. */
2383 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2384 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2385 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2386 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2387 NUM_BANKS(ADDR_SURF_16_BANK) |
2388 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2389 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2390 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2392 case 15: /* Thin 16 bpp. */
2393 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2394 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2395 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2396 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2397 NUM_BANKS(ADDR_SURF_16_BANK) |
2398 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2399 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2400 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2402 case 16: /* Thin 32 bpp. */
2403 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2404 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2405 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2406 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2407 NUM_BANKS(ADDR_SURF_16_BANK) |
2408 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2409 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2410 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2412 case 17: /* Thin 64 bpp. */
2413 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2414 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2415 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2416 TILE_SPLIT(split_equal_to_row_size) |
2417 NUM_BANKS(ADDR_SURF_16_BANK) |
2418 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2419 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2420 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2422 case 21: /* 8 bpp PRT. */
2423 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2424 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2425 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2426 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2427 NUM_BANKS(ADDR_SURF_16_BANK) |
2428 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2429 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2430 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2432 case 22: /* 16 bpp PRT */
2433 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2434 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2435 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2436 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2437 NUM_BANKS(ADDR_SURF_16_BANK) |
2438 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2439 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2440 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2442 case 23: /* 32 bpp PRT */
2443 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2444 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2445 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2446 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2447 NUM_BANKS(ADDR_SURF_16_BANK) |
2448 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2449 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2450 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2452 case 24: /* 64 bpp PRT */
2453 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2454 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2455 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2456 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2457 NUM_BANKS(ADDR_SURF_16_BANK) |
2458 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2459 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2460 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2462 case 25: /* 128 bpp PRT */
2463 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2464 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2465 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2466 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2467 NUM_BANKS(ADDR_SURF_8_BANK) |
2468 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2469 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2470 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2476 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2477 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2479 } else if ((rdev->family == CHIP_VERDE) ||
2480 (rdev->family == CHIP_OLAND) ||
2481 (rdev->family == CHIP_HAINAN)) {
2482 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2483 switch (reg_offset) {
2484 case 0: /* non-AA compressed depth or any compressed stencil */
2485 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2486 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2487 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2488 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2489 NUM_BANKS(ADDR_SURF_16_BANK) |
2490 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2491 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2492 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2494 case 1: /* 2xAA/4xAA compressed depth only */
2495 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2496 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2497 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2498 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2499 NUM_BANKS(ADDR_SURF_16_BANK) |
2500 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2501 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2502 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2504 case 2: /* 8xAA compressed depth only */
2505 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2506 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2507 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2508 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2509 NUM_BANKS(ADDR_SURF_16_BANK) |
2510 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2511 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2512 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2514 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2515 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2516 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2517 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2518 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2519 NUM_BANKS(ADDR_SURF_16_BANK) |
2520 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2521 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2522 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2524 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2526 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2528 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2529 NUM_BANKS(ADDR_SURF_16_BANK) |
2530 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2531 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2532 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2534 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2535 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2536 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2537 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2538 TILE_SPLIT(split_equal_to_row_size) |
2539 NUM_BANKS(ADDR_SURF_16_BANK) |
2540 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2541 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2542 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2544 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2545 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2546 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2547 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2548 TILE_SPLIT(split_equal_to_row_size) |
2549 NUM_BANKS(ADDR_SURF_16_BANK) |
2550 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2551 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2552 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2554 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2555 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2556 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2557 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2558 TILE_SPLIT(split_equal_to_row_size) |
2559 NUM_BANKS(ADDR_SURF_16_BANK) |
2560 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2561 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2562 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2564 case 8: /* 1D and 1D Array Surfaces */
2565 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2566 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2567 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2568 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2569 NUM_BANKS(ADDR_SURF_16_BANK) |
2570 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2571 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2572 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2574 case 9: /* Displayable maps. */
2575 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2576 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2577 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2578 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2579 NUM_BANKS(ADDR_SURF_16_BANK) |
2580 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2581 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2582 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2584 case 10: /* Display 8bpp. */
2585 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2586 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2587 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2588 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2589 NUM_BANKS(ADDR_SURF_16_BANK) |
2590 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2591 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2592 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2594 case 11: /* Display 16bpp. */
2595 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2596 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2597 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2598 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2599 NUM_BANKS(ADDR_SURF_16_BANK) |
2600 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2601 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2602 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2604 case 12: /* Display 32bpp. */
2605 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2606 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2607 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2608 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2609 NUM_BANKS(ADDR_SURF_16_BANK) |
2610 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2611 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2612 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2614 case 13: /* Thin. */
2615 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2616 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2617 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2618 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2619 NUM_BANKS(ADDR_SURF_16_BANK) |
2620 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2621 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2622 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2624 case 14: /* Thin 8 bpp. */
2625 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2626 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2627 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2628 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2629 NUM_BANKS(ADDR_SURF_16_BANK) |
2630 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2631 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2632 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2634 case 15: /* Thin 16 bpp. */
2635 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2636 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2637 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2638 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2639 NUM_BANKS(ADDR_SURF_16_BANK) |
2640 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2641 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2642 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2644 case 16: /* Thin 32 bpp. */
2645 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2646 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2647 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2648 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2649 NUM_BANKS(ADDR_SURF_16_BANK) |
2650 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2651 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2652 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2654 case 17: /* Thin 64 bpp. */
2655 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2656 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2657 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2658 TILE_SPLIT(split_equal_to_row_size) |
2659 NUM_BANKS(ADDR_SURF_16_BANK) |
2660 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2661 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2662 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2664 case 21: /* 8 bpp PRT. */
2665 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2666 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2667 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2668 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2669 NUM_BANKS(ADDR_SURF_16_BANK) |
2670 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2671 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2672 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2674 case 22: /* 16 bpp PRT */
2675 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2676 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2677 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2678 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2679 NUM_BANKS(ADDR_SURF_16_BANK) |
2680 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2681 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2682 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2684 case 23: /* 32 bpp PRT */
2685 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2686 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2687 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2688 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2689 NUM_BANKS(ADDR_SURF_16_BANK) |
2690 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2694 case 24: /* 64 bpp PRT */
2695 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2696 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2697 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2698 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2699 NUM_BANKS(ADDR_SURF_16_BANK) |
2700 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2701 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2702 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2704 case 25: /* 128 bpp PRT */
2705 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2706 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2707 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2708 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2709 NUM_BANKS(ADDR_SURF_8_BANK) |
2710 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2711 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2712 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2718 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2719 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2722 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2725 static void si_select_se_sh(struct radeon_device *rdev,
2726 u32 se_num, u32 sh_num)
2728 u32 data = INSTANCE_BROADCAST_WRITES;
2730 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2731 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2732 else if (se_num == 0xffffffff)
2733 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2734 else if (sh_num == 0xffffffff)
2735 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2737 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2738 WREG32(GRBM_GFX_INDEX, data);
2741 static u32 si_create_bitmask(u32 bit_width)
2745 for (i = 0; i < bit_width; i++) {
2752 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2756 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2758 data &= INACTIVE_CUS_MASK;
2761 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2763 data >>= INACTIVE_CUS_SHIFT;
2765 mask = si_create_bitmask(cu_per_sh);
2767 return ~data & mask;
2770 static void si_setup_spi(struct radeon_device *rdev,
2771 u32 se_num, u32 sh_per_se,
2775 u32 data, mask, active_cu;
2777 for (i = 0; i < se_num; i++) {
2778 for (j = 0; j < sh_per_se; j++) {
2779 si_select_se_sh(rdev, i, j);
2780 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2781 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2784 for (k = 0; k < 16; k++) {
2786 if (active_cu & mask) {
2788 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2794 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2797 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2798 u32 max_rb_num, u32 se_num,
2803 data = RREG32(CC_RB_BACKEND_DISABLE);
2805 data &= BACKEND_DISABLE_MASK;
2808 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2810 data >>= BACKEND_DISABLE_SHIFT;
2812 mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2817 static void si_setup_rb(struct radeon_device *rdev,
2818 u32 se_num, u32 sh_per_se,
2823 u32 disabled_rbs = 0;
2824 u32 enabled_rbs = 0;
2826 for (i = 0; i < se_num; i++) {
2827 for (j = 0; j < sh_per_se; j++) {
2828 si_select_se_sh(rdev, i, j);
2829 data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2830 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2833 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2836 for (i = 0; i < max_rb_num; i++) {
2837 if (!(disabled_rbs & mask))
2838 enabled_rbs |= mask;
2842 for (i = 0; i < se_num; i++) {
2843 si_select_se_sh(rdev, i, 0xffffffff);
2845 for (j = 0; j < sh_per_se; j++) {
2846 switch (enabled_rbs & 3) {
2848 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2851 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2855 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2860 WREG32(PA_SC_RASTER_CONFIG, data);
2862 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2865 static void si_gpu_init(struct radeon_device *rdev)
2867 u32 gb_addr_config = 0;
2868 u32 mc_shared_chmap, mc_arb_ramcfg;
2870 u32 hdp_host_path_cntl;
2874 switch (rdev->family) {
2876 rdev->config.si.max_shader_engines = 2;
2877 rdev->config.si.max_tile_pipes = 12;
2878 rdev->config.si.max_cu_per_sh = 8;
2879 rdev->config.si.max_sh_per_se = 2;
2880 rdev->config.si.max_backends_per_se = 4;
2881 rdev->config.si.max_texture_channel_caches = 12;
2882 rdev->config.si.max_gprs = 256;
2883 rdev->config.si.max_gs_threads = 32;
2884 rdev->config.si.max_hw_contexts = 8;
2886 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2887 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2888 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2889 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2890 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2893 rdev->config.si.max_shader_engines = 2;
2894 rdev->config.si.max_tile_pipes = 8;
2895 rdev->config.si.max_cu_per_sh = 5;
2896 rdev->config.si.max_sh_per_se = 2;
2897 rdev->config.si.max_backends_per_se = 4;
2898 rdev->config.si.max_texture_channel_caches = 8;
2899 rdev->config.si.max_gprs = 256;
2900 rdev->config.si.max_gs_threads = 32;
2901 rdev->config.si.max_hw_contexts = 8;
2903 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2904 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2905 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2906 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2907 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2911 rdev->config.si.max_shader_engines = 1;
2912 rdev->config.si.max_tile_pipes = 4;
2913 rdev->config.si.max_cu_per_sh = 5;
2914 rdev->config.si.max_sh_per_se = 2;
2915 rdev->config.si.max_backends_per_se = 4;
2916 rdev->config.si.max_texture_channel_caches = 4;
2917 rdev->config.si.max_gprs = 256;
2918 rdev->config.si.max_gs_threads = 32;
2919 rdev->config.si.max_hw_contexts = 8;
2921 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2922 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2923 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2924 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2925 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2928 rdev->config.si.max_shader_engines = 1;
2929 rdev->config.si.max_tile_pipes = 4;
2930 rdev->config.si.max_cu_per_sh = 6;
2931 rdev->config.si.max_sh_per_se = 1;
2932 rdev->config.si.max_backends_per_se = 2;
2933 rdev->config.si.max_texture_channel_caches = 4;
2934 rdev->config.si.max_gprs = 256;
2935 rdev->config.si.max_gs_threads = 16;
2936 rdev->config.si.max_hw_contexts = 8;
2938 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2939 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2940 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2941 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2942 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2945 rdev->config.si.max_shader_engines = 1;
2946 rdev->config.si.max_tile_pipes = 4;
2947 rdev->config.si.max_cu_per_sh = 5;
2948 rdev->config.si.max_sh_per_se = 1;
2949 rdev->config.si.max_backends_per_se = 1;
2950 rdev->config.si.max_texture_channel_caches = 2;
2951 rdev->config.si.max_gprs = 256;
2952 rdev->config.si.max_gs_threads = 16;
2953 rdev->config.si.max_hw_contexts = 8;
2955 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2956 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2957 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2958 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2959 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
2963 /* Initialize HDP */
2964 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2965 WREG32((0x2c14 + j), 0x00000000);
2966 WREG32((0x2c18 + j), 0x00000000);
2967 WREG32((0x2c1c + j), 0x00000000);
2968 WREG32((0x2c20 + j), 0x00000000);
2969 WREG32((0x2c24 + j), 0x00000000);
2972 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2974 evergreen_fix_pci_max_read_req_size(rdev);
2976 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2978 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2979 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2981 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
2982 rdev->config.si.mem_max_burst_length_bytes = 256;
2983 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2984 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2985 if (rdev->config.si.mem_row_size_in_kb > 4)
2986 rdev->config.si.mem_row_size_in_kb = 4;
2987 /* XXX use MC settings? */
2988 rdev->config.si.shader_engine_tile_size = 32;
2989 rdev->config.si.num_gpus = 1;
2990 rdev->config.si.multi_gpu_tile_size = 64;
2992 /* fix up row size */
2993 gb_addr_config &= ~ROW_SIZE_MASK;
2994 switch (rdev->config.si.mem_row_size_in_kb) {
2997 gb_addr_config |= ROW_SIZE(0);
3000 gb_addr_config |= ROW_SIZE(1);
3003 gb_addr_config |= ROW_SIZE(2);
3007 /* setup tiling info dword. gb_addr_config is not adequate since it does
3008 * not have bank info, so create a custom tiling dword.
3009 * bits 3:0 num_pipes
3010 * bits 7:4 num_banks
3011 * bits 11:8 group_size
3012 * bits 15:12 row_size
3014 rdev->config.si.tile_config = 0;
3015 switch (rdev->config.si.num_tile_pipes) {
3017 rdev->config.si.tile_config |= (0 << 0);
3020 rdev->config.si.tile_config |= (1 << 0);
3023 rdev->config.si.tile_config |= (2 << 0);
3027 /* XXX what about 12? */
3028 rdev->config.si.tile_config |= (3 << 0);
3031 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3032 case 0: /* four banks */
3033 rdev->config.si.tile_config |= 0 << 4;
3035 case 1: /* eight banks */
3036 rdev->config.si.tile_config |= 1 << 4;
3038 case 2: /* sixteen banks */
3040 rdev->config.si.tile_config |= 2 << 4;
3043 rdev->config.si.tile_config |=
3044 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3045 rdev->config.si.tile_config |=
3046 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3048 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3049 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3050 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3051 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3052 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3053 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3054 if (rdev->has_uvd) {
3055 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3056 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3057 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3060 si_tiling_mode_table_init(rdev);
3062 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3063 rdev->config.si.max_sh_per_se,
3064 rdev->config.si.max_backends_per_se);
3066 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3067 rdev->config.si.max_sh_per_se,
3068 rdev->config.si.max_cu_per_sh);
3071 /* set HW defaults for 3D engine */
3072 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3073 ROQ_IB2_START(0x2b)));
3074 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3076 sx_debug_1 = RREG32(SX_DEBUG_1);
3077 WREG32(SX_DEBUG_1, sx_debug_1);
3079 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3081 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3082 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3083 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3084 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3086 WREG32(VGT_NUM_INSTANCES, 1);
3088 WREG32(CP_PERFMON_CNTL, 0);
3090 WREG32(SQ_CONFIG, 0);
3092 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3093 FORCE_EOV_MAX_REZ_CNT(255)));
3095 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3096 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3098 WREG32(VGT_GS_VERTEX_REUSE, 16);
3099 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3101 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3102 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3103 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3104 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3105 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3106 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3107 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3108 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3110 tmp = RREG32(HDP_MISC_CNTL);
3111 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3112 WREG32(HDP_MISC_CNTL, tmp);
3114 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3115 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3117 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3123 * GPU scratch registers helpers function.
3125 static void si_scratch_init(struct radeon_device *rdev)
3129 rdev->scratch.num_reg = 7;
3130 rdev->scratch.reg_base = SCRATCH_REG0;
3131 for (i = 0; i < rdev->scratch.num_reg; i++) {
3132 rdev->scratch.free[i] = true;
3133 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3137 void si_fence_ring_emit(struct radeon_device *rdev,
3138 struct radeon_fence *fence)
3140 struct radeon_ring *ring = &rdev->ring[fence->ring];
3141 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3143 /* flush read cache over gart */
3144 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3145 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3146 radeon_ring_write(ring, 0);
3147 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3148 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3149 PACKET3_TC_ACTION_ENA |
3150 PACKET3_SH_KCACHE_ACTION_ENA |
3151 PACKET3_SH_ICACHE_ACTION_ENA);
3152 radeon_ring_write(ring, 0xFFFFFFFF);
3153 radeon_ring_write(ring, 0);
3154 radeon_ring_write(ring, 10); /* poll interval */
3155 /* EVENT_WRITE_EOP - flush caches, send int */
3156 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3157 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3158 radeon_ring_write(ring, addr & 0xffffffff);
3159 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3160 radeon_ring_write(ring, fence->seq);
3161 radeon_ring_write(ring, 0);
3167 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3169 struct radeon_ring *ring = &rdev->ring[ib->ring];
3172 if (ib->is_const_ib) {
3173 /* set switch buffer packet before const IB */
3174 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3175 radeon_ring_write(ring, 0);
3177 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3180 if (ring->rptr_save_reg) {
3181 next_rptr = ring->wptr + 3 + 4 + 8;
3182 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3183 radeon_ring_write(ring, ((ring->rptr_save_reg -
3184 PACKET3_SET_CONFIG_REG_START) >> 2));
3185 radeon_ring_write(ring, next_rptr);
3186 } else if (rdev->wb.enabled) {
3187 next_rptr = ring->wptr + 5 + 4 + 8;
3188 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3189 radeon_ring_write(ring, (1 << 8));
3190 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3191 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3192 radeon_ring_write(ring, next_rptr);
3195 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3198 radeon_ring_write(ring, header);
3199 radeon_ring_write(ring,
3203 (ib->gpu_addr & 0xFFFFFFFC));
3204 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3205 radeon_ring_write(ring, ib->length_dw |
3206 (ib->vm ? (ib->vm->id << 24) : 0));
3208 if (!ib->is_const_ib) {
3209 /* flush read cache over gart for this vmid */
3210 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3211 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3212 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
3213 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3214 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3215 PACKET3_TC_ACTION_ENA |
3216 PACKET3_SH_KCACHE_ACTION_ENA |
3217 PACKET3_SH_ICACHE_ACTION_ENA);
3218 radeon_ring_write(ring, 0xFFFFFFFF);
3219 radeon_ring_write(ring, 0);
3220 radeon_ring_write(ring, 10); /* poll interval */
3227 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3230 WREG32(CP_ME_CNTL, 0);
3232 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3233 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3234 WREG32(SCRATCH_UMSK, 0);
3235 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3236 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3237 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3242 static int si_cp_load_microcode(struct radeon_device *rdev)
3244 const __be32 *fw_data;
3247 if (!rdev->me_fw || !rdev->pfp_fw)
3250 si_cp_enable(rdev, false);
3253 fw_data = (const __be32 *)rdev->pfp_fw->data;
3254 WREG32(CP_PFP_UCODE_ADDR, 0);
3255 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3256 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3257 WREG32(CP_PFP_UCODE_ADDR, 0);
3260 fw_data = (const __be32 *)rdev->ce_fw->data;
3261 WREG32(CP_CE_UCODE_ADDR, 0);
3262 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3263 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3264 WREG32(CP_CE_UCODE_ADDR, 0);
3267 fw_data = (const __be32 *)rdev->me_fw->data;
3268 WREG32(CP_ME_RAM_WADDR, 0);
3269 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3270 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3271 WREG32(CP_ME_RAM_WADDR, 0);
3273 WREG32(CP_PFP_UCODE_ADDR, 0);
3274 WREG32(CP_CE_UCODE_ADDR, 0);
3275 WREG32(CP_ME_RAM_WADDR, 0);
3276 WREG32(CP_ME_RAM_RADDR, 0);
3280 static int si_cp_start(struct radeon_device *rdev)
3282 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3285 r = radeon_ring_lock(rdev, ring, 7 + 4);
3287 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3291 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3292 radeon_ring_write(ring, 0x1);
3293 radeon_ring_write(ring, 0x0);
3294 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3295 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3296 radeon_ring_write(ring, 0);
3297 radeon_ring_write(ring, 0);
3299 /* init the CE partitions */
3300 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3301 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3302 radeon_ring_write(ring, 0xc000);
3303 radeon_ring_write(ring, 0xe000);
3304 radeon_ring_unlock_commit(rdev, ring);
3306 si_cp_enable(rdev, true);
3308 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3310 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3314 /* setup clear context state */
3315 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3316 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3318 for (i = 0; i < si_default_size; i++)
3319 radeon_ring_write(ring, si_default_state[i]);
3321 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3322 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3324 /* set clear context state */
3325 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3326 radeon_ring_write(ring, 0);
3328 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3329 radeon_ring_write(ring, 0x00000316);
3330 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3331 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3333 radeon_ring_unlock_commit(rdev, ring);
3335 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3336 ring = &rdev->ring[i];
3337 r = radeon_ring_lock(rdev, ring, 2);
3339 /* clear the compute context state */
3340 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3341 radeon_ring_write(ring, 0);
3343 radeon_ring_unlock_commit(rdev, ring);
3349 static void si_cp_fini(struct radeon_device *rdev)
3351 struct radeon_ring *ring;
3352 si_cp_enable(rdev, false);
3354 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3355 radeon_ring_fini(rdev, ring);
3356 radeon_scratch_free(rdev, ring->rptr_save_reg);
3358 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3359 radeon_ring_fini(rdev, ring);
3360 radeon_scratch_free(rdev, ring->rptr_save_reg);
3362 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3363 radeon_ring_fini(rdev, ring);
3364 radeon_scratch_free(rdev, ring->rptr_save_reg);
3367 static int si_cp_resume(struct radeon_device *rdev)
3369 struct radeon_ring *ring;
3374 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3375 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3380 RREG32(GRBM_SOFT_RESET);
3382 WREG32(GRBM_SOFT_RESET, 0);
3383 RREG32(GRBM_SOFT_RESET);
3385 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3386 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3388 /* Set the write pointer delay */
3389 WREG32(CP_RB_WPTR_DELAY, 0);
3391 WREG32(CP_DEBUG, 0);
3392 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3394 /* ring 0 - compute and gfx */
3395 /* Set ring buffer size */
3396 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3397 rb_bufsz = drm_order(ring->ring_size / 8);
3398 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3400 tmp |= BUF_SWAP_32BIT;
3402 WREG32(CP_RB0_CNTL, tmp);
3404 /* Initialize the ring buffer's read and write pointers */
3405 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3407 WREG32(CP_RB0_WPTR, ring->wptr);
3409 /* set the wb address whether it's enabled or not */
3410 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3411 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3413 if (rdev->wb.enabled)
3414 WREG32(SCRATCH_UMSK, 0xff);
3416 tmp |= RB_NO_UPDATE;
3417 WREG32(SCRATCH_UMSK, 0);
3421 WREG32(CP_RB0_CNTL, tmp);
3423 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3425 ring->rptr = RREG32(CP_RB0_RPTR);
3427 /* ring1 - compute only */
3428 /* Set ring buffer size */
3429 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3430 rb_bufsz = drm_order(ring->ring_size / 8);
3431 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3433 tmp |= BUF_SWAP_32BIT;
3435 WREG32(CP_RB1_CNTL, tmp);
3437 /* Initialize the ring buffer's read and write pointers */
3438 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3440 WREG32(CP_RB1_WPTR, ring->wptr);
3442 /* set the wb address whether it's enabled or not */
3443 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3444 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3447 WREG32(CP_RB1_CNTL, tmp);
3449 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3451 ring->rptr = RREG32(CP_RB1_RPTR);
3453 /* ring2 - compute only */
3454 /* Set ring buffer size */
3455 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3456 rb_bufsz = drm_order(ring->ring_size / 8);
3457 tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3459 tmp |= BUF_SWAP_32BIT;
3461 WREG32(CP_RB2_CNTL, tmp);
3463 /* Initialize the ring buffer's read and write pointers */
3464 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3466 WREG32(CP_RB2_WPTR, ring->wptr);
3468 /* set the wb address whether it's enabled or not */
3469 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3470 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3473 WREG32(CP_RB2_CNTL, tmp);
3475 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3477 ring->rptr = RREG32(CP_RB2_RPTR);
3479 /* start the rings */
3481 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3482 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3483 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3484 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3486 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3487 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3488 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3491 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3493 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3495 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3497 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3503 static u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3509 tmp = RREG32(GRBM_STATUS);
3510 if (tmp & (PA_BUSY | SC_BUSY |
3511 BCI_BUSY | SX_BUSY |
3512 TA_BUSY | VGT_BUSY |
3514 GDS_BUSY | SPI_BUSY |
3515 IA_BUSY | IA_BUSY_NO_DMA))
3516 reset_mask |= RADEON_RESET_GFX;
3518 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3519 CP_BUSY | CP_COHERENCY_BUSY))
3520 reset_mask |= RADEON_RESET_CP;
3522 if (tmp & GRBM_EE_BUSY)
3523 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3526 tmp = RREG32(GRBM_STATUS2);
3527 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3528 reset_mask |= RADEON_RESET_RLC;
3530 /* DMA_STATUS_REG 0 */
3531 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3532 if (!(tmp & DMA_IDLE))
3533 reset_mask |= RADEON_RESET_DMA;
3535 /* DMA_STATUS_REG 1 */
3536 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3537 if (!(tmp & DMA_IDLE))
3538 reset_mask |= RADEON_RESET_DMA1;
3541 tmp = RREG32(SRBM_STATUS2);
3543 reset_mask |= RADEON_RESET_DMA;
3545 if (tmp & DMA1_BUSY)
3546 reset_mask |= RADEON_RESET_DMA1;
3549 tmp = RREG32(SRBM_STATUS);
3552 reset_mask |= RADEON_RESET_IH;
3555 reset_mask |= RADEON_RESET_SEM;
3557 if (tmp & GRBM_RQ_PENDING)
3558 reset_mask |= RADEON_RESET_GRBM;
3561 reset_mask |= RADEON_RESET_VMC;
3563 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3564 MCC_BUSY | MCD_BUSY))
3565 reset_mask |= RADEON_RESET_MC;
3567 if (evergreen_is_display_hung(rdev))
3568 reset_mask |= RADEON_RESET_DISPLAY;
3571 tmp = RREG32(VM_L2_STATUS);
3573 reset_mask |= RADEON_RESET_VMC;
3575 /* Skip MC reset as it's mostly likely not hung, just busy */
3576 if (reset_mask & RADEON_RESET_MC) {
3577 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3578 reset_mask &= ~RADEON_RESET_MC;
3584 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3586 struct evergreen_mc_save save;
3587 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3590 if (reset_mask == 0)
3593 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3595 evergreen_print_gpu_status_regs(rdev);
3596 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3597 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3598 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3599 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3601 /* Disable CP parsing/prefetching */
3602 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3604 if (reset_mask & RADEON_RESET_DMA) {
3606 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3607 tmp &= ~DMA_RB_ENABLE;
3608 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3610 if (reset_mask & RADEON_RESET_DMA1) {
3612 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3613 tmp &= ~DMA_RB_ENABLE;
3614 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3619 evergreen_mc_stop(rdev, &save);
3620 if (evergreen_mc_wait_for_idle(rdev)) {
3621 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3624 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3625 grbm_soft_reset = SOFT_RESET_CB |
3639 if (reset_mask & RADEON_RESET_CP) {
3640 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3642 srbm_soft_reset |= SOFT_RESET_GRBM;
3645 if (reset_mask & RADEON_RESET_DMA)
3646 srbm_soft_reset |= SOFT_RESET_DMA;
3648 if (reset_mask & RADEON_RESET_DMA1)
3649 srbm_soft_reset |= SOFT_RESET_DMA1;
3651 if (reset_mask & RADEON_RESET_DISPLAY)
3652 srbm_soft_reset |= SOFT_RESET_DC;
3654 if (reset_mask & RADEON_RESET_RLC)
3655 grbm_soft_reset |= SOFT_RESET_RLC;
3657 if (reset_mask & RADEON_RESET_SEM)
3658 srbm_soft_reset |= SOFT_RESET_SEM;
3660 if (reset_mask & RADEON_RESET_IH)
3661 srbm_soft_reset |= SOFT_RESET_IH;
3663 if (reset_mask & RADEON_RESET_GRBM)
3664 srbm_soft_reset |= SOFT_RESET_GRBM;
3666 if (reset_mask & RADEON_RESET_VMC)
3667 srbm_soft_reset |= SOFT_RESET_VMC;
3669 if (reset_mask & RADEON_RESET_MC)
3670 srbm_soft_reset |= SOFT_RESET_MC;
3672 if (grbm_soft_reset) {
3673 tmp = RREG32(GRBM_SOFT_RESET);
3674 tmp |= grbm_soft_reset;
3675 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3676 WREG32(GRBM_SOFT_RESET, tmp);
3677 tmp = RREG32(GRBM_SOFT_RESET);
3681 tmp &= ~grbm_soft_reset;
3682 WREG32(GRBM_SOFT_RESET, tmp);
3683 tmp = RREG32(GRBM_SOFT_RESET);
3686 if (srbm_soft_reset) {
3687 tmp = RREG32(SRBM_SOFT_RESET);
3688 tmp |= srbm_soft_reset;
3689 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3690 WREG32(SRBM_SOFT_RESET, tmp);
3691 tmp = RREG32(SRBM_SOFT_RESET);
3695 tmp &= ~srbm_soft_reset;
3696 WREG32(SRBM_SOFT_RESET, tmp);
3697 tmp = RREG32(SRBM_SOFT_RESET);
3700 /* Wait a little for things to settle down */
3703 evergreen_mc_resume(rdev, &save);
3706 evergreen_print_gpu_status_regs(rdev);
3709 int si_asic_reset(struct radeon_device *rdev)
3713 reset_mask = si_gpu_check_soft_reset(rdev);
3716 r600_set_bios_scratch_engine_hung(rdev, true);
3718 si_gpu_soft_reset(rdev, reset_mask);
3720 reset_mask = si_gpu_check_soft_reset(rdev);
3723 r600_set_bios_scratch_engine_hung(rdev, false);
3729 * si_gfx_is_lockup - Check if the GFX engine is locked up
3731 * @rdev: radeon_device pointer
3732 * @ring: radeon_ring structure holding ring information
3734 * Check if the GFX engine is locked up.
3735 * Returns true if the engine appears to be locked up, false if not.
3737 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3739 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3741 if (!(reset_mask & (RADEON_RESET_GFX |
3742 RADEON_RESET_COMPUTE |
3743 RADEON_RESET_CP))) {
3744 radeon_ring_lockup_update(ring);
3747 /* force CP activities */
3748 radeon_ring_force_activity(rdev, ring);
3749 return radeon_ring_test_lockup(rdev, ring);
3753 * si_dma_is_lockup - Check if the DMA engine is locked up
3755 * @rdev: radeon_device pointer
3756 * @ring: radeon_ring structure holding ring information
3758 * Check if the async DMA engine is locked up.
3759 * Returns true if the engine appears to be locked up, false if not.
3761 bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3763 u32 reset_mask = si_gpu_check_soft_reset(rdev);
3766 if (ring->idx == R600_RING_TYPE_DMA_INDEX)
3767 mask = RADEON_RESET_DMA;
3769 mask = RADEON_RESET_DMA1;
3771 if (!(reset_mask & mask)) {
3772 radeon_ring_lockup_update(ring);
3775 /* force ring activities */
3776 radeon_ring_force_activity(rdev, ring);
3777 return radeon_ring_test_lockup(rdev, ring);
3781 static void si_mc_program(struct radeon_device *rdev)
3783 struct evergreen_mc_save save;
3787 /* Initialize HDP */
3788 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3789 WREG32((0x2c14 + j), 0x00000000);
3790 WREG32((0x2c18 + j), 0x00000000);
3791 WREG32((0x2c1c + j), 0x00000000);
3792 WREG32((0x2c20 + j), 0x00000000);
3793 WREG32((0x2c24 + j), 0x00000000);
3795 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3797 evergreen_mc_stop(rdev, &save);
3798 if (radeon_mc_wait_for_idle(rdev)) {
3799 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3801 if (!ASIC_IS_NODCE(rdev))
3802 /* Lockout access through VGA aperture*/
3803 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3804 /* Update configuration */
3805 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3806 rdev->mc.vram_start >> 12);
3807 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3808 rdev->mc.vram_end >> 12);
3809 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3810 rdev->vram_scratch.gpu_addr >> 12);
3811 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3812 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3813 WREG32(MC_VM_FB_LOCATION, tmp);
3814 /* XXX double check these! */
3815 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3816 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3817 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3818 WREG32(MC_VM_AGP_BASE, 0);
3819 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3820 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3821 if (radeon_mc_wait_for_idle(rdev)) {
3822 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3824 evergreen_mc_resume(rdev, &save);
3825 if (!ASIC_IS_NODCE(rdev)) {
3826 /* we need to own VRAM, so turn off the VGA renderer here
3827 * to stop it overwriting our objects */
3828 rv515_vga_render_disable(rdev);
3832 void si_vram_gtt_location(struct radeon_device *rdev,
3833 struct radeon_mc *mc)
3835 if (mc->mc_vram_size > 0xFFC0000000ULL) {
3836 /* leave room for at least 1024M GTT */
3837 dev_warn(rdev->dev, "limiting VRAM\n");
3838 mc->real_vram_size = 0xFFC0000000ULL;
3839 mc->mc_vram_size = 0xFFC0000000ULL;
3841 radeon_vram_location(rdev, &rdev->mc, 0);
3842 rdev->mc.gtt_base_align = 0;
3843 radeon_gtt_location(rdev, mc);
3846 static int si_mc_init(struct radeon_device *rdev)
3849 int chansize, numchan;
3851 /* Get VRAM informations */
3852 rdev->mc.vram_is_ddr = true;
3853 tmp = RREG32(MC_ARB_RAMCFG);
3854 if (tmp & CHANSIZE_OVERRIDE) {
3856 } else if (tmp & CHANSIZE_MASK) {
3861 tmp = RREG32(MC_SHARED_CHMAP);
3862 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3892 rdev->mc.vram_width = numchan * chansize;
3893 /* Could aper size report 0 ? */
3894 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3895 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3896 /* size in MB on si */
3897 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3898 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3899 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3900 si_vram_gtt_location(rdev, &rdev->mc);
3901 radeon_update_bandwidth_info(rdev);
3909 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3911 /* flush hdp cache */
3912 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3914 /* bits 0-15 are the VM contexts0-15 */
3915 WREG32(VM_INVALIDATE_REQUEST, 1);
3918 static int si_pcie_gart_enable(struct radeon_device *rdev)
3922 if (rdev->gart.robj == NULL) {
3923 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3926 r = radeon_gart_table_vram_pin(rdev);
3929 radeon_gart_restore(rdev);
3930 /* Setup TLB control */
3931 WREG32(MC_VM_MX_L1_TLB_CNTL,
3934 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3935 ENABLE_ADVANCED_DRIVER_MODEL |
3936 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3937 /* Setup L2 cache */
3938 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3939 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3940 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3941 EFFECTIVE_L2_QUEUE_SIZE(7) |
3942 CONTEXT1_IDENTITY_ACCESS_MODE(1));
3943 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3944 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3945 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3946 /* setup context0 */
3947 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3948 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3949 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3950 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3951 (u32)(rdev->dummy_page.addr >> 12));
3952 WREG32(VM_CONTEXT0_CNTL2, 0);
3953 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3954 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3960 /* empty context1-15 */
3961 /* set vm size, must be a multiple of 4 */
3962 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
3963 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
3964 /* Assign the pt base to something valid for now; the pts used for
3965 * the VMs are determined by the application and setup and assigned
3966 * on the fly in the vm part of radeon_gart.c
3968 for (i = 1; i < 16; i++) {
3970 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3971 rdev->gart.table_addr >> 12);
3973 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3974 rdev->gart.table_addr >> 12);
3977 /* enable context1-15 */
3978 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3979 (u32)(rdev->dummy_page.addr >> 12));
3980 WREG32(VM_CONTEXT1_CNTL2, 4);
3981 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
3982 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3983 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3984 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3985 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3986 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3987 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3988 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3989 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3990 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3991 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3992 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3993 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
3995 si_pcie_gart_tlb_flush(rdev);
3996 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3997 (unsigned)(rdev->mc.gtt_size >> 20),
3998 (unsigned long long)rdev->gart.table_addr);
3999 rdev->gart.ready = true;
4003 static void si_pcie_gart_disable(struct radeon_device *rdev)
4005 /* Disable all tables */
4006 WREG32(VM_CONTEXT0_CNTL, 0);
4007 WREG32(VM_CONTEXT1_CNTL, 0);
4008 /* Setup TLB control */
4009 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4010 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4011 /* Setup L2 cache */
4012 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4013 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4014 EFFECTIVE_L2_QUEUE_SIZE(7) |
4015 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4016 WREG32(VM_L2_CNTL2, 0);
4017 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4018 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4019 radeon_gart_table_vram_unpin(rdev);
4022 static void si_pcie_gart_fini(struct radeon_device *rdev)
4024 si_pcie_gart_disable(rdev);
4025 radeon_gart_table_vram_free(rdev);
4026 radeon_gart_fini(rdev);
4030 static bool si_vm_reg_valid(u32 reg)
4032 /* context regs are fine */
4036 /* check config regs */
4038 case GRBM_GFX_INDEX:
4039 case CP_STRMOUT_CNTL:
4040 case VGT_VTX_VECT_EJECT_REG:
4041 case VGT_CACHE_INVALIDATION:
4042 case VGT_ESGS_RING_SIZE:
4043 case VGT_GSVS_RING_SIZE:
4044 case VGT_GS_VERTEX_REUSE:
4045 case VGT_PRIMITIVE_TYPE:
4046 case VGT_INDEX_TYPE:
4047 case VGT_NUM_INDICES:
4048 case VGT_NUM_INSTANCES:
4049 case VGT_TF_RING_SIZE:
4050 case VGT_HS_OFFCHIP_PARAM:
4051 case VGT_TF_MEMORY_BASE:
4053 case PA_SU_LINE_STIPPLE_VALUE:
4054 case PA_SC_LINE_STIPPLE_STATE:
4057 case SPI_STATIC_THREAD_MGMT_1:
4058 case SPI_STATIC_THREAD_MGMT_2:
4059 case SPI_STATIC_THREAD_MGMT_3:
4060 case SPI_PS_MAX_WAVE_ID:
4061 case SPI_CONFIG_CNTL:
4062 case SPI_CONFIG_CNTL_1:
4066 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4071 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4072 u32 *ib, struct radeon_cs_packet *pkt)
4074 switch (pkt->opcode) {
4076 case PACKET3_SET_BASE:
4077 case PACKET3_SET_CE_DE_COUNTERS:
4078 case PACKET3_LOAD_CONST_RAM:
4079 case PACKET3_WRITE_CONST_RAM:
4080 case PACKET3_WRITE_CONST_RAM_OFFSET:
4081 case PACKET3_DUMP_CONST_RAM:
4082 case PACKET3_INCREMENT_CE_COUNTER:
4083 case PACKET3_WAIT_ON_DE_COUNTER:
4084 case PACKET3_CE_WRITE:
4087 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4093 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4094 u32 *ib, struct radeon_cs_packet *pkt)
4096 u32 idx = pkt->idx + 1;
4097 u32 idx_value = ib[idx];
4098 u32 start_reg, end_reg, reg, i;
4101 switch (pkt->opcode) {
4103 case PACKET3_SET_BASE:
4104 case PACKET3_CLEAR_STATE:
4105 case PACKET3_INDEX_BUFFER_SIZE:
4106 case PACKET3_DISPATCH_DIRECT:
4107 case PACKET3_DISPATCH_INDIRECT:
4108 case PACKET3_ALLOC_GDS:
4109 case PACKET3_WRITE_GDS_RAM:
4110 case PACKET3_ATOMIC_GDS:
4111 case PACKET3_ATOMIC:
4112 case PACKET3_OCCLUSION_QUERY:
4113 case PACKET3_SET_PREDICATION:
4114 case PACKET3_COND_EXEC:
4115 case PACKET3_PRED_EXEC:
4116 case PACKET3_DRAW_INDIRECT:
4117 case PACKET3_DRAW_INDEX_INDIRECT:
4118 case PACKET3_INDEX_BASE:
4119 case PACKET3_DRAW_INDEX_2:
4120 case PACKET3_CONTEXT_CONTROL:
4121 case PACKET3_INDEX_TYPE:
4122 case PACKET3_DRAW_INDIRECT_MULTI:
4123 case PACKET3_DRAW_INDEX_AUTO:
4124 case PACKET3_DRAW_INDEX_IMMD:
4125 case PACKET3_NUM_INSTANCES:
4126 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4127 case PACKET3_STRMOUT_BUFFER_UPDATE:
4128 case PACKET3_DRAW_INDEX_OFFSET_2:
4129 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4130 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4131 case PACKET3_MPEG_INDEX:
4132 case PACKET3_WAIT_REG_MEM:
4133 case PACKET3_MEM_WRITE:
4134 case PACKET3_PFP_SYNC_ME:
4135 case PACKET3_SURFACE_SYNC:
4136 case PACKET3_EVENT_WRITE:
4137 case PACKET3_EVENT_WRITE_EOP:
4138 case PACKET3_EVENT_WRITE_EOS:
4139 case PACKET3_SET_CONTEXT_REG:
4140 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4141 case PACKET3_SET_SH_REG:
4142 case PACKET3_SET_SH_REG_OFFSET:
4143 case PACKET3_INCREMENT_DE_COUNTER:
4144 case PACKET3_WAIT_ON_CE_COUNTER:
4145 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4146 case PACKET3_ME_WRITE:
4148 case PACKET3_COPY_DATA:
4149 if ((idx_value & 0xf00) == 0) {
4150 reg = ib[idx + 3] * 4;
4151 if (!si_vm_reg_valid(reg))
4155 case PACKET3_WRITE_DATA:
4156 if ((idx_value & 0xf00) == 0) {
4157 start_reg = ib[idx + 1] * 4;
4158 if (idx_value & 0x10000) {
4159 if (!si_vm_reg_valid(start_reg))
4162 for (i = 0; i < (pkt->count - 2); i++) {
4163 reg = start_reg + (4 * i);
4164 if (!si_vm_reg_valid(reg))
4170 case PACKET3_COND_WRITE:
4171 if (idx_value & 0x100) {
4172 reg = ib[idx + 5] * 4;
4173 if (!si_vm_reg_valid(reg))
4177 case PACKET3_COPY_DW:
4178 if (idx_value & 0x2) {
4179 reg = ib[idx + 3] * 4;
4180 if (!si_vm_reg_valid(reg))
4184 case PACKET3_SET_CONFIG_REG:
4185 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4186 end_reg = 4 * pkt->count + start_reg - 4;
4187 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4188 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4189 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4190 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4193 for (i = 0; i < pkt->count; i++) {
4194 reg = start_reg + (4 * i);
4195 if (!si_vm_reg_valid(reg))
4199 case PACKET3_CP_DMA:
4200 command = ib[idx + 4];
4202 if (command & PACKET3_CP_DMA_CMD_SAS) {
4203 /* src address space is register */
4204 if (((info & 0x60000000) >> 29) == 0) {
4205 start_reg = idx_value << 2;
4206 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4208 if (!si_vm_reg_valid(reg)) {
4209 DRM_ERROR("CP DMA Bad SRC register\n");
4213 for (i = 0; i < (command & 0x1fffff); i++) {
4214 reg = start_reg + (4 * i);
4215 if (!si_vm_reg_valid(reg)) {
4216 DRM_ERROR("CP DMA Bad SRC register\n");
4223 if (command & PACKET3_CP_DMA_CMD_DAS) {
4224 /* dst address space is register */
4225 if (((info & 0x00300000) >> 20) == 0) {
4226 start_reg = ib[idx + 2];
4227 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4229 if (!si_vm_reg_valid(reg)) {
4230 DRM_ERROR("CP DMA Bad DST register\n");
4234 for (i = 0; i < (command & 0x1fffff); i++) {
4235 reg = start_reg + (4 * i);
4236 if (!si_vm_reg_valid(reg)) {
4237 DRM_ERROR("CP DMA Bad DST register\n");
4246 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4252 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4253 u32 *ib, struct radeon_cs_packet *pkt)
4255 u32 idx = pkt->idx + 1;
4256 u32 idx_value = ib[idx];
4257 u32 start_reg, reg, i;
4259 switch (pkt->opcode) {
4261 case PACKET3_SET_BASE:
4262 case PACKET3_CLEAR_STATE:
4263 case PACKET3_DISPATCH_DIRECT:
4264 case PACKET3_DISPATCH_INDIRECT:
4265 case PACKET3_ALLOC_GDS:
4266 case PACKET3_WRITE_GDS_RAM:
4267 case PACKET3_ATOMIC_GDS:
4268 case PACKET3_ATOMIC:
4269 case PACKET3_OCCLUSION_QUERY:
4270 case PACKET3_SET_PREDICATION:
4271 case PACKET3_COND_EXEC:
4272 case PACKET3_PRED_EXEC:
4273 case PACKET3_CONTEXT_CONTROL:
4274 case PACKET3_STRMOUT_BUFFER_UPDATE:
4275 case PACKET3_WAIT_REG_MEM:
4276 case PACKET3_MEM_WRITE:
4277 case PACKET3_PFP_SYNC_ME:
4278 case PACKET3_SURFACE_SYNC:
4279 case PACKET3_EVENT_WRITE:
4280 case PACKET3_EVENT_WRITE_EOP:
4281 case PACKET3_EVENT_WRITE_EOS:
4282 case PACKET3_SET_CONTEXT_REG:
4283 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4284 case PACKET3_SET_SH_REG:
4285 case PACKET3_SET_SH_REG_OFFSET:
4286 case PACKET3_INCREMENT_DE_COUNTER:
4287 case PACKET3_WAIT_ON_CE_COUNTER:
4288 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4289 case PACKET3_ME_WRITE:
4291 case PACKET3_COPY_DATA:
4292 if ((idx_value & 0xf00) == 0) {
4293 reg = ib[idx + 3] * 4;
4294 if (!si_vm_reg_valid(reg))
4298 case PACKET3_WRITE_DATA:
4299 if ((idx_value & 0xf00) == 0) {
4300 start_reg = ib[idx + 1] * 4;
4301 if (idx_value & 0x10000) {
4302 if (!si_vm_reg_valid(start_reg))
4305 for (i = 0; i < (pkt->count - 2); i++) {
4306 reg = start_reg + (4 * i);
4307 if (!si_vm_reg_valid(reg))
4313 case PACKET3_COND_WRITE:
4314 if (idx_value & 0x100) {
4315 reg = ib[idx + 5] * 4;
4316 if (!si_vm_reg_valid(reg))
4320 case PACKET3_COPY_DW:
4321 if (idx_value & 0x2) {
4322 reg = ib[idx + 3] * 4;
4323 if (!si_vm_reg_valid(reg))
4328 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4334 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4338 struct radeon_cs_packet pkt;
4342 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4343 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4346 case RADEON_PACKET_TYPE0:
4347 dev_err(rdev->dev, "Packet0 not allowed!\n");
4350 case RADEON_PACKET_TYPE2:
4353 case RADEON_PACKET_TYPE3:
4354 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4355 if (ib->is_const_ib)
4356 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4359 case RADEON_RING_TYPE_GFX_INDEX:
4360 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4362 case CAYMAN_RING_TYPE_CP1_INDEX:
4363 case CAYMAN_RING_TYPE_CP2_INDEX:
4364 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4367 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4372 idx += pkt.count + 2;
4375 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4381 } while (idx < ib->length_dw);
4389 int si_vm_init(struct radeon_device *rdev)
4392 rdev->vm_manager.nvm = 16;
4393 /* base offset of vram pages */
4394 rdev->vm_manager.vram_base_offset = 0;
4399 void si_vm_fini(struct radeon_device *rdev)
4404 * si_vm_set_page - update the page tables using the CP
4406 * @rdev: radeon_device pointer
4407 * @ib: indirect buffer to fill with commands
4408 * @pe: addr of the page entry
4409 * @addr: dst addr to write into pe
4410 * @count: number of page entries to update
4411 * @incr: increase next addr by incr bytes
4412 * @flags: access flags
4414 * Update the page tables using the CP (SI).
4416 void si_vm_set_page(struct radeon_device *rdev,
4417 struct radeon_ib *ib,
4419 uint64_t addr, unsigned count,
4420 uint32_t incr, uint32_t flags)
4422 uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
4426 if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
4428 ndw = 2 + count * 2;
4432 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
4433 ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
4434 WRITE_DATA_DST_SEL(1));
4435 ib->ptr[ib->length_dw++] = pe;
4436 ib->ptr[ib->length_dw++] = upper_32_bits(pe);
4437 for (; ndw > 2; ndw -= 2, --count, pe += 8) {
4438 if (flags & RADEON_VM_PAGE_SYSTEM) {
4439 value = radeon_vm_map_gart(rdev, addr);
4440 value &= 0xFFFFFFFFFFFFF000ULL;
4441 } else if (flags & RADEON_VM_PAGE_VALID) {
4447 value |= r600_flags;
4448 ib->ptr[ib->length_dw++] = value;
4449 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4454 if (flags & RADEON_VM_PAGE_SYSTEM) {
4460 /* for non-physically contiguous pages (system) */
4461 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
4462 ib->ptr[ib->length_dw++] = pe;
4463 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
4464 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
4465 if (flags & RADEON_VM_PAGE_SYSTEM) {
4466 value = radeon_vm_map_gart(rdev, addr);
4467 value &= 0xFFFFFFFFFFFFF000ULL;
4468 } else if (flags & RADEON_VM_PAGE_VALID) {
4474 value |= r600_flags;
4475 ib->ptr[ib->length_dw++] = value;
4476 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4485 if (flags & RADEON_VM_PAGE_VALID)
4489 /* for physically contiguous pages (vram) */
4490 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
4491 ib->ptr[ib->length_dw++] = pe; /* dst addr */
4492 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
4493 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
4494 ib->ptr[ib->length_dw++] = 0;
4495 ib->ptr[ib->length_dw++] = value; /* value */
4496 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4497 ib->ptr[ib->length_dw++] = incr; /* increment size */
4498 ib->ptr[ib->length_dw++] = 0;
4500 addr += (ndw / 2) * incr;
4504 while (ib->length_dw & 0x7)
4505 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
4509 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4511 struct radeon_ring *ring = &rdev->ring[ridx];
4516 /* write new base address */
4517 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4518 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4519 WRITE_DATA_DST_SEL(0)));
4522 radeon_ring_write(ring,
4523 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4525 radeon_ring_write(ring,
4526 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4528 radeon_ring_write(ring, 0);
4529 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4531 /* flush hdp cache */
4532 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4533 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4534 WRITE_DATA_DST_SEL(0)));
4535 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4536 radeon_ring_write(ring, 0);
4537 radeon_ring_write(ring, 0x1);
4539 /* bits 0-15 are the VM contexts0-15 */
4540 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4541 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4542 WRITE_DATA_DST_SEL(0)));
4543 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4544 radeon_ring_write(ring, 0);
4545 radeon_ring_write(ring, 1 << vm->id);
4547 /* sync PFP to ME, otherwise we might get invalid PFP reads */
4548 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4549 radeon_ring_write(ring, 0x0);
4552 void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4554 struct radeon_ring *ring = &rdev->ring[ridx];
4559 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4561 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
4563 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2));
4565 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4567 /* flush hdp cache */
4568 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4569 radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4570 radeon_ring_write(ring, 1);
4572 /* bits 0-7 are the VM contexts0-7 */
4573 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4574 radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
4575 radeon_ring_write(ring, 1 << vm->id);
4579 * Power and clock gating
4581 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
4585 for (i = 0; i < rdev->usec_timeout; i++) {
4586 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
4591 for (i = 0; i < rdev->usec_timeout; i++) {
4592 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
4598 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
4601 u32 tmp = RREG32(CP_INT_CNTL_RING0);
4606 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4608 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4609 WREG32(CP_INT_CNTL_RING0, tmp);
4612 /* read a gfx register */
4613 tmp = RREG32(DB_DEPTH_INFO);
4615 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
4616 for (i = 0; i < rdev->usec_timeout; i++) {
4617 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
4624 static void si_set_uvd_dcm(struct radeon_device *rdev,
4629 tmp = RREG32(UVD_CGC_CTRL);
4630 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
4631 tmp |= DCM | CG_DT(1) | CLK_OD(4);
4635 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
4641 WREG32(UVD_CGC_CTRL, tmp);
4642 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
4645 static void si_init_uvd_internal_cg(struct radeon_device *rdev)
4647 bool hw_mode = true;
4650 si_set_uvd_dcm(rdev, false);
4652 u32 tmp = RREG32(UVD_CGC_CTRL);
4654 WREG32(UVD_CGC_CTRL, tmp);
4658 static u32 si_halt_rlc(struct radeon_device *rdev)
4662 orig = data = RREG32(RLC_CNTL);
4664 if (data & RLC_ENABLE) {
4665 data &= ~RLC_ENABLE;
4666 WREG32(RLC_CNTL, data);
4668 si_wait_for_rlc_serdes(rdev);
4674 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
4678 tmp = RREG32(RLC_CNTL);
4680 WREG32(RLC_CNTL, rlc);
4683 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
4687 orig = data = RREG32(DMA_PG);
4689 data |= PG_CNTL_ENABLE;
4691 data &= ~PG_CNTL_ENABLE;
4693 WREG32(DMA_PG, data);
4696 static void si_init_dma_pg(struct radeon_device *rdev)
4700 WREG32(DMA_PGFSM_WRITE, 0x00002000);
4701 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
4703 for (tmp = 0; tmp < 5; tmp++)
4704 WREG32(DMA_PGFSM_WRITE, 0);
4707 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
4713 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
4714 WREG32(RLC_TTOP_D, tmp);
4716 tmp = RREG32(RLC_PG_CNTL);
4717 tmp |= GFX_PG_ENABLE;
4718 WREG32(RLC_PG_CNTL, tmp);
4720 tmp = RREG32(RLC_AUTO_PG_CTRL);
4722 WREG32(RLC_AUTO_PG_CTRL, tmp);
4724 tmp = RREG32(RLC_AUTO_PG_CTRL);
4726 WREG32(RLC_AUTO_PG_CTRL, tmp);
4728 tmp = RREG32(DB_RENDER_CONTROL);
4732 static void si_init_gfx_cgpg(struct radeon_device *rdev)
4736 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4738 tmp = RREG32(RLC_PG_CNTL);
4740 WREG32(RLC_PG_CNTL, tmp);
4742 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4744 tmp = RREG32(RLC_AUTO_PG_CTRL);
4746 tmp &= ~GRBM_REG_SGIT_MASK;
4747 tmp |= GRBM_REG_SGIT(0x700);
4748 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
4749 WREG32(RLC_AUTO_PG_CTRL, tmp);
4752 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
4754 u32 mask = 0, tmp, tmp1;
4757 si_select_se_sh(rdev, se, sh);
4758 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
4759 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
4760 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
4767 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
4772 return (~tmp) & mask;
4775 static void si_init_ao_cu_mask(struct radeon_device *rdev)
4777 u32 i, j, k, active_cu_number = 0;
4778 u32 mask, counter, cu_bitmap;
4781 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
4782 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
4786 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
4787 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
4795 active_cu_number += counter;
4796 tmp |= (cu_bitmap << (i * 16 + j * 8));
4800 WREG32(RLC_PG_AO_CU_MASK, tmp);
4802 tmp = RREG32(RLC_MAX_PG_CU);
4803 tmp &= ~MAX_PU_CU_MASK;
4804 tmp |= MAX_PU_CU(active_cu_number);
4805 WREG32(RLC_MAX_PG_CU, tmp);
4808 static void si_enable_cgcg(struct radeon_device *rdev,
4811 u32 data, orig, tmp;
4813 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
4815 si_enable_gui_idle_interrupt(rdev, enable);
4818 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
4820 tmp = si_halt_rlc(rdev);
4822 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4823 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4824 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
4826 si_wait_for_rlc_serdes(rdev);
4828 si_update_rlc(rdev, tmp);
4830 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
4832 data |= CGCG_EN | CGLS_EN;
4834 RREG32(CB_CGTT_SCLK_CTRL);
4835 RREG32(CB_CGTT_SCLK_CTRL);
4836 RREG32(CB_CGTT_SCLK_CTRL);
4837 RREG32(CB_CGTT_SCLK_CTRL);
4839 data &= ~(CGCG_EN | CGLS_EN);
4843 WREG32(RLC_CGCG_CGLS_CTRL, data);
4846 static void si_enable_mgcg(struct radeon_device *rdev,
4849 u32 data, orig, tmp = 0;
4852 orig = data = RREG32(CGTS_SM_CTRL_REG);
4855 WREG32(CGTS_SM_CTRL_REG, data);
4857 orig = data = RREG32(CP_MEM_SLP_CNTL);
4858 data |= CP_MEM_LS_EN;
4860 WREG32(CP_MEM_SLP_CNTL, data);
4862 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4865 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
4867 tmp = si_halt_rlc(rdev);
4869 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4870 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4871 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
4873 si_update_rlc(rdev, tmp);
4875 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4878 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
4880 data = RREG32(CP_MEM_SLP_CNTL);
4881 if (data & CP_MEM_LS_EN) {
4882 data &= ~CP_MEM_LS_EN;
4883 WREG32(CP_MEM_SLP_CNTL, data);
4885 orig = data = RREG32(CGTS_SM_CTRL_REG);
4886 data |= LS_OVERRIDE | OVERRIDE;
4888 WREG32(CGTS_SM_CTRL_REG, data);
4890 tmp = si_halt_rlc(rdev);
4892 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
4893 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
4894 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
4896 si_update_rlc(rdev, tmp);
4900 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
4903 u32 orig, data, tmp;
4906 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
4908 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
4910 orig = data = RREG32(UVD_CGC_CTRL);
4913 WREG32(UVD_CGC_CTRL, data);
4915 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
4916 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
4918 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
4920 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
4922 orig = data = RREG32(UVD_CGC_CTRL);
4925 WREG32(UVD_CGC_CTRL, data);
4927 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
4928 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
4932 static const u32 mc_cg_registers[] =
4945 static void si_enable_mc_ls(struct radeon_device *rdev,
4951 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
4952 orig = data = RREG32(mc_cg_registers[i]);
4954 data |= MC_LS_ENABLE;
4956 data &= ~MC_LS_ENABLE;
4958 WREG32(mc_cg_registers[i], data);
4963 static void si_init_cg(struct radeon_device *rdev)
4965 bool has_uvd = true;
4967 si_enable_mgcg(rdev, true);
4968 si_enable_cgcg(rdev, true);
4969 /* disable MC LS on Tahiti */
4970 if (rdev->family == CHIP_TAHITI)
4971 si_enable_mc_ls(rdev, false);
4973 si_enable_uvd_mgcg(rdev, true);
4974 si_init_uvd_internal_cg(rdev);
4978 static void si_fini_cg(struct radeon_device *rdev)
4980 bool has_uvd = true;
4983 si_enable_uvd_mgcg(rdev, false);
4984 si_enable_cgcg(rdev, false);
4985 si_enable_mgcg(rdev, false);
4988 static void si_init_pg(struct radeon_device *rdev)
4990 bool has_pg = false;
4992 /* only cape verde supports PG */
4993 if (rdev->family == CHIP_VERDE)
4997 si_init_ao_cu_mask(rdev);
4998 si_init_dma_pg(rdev);
4999 si_enable_dma_pg(rdev, true);
5000 si_init_gfx_cgpg(rdev);
5001 si_enable_gfx_cgpg(rdev, true);
5003 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5004 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5008 static void si_fini_pg(struct radeon_device *rdev)
5010 bool has_pg = false;
5012 /* only cape verde supports PG */
5013 if (rdev->family == CHIP_VERDE)
5017 si_enable_dma_pg(rdev, false);
5018 si_enable_gfx_cgpg(rdev, false);
5025 void si_rlc_fini(struct radeon_device *rdev)
5029 /* save restore block */
5030 if (rdev->rlc.save_restore_obj) {
5031 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
5032 if (unlikely(r != 0))
5033 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
5034 radeon_bo_unpin(rdev->rlc.save_restore_obj);
5035 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
5037 radeon_bo_unref(&rdev->rlc.save_restore_obj);
5038 rdev->rlc.save_restore_obj = NULL;
5041 /* clear state block */
5042 if (rdev->rlc.clear_state_obj) {
5043 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
5044 if (unlikely(r != 0))
5045 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
5046 radeon_bo_unpin(rdev->rlc.clear_state_obj);
5047 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
5049 radeon_bo_unref(&rdev->rlc.clear_state_obj);
5050 rdev->rlc.clear_state_obj = NULL;
5054 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
5056 int si_rlc_init(struct radeon_device *rdev)
5058 volatile u32 *dst_ptr;
5059 u32 dws, data, i, j, k, reg_num;
5060 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index;
5061 u64 reg_list_mc_addr;
5062 const struct cs_section_def *cs_data = si_cs_data;
5065 /* save restore block */
5066 if (rdev->rlc.save_restore_obj == NULL) {
5067 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
5068 RADEON_GEM_DOMAIN_VRAM, NULL,
5069 &rdev->rlc.save_restore_obj);
5071 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
5076 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
5077 if (unlikely(r != 0)) {
5081 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
5082 &rdev->rlc.save_restore_gpu_addr);
5084 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
5085 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
5090 if (rdev->family == CHIP_VERDE) {
5091 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
5093 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
5097 /* write the sr buffer */
5098 dst_ptr = rdev->rlc.sr_ptr;
5099 for (i = 0; i < ARRAY_SIZE(verde_rlc_save_restore_register_list); i++) {
5100 dst_ptr[i] = verde_rlc_save_restore_register_list[i];
5102 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
5104 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
5106 /* clear state block */
5109 for (i = 0; cs_data[i].section != NULL; i++) {
5110 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
5112 dws += cs_data[i].section[j].reg_count;
5115 reg_list_blk_index = (3 * reg_list_num + 2);
5116 dws += reg_list_blk_index;
5118 if (rdev->rlc.clear_state_obj == NULL) {
5119 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
5120 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
5122 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
5127 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
5128 if (unlikely(r != 0)) {
5132 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
5133 &rdev->rlc.clear_state_gpu_addr);
5136 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
5137 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
5141 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
5143 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
5147 /* set up the cs buffer */
5148 dst_ptr = rdev->rlc.cs_ptr;
5149 reg_list_hdr_blk_index = 0;
5150 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
5151 data = upper_32_bits(reg_list_mc_addr);
5152 dst_ptr[reg_list_hdr_blk_index] = data;
5153 reg_list_hdr_blk_index++;
5154 for (i = 0; cs_data[i].section != NULL; i++) {
5155 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
5156 reg_num = cs_data[i].section[j].reg_count;
5157 data = reg_list_mc_addr & 0xffffffff;
5158 dst_ptr[reg_list_hdr_blk_index] = data;
5159 reg_list_hdr_blk_index++;
5161 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
5162 dst_ptr[reg_list_hdr_blk_index] = data;
5163 reg_list_hdr_blk_index++;
5165 data = 0x08000000 | (reg_num * 4);
5166 dst_ptr[reg_list_hdr_blk_index] = data;
5167 reg_list_hdr_blk_index++;
5169 for (k = 0; k < reg_num; k++) {
5170 data = cs_data[i].section[j].extent[k];
5171 dst_ptr[reg_list_blk_index + k] = data;
5173 reg_list_mc_addr += reg_num * 4;
5174 reg_list_blk_index += reg_num;
5177 dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
5179 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
5180 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
5185 static void si_rlc_reset(struct radeon_device *rdev)
5187 u32 tmp = RREG32(GRBM_SOFT_RESET);
5189 tmp |= SOFT_RESET_RLC;
5190 WREG32(GRBM_SOFT_RESET, tmp);
5192 tmp &= ~SOFT_RESET_RLC;
5193 WREG32(GRBM_SOFT_RESET, tmp);
5197 static void si_rlc_stop(struct radeon_device *rdev)
5199 WREG32(RLC_CNTL, 0);
5201 si_enable_gui_idle_interrupt(rdev, false);
5203 si_wait_for_rlc_serdes(rdev);
5206 static void si_rlc_start(struct radeon_device *rdev)
5208 WREG32(RLC_CNTL, RLC_ENABLE);
5210 si_enable_gui_idle_interrupt(rdev, true);
5215 static bool si_lbpw_supported(struct radeon_device *rdev)
5219 /* Enable LBPW only for DDR3 */
5220 tmp = RREG32(MC_SEQ_MISC0);
5221 if ((tmp & 0xF0000000) == 0xB0000000)
5226 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5230 tmp = RREG32(RLC_LB_CNTL);
5232 tmp |= LOAD_BALANCE_ENABLE;
5234 tmp &= ~LOAD_BALANCE_ENABLE;
5235 WREG32(RLC_LB_CNTL, tmp);
5238 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5239 WREG32(SPI_LB_CU_MASK, 0x00ff);
5243 static int si_rlc_resume(struct radeon_device *rdev)
5246 const __be32 *fw_data;
5259 WREG32(RLC_RL_BASE, 0);
5260 WREG32(RLC_RL_SIZE, 0);
5261 WREG32(RLC_LB_CNTL, 0);
5262 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5263 WREG32(RLC_LB_CNTR_INIT, 0);
5264 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5266 WREG32(RLC_MC_CNTL, 0);
5267 WREG32(RLC_UCODE_CNTL, 0);
5269 fw_data = (const __be32 *)rdev->rlc_fw->data;
5270 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5271 WREG32(RLC_UCODE_ADDR, i);
5272 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5274 WREG32(RLC_UCODE_ADDR, 0);
5276 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5283 static void si_enable_interrupts(struct radeon_device *rdev)
5285 u32 ih_cntl = RREG32(IH_CNTL);
5286 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5288 ih_cntl |= ENABLE_INTR;
5289 ih_rb_cntl |= IH_RB_ENABLE;
5290 WREG32(IH_CNTL, ih_cntl);
5291 WREG32(IH_RB_CNTL, ih_rb_cntl);
5292 rdev->ih.enabled = true;
5295 static void si_disable_interrupts(struct radeon_device *rdev)
5297 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5298 u32 ih_cntl = RREG32(IH_CNTL);
5300 ih_rb_cntl &= ~IH_RB_ENABLE;
5301 ih_cntl &= ~ENABLE_INTR;
5302 WREG32(IH_RB_CNTL, ih_rb_cntl);
5303 WREG32(IH_CNTL, ih_cntl);
5304 /* set rptr, wptr to 0 */
5305 WREG32(IH_RB_RPTR, 0);
5306 WREG32(IH_RB_WPTR, 0);
5307 rdev->ih.enabled = false;
5311 static void si_disable_interrupt_state(struct radeon_device *rdev)
5315 WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5316 WREG32(CP_INT_CNTL_RING1, 0);
5317 WREG32(CP_INT_CNTL_RING2, 0);
5318 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5319 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5320 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5321 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5322 WREG32(GRBM_INT_CNTL, 0);
5323 if (rdev->num_crtc >= 2) {
5324 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5325 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5327 if (rdev->num_crtc >= 4) {
5328 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5329 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5331 if (rdev->num_crtc >= 6) {
5332 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5333 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5336 if (rdev->num_crtc >= 2) {
5337 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5338 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5340 if (rdev->num_crtc >= 4) {
5341 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5342 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5344 if (rdev->num_crtc >= 6) {
5345 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5346 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5349 if (!ASIC_IS_NODCE(rdev)) {
5350 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
5352 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5353 WREG32(DC_HPD1_INT_CONTROL, tmp);
5354 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5355 WREG32(DC_HPD2_INT_CONTROL, tmp);
5356 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5357 WREG32(DC_HPD3_INT_CONTROL, tmp);
5358 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5359 WREG32(DC_HPD4_INT_CONTROL, tmp);
5360 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5361 WREG32(DC_HPD5_INT_CONTROL, tmp);
5362 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5363 WREG32(DC_HPD6_INT_CONTROL, tmp);
5367 static int si_irq_init(struct radeon_device *rdev)
5371 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5374 ret = r600_ih_ring_alloc(rdev);
5379 si_disable_interrupts(rdev);
5382 ret = si_rlc_resume(rdev);
5384 r600_ih_ring_fini(rdev);
5388 /* setup interrupt control */
5389 /* set dummy read address to ring address */
5390 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5391 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5392 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5393 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5395 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5396 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5397 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5398 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5400 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5401 rb_bufsz = drm_order(rdev->ih.ring_size / 4);
5403 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5404 IH_WPTR_OVERFLOW_CLEAR |
5407 if (rdev->wb.enabled)
5408 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5410 /* set the writeback address whether it's enabled or not */
5411 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5412 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5414 WREG32(IH_RB_CNTL, ih_rb_cntl);
5416 /* set rptr, wptr to 0 */
5417 WREG32(IH_RB_RPTR, 0);
5418 WREG32(IH_RB_WPTR, 0);
5420 /* Default settings for IH_CNTL (disabled at first) */
5421 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5422 /* RPTR_REARM only works if msi's are enabled */
5423 if (rdev->msi_enabled)
5424 ih_cntl |= RPTR_REARM;
5425 WREG32(IH_CNTL, ih_cntl);
5427 /* force the active interrupt state to all disabled */
5428 si_disable_interrupt_state(rdev);
5430 pci_set_master(rdev->pdev);
5433 si_enable_interrupts(rdev);
5438 int si_irq_set(struct radeon_device *rdev)
5440 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
5441 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
5442 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
5443 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
5444 u32 grbm_int_cntl = 0;
5445 u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
5446 u32 dma_cntl, dma_cntl1;
5447 u32 thermal_int = 0;
5449 if (!rdev->irq.installed) {
5450 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
5453 /* don't enable anything if the ih is disabled */
5454 if (!rdev->ih.enabled) {
5455 si_disable_interrupts(rdev);
5456 /* force the active interrupt state to all disabled */
5457 si_disable_interrupt_state(rdev);
5461 if (!ASIC_IS_NODCE(rdev)) {
5462 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
5463 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
5464 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
5465 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
5466 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
5467 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
5470 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5471 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5473 thermal_int = RREG32(CG_THERMAL_INT) &
5474 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
5476 /* enable CP interrupts on all rings */
5477 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
5478 DRM_DEBUG("si_irq_set: sw int gfx\n");
5479 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
5481 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
5482 DRM_DEBUG("si_irq_set: sw int cp1\n");
5483 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
5485 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
5486 DRM_DEBUG("si_irq_set: sw int cp2\n");
5487 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
5489 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
5490 DRM_DEBUG("si_irq_set: sw int dma\n");
5491 dma_cntl |= TRAP_ENABLE;
5494 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
5495 DRM_DEBUG("si_irq_set: sw int dma1\n");
5496 dma_cntl1 |= TRAP_ENABLE;
5498 if (rdev->irq.crtc_vblank_int[0] ||
5499 atomic_read(&rdev->irq.pflip[0])) {
5500 DRM_DEBUG("si_irq_set: vblank 0\n");
5501 crtc1 |= VBLANK_INT_MASK;
5503 if (rdev->irq.crtc_vblank_int[1] ||
5504 atomic_read(&rdev->irq.pflip[1])) {
5505 DRM_DEBUG("si_irq_set: vblank 1\n");
5506 crtc2 |= VBLANK_INT_MASK;
5508 if (rdev->irq.crtc_vblank_int[2] ||
5509 atomic_read(&rdev->irq.pflip[2])) {
5510 DRM_DEBUG("si_irq_set: vblank 2\n");
5511 crtc3 |= VBLANK_INT_MASK;
5513 if (rdev->irq.crtc_vblank_int[3] ||
5514 atomic_read(&rdev->irq.pflip[3])) {
5515 DRM_DEBUG("si_irq_set: vblank 3\n");
5516 crtc4 |= VBLANK_INT_MASK;
5518 if (rdev->irq.crtc_vblank_int[4] ||
5519 atomic_read(&rdev->irq.pflip[4])) {
5520 DRM_DEBUG("si_irq_set: vblank 4\n");
5521 crtc5 |= VBLANK_INT_MASK;
5523 if (rdev->irq.crtc_vblank_int[5] ||
5524 atomic_read(&rdev->irq.pflip[5])) {
5525 DRM_DEBUG("si_irq_set: vblank 5\n");
5526 crtc6 |= VBLANK_INT_MASK;
5528 if (rdev->irq.hpd[0]) {
5529 DRM_DEBUG("si_irq_set: hpd 1\n");
5530 hpd1 |= DC_HPDx_INT_EN;
5532 if (rdev->irq.hpd[1]) {
5533 DRM_DEBUG("si_irq_set: hpd 2\n");
5534 hpd2 |= DC_HPDx_INT_EN;
5536 if (rdev->irq.hpd[2]) {
5537 DRM_DEBUG("si_irq_set: hpd 3\n");
5538 hpd3 |= DC_HPDx_INT_EN;
5540 if (rdev->irq.hpd[3]) {
5541 DRM_DEBUG("si_irq_set: hpd 4\n");
5542 hpd4 |= DC_HPDx_INT_EN;
5544 if (rdev->irq.hpd[4]) {
5545 DRM_DEBUG("si_irq_set: hpd 5\n");
5546 hpd5 |= DC_HPDx_INT_EN;
5548 if (rdev->irq.hpd[5]) {
5549 DRM_DEBUG("si_irq_set: hpd 6\n");
5550 hpd6 |= DC_HPDx_INT_EN;
5553 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
5554 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
5555 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
5557 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
5558 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
5560 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
5562 if (rdev->irq.dpm_thermal) {
5563 DRM_DEBUG("dpm thermal\n");
5564 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
5567 if (rdev->num_crtc >= 2) {
5568 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
5569 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
5571 if (rdev->num_crtc >= 4) {
5572 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
5573 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
5575 if (rdev->num_crtc >= 6) {
5576 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
5577 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
5580 if (rdev->num_crtc >= 2) {
5581 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
5582 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
5584 if (rdev->num_crtc >= 4) {
5585 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
5586 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
5588 if (rdev->num_crtc >= 6) {
5589 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
5590 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
5593 if (!ASIC_IS_NODCE(rdev)) {
5594 WREG32(DC_HPD1_INT_CONTROL, hpd1);
5595 WREG32(DC_HPD2_INT_CONTROL, hpd2);
5596 WREG32(DC_HPD3_INT_CONTROL, hpd3);
5597 WREG32(DC_HPD4_INT_CONTROL, hpd4);
5598 WREG32(DC_HPD5_INT_CONTROL, hpd5);
5599 WREG32(DC_HPD6_INT_CONTROL, hpd6);
5602 WREG32(CG_THERMAL_INT, thermal_int);
5607 static inline void si_irq_ack(struct radeon_device *rdev)
5611 if (ASIC_IS_NODCE(rdev))
5614 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
5615 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
5616 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
5617 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
5618 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
5619 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
5620 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
5621 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
5622 if (rdev->num_crtc >= 4) {
5623 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
5624 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
5626 if (rdev->num_crtc >= 6) {
5627 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
5628 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
5631 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
5632 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5633 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
5634 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5635 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
5636 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
5637 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
5638 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
5639 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
5640 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
5641 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
5642 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
5644 if (rdev->num_crtc >= 4) {
5645 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
5646 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5647 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
5648 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5649 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
5650 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
5651 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
5652 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
5653 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
5654 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
5655 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
5656 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
5659 if (rdev->num_crtc >= 6) {
5660 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
5661 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5662 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
5663 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
5664 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
5665 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
5666 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
5667 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
5668 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
5669 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
5670 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
5671 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
5674 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5675 tmp = RREG32(DC_HPD1_INT_CONTROL);
5676 tmp |= DC_HPDx_INT_ACK;
5677 WREG32(DC_HPD1_INT_CONTROL, tmp);
5679 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5680 tmp = RREG32(DC_HPD2_INT_CONTROL);
5681 tmp |= DC_HPDx_INT_ACK;
5682 WREG32(DC_HPD2_INT_CONTROL, tmp);
5684 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5685 tmp = RREG32(DC_HPD3_INT_CONTROL);
5686 tmp |= DC_HPDx_INT_ACK;
5687 WREG32(DC_HPD3_INT_CONTROL, tmp);
5689 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5690 tmp = RREG32(DC_HPD4_INT_CONTROL);
5691 tmp |= DC_HPDx_INT_ACK;
5692 WREG32(DC_HPD4_INT_CONTROL, tmp);
5694 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5695 tmp = RREG32(DC_HPD5_INT_CONTROL);
5696 tmp |= DC_HPDx_INT_ACK;
5697 WREG32(DC_HPD5_INT_CONTROL, tmp);
5699 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5700 tmp = RREG32(DC_HPD5_INT_CONTROL);
5701 tmp |= DC_HPDx_INT_ACK;
5702 WREG32(DC_HPD6_INT_CONTROL, tmp);
5706 static void si_irq_disable(struct radeon_device *rdev)
5708 si_disable_interrupts(rdev);
5709 /* Wait and acknowledge irq */
5712 si_disable_interrupt_state(rdev);
5715 static void si_irq_suspend(struct radeon_device *rdev)
5717 si_irq_disable(rdev);
5721 static void si_irq_fini(struct radeon_device *rdev)
5723 si_irq_suspend(rdev);
5724 r600_ih_ring_fini(rdev);
5727 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
5731 if (rdev->wb.enabled)
5732 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
5734 wptr = RREG32(IH_RB_WPTR);
5736 if (wptr & RB_OVERFLOW) {
5737 /* When a ring buffer overflow happen start parsing interrupt
5738 * from the last not overwritten vector (wptr + 16). Hopefully
5739 * this should allow us to catchup.
5741 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
5742 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
5743 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
5744 tmp = RREG32(IH_RB_CNTL);
5745 tmp |= IH_WPTR_OVERFLOW_CLEAR;
5746 WREG32(IH_RB_CNTL, tmp);
5748 return (wptr & rdev->ih.ptr_mask);
5752 * Each IV ring entry is 128 bits:
5753 * [7:0] - interrupt source id
5755 * [59:32] - interrupt source data
5756 * [63:60] - reserved
5759 * [127:80] - reserved
5761 int si_irq_process(struct radeon_device *rdev)
5765 u32 src_id, src_data, ring_id;
5767 bool queue_hotplug = false;
5768 bool queue_thermal = false;
5770 if (!rdev->ih.enabled || rdev->shutdown)
5773 wptr = si_get_ih_wptr(rdev);
5776 /* is somebody else already processing irqs? */
5777 if (atomic_xchg(&rdev->ih.lock, 1))
5780 rptr = rdev->ih.rptr;
5781 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
5783 /* Order reading of wptr vs. reading of IH ring data */
5786 /* display interrupts */
5789 while (rptr != wptr) {
5790 /* wptr/rptr are in bytes! */
5791 ring_index = rptr / 4;
5792 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
5793 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
5794 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
5797 case 1: /* D1 vblank/vline */
5799 case 0: /* D1 vblank */
5800 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
5801 if (rdev->irq.crtc_vblank_int[0]) {
5802 drm_handle_vblank(rdev->ddev, 0);
5803 rdev->pm.vblank_sync = true;
5804 wake_up(&rdev->irq.vblank_queue);
5806 if (atomic_read(&rdev->irq.pflip[0]))
5807 radeon_crtc_handle_flip(rdev, 0);
5808 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
5809 DRM_DEBUG("IH: D1 vblank\n");
5812 case 1: /* D1 vline */
5813 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
5814 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
5815 DRM_DEBUG("IH: D1 vline\n");
5819 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5823 case 2: /* D2 vblank/vline */
5825 case 0: /* D2 vblank */
5826 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
5827 if (rdev->irq.crtc_vblank_int[1]) {
5828 drm_handle_vblank(rdev->ddev, 1);
5829 rdev->pm.vblank_sync = true;
5830 wake_up(&rdev->irq.vblank_queue);
5832 if (atomic_read(&rdev->irq.pflip[1]))
5833 radeon_crtc_handle_flip(rdev, 1);
5834 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
5835 DRM_DEBUG("IH: D2 vblank\n");
5838 case 1: /* D2 vline */
5839 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
5840 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
5841 DRM_DEBUG("IH: D2 vline\n");
5845 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5849 case 3: /* D3 vblank/vline */
5851 case 0: /* D3 vblank */
5852 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
5853 if (rdev->irq.crtc_vblank_int[2]) {
5854 drm_handle_vblank(rdev->ddev, 2);
5855 rdev->pm.vblank_sync = true;
5856 wake_up(&rdev->irq.vblank_queue);
5858 if (atomic_read(&rdev->irq.pflip[2]))
5859 radeon_crtc_handle_flip(rdev, 2);
5860 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
5861 DRM_DEBUG("IH: D3 vblank\n");
5864 case 1: /* D3 vline */
5865 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
5866 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
5867 DRM_DEBUG("IH: D3 vline\n");
5871 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5875 case 4: /* D4 vblank/vline */
5877 case 0: /* D4 vblank */
5878 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
5879 if (rdev->irq.crtc_vblank_int[3]) {
5880 drm_handle_vblank(rdev->ddev, 3);
5881 rdev->pm.vblank_sync = true;
5882 wake_up(&rdev->irq.vblank_queue);
5884 if (atomic_read(&rdev->irq.pflip[3]))
5885 radeon_crtc_handle_flip(rdev, 3);
5886 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
5887 DRM_DEBUG("IH: D4 vblank\n");
5890 case 1: /* D4 vline */
5891 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
5892 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
5893 DRM_DEBUG("IH: D4 vline\n");
5897 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5901 case 5: /* D5 vblank/vline */
5903 case 0: /* D5 vblank */
5904 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
5905 if (rdev->irq.crtc_vblank_int[4]) {
5906 drm_handle_vblank(rdev->ddev, 4);
5907 rdev->pm.vblank_sync = true;
5908 wake_up(&rdev->irq.vblank_queue);
5910 if (atomic_read(&rdev->irq.pflip[4]))
5911 radeon_crtc_handle_flip(rdev, 4);
5912 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
5913 DRM_DEBUG("IH: D5 vblank\n");
5916 case 1: /* D5 vline */
5917 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
5918 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
5919 DRM_DEBUG("IH: D5 vline\n");
5923 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5927 case 6: /* D6 vblank/vline */
5929 case 0: /* D6 vblank */
5930 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
5931 if (rdev->irq.crtc_vblank_int[5]) {
5932 drm_handle_vblank(rdev->ddev, 5);
5933 rdev->pm.vblank_sync = true;
5934 wake_up(&rdev->irq.vblank_queue);
5936 if (atomic_read(&rdev->irq.pflip[5]))
5937 radeon_crtc_handle_flip(rdev, 5);
5938 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
5939 DRM_DEBUG("IH: D6 vblank\n");
5942 case 1: /* D6 vline */
5943 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
5944 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
5945 DRM_DEBUG("IH: D6 vline\n");
5949 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5953 case 42: /* HPD hotplug */
5956 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
5957 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
5958 queue_hotplug = true;
5959 DRM_DEBUG("IH: HPD1\n");
5963 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
5964 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
5965 queue_hotplug = true;
5966 DRM_DEBUG("IH: HPD2\n");
5970 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
5971 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
5972 queue_hotplug = true;
5973 DRM_DEBUG("IH: HPD3\n");
5977 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5978 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5979 queue_hotplug = true;
5980 DRM_DEBUG("IH: HPD4\n");
5984 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5985 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5986 queue_hotplug = true;
5987 DRM_DEBUG("IH: HPD5\n");
5991 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5992 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5993 queue_hotplug = true;
5994 DRM_DEBUG("IH: HPD6\n");
5998 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6004 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6005 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6006 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
6007 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6008 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
6009 /* reset addr and status */
6010 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6012 case 176: /* RINGID0 CP_INT */
6013 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6015 case 177: /* RINGID1 CP_INT */
6016 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6018 case 178: /* RINGID2 CP_INT */
6019 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6021 case 181: /* CP EOP event */
6022 DRM_DEBUG("IH: CP EOP\n");
6025 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6028 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6031 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6035 case 224: /* DMA trap event */
6036 DRM_DEBUG("IH: DMA trap\n");
6037 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6039 case 230: /* thermal low to high */
6040 DRM_DEBUG("IH: thermal low to high\n");
6041 rdev->pm.dpm.thermal.high_to_low = false;
6042 queue_thermal = true;
6044 case 231: /* thermal high to low */
6045 DRM_DEBUG("IH: thermal high to low\n");
6046 rdev->pm.dpm.thermal.high_to_low = true;
6047 queue_thermal = true;
6049 case 233: /* GUI IDLE */
6050 DRM_DEBUG("IH: GUI idle\n");
6052 case 244: /* DMA trap event */
6053 DRM_DEBUG("IH: DMA1 trap\n");
6054 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6057 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6061 /* wptr/rptr are in bytes! */
6063 rptr &= rdev->ih.ptr_mask;
6066 schedule_work(&rdev->hotplug_work);
6067 if (queue_thermal && rdev->pm.dpm_enabled)
6068 schedule_work(&rdev->pm.dpm.thermal.work);
6069 rdev->ih.rptr = rptr;
6070 WREG32(IH_RB_RPTR, rdev->ih.rptr);
6071 atomic_set(&rdev->ih.lock, 0);
6073 /* make sure wptr hasn't changed while processing */
6074 wptr = si_get_ih_wptr(rdev);
6082 * si_copy_dma - copy pages using the DMA engine
6084 * @rdev: radeon_device pointer
6085 * @src_offset: src GPU address
6086 * @dst_offset: dst GPU address
6087 * @num_gpu_pages: number of GPU pages to xfer
6088 * @fence: radeon fence object
6090 * Copy GPU paging using the DMA engine (SI).
6091 * Used by the radeon ttm implementation to move pages if
6092 * registered as the asic copy callback.
6094 int si_copy_dma(struct radeon_device *rdev,
6095 uint64_t src_offset, uint64_t dst_offset,
6096 unsigned num_gpu_pages,
6097 struct radeon_fence **fence)
6099 struct radeon_semaphore *sem = NULL;
6100 int ring_index = rdev->asic->copy.dma_ring_index;
6101 struct radeon_ring *ring = &rdev->ring[ring_index];
6102 u32 size_in_bytes, cur_size_in_bytes;
6106 r = radeon_semaphore_create(rdev, &sem);
6108 DRM_ERROR("radeon: moving bo (%d).\n", r);
6112 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
6113 num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
6114 r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
6116 DRM_ERROR("radeon: moving bo (%d).\n", r);
6117 radeon_semaphore_free(rdev, &sem, NULL);
6121 if (radeon_fence_need_sync(*fence, ring->idx)) {
6122 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
6124 radeon_fence_note_sync(*fence, ring->idx);
6126 radeon_semaphore_free(rdev, &sem, NULL);
6129 for (i = 0; i < num_loops; i++) {
6130 cur_size_in_bytes = size_in_bytes;
6131 if (cur_size_in_bytes > 0xFFFFF)
6132 cur_size_in_bytes = 0xFFFFF;
6133 size_in_bytes -= cur_size_in_bytes;
6134 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
6135 radeon_ring_write(ring, dst_offset & 0xffffffff);
6136 radeon_ring_write(ring, src_offset & 0xffffffff);
6137 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
6138 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
6139 src_offset += cur_size_in_bytes;
6140 dst_offset += cur_size_in_bytes;
6143 r = radeon_fence_emit(rdev, fence, ring->idx);
6145 radeon_ring_unlock_undo(rdev, ring);
6149 radeon_ring_unlock_commit(rdev, ring);
6150 radeon_semaphore_free(rdev, &sem, *fence);
6156 * startup/shutdown callbacks
6158 static int si_startup(struct radeon_device *rdev)
6160 struct radeon_ring *ring;
6163 /* enable pcie gen2/3 link */
6164 si_pcie_gen3_enable(rdev);
6166 si_program_aspm(rdev);
6168 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6169 !rdev->rlc_fw || !rdev->mc_fw) {
6170 r = si_init_microcode(rdev);
6172 DRM_ERROR("Failed to load firmware!\n");
6177 r = si_mc_load_microcode(rdev);
6179 DRM_ERROR("Failed to load MC firmware!\n");
6183 r = r600_vram_scratch_init(rdev);
6187 si_mc_program(rdev);
6188 r = si_pcie_gart_enable(rdev);
6193 /* allocate rlc buffers */
6194 r = si_rlc_init(rdev);
6196 DRM_ERROR("Failed to init rlc BOs!\n");
6200 /* allocate wb buffer */
6201 r = radeon_wb_init(rdev);
6205 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6207 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6211 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6213 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6217 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6219 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6223 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6225 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6229 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6231 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6235 if (rdev->has_uvd) {
6236 r = rv770_uvd_resume(rdev);
6238 r = radeon_fence_driver_start_ring(rdev,
6239 R600_RING_TYPE_UVD_INDEX);
6241 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6244 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6248 if (!rdev->irq.installed) {
6249 r = radeon_irq_kms_init(rdev);
6254 r = si_irq_init(rdev);
6256 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6257 radeon_irq_kms_fini(rdev);
6262 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6263 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6264 CP_RB0_RPTR, CP_RB0_WPTR,
6265 0, 0xfffff, RADEON_CP_PACKET2);
6269 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6270 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6271 CP_RB1_RPTR, CP_RB1_WPTR,
6272 0, 0xfffff, RADEON_CP_PACKET2);
6276 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6277 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6278 CP_RB2_RPTR, CP_RB2_WPTR,
6279 0, 0xfffff, RADEON_CP_PACKET2);
6283 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6284 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6285 DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
6286 DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
6287 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6291 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6292 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6293 DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
6294 DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
6295 2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6299 r = si_cp_load_microcode(rdev);
6302 r = si_cp_resume(rdev);
6306 r = cayman_dma_resume(rdev);
6310 if (rdev->has_uvd) {
6311 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6312 if (ring->ring_size) {
6313 r = radeon_ring_init(rdev, ring, ring->ring_size,
6314 R600_WB_UVD_RPTR_OFFSET,
6315 UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
6316 0, 0xfffff, RADEON_CP_PACKET2);
6318 r = r600_uvd_init(rdev);
6320 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6324 r = radeon_ib_pool_init(rdev);
6326 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6330 r = radeon_vm_manager_init(rdev);
6332 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6339 int si_resume(struct radeon_device *rdev)
6343 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6344 * posting will perform necessary task to bring back GPU into good
6348 atom_asic_init(rdev->mode_info.atom_context);
6350 /* init golden registers */
6351 si_init_golden_registers(rdev);
6353 rdev->accel_working = true;
6354 r = si_startup(rdev);
6356 DRM_ERROR("si startup failed on resume\n");
6357 rdev->accel_working = false;
6365 int si_suspend(struct radeon_device *rdev)
6367 radeon_vm_manager_fini(rdev);
6368 si_cp_enable(rdev, false);
6369 cayman_dma_stop(rdev);
6370 if (rdev->has_uvd) {
6371 r600_uvd_rbc_stop(rdev);
6372 radeon_uvd_suspend(rdev);
6374 si_irq_suspend(rdev);
6375 radeon_wb_disable(rdev);
6376 si_pcie_gart_disable(rdev);
6380 /* Plan is to move initialization in that function and use
6381 * helper function so that radeon_device_init pretty much
6382 * do nothing more than calling asic specific function. This
6383 * should also allow to remove a bunch of callback function
6386 int si_init(struct radeon_device *rdev)
6388 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6392 if (!radeon_get_bios(rdev)) {
6393 if (ASIC_IS_AVIVO(rdev))
6396 /* Must be an ATOMBIOS */
6397 if (!rdev->is_atom_bios) {
6398 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6401 r = radeon_atombios_init(rdev);
6405 /* Post card if necessary */
6406 if (!radeon_card_posted(rdev)) {
6408 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6411 DRM_INFO("GPU not posted. posting now...\n");
6412 atom_asic_init(rdev->mode_info.atom_context);
6414 /* init golden registers */
6415 si_init_golden_registers(rdev);
6416 /* Initialize scratch registers */
6417 si_scratch_init(rdev);
6418 /* Initialize surface registers */
6419 radeon_surface_init(rdev);
6420 /* Initialize clocks */
6421 radeon_get_clock_info(rdev->ddev);
6424 r = radeon_fence_driver_init(rdev);
6428 /* initialize memory controller */
6429 r = si_mc_init(rdev);
6432 /* Memory manager */
6433 r = radeon_bo_init(rdev);
6437 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6438 ring->ring_obj = NULL;
6439 r600_ring_init(rdev, ring, 1024 * 1024);
6441 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6442 ring->ring_obj = NULL;
6443 r600_ring_init(rdev, ring, 1024 * 1024);
6445 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6446 ring->ring_obj = NULL;
6447 r600_ring_init(rdev, ring, 1024 * 1024);
6449 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6450 ring->ring_obj = NULL;
6451 r600_ring_init(rdev, ring, 64 * 1024);
6453 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6454 ring->ring_obj = NULL;
6455 r600_ring_init(rdev, ring, 64 * 1024);
6457 if (rdev->has_uvd) {
6458 r = radeon_uvd_init(rdev);
6460 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6461 ring->ring_obj = NULL;
6462 r600_ring_init(rdev, ring, 4096);
6466 rdev->ih.ring_obj = NULL;
6467 r600_ih_ring_init(rdev, 64 * 1024);
6469 r = r600_pcie_gart_init(rdev);
6473 rdev->accel_working = true;
6474 r = si_startup(rdev);
6476 dev_err(rdev->dev, "disabling GPU acceleration\n");
6478 cayman_dma_fini(rdev);
6481 radeon_wb_fini(rdev);
6482 radeon_ib_pool_fini(rdev);
6483 radeon_vm_manager_fini(rdev);
6484 radeon_irq_kms_fini(rdev);
6485 si_pcie_gart_fini(rdev);
6486 rdev->accel_working = false;
6489 /* Don't start up if the MC ucode is missing.
6490 * The default clocks and voltages before the MC ucode
6491 * is loaded are not suffient for advanced operations.
6494 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6501 void si_fini(struct radeon_device *rdev)
6504 cayman_dma_fini(rdev);
6509 radeon_wb_fini(rdev);
6510 radeon_vm_manager_fini(rdev);
6511 radeon_ib_pool_fini(rdev);
6512 radeon_irq_kms_fini(rdev);
6514 radeon_uvd_fini(rdev);
6515 si_pcie_gart_fini(rdev);
6516 r600_vram_scratch_fini(rdev);
6517 radeon_gem_fini(rdev);
6518 radeon_fence_driver_fini(rdev);
6519 radeon_bo_fini(rdev);
6520 radeon_atombios_fini(rdev);
6526 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6528 * @rdev: radeon_device pointer
6530 * Fetches a GPU clock counter snapshot (SI).
6531 * Returns the 64 bit clock counter snapshot.
6533 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6537 mutex_lock(&rdev->gpu_clock_mutex);
6538 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6539 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6540 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6541 mutex_unlock(&rdev->gpu_clock_mutex);
6545 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6547 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6550 /* bypass vclk and dclk with bclk */
6551 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6552 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6553 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6555 /* put PLL in bypass mode */
6556 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6558 if (!vclk || !dclk) {
6559 /* keep the Bypass mode, put PLL to sleep */
6560 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6564 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6565 16384, 0x03FFFFFF, 0, 128, 5,
6566 &fb_div, &vclk_div, &dclk_div);
6570 /* set RESET_ANTI_MUX to 0 */
6571 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
6573 /* set VCO_MODE to 1 */
6574 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
6576 /* toggle UPLL_SLEEP to 1 then back to 0 */
6577 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
6578 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
6580 /* deassert UPLL_RESET */
6581 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6585 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6589 /* assert UPLL_RESET again */
6590 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
6592 /* disable spread spectrum. */
6593 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
6595 /* set feedback divider */
6596 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
6598 /* set ref divider to 0 */
6599 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
6601 if (fb_div < 307200)
6602 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
6604 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
6606 /* set PDIV_A and PDIV_B */
6607 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6608 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
6609 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
6611 /* give the PLL some time to settle */
6614 /* deassert PLL_RESET */
6615 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
6619 /* switch from bypass mode to normal mode */
6620 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
6622 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
6626 /* switch VCLK and DCLK selection */
6627 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6628 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
6629 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6636 static void si_pcie_gen3_enable(struct radeon_device *rdev)
6638 struct pci_dev *root = rdev->pdev->bus->self;
6639 int bridge_pos, gpu_pos;
6640 u32 speed_cntl, mask, current_data_rate;
6644 if (radeon_pcie_gen2 == 0)
6647 if (rdev->flags & RADEON_IS_IGP)
6650 if (!(rdev->flags & RADEON_IS_PCIE))
6653 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
6657 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
6660 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6661 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
6662 LC_CURRENT_DATA_RATE_SHIFT;
6663 if (mask & DRM_PCIE_SPEED_80) {
6664 if (current_data_rate == 2) {
6665 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
6668 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
6669 } else if (mask & DRM_PCIE_SPEED_50) {
6670 if (current_data_rate == 1) {
6671 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
6674 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
6677 bridge_pos = pci_pcie_cap(root);
6681 gpu_pos = pci_pcie_cap(rdev->pdev);
6685 if (mask & DRM_PCIE_SPEED_80) {
6686 /* re-try equalization if gen3 is not already enabled */
6687 if (current_data_rate != 2) {
6688 u16 bridge_cfg, gpu_cfg;
6689 u16 bridge_cfg2, gpu_cfg2;
6690 u32 max_lw, current_lw, tmp;
6692 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6693 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6695 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
6696 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6698 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
6699 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6701 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
6702 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
6703 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
6705 if (current_lw < max_lw) {
6706 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6707 if (tmp & LC_RENEGOTIATION_SUPPORT) {
6708 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
6709 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
6710 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
6711 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
6715 for (i = 0; i < 10; i++) {
6717 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
6718 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
6721 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
6722 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
6724 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
6725 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
6727 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6728 tmp |= LC_SET_QUIESCE;
6729 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6731 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6733 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6738 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
6739 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6740 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
6741 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
6743 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
6744 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
6745 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
6746 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
6749 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
6750 tmp16 &= ~((1 << 4) | (7 << 9));
6751 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
6752 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
6754 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6755 tmp16 &= ~((1 << 4) | (7 << 9));
6756 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
6757 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6759 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
6760 tmp &= ~LC_SET_QUIESCE;
6761 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
6766 /* set the link speed */
6767 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
6768 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
6769 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6771 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
6773 if (mask & DRM_PCIE_SPEED_80)
6774 tmp16 |= 3; /* gen3 */
6775 else if (mask & DRM_PCIE_SPEED_50)
6776 tmp16 |= 2; /* gen2 */
6778 tmp16 |= 1; /* gen1 */
6779 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
6781 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6782 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
6783 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
6785 for (i = 0; i < rdev->usec_timeout; i++) {
6786 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
6787 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
6793 static void si_program_aspm(struct radeon_device *rdev)
6796 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
6797 bool disable_clkreq = false;
6799 if (!(rdev->flags & RADEON_IS_PCIE))
6802 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6803 data &= ~LC_XMIT_N_FTS_MASK;
6804 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
6806 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
6808 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
6809 data |= LC_GO_TO_RECOVERY;
6811 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
6813 orig = data = RREG32_PCIE(PCIE_P_CNTL);
6814 data |= P_IGNORE_EDB_ERR;
6816 WREG32_PCIE(PCIE_P_CNTL, data);
6818 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6819 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
6820 data |= LC_PMI_TO_L1_DIS;
6822 data |= LC_L0S_INACTIVITY(7);
6825 data |= LC_L1_INACTIVITY(7);
6826 data &= ~LC_PMI_TO_L1_DIS;
6828 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6830 if (!disable_plloff_in_l1) {
6831 bool clk_req_support;
6833 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6834 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6835 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6837 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6839 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6840 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6841 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6843 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6845 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6846 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
6847 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
6849 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6851 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6852 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
6853 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
6855 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6857 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
6858 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
6859 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6861 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
6863 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
6864 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6866 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
6868 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
6869 data &= ~PLL_RAMP_UP_TIME_2_MASK;
6871 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
6873 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
6874 data &= ~PLL_RAMP_UP_TIME_3_MASK;
6876 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
6878 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
6879 data &= ~PLL_RAMP_UP_TIME_0_MASK;
6881 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
6883 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
6884 data &= ~PLL_RAMP_UP_TIME_1_MASK;
6886 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
6888 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
6889 data &= ~PLL_RAMP_UP_TIME_2_MASK;
6891 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
6893 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
6894 data &= ~PLL_RAMP_UP_TIME_3_MASK;
6896 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
6898 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
6899 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
6900 data |= LC_DYN_LANES_PWR_STATE(3);
6902 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
6904 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
6905 data &= ~LS2_EXIT_TIME_MASK;
6906 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
6907 data |= LS2_EXIT_TIME(5);
6909 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
6911 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
6912 data &= ~LS2_EXIT_TIME_MASK;
6913 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
6914 data |= LS2_EXIT_TIME(5);
6916 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
6918 if (!disable_clkreq) {
6919 struct pci_dev *root = rdev->pdev->bus->self;
6922 clk_req_support = false;
6923 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
6924 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
6925 clk_req_support = true;
6927 clk_req_support = false;
6930 if (clk_req_support) {
6931 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
6932 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
6934 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
6936 orig = data = RREG32(THM_CLK_CNTL);
6937 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
6938 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
6940 WREG32(THM_CLK_CNTL, data);
6942 orig = data = RREG32(MISC_CLK_CNTL);
6943 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
6944 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
6946 WREG32(MISC_CLK_CNTL, data);
6948 orig = data = RREG32(CG_CLKPIN_CNTL);
6949 data &= ~BCLK_AS_XCLK;
6951 WREG32(CG_CLKPIN_CNTL, data);
6953 orig = data = RREG32(CG_CLKPIN_CNTL_2);
6954 data &= ~FORCE_BIF_REFCLK_EN;
6956 WREG32(CG_CLKPIN_CNTL_2, data);
6958 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
6959 data &= ~MPLL_CLKOUT_SEL_MASK;
6960 data |= MPLL_CLKOUT_SEL(4);
6962 WREG32(MPLL_BYPASSCLK_SEL, data);
6964 orig = data = RREG32(SPLL_CNTL_MODE);
6965 data &= ~SPLL_REFCLK_SEL_MASK;
6967 WREG32(SPLL_CNTL_MODE, data);
6972 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
6975 orig = data = RREG32_PCIE(PCIE_CNTL2);
6976 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
6978 WREG32_PCIE(PCIE_CNTL2, data);
6981 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
6982 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
6983 data = RREG32_PCIE(PCIE_LC_STATUS1);
6984 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
6985 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
6986 data &= ~LC_L0S_INACTIVITY_MASK;
6988 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);