2 * Copyright 2011 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
33 #include "si_blit_shaders.h"
34 #include "clearstate_si.h"
35 #include "radeon_ucode.h"
38 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
39 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
40 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
41 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
42 MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
43 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
44 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
46 MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
47 MODULE_FIRMWARE("radeon/tahiti_me.bin");
48 MODULE_FIRMWARE("radeon/tahiti_ce.bin");
49 MODULE_FIRMWARE("radeon/tahiti_mc.bin");
50 MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
51 MODULE_FIRMWARE("radeon/tahiti_smc.bin");
53 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
54 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
55 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
56 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
57 MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
58 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
59 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
61 MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
62 MODULE_FIRMWARE("radeon/pitcairn_me.bin");
63 MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
64 MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
65 MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
66 MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
68 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
69 MODULE_FIRMWARE("radeon/VERDE_me.bin");
70 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
71 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
72 MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
73 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
74 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
76 MODULE_FIRMWARE("radeon/verde_pfp.bin");
77 MODULE_FIRMWARE("radeon/verde_me.bin");
78 MODULE_FIRMWARE("radeon/verde_ce.bin");
79 MODULE_FIRMWARE("radeon/verde_mc.bin");
80 MODULE_FIRMWARE("radeon/verde_rlc.bin");
81 MODULE_FIRMWARE("radeon/verde_smc.bin");
83 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
84 MODULE_FIRMWARE("radeon/OLAND_me.bin");
85 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
86 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
87 MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
88 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
89 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
91 MODULE_FIRMWARE("radeon/oland_pfp.bin");
92 MODULE_FIRMWARE("radeon/oland_me.bin");
93 MODULE_FIRMWARE("radeon/oland_ce.bin");
94 MODULE_FIRMWARE("radeon/oland_mc.bin");
95 MODULE_FIRMWARE("radeon/oland_rlc.bin");
96 MODULE_FIRMWARE("radeon/oland_smc.bin");
98 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
99 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
100 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
101 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
102 MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
103 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
104 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
106 MODULE_FIRMWARE("radeon/hainan_pfp.bin");
107 MODULE_FIRMWARE("radeon/hainan_me.bin");
108 MODULE_FIRMWARE("radeon/hainan_ce.bin");
109 MODULE_FIRMWARE("radeon/hainan_mc.bin");
110 MODULE_FIRMWARE("radeon/hainan_rlc.bin");
111 MODULE_FIRMWARE("radeon/hainan_smc.bin");
113 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
114 static void si_pcie_gen3_enable(struct radeon_device *rdev);
115 static void si_program_aspm(struct radeon_device *rdev);
116 extern void sumo_rlc_fini(struct radeon_device *rdev);
117 extern int sumo_rlc_init(struct radeon_device *rdev);
118 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
119 extern void r600_ih_ring_fini(struct radeon_device *rdev);
120 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
121 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
122 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
123 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
124 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
125 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
126 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
128 static void si_init_pg(struct radeon_device *rdev);
129 static void si_init_cg(struct radeon_device *rdev);
130 static void si_fini_pg(struct radeon_device *rdev);
131 static void si_fini_cg(struct radeon_device *rdev);
132 static void si_rlc_stop(struct radeon_device *rdev);
134 static const u32 verde_rlc_save_restore_register_list[] =
136 (0x8000 << 16) | (0x98f4 >> 2),
138 (0x8040 << 16) | (0x98f4 >> 2),
140 (0x8000 << 16) | (0xe80 >> 2),
142 (0x8040 << 16) | (0xe80 >> 2),
144 (0x8000 << 16) | (0x89bc >> 2),
146 (0x8040 << 16) | (0x89bc >> 2),
148 (0x8000 << 16) | (0x8c1c >> 2),
150 (0x8040 << 16) | (0x8c1c >> 2),
152 (0x9c00 << 16) | (0x98f0 >> 2),
154 (0x9c00 << 16) | (0xe7c >> 2),
156 (0x8000 << 16) | (0x9148 >> 2),
158 (0x8040 << 16) | (0x9148 >> 2),
160 (0x9c00 << 16) | (0x9150 >> 2),
162 (0x9c00 << 16) | (0x897c >> 2),
164 (0x9c00 << 16) | (0x8d8c >> 2),
166 (0x9c00 << 16) | (0xac54 >> 2),
169 (0x9c00 << 16) | (0x98f8 >> 2),
171 (0x9c00 << 16) | (0x9910 >> 2),
173 (0x9c00 << 16) | (0x9914 >> 2),
175 (0x9c00 << 16) | (0x9918 >> 2),
177 (0x9c00 << 16) | (0x991c >> 2),
179 (0x9c00 << 16) | (0x9920 >> 2),
181 (0x9c00 << 16) | (0x9924 >> 2),
183 (0x9c00 << 16) | (0x9928 >> 2),
185 (0x9c00 << 16) | (0x992c >> 2),
187 (0x9c00 << 16) | (0x9930 >> 2),
189 (0x9c00 << 16) | (0x9934 >> 2),
191 (0x9c00 << 16) | (0x9938 >> 2),
193 (0x9c00 << 16) | (0x993c >> 2),
195 (0x9c00 << 16) | (0x9940 >> 2),
197 (0x9c00 << 16) | (0x9944 >> 2),
199 (0x9c00 << 16) | (0x9948 >> 2),
201 (0x9c00 << 16) | (0x994c >> 2),
203 (0x9c00 << 16) | (0x9950 >> 2),
205 (0x9c00 << 16) | (0x9954 >> 2),
207 (0x9c00 << 16) | (0x9958 >> 2),
209 (0x9c00 << 16) | (0x995c >> 2),
211 (0x9c00 << 16) | (0x9960 >> 2),
213 (0x9c00 << 16) | (0x9964 >> 2),
215 (0x9c00 << 16) | (0x9968 >> 2),
217 (0x9c00 << 16) | (0x996c >> 2),
219 (0x9c00 << 16) | (0x9970 >> 2),
221 (0x9c00 << 16) | (0x9974 >> 2),
223 (0x9c00 << 16) | (0x9978 >> 2),
225 (0x9c00 << 16) | (0x997c >> 2),
227 (0x9c00 << 16) | (0x9980 >> 2),
229 (0x9c00 << 16) | (0x9984 >> 2),
231 (0x9c00 << 16) | (0x9988 >> 2),
233 (0x9c00 << 16) | (0x998c >> 2),
235 (0x9c00 << 16) | (0x8c00 >> 2),
237 (0x9c00 << 16) | (0x8c14 >> 2),
239 (0x9c00 << 16) | (0x8c04 >> 2),
241 (0x9c00 << 16) | (0x8c08 >> 2),
243 (0x8000 << 16) | (0x9b7c >> 2),
245 (0x8040 << 16) | (0x9b7c >> 2),
247 (0x8000 << 16) | (0xe84 >> 2),
249 (0x8040 << 16) | (0xe84 >> 2),
251 (0x8000 << 16) | (0x89c0 >> 2),
253 (0x8040 << 16) | (0x89c0 >> 2),
255 (0x8000 << 16) | (0x914c >> 2),
257 (0x8040 << 16) | (0x914c >> 2),
259 (0x8000 << 16) | (0x8c20 >> 2),
261 (0x8040 << 16) | (0x8c20 >> 2),
263 (0x8000 << 16) | (0x9354 >> 2),
265 (0x8040 << 16) | (0x9354 >> 2),
267 (0x9c00 << 16) | (0x9060 >> 2),
269 (0x9c00 << 16) | (0x9364 >> 2),
271 (0x9c00 << 16) | (0x9100 >> 2),
273 (0x9c00 << 16) | (0x913c >> 2),
275 (0x8000 << 16) | (0x90e0 >> 2),
277 (0x8000 << 16) | (0x90e4 >> 2),
279 (0x8000 << 16) | (0x90e8 >> 2),
281 (0x8040 << 16) | (0x90e0 >> 2),
283 (0x8040 << 16) | (0x90e4 >> 2),
285 (0x8040 << 16) | (0x90e8 >> 2),
287 (0x9c00 << 16) | (0x8bcc >> 2),
289 (0x9c00 << 16) | (0x8b24 >> 2),
291 (0x9c00 << 16) | (0x88c4 >> 2),
293 (0x9c00 << 16) | (0x8e50 >> 2),
295 (0x9c00 << 16) | (0x8c0c >> 2),
297 (0x9c00 << 16) | (0x8e58 >> 2),
299 (0x9c00 << 16) | (0x8e5c >> 2),
301 (0x9c00 << 16) | (0x9508 >> 2),
303 (0x9c00 << 16) | (0x950c >> 2),
305 (0x9c00 << 16) | (0x9494 >> 2),
307 (0x9c00 << 16) | (0xac0c >> 2),
309 (0x9c00 << 16) | (0xac10 >> 2),
311 (0x9c00 << 16) | (0xac14 >> 2),
313 (0x9c00 << 16) | (0xae00 >> 2),
315 (0x9c00 << 16) | (0xac08 >> 2),
317 (0x9c00 << 16) | (0x88d4 >> 2),
319 (0x9c00 << 16) | (0x88c8 >> 2),
321 (0x9c00 << 16) | (0x88cc >> 2),
323 (0x9c00 << 16) | (0x89b0 >> 2),
325 (0x9c00 << 16) | (0x8b10 >> 2),
327 (0x9c00 << 16) | (0x8a14 >> 2),
329 (0x9c00 << 16) | (0x9830 >> 2),
331 (0x9c00 << 16) | (0x9834 >> 2),
333 (0x9c00 << 16) | (0x9838 >> 2),
335 (0x9c00 << 16) | (0x9a10 >> 2),
337 (0x8000 << 16) | (0x9870 >> 2),
339 (0x8000 << 16) | (0x9874 >> 2),
341 (0x8001 << 16) | (0x9870 >> 2),
343 (0x8001 << 16) | (0x9874 >> 2),
345 (0x8040 << 16) | (0x9870 >> 2),
347 (0x8040 << 16) | (0x9874 >> 2),
349 (0x8041 << 16) | (0x9870 >> 2),
351 (0x8041 << 16) | (0x9874 >> 2),
356 static const u32 tahiti_golden_rlc_registers[] =
358 0xc424, 0xffffffff, 0x00601005,
359 0xc47c, 0xffffffff, 0x10104040,
360 0xc488, 0xffffffff, 0x0100000a,
361 0xc314, 0xffffffff, 0x00000800,
362 0xc30c, 0xffffffff, 0x800000f4,
363 0xf4a8, 0xffffffff, 0x00000000
366 static const u32 tahiti_golden_registers[] =
368 0x9a10, 0x00010000, 0x00018208,
369 0x9830, 0xffffffff, 0x00000000,
370 0x9834, 0xf00fffff, 0x00000400,
371 0x9838, 0x0002021c, 0x00020200,
372 0xc78, 0x00000080, 0x00000000,
373 0xd030, 0x000300c0, 0x00800040,
374 0xd830, 0x000300c0, 0x00800040,
375 0x5bb0, 0x000000f0, 0x00000070,
376 0x5bc0, 0x00200000, 0x50100000,
377 0x7030, 0x31000311, 0x00000011,
378 0x277c, 0x00000003, 0x000007ff,
379 0x240c, 0x000007ff, 0x00000000,
380 0x8a14, 0xf000001f, 0x00000007,
381 0x8b24, 0xffffffff, 0x00ffffff,
382 0x8b10, 0x0000ff0f, 0x00000000,
383 0x28a4c, 0x07ffffff, 0x4e000000,
384 0x28350, 0x3f3f3fff, 0x2a00126a,
385 0x30, 0x000000ff, 0x0040,
386 0x34, 0x00000040, 0x00004040,
387 0x9100, 0x07ffffff, 0x03000000,
388 0x8e88, 0x01ff1f3f, 0x00000000,
389 0x8e84, 0x01ff1f3f, 0x00000000,
390 0x9060, 0x0000007f, 0x00000020,
391 0x9508, 0x00010000, 0x00010000,
392 0xac14, 0x00000200, 0x000002fb,
393 0xac10, 0xffffffff, 0x0000543b,
394 0xac0c, 0xffffffff, 0xa9210876,
395 0x88d0, 0xffffffff, 0x000fff40,
396 0x88d4, 0x0000001f, 0x00000010,
397 0x1410, 0x20000000, 0x20fffed8,
398 0x15c0, 0x000c0fc0, 0x000c0400
401 static const u32 tahiti_golden_registers2[] =
403 0xc64, 0x00000001, 0x00000001
406 static const u32 pitcairn_golden_rlc_registers[] =
408 0xc424, 0xffffffff, 0x00601004,
409 0xc47c, 0xffffffff, 0x10102020,
410 0xc488, 0xffffffff, 0x01000020,
411 0xc314, 0xffffffff, 0x00000800,
412 0xc30c, 0xffffffff, 0x800000a4
415 static const u32 pitcairn_golden_registers[] =
417 0x9a10, 0x00010000, 0x00018208,
418 0x9830, 0xffffffff, 0x00000000,
419 0x9834, 0xf00fffff, 0x00000400,
420 0x9838, 0x0002021c, 0x00020200,
421 0xc78, 0x00000080, 0x00000000,
422 0xd030, 0x000300c0, 0x00800040,
423 0xd830, 0x000300c0, 0x00800040,
424 0x5bb0, 0x000000f0, 0x00000070,
425 0x5bc0, 0x00200000, 0x50100000,
426 0x7030, 0x31000311, 0x00000011,
427 0x2ae4, 0x00073ffe, 0x000022a2,
428 0x240c, 0x000007ff, 0x00000000,
429 0x8a14, 0xf000001f, 0x00000007,
430 0x8b24, 0xffffffff, 0x00ffffff,
431 0x8b10, 0x0000ff0f, 0x00000000,
432 0x28a4c, 0x07ffffff, 0x4e000000,
433 0x28350, 0x3f3f3fff, 0x2a00126a,
434 0x30, 0x000000ff, 0x0040,
435 0x34, 0x00000040, 0x00004040,
436 0x9100, 0x07ffffff, 0x03000000,
437 0x9060, 0x0000007f, 0x00000020,
438 0x9508, 0x00010000, 0x00010000,
439 0xac14, 0x000003ff, 0x000000f7,
440 0xac10, 0xffffffff, 0x00000000,
441 0xac0c, 0xffffffff, 0x32761054,
442 0x88d4, 0x0000001f, 0x00000010,
443 0x15c0, 0x000c0fc0, 0x000c0400
446 static const u32 verde_golden_rlc_registers[] =
448 0xc424, 0xffffffff, 0x033f1005,
449 0xc47c, 0xffffffff, 0x10808020,
450 0xc488, 0xffffffff, 0x00800008,
451 0xc314, 0xffffffff, 0x00001000,
452 0xc30c, 0xffffffff, 0x80010014
455 static const u32 verde_golden_registers[] =
457 0x9a10, 0x00010000, 0x00018208,
458 0x9830, 0xffffffff, 0x00000000,
459 0x9834, 0xf00fffff, 0x00000400,
460 0x9838, 0x0002021c, 0x00020200,
461 0xc78, 0x00000080, 0x00000000,
462 0xd030, 0x000300c0, 0x00800040,
463 0xd030, 0x000300c0, 0x00800040,
464 0xd830, 0x000300c0, 0x00800040,
465 0xd830, 0x000300c0, 0x00800040,
466 0x5bb0, 0x000000f0, 0x00000070,
467 0x5bc0, 0x00200000, 0x50100000,
468 0x7030, 0x31000311, 0x00000011,
469 0x2ae4, 0x00073ffe, 0x000022a2,
470 0x2ae4, 0x00073ffe, 0x000022a2,
471 0x2ae4, 0x00073ffe, 0x000022a2,
472 0x240c, 0x000007ff, 0x00000000,
473 0x240c, 0x000007ff, 0x00000000,
474 0x240c, 0x000007ff, 0x00000000,
475 0x8a14, 0xf000001f, 0x00000007,
476 0x8a14, 0xf000001f, 0x00000007,
477 0x8a14, 0xf000001f, 0x00000007,
478 0x8b24, 0xffffffff, 0x00ffffff,
479 0x8b10, 0x0000ff0f, 0x00000000,
480 0x28a4c, 0x07ffffff, 0x4e000000,
481 0x28350, 0x3f3f3fff, 0x0000124a,
482 0x28350, 0x3f3f3fff, 0x0000124a,
483 0x28350, 0x3f3f3fff, 0x0000124a,
484 0x30, 0x000000ff, 0x0040,
485 0x34, 0x00000040, 0x00004040,
486 0x9100, 0x07ffffff, 0x03000000,
487 0x9100, 0x07ffffff, 0x03000000,
488 0x8e88, 0x01ff1f3f, 0x00000000,
489 0x8e88, 0x01ff1f3f, 0x00000000,
490 0x8e88, 0x01ff1f3f, 0x00000000,
491 0x8e84, 0x01ff1f3f, 0x00000000,
492 0x8e84, 0x01ff1f3f, 0x00000000,
493 0x8e84, 0x01ff1f3f, 0x00000000,
494 0x9060, 0x0000007f, 0x00000020,
495 0x9508, 0x00010000, 0x00010000,
496 0xac14, 0x000003ff, 0x00000003,
497 0xac14, 0x000003ff, 0x00000003,
498 0xac14, 0x000003ff, 0x00000003,
499 0xac10, 0xffffffff, 0x00000000,
500 0xac10, 0xffffffff, 0x00000000,
501 0xac10, 0xffffffff, 0x00000000,
502 0xac0c, 0xffffffff, 0x00001032,
503 0xac0c, 0xffffffff, 0x00001032,
504 0xac0c, 0xffffffff, 0x00001032,
505 0x88d4, 0x0000001f, 0x00000010,
506 0x88d4, 0x0000001f, 0x00000010,
507 0x88d4, 0x0000001f, 0x00000010,
508 0x15c0, 0x000c0fc0, 0x000c0400
511 static const u32 oland_golden_rlc_registers[] =
513 0xc424, 0xffffffff, 0x00601005,
514 0xc47c, 0xffffffff, 0x10104040,
515 0xc488, 0xffffffff, 0x0100000a,
516 0xc314, 0xffffffff, 0x00000800,
517 0xc30c, 0xffffffff, 0x800000f4
520 static const u32 oland_golden_registers[] =
522 0x9a10, 0x00010000, 0x00018208,
523 0x9830, 0xffffffff, 0x00000000,
524 0x9834, 0xf00fffff, 0x00000400,
525 0x9838, 0x0002021c, 0x00020200,
526 0xc78, 0x00000080, 0x00000000,
527 0xd030, 0x000300c0, 0x00800040,
528 0xd830, 0x000300c0, 0x00800040,
529 0x5bb0, 0x000000f0, 0x00000070,
530 0x5bc0, 0x00200000, 0x50100000,
531 0x7030, 0x31000311, 0x00000011,
532 0x2ae4, 0x00073ffe, 0x000022a2,
533 0x240c, 0x000007ff, 0x00000000,
534 0x8a14, 0xf000001f, 0x00000007,
535 0x8b24, 0xffffffff, 0x00ffffff,
536 0x8b10, 0x0000ff0f, 0x00000000,
537 0x28a4c, 0x07ffffff, 0x4e000000,
538 0x28350, 0x3f3f3fff, 0x00000082,
539 0x30, 0x000000ff, 0x0040,
540 0x34, 0x00000040, 0x00004040,
541 0x9100, 0x07ffffff, 0x03000000,
542 0x9060, 0x0000007f, 0x00000020,
543 0x9508, 0x00010000, 0x00010000,
544 0xac14, 0x000003ff, 0x000000f3,
545 0xac10, 0xffffffff, 0x00000000,
546 0xac0c, 0xffffffff, 0x00003210,
547 0x88d4, 0x0000001f, 0x00000010,
548 0x15c0, 0x000c0fc0, 0x000c0400
551 static const u32 hainan_golden_registers[] =
553 0x9a10, 0x00010000, 0x00018208,
554 0x9830, 0xffffffff, 0x00000000,
555 0x9834, 0xf00fffff, 0x00000400,
556 0x9838, 0x0002021c, 0x00020200,
557 0xd0c0, 0xff000fff, 0x00000100,
558 0xd030, 0x000300c0, 0x00800040,
559 0xd8c0, 0xff000fff, 0x00000100,
560 0xd830, 0x000300c0, 0x00800040,
561 0x2ae4, 0x00073ffe, 0x000022a2,
562 0x240c, 0x000007ff, 0x00000000,
563 0x8a14, 0xf000001f, 0x00000007,
564 0x8b24, 0xffffffff, 0x00ffffff,
565 0x8b10, 0x0000ff0f, 0x00000000,
566 0x28a4c, 0x07ffffff, 0x4e000000,
567 0x28350, 0x3f3f3fff, 0x00000000,
568 0x30, 0x000000ff, 0x0040,
569 0x34, 0x00000040, 0x00004040,
570 0x9100, 0x03e00000, 0x03600000,
571 0x9060, 0x0000007f, 0x00000020,
572 0x9508, 0x00010000, 0x00010000,
573 0xac14, 0x000003ff, 0x000000f1,
574 0xac10, 0xffffffff, 0x00000000,
575 0xac0c, 0xffffffff, 0x00003210,
576 0x88d4, 0x0000001f, 0x00000010,
577 0x15c0, 0x000c0fc0, 0x000c0400
580 static const u32 hainan_golden_registers2[] =
582 0x98f8, 0xffffffff, 0x02010001
585 static const u32 tahiti_mgcg_cgcg_init[] =
587 0xc400, 0xffffffff, 0xfffffffc,
588 0x802c, 0xffffffff, 0xe0000000,
589 0x9a60, 0xffffffff, 0x00000100,
590 0x92a4, 0xffffffff, 0x00000100,
591 0xc164, 0xffffffff, 0x00000100,
592 0x9774, 0xffffffff, 0x00000100,
593 0x8984, 0xffffffff, 0x06000100,
594 0x8a18, 0xffffffff, 0x00000100,
595 0x92a0, 0xffffffff, 0x00000100,
596 0xc380, 0xffffffff, 0x00000100,
597 0x8b28, 0xffffffff, 0x00000100,
598 0x9144, 0xffffffff, 0x00000100,
599 0x8d88, 0xffffffff, 0x00000100,
600 0x8d8c, 0xffffffff, 0x00000100,
601 0x9030, 0xffffffff, 0x00000100,
602 0x9034, 0xffffffff, 0x00000100,
603 0x9038, 0xffffffff, 0x00000100,
604 0x903c, 0xffffffff, 0x00000100,
605 0xad80, 0xffffffff, 0x00000100,
606 0xac54, 0xffffffff, 0x00000100,
607 0x897c, 0xffffffff, 0x06000100,
608 0x9868, 0xffffffff, 0x00000100,
609 0x9510, 0xffffffff, 0x00000100,
610 0xaf04, 0xffffffff, 0x00000100,
611 0xae04, 0xffffffff, 0x00000100,
612 0x949c, 0xffffffff, 0x00000100,
613 0x802c, 0xffffffff, 0xe0000000,
614 0x9160, 0xffffffff, 0x00010000,
615 0x9164, 0xffffffff, 0x00030002,
616 0x9168, 0xffffffff, 0x00040007,
617 0x916c, 0xffffffff, 0x00060005,
618 0x9170, 0xffffffff, 0x00090008,
619 0x9174, 0xffffffff, 0x00020001,
620 0x9178, 0xffffffff, 0x00040003,
621 0x917c, 0xffffffff, 0x00000007,
622 0x9180, 0xffffffff, 0x00060005,
623 0x9184, 0xffffffff, 0x00090008,
624 0x9188, 0xffffffff, 0x00030002,
625 0x918c, 0xffffffff, 0x00050004,
626 0x9190, 0xffffffff, 0x00000008,
627 0x9194, 0xffffffff, 0x00070006,
628 0x9198, 0xffffffff, 0x000a0009,
629 0x919c, 0xffffffff, 0x00040003,
630 0x91a0, 0xffffffff, 0x00060005,
631 0x91a4, 0xffffffff, 0x00000009,
632 0x91a8, 0xffffffff, 0x00080007,
633 0x91ac, 0xffffffff, 0x000b000a,
634 0x91b0, 0xffffffff, 0x00050004,
635 0x91b4, 0xffffffff, 0x00070006,
636 0x91b8, 0xffffffff, 0x0008000b,
637 0x91bc, 0xffffffff, 0x000a0009,
638 0x91c0, 0xffffffff, 0x000d000c,
639 0x91c4, 0xffffffff, 0x00060005,
640 0x91c8, 0xffffffff, 0x00080007,
641 0x91cc, 0xffffffff, 0x0000000b,
642 0x91d0, 0xffffffff, 0x000a0009,
643 0x91d4, 0xffffffff, 0x000d000c,
644 0x91d8, 0xffffffff, 0x00070006,
645 0x91dc, 0xffffffff, 0x00090008,
646 0x91e0, 0xffffffff, 0x0000000c,
647 0x91e4, 0xffffffff, 0x000b000a,
648 0x91e8, 0xffffffff, 0x000e000d,
649 0x91ec, 0xffffffff, 0x00080007,
650 0x91f0, 0xffffffff, 0x000a0009,
651 0x91f4, 0xffffffff, 0x0000000d,
652 0x91f8, 0xffffffff, 0x000c000b,
653 0x91fc, 0xffffffff, 0x000f000e,
654 0x9200, 0xffffffff, 0x00090008,
655 0x9204, 0xffffffff, 0x000b000a,
656 0x9208, 0xffffffff, 0x000c000f,
657 0x920c, 0xffffffff, 0x000e000d,
658 0x9210, 0xffffffff, 0x00110010,
659 0x9214, 0xffffffff, 0x000a0009,
660 0x9218, 0xffffffff, 0x000c000b,
661 0x921c, 0xffffffff, 0x0000000f,
662 0x9220, 0xffffffff, 0x000e000d,
663 0x9224, 0xffffffff, 0x00110010,
664 0x9228, 0xffffffff, 0x000b000a,
665 0x922c, 0xffffffff, 0x000d000c,
666 0x9230, 0xffffffff, 0x00000010,
667 0x9234, 0xffffffff, 0x000f000e,
668 0x9238, 0xffffffff, 0x00120011,
669 0x923c, 0xffffffff, 0x000c000b,
670 0x9240, 0xffffffff, 0x000e000d,
671 0x9244, 0xffffffff, 0x00000011,
672 0x9248, 0xffffffff, 0x0010000f,
673 0x924c, 0xffffffff, 0x00130012,
674 0x9250, 0xffffffff, 0x000d000c,
675 0x9254, 0xffffffff, 0x000f000e,
676 0x9258, 0xffffffff, 0x00100013,
677 0x925c, 0xffffffff, 0x00120011,
678 0x9260, 0xffffffff, 0x00150014,
679 0x9264, 0xffffffff, 0x000e000d,
680 0x9268, 0xffffffff, 0x0010000f,
681 0x926c, 0xffffffff, 0x00000013,
682 0x9270, 0xffffffff, 0x00120011,
683 0x9274, 0xffffffff, 0x00150014,
684 0x9278, 0xffffffff, 0x000f000e,
685 0x927c, 0xffffffff, 0x00110010,
686 0x9280, 0xffffffff, 0x00000014,
687 0x9284, 0xffffffff, 0x00130012,
688 0x9288, 0xffffffff, 0x00160015,
689 0x928c, 0xffffffff, 0x0010000f,
690 0x9290, 0xffffffff, 0x00120011,
691 0x9294, 0xffffffff, 0x00000015,
692 0x9298, 0xffffffff, 0x00140013,
693 0x929c, 0xffffffff, 0x00170016,
694 0x9150, 0xffffffff, 0x96940200,
695 0x8708, 0xffffffff, 0x00900100,
696 0xc478, 0xffffffff, 0x00000080,
697 0xc404, 0xffffffff, 0x0020003f,
698 0x30, 0xffffffff, 0x0000001c,
699 0x34, 0x000f0000, 0x000f0000,
700 0x160c, 0xffffffff, 0x00000100,
701 0x1024, 0xffffffff, 0x00000100,
702 0x102c, 0x00000101, 0x00000000,
703 0x20a8, 0xffffffff, 0x00000104,
704 0x264c, 0x000c0000, 0x000c0000,
705 0x2648, 0x000c0000, 0x000c0000,
706 0x55e4, 0xff000fff, 0x00000100,
707 0x55e8, 0x00000001, 0x00000001,
708 0x2f50, 0x00000001, 0x00000001,
709 0x30cc, 0xc0000fff, 0x00000104,
710 0xc1e4, 0x00000001, 0x00000001,
711 0xd0c0, 0xfffffff0, 0x00000100,
712 0xd8c0, 0xfffffff0, 0x00000100
715 static const u32 pitcairn_mgcg_cgcg_init[] =
717 0xc400, 0xffffffff, 0xfffffffc,
718 0x802c, 0xffffffff, 0xe0000000,
719 0x9a60, 0xffffffff, 0x00000100,
720 0x92a4, 0xffffffff, 0x00000100,
721 0xc164, 0xffffffff, 0x00000100,
722 0x9774, 0xffffffff, 0x00000100,
723 0x8984, 0xffffffff, 0x06000100,
724 0x8a18, 0xffffffff, 0x00000100,
725 0x92a0, 0xffffffff, 0x00000100,
726 0xc380, 0xffffffff, 0x00000100,
727 0x8b28, 0xffffffff, 0x00000100,
728 0x9144, 0xffffffff, 0x00000100,
729 0x8d88, 0xffffffff, 0x00000100,
730 0x8d8c, 0xffffffff, 0x00000100,
731 0x9030, 0xffffffff, 0x00000100,
732 0x9034, 0xffffffff, 0x00000100,
733 0x9038, 0xffffffff, 0x00000100,
734 0x903c, 0xffffffff, 0x00000100,
735 0xad80, 0xffffffff, 0x00000100,
736 0xac54, 0xffffffff, 0x00000100,
737 0x897c, 0xffffffff, 0x06000100,
738 0x9868, 0xffffffff, 0x00000100,
739 0x9510, 0xffffffff, 0x00000100,
740 0xaf04, 0xffffffff, 0x00000100,
741 0xae04, 0xffffffff, 0x00000100,
742 0x949c, 0xffffffff, 0x00000100,
743 0x802c, 0xffffffff, 0xe0000000,
744 0x9160, 0xffffffff, 0x00010000,
745 0x9164, 0xffffffff, 0x00030002,
746 0x9168, 0xffffffff, 0x00040007,
747 0x916c, 0xffffffff, 0x00060005,
748 0x9170, 0xffffffff, 0x00090008,
749 0x9174, 0xffffffff, 0x00020001,
750 0x9178, 0xffffffff, 0x00040003,
751 0x917c, 0xffffffff, 0x00000007,
752 0x9180, 0xffffffff, 0x00060005,
753 0x9184, 0xffffffff, 0x00090008,
754 0x9188, 0xffffffff, 0x00030002,
755 0x918c, 0xffffffff, 0x00050004,
756 0x9190, 0xffffffff, 0x00000008,
757 0x9194, 0xffffffff, 0x00070006,
758 0x9198, 0xffffffff, 0x000a0009,
759 0x919c, 0xffffffff, 0x00040003,
760 0x91a0, 0xffffffff, 0x00060005,
761 0x91a4, 0xffffffff, 0x00000009,
762 0x91a8, 0xffffffff, 0x00080007,
763 0x91ac, 0xffffffff, 0x000b000a,
764 0x91b0, 0xffffffff, 0x00050004,
765 0x91b4, 0xffffffff, 0x00070006,
766 0x91b8, 0xffffffff, 0x0008000b,
767 0x91bc, 0xffffffff, 0x000a0009,
768 0x91c0, 0xffffffff, 0x000d000c,
769 0x9200, 0xffffffff, 0x00090008,
770 0x9204, 0xffffffff, 0x000b000a,
771 0x9208, 0xffffffff, 0x000c000f,
772 0x920c, 0xffffffff, 0x000e000d,
773 0x9210, 0xffffffff, 0x00110010,
774 0x9214, 0xffffffff, 0x000a0009,
775 0x9218, 0xffffffff, 0x000c000b,
776 0x921c, 0xffffffff, 0x0000000f,
777 0x9220, 0xffffffff, 0x000e000d,
778 0x9224, 0xffffffff, 0x00110010,
779 0x9228, 0xffffffff, 0x000b000a,
780 0x922c, 0xffffffff, 0x000d000c,
781 0x9230, 0xffffffff, 0x00000010,
782 0x9234, 0xffffffff, 0x000f000e,
783 0x9238, 0xffffffff, 0x00120011,
784 0x923c, 0xffffffff, 0x000c000b,
785 0x9240, 0xffffffff, 0x000e000d,
786 0x9244, 0xffffffff, 0x00000011,
787 0x9248, 0xffffffff, 0x0010000f,
788 0x924c, 0xffffffff, 0x00130012,
789 0x9250, 0xffffffff, 0x000d000c,
790 0x9254, 0xffffffff, 0x000f000e,
791 0x9258, 0xffffffff, 0x00100013,
792 0x925c, 0xffffffff, 0x00120011,
793 0x9260, 0xffffffff, 0x00150014,
794 0x9150, 0xffffffff, 0x96940200,
795 0x8708, 0xffffffff, 0x00900100,
796 0xc478, 0xffffffff, 0x00000080,
797 0xc404, 0xffffffff, 0x0020003f,
798 0x30, 0xffffffff, 0x0000001c,
799 0x34, 0x000f0000, 0x000f0000,
800 0x160c, 0xffffffff, 0x00000100,
801 0x1024, 0xffffffff, 0x00000100,
802 0x102c, 0x00000101, 0x00000000,
803 0x20a8, 0xffffffff, 0x00000104,
804 0x55e4, 0xff000fff, 0x00000100,
805 0x55e8, 0x00000001, 0x00000001,
806 0x2f50, 0x00000001, 0x00000001,
807 0x30cc, 0xc0000fff, 0x00000104,
808 0xc1e4, 0x00000001, 0x00000001,
809 0xd0c0, 0xfffffff0, 0x00000100,
810 0xd8c0, 0xfffffff0, 0x00000100
813 static const u32 verde_mgcg_cgcg_init[] =
815 0xc400, 0xffffffff, 0xfffffffc,
816 0x802c, 0xffffffff, 0xe0000000,
817 0x9a60, 0xffffffff, 0x00000100,
818 0x92a4, 0xffffffff, 0x00000100,
819 0xc164, 0xffffffff, 0x00000100,
820 0x9774, 0xffffffff, 0x00000100,
821 0x8984, 0xffffffff, 0x06000100,
822 0x8a18, 0xffffffff, 0x00000100,
823 0x92a0, 0xffffffff, 0x00000100,
824 0xc380, 0xffffffff, 0x00000100,
825 0x8b28, 0xffffffff, 0x00000100,
826 0x9144, 0xffffffff, 0x00000100,
827 0x8d88, 0xffffffff, 0x00000100,
828 0x8d8c, 0xffffffff, 0x00000100,
829 0x9030, 0xffffffff, 0x00000100,
830 0x9034, 0xffffffff, 0x00000100,
831 0x9038, 0xffffffff, 0x00000100,
832 0x903c, 0xffffffff, 0x00000100,
833 0xad80, 0xffffffff, 0x00000100,
834 0xac54, 0xffffffff, 0x00000100,
835 0x897c, 0xffffffff, 0x06000100,
836 0x9868, 0xffffffff, 0x00000100,
837 0x9510, 0xffffffff, 0x00000100,
838 0xaf04, 0xffffffff, 0x00000100,
839 0xae04, 0xffffffff, 0x00000100,
840 0x949c, 0xffffffff, 0x00000100,
841 0x802c, 0xffffffff, 0xe0000000,
842 0x9160, 0xffffffff, 0x00010000,
843 0x9164, 0xffffffff, 0x00030002,
844 0x9168, 0xffffffff, 0x00040007,
845 0x916c, 0xffffffff, 0x00060005,
846 0x9170, 0xffffffff, 0x00090008,
847 0x9174, 0xffffffff, 0x00020001,
848 0x9178, 0xffffffff, 0x00040003,
849 0x917c, 0xffffffff, 0x00000007,
850 0x9180, 0xffffffff, 0x00060005,
851 0x9184, 0xffffffff, 0x00090008,
852 0x9188, 0xffffffff, 0x00030002,
853 0x918c, 0xffffffff, 0x00050004,
854 0x9190, 0xffffffff, 0x00000008,
855 0x9194, 0xffffffff, 0x00070006,
856 0x9198, 0xffffffff, 0x000a0009,
857 0x919c, 0xffffffff, 0x00040003,
858 0x91a0, 0xffffffff, 0x00060005,
859 0x91a4, 0xffffffff, 0x00000009,
860 0x91a8, 0xffffffff, 0x00080007,
861 0x91ac, 0xffffffff, 0x000b000a,
862 0x91b0, 0xffffffff, 0x00050004,
863 0x91b4, 0xffffffff, 0x00070006,
864 0x91b8, 0xffffffff, 0x0008000b,
865 0x91bc, 0xffffffff, 0x000a0009,
866 0x91c0, 0xffffffff, 0x000d000c,
867 0x9200, 0xffffffff, 0x00090008,
868 0x9204, 0xffffffff, 0x000b000a,
869 0x9208, 0xffffffff, 0x000c000f,
870 0x920c, 0xffffffff, 0x000e000d,
871 0x9210, 0xffffffff, 0x00110010,
872 0x9214, 0xffffffff, 0x000a0009,
873 0x9218, 0xffffffff, 0x000c000b,
874 0x921c, 0xffffffff, 0x0000000f,
875 0x9220, 0xffffffff, 0x000e000d,
876 0x9224, 0xffffffff, 0x00110010,
877 0x9228, 0xffffffff, 0x000b000a,
878 0x922c, 0xffffffff, 0x000d000c,
879 0x9230, 0xffffffff, 0x00000010,
880 0x9234, 0xffffffff, 0x000f000e,
881 0x9238, 0xffffffff, 0x00120011,
882 0x923c, 0xffffffff, 0x000c000b,
883 0x9240, 0xffffffff, 0x000e000d,
884 0x9244, 0xffffffff, 0x00000011,
885 0x9248, 0xffffffff, 0x0010000f,
886 0x924c, 0xffffffff, 0x00130012,
887 0x9250, 0xffffffff, 0x000d000c,
888 0x9254, 0xffffffff, 0x000f000e,
889 0x9258, 0xffffffff, 0x00100013,
890 0x925c, 0xffffffff, 0x00120011,
891 0x9260, 0xffffffff, 0x00150014,
892 0x9150, 0xffffffff, 0x96940200,
893 0x8708, 0xffffffff, 0x00900100,
894 0xc478, 0xffffffff, 0x00000080,
895 0xc404, 0xffffffff, 0x0020003f,
896 0x30, 0xffffffff, 0x0000001c,
897 0x34, 0x000f0000, 0x000f0000,
898 0x160c, 0xffffffff, 0x00000100,
899 0x1024, 0xffffffff, 0x00000100,
900 0x102c, 0x00000101, 0x00000000,
901 0x20a8, 0xffffffff, 0x00000104,
902 0x264c, 0x000c0000, 0x000c0000,
903 0x2648, 0x000c0000, 0x000c0000,
904 0x55e4, 0xff000fff, 0x00000100,
905 0x55e8, 0x00000001, 0x00000001,
906 0x2f50, 0x00000001, 0x00000001,
907 0x30cc, 0xc0000fff, 0x00000104,
908 0xc1e4, 0x00000001, 0x00000001,
909 0xd0c0, 0xfffffff0, 0x00000100,
910 0xd8c0, 0xfffffff0, 0x00000100
913 static const u32 oland_mgcg_cgcg_init[] =
915 0xc400, 0xffffffff, 0xfffffffc,
916 0x802c, 0xffffffff, 0xe0000000,
917 0x9a60, 0xffffffff, 0x00000100,
918 0x92a4, 0xffffffff, 0x00000100,
919 0xc164, 0xffffffff, 0x00000100,
920 0x9774, 0xffffffff, 0x00000100,
921 0x8984, 0xffffffff, 0x06000100,
922 0x8a18, 0xffffffff, 0x00000100,
923 0x92a0, 0xffffffff, 0x00000100,
924 0xc380, 0xffffffff, 0x00000100,
925 0x8b28, 0xffffffff, 0x00000100,
926 0x9144, 0xffffffff, 0x00000100,
927 0x8d88, 0xffffffff, 0x00000100,
928 0x8d8c, 0xffffffff, 0x00000100,
929 0x9030, 0xffffffff, 0x00000100,
930 0x9034, 0xffffffff, 0x00000100,
931 0x9038, 0xffffffff, 0x00000100,
932 0x903c, 0xffffffff, 0x00000100,
933 0xad80, 0xffffffff, 0x00000100,
934 0xac54, 0xffffffff, 0x00000100,
935 0x897c, 0xffffffff, 0x06000100,
936 0x9868, 0xffffffff, 0x00000100,
937 0x9510, 0xffffffff, 0x00000100,
938 0xaf04, 0xffffffff, 0x00000100,
939 0xae04, 0xffffffff, 0x00000100,
940 0x949c, 0xffffffff, 0x00000100,
941 0x802c, 0xffffffff, 0xe0000000,
942 0x9160, 0xffffffff, 0x00010000,
943 0x9164, 0xffffffff, 0x00030002,
944 0x9168, 0xffffffff, 0x00040007,
945 0x916c, 0xffffffff, 0x00060005,
946 0x9170, 0xffffffff, 0x00090008,
947 0x9174, 0xffffffff, 0x00020001,
948 0x9178, 0xffffffff, 0x00040003,
949 0x917c, 0xffffffff, 0x00000007,
950 0x9180, 0xffffffff, 0x00060005,
951 0x9184, 0xffffffff, 0x00090008,
952 0x9188, 0xffffffff, 0x00030002,
953 0x918c, 0xffffffff, 0x00050004,
954 0x9190, 0xffffffff, 0x00000008,
955 0x9194, 0xffffffff, 0x00070006,
956 0x9198, 0xffffffff, 0x000a0009,
957 0x919c, 0xffffffff, 0x00040003,
958 0x91a0, 0xffffffff, 0x00060005,
959 0x91a4, 0xffffffff, 0x00000009,
960 0x91a8, 0xffffffff, 0x00080007,
961 0x91ac, 0xffffffff, 0x000b000a,
962 0x91b0, 0xffffffff, 0x00050004,
963 0x91b4, 0xffffffff, 0x00070006,
964 0x91b8, 0xffffffff, 0x0008000b,
965 0x91bc, 0xffffffff, 0x000a0009,
966 0x91c0, 0xffffffff, 0x000d000c,
967 0x91c4, 0xffffffff, 0x00060005,
968 0x91c8, 0xffffffff, 0x00080007,
969 0x91cc, 0xffffffff, 0x0000000b,
970 0x91d0, 0xffffffff, 0x000a0009,
971 0x91d4, 0xffffffff, 0x000d000c,
972 0x9150, 0xffffffff, 0x96940200,
973 0x8708, 0xffffffff, 0x00900100,
974 0xc478, 0xffffffff, 0x00000080,
975 0xc404, 0xffffffff, 0x0020003f,
976 0x30, 0xffffffff, 0x0000001c,
977 0x34, 0x000f0000, 0x000f0000,
978 0x160c, 0xffffffff, 0x00000100,
979 0x1024, 0xffffffff, 0x00000100,
980 0x102c, 0x00000101, 0x00000000,
981 0x20a8, 0xffffffff, 0x00000104,
982 0x264c, 0x000c0000, 0x000c0000,
983 0x2648, 0x000c0000, 0x000c0000,
984 0x55e4, 0xff000fff, 0x00000100,
985 0x55e8, 0x00000001, 0x00000001,
986 0x2f50, 0x00000001, 0x00000001,
987 0x30cc, 0xc0000fff, 0x00000104,
988 0xc1e4, 0x00000001, 0x00000001,
989 0xd0c0, 0xfffffff0, 0x00000100,
990 0xd8c0, 0xfffffff0, 0x00000100
993 static const u32 hainan_mgcg_cgcg_init[] =
995 0xc400, 0xffffffff, 0xfffffffc,
996 0x802c, 0xffffffff, 0xe0000000,
997 0x9a60, 0xffffffff, 0x00000100,
998 0x92a4, 0xffffffff, 0x00000100,
999 0xc164, 0xffffffff, 0x00000100,
1000 0x9774, 0xffffffff, 0x00000100,
1001 0x8984, 0xffffffff, 0x06000100,
1002 0x8a18, 0xffffffff, 0x00000100,
1003 0x92a0, 0xffffffff, 0x00000100,
1004 0xc380, 0xffffffff, 0x00000100,
1005 0x8b28, 0xffffffff, 0x00000100,
1006 0x9144, 0xffffffff, 0x00000100,
1007 0x8d88, 0xffffffff, 0x00000100,
1008 0x8d8c, 0xffffffff, 0x00000100,
1009 0x9030, 0xffffffff, 0x00000100,
1010 0x9034, 0xffffffff, 0x00000100,
1011 0x9038, 0xffffffff, 0x00000100,
1012 0x903c, 0xffffffff, 0x00000100,
1013 0xad80, 0xffffffff, 0x00000100,
1014 0xac54, 0xffffffff, 0x00000100,
1015 0x897c, 0xffffffff, 0x06000100,
1016 0x9868, 0xffffffff, 0x00000100,
1017 0x9510, 0xffffffff, 0x00000100,
1018 0xaf04, 0xffffffff, 0x00000100,
1019 0xae04, 0xffffffff, 0x00000100,
1020 0x949c, 0xffffffff, 0x00000100,
1021 0x802c, 0xffffffff, 0xe0000000,
1022 0x9160, 0xffffffff, 0x00010000,
1023 0x9164, 0xffffffff, 0x00030002,
1024 0x9168, 0xffffffff, 0x00040007,
1025 0x916c, 0xffffffff, 0x00060005,
1026 0x9170, 0xffffffff, 0x00090008,
1027 0x9174, 0xffffffff, 0x00020001,
1028 0x9178, 0xffffffff, 0x00040003,
1029 0x917c, 0xffffffff, 0x00000007,
1030 0x9180, 0xffffffff, 0x00060005,
1031 0x9184, 0xffffffff, 0x00090008,
1032 0x9188, 0xffffffff, 0x00030002,
1033 0x918c, 0xffffffff, 0x00050004,
1034 0x9190, 0xffffffff, 0x00000008,
1035 0x9194, 0xffffffff, 0x00070006,
1036 0x9198, 0xffffffff, 0x000a0009,
1037 0x919c, 0xffffffff, 0x00040003,
1038 0x91a0, 0xffffffff, 0x00060005,
1039 0x91a4, 0xffffffff, 0x00000009,
1040 0x91a8, 0xffffffff, 0x00080007,
1041 0x91ac, 0xffffffff, 0x000b000a,
1042 0x91b0, 0xffffffff, 0x00050004,
1043 0x91b4, 0xffffffff, 0x00070006,
1044 0x91b8, 0xffffffff, 0x0008000b,
1045 0x91bc, 0xffffffff, 0x000a0009,
1046 0x91c0, 0xffffffff, 0x000d000c,
1047 0x91c4, 0xffffffff, 0x00060005,
1048 0x91c8, 0xffffffff, 0x00080007,
1049 0x91cc, 0xffffffff, 0x0000000b,
1050 0x91d0, 0xffffffff, 0x000a0009,
1051 0x91d4, 0xffffffff, 0x000d000c,
1052 0x9150, 0xffffffff, 0x96940200,
1053 0x8708, 0xffffffff, 0x00900100,
1054 0xc478, 0xffffffff, 0x00000080,
1055 0xc404, 0xffffffff, 0x0020003f,
1056 0x30, 0xffffffff, 0x0000001c,
1057 0x34, 0x000f0000, 0x000f0000,
1058 0x160c, 0xffffffff, 0x00000100,
1059 0x1024, 0xffffffff, 0x00000100,
1060 0x20a8, 0xffffffff, 0x00000104,
1061 0x264c, 0x000c0000, 0x000c0000,
1062 0x2648, 0x000c0000, 0x000c0000,
1063 0x2f50, 0x00000001, 0x00000001,
1064 0x30cc, 0xc0000fff, 0x00000104,
1065 0xc1e4, 0x00000001, 0x00000001,
1066 0xd0c0, 0xfffffff0, 0x00000100,
1067 0xd8c0, 0xfffffff0, 0x00000100
1070 static u32 verde_pg_init[] =
1072 0x353c, 0xffffffff, 0x40000,
1073 0x3538, 0xffffffff, 0x200010ff,
1074 0x353c, 0xffffffff, 0x0,
1075 0x353c, 0xffffffff, 0x0,
1076 0x353c, 0xffffffff, 0x0,
1077 0x353c, 0xffffffff, 0x0,
1078 0x353c, 0xffffffff, 0x0,
1079 0x353c, 0xffffffff, 0x7007,
1080 0x3538, 0xffffffff, 0x300010ff,
1081 0x353c, 0xffffffff, 0x0,
1082 0x353c, 0xffffffff, 0x0,
1083 0x353c, 0xffffffff, 0x0,
1084 0x353c, 0xffffffff, 0x0,
1085 0x353c, 0xffffffff, 0x0,
1086 0x353c, 0xffffffff, 0x400000,
1087 0x3538, 0xffffffff, 0x100010ff,
1088 0x353c, 0xffffffff, 0x0,
1089 0x353c, 0xffffffff, 0x0,
1090 0x353c, 0xffffffff, 0x0,
1091 0x353c, 0xffffffff, 0x0,
1092 0x353c, 0xffffffff, 0x0,
1093 0x353c, 0xffffffff, 0x120200,
1094 0x3538, 0xffffffff, 0x500010ff,
1095 0x353c, 0xffffffff, 0x0,
1096 0x353c, 0xffffffff, 0x0,
1097 0x353c, 0xffffffff, 0x0,
1098 0x353c, 0xffffffff, 0x0,
1099 0x353c, 0xffffffff, 0x0,
1100 0x353c, 0xffffffff, 0x1e1e16,
1101 0x3538, 0xffffffff, 0x600010ff,
1102 0x353c, 0xffffffff, 0x0,
1103 0x353c, 0xffffffff, 0x0,
1104 0x353c, 0xffffffff, 0x0,
1105 0x353c, 0xffffffff, 0x0,
1106 0x353c, 0xffffffff, 0x0,
1107 0x353c, 0xffffffff, 0x171f1e,
1108 0x3538, 0xffffffff, 0x700010ff,
1109 0x353c, 0xffffffff, 0x0,
1110 0x353c, 0xffffffff, 0x0,
1111 0x353c, 0xffffffff, 0x0,
1112 0x353c, 0xffffffff, 0x0,
1113 0x353c, 0xffffffff, 0x0,
1114 0x353c, 0xffffffff, 0x0,
1115 0x3538, 0xffffffff, 0x9ff,
1116 0x3500, 0xffffffff, 0x0,
1117 0x3504, 0xffffffff, 0x10000800,
1118 0x3504, 0xffffffff, 0xf,
1119 0x3504, 0xffffffff, 0xf,
1120 0x3500, 0xffffffff, 0x4,
1121 0x3504, 0xffffffff, 0x1000051e,
1122 0x3504, 0xffffffff, 0xffff,
1123 0x3504, 0xffffffff, 0xffff,
1124 0x3500, 0xffffffff, 0x8,
1125 0x3504, 0xffffffff, 0x80500,
1126 0x3500, 0xffffffff, 0x12,
1127 0x3504, 0xffffffff, 0x9050c,
1128 0x3500, 0xffffffff, 0x1d,
1129 0x3504, 0xffffffff, 0xb052c,
1130 0x3500, 0xffffffff, 0x2a,
1131 0x3504, 0xffffffff, 0x1053e,
1132 0x3500, 0xffffffff, 0x2d,
1133 0x3504, 0xffffffff, 0x10546,
1134 0x3500, 0xffffffff, 0x30,
1135 0x3504, 0xffffffff, 0xa054e,
1136 0x3500, 0xffffffff, 0x3c,
1137 0x3504, 0xffffffff, 0x1055f,
1138 0x3500, 0xffffffff, 0x3f,
1139 0x3504, 0xffffffff, 0x10567,
1140 0x3500, 0xffffffff, 0x42,
1141 0x3504, 0xffffffff, 0x1056f,
1142 0x3500, 0xffffffff, 0x45,
1143 0x3504, 0xffffffff, 0x10572,
1144 0x3500, 0xffffffff, 0x48,
1145 0x3504, 0xffffffff, 0x20575,
1146 0x3500, 0xffffffff, 0x4c,
1147 0x3504, 0xffffffff, 0x190801,
1148 0x3500, 0xffffffff, 0x67,
1149 0x3504, 0xffffffff, 0x1082a,
1150 0x3500, 0xffffffff, 0x6a,
1151 0x3504, 0xffffffff, 0x1b082d,
1152 0x3500, 0xffffffff, 0x87,
1153 0x3504, 0xffffffff, 0x310851,
1154 0x3500, 0xffffffff, 0xba,
1155 0x3504, 0xffffffff, 0x891,
1156 0x3500, 0xffffffff, 0xbc,
1157 0x3504, 0xffffffff, 0x893,
1158 0x3500, 0xffffffff, 0xbe,
1159 0x3504, 0xffffffff, 0x20895,
1160 0x3500, 0xffffffff, 0xc2,
1161 0x3504, 0xffffffff, 0x20899,
1162 0x3500, 0xffffffff, 0xc6,
1163 0x3504, 0xffffffff, 0x2089d,
1164 0x3500, 0xffffffff, 0xca,
1165 0x3504, 0xffffffff, 0x8a1,
1166 0x3500, 0xffffffff, 0xcc,
1167 0x3504, 0xffffffff, 0x8a3,
1168 0x3500, 0xffffffff, 0xce,
1169 0x3504, 0xffffffff, 0x308a5,
1170 0x3500, 0xffffffff, 0xd3,
1171 0x3504, 0xffffffff, 0x6d08cd,
1172 0x3500, 0xffffffff, 0x142,
1173 0x3504, 0xffffffff, 0x2000095a,
1174 0x3504, 0xffffffff, 0x1,
1175 0x3500, 0xffffffff, 0x144,
1176 0x3504, 0xffffffff, 0x301f095b,
1177 0x3500, 0xffffffff, 0x165,
1178 0x3504, 0xffffffff, 0xc094d,
1179 0x3500, 0xffffffff, 0x173,
1180 0x3504, 0xffffffff, 0xf096d,
1181 0x3500, 0xffffffff, 0x184,
1182 0x3504, 0xffffffff, 0x15097f,
1183 0x3500, 0xffffffff, 0x19b,
1184 0x3504, 0xffffffff, 0xc0998,
1185 0x3500, 0xffffffff, 0x1a9,
1186 0x3504, 0xffffffff, 0x409a7,
1187 0x3500, 0xffffffff, 0x1af,
1188 0x3504, 0xffffffff, 0xcdc,
1189 0x3500, 0xffffffff, 0x1b1,
1190 0x3504, 0xffffffff, 0x800,
1191 0x3508, 0xffffffff, 0x6c9b2000,
1192 0x3510, 0xfc00, 0x2000,
1193 0x3544, 0xffffffff, 0xfc0,
1194 0x28d4, 0x00000100, 0x100
1197 static void si_init_golden_registers(struct radeon_device *rdev)
1199 switch (rdev->family) {
1201 radeon_program_register_sequence(rdev,
1202 tahiti_golden_registers,
1203 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1204 radeon_program_register_sequence(rdev,
1205 tahiti_golden_rlc_registers,
1206 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1207 radeon_program_register_sequence(rdev,
1208 tahiti_mgcg_cgcg_init,
1209 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1210 radeon_program_register_sequence(rdev,
1211 tahiti_golden_registers2,
1212 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1215 radeon_program_register_sequence(rdev,
1216 pitcairn_golden_registers,
1217 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1218 radeon_program_register_sequence(rdev,
1219 pitcairn_golden_rlc_registers,
1220 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1221 radeon_program_register_sequence(rdev,
1222 pitcairn_mgcg_cgcg_init,
1223 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1226 radeon_program_register_sequence(rdev,
1227 verde_golden_registers,
1228 (const u32)ARRAY_SIZE(verde_golden_registers));
1229 radeon_program_register_sequence(rdev,
1230 verde_golden_rlc_registers,
1231 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1232 radeon_program_register_sequence(rdev,
1233 verde_mgcg_cgcg_init,
1234 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1235 radeon_program_register_sequence(rdev,
1237 (const u32)ARRAY_SIZE(verde_pg_init));
1240 radeon_program_register_sequence(rdev,
1241 oland_golden_registers,
1242 (const u32)ARRAY_SIZE(oland_golden_registers));
1243 radeon_program_register_sequence(rdev,
1244 oland_golden_rlc_registers,
1245 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1246 radeon_program_register_sequence(rdev,
1247 oland_mgcg_cgcg_init,
1248 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1251 radeon_program_register_sequence(rdev,
1252 hainan_golden_registers,
1253 (const u32)ARRAY_SIZE(hainan_golden_registers));
1254 radeon_program_register_sequence(rdev,
1255 hainan_golden_registers2,
1256 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1257 radeon_program_register_sequence(rdev,
1258 hainan_mgcg_cgcg_init,
1259 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1266 #define PCIE_BUS_CLK 10000
1267 #define TCLK (PCIE_BUS_CLK / 10)
1270 * si_get_xclk - get the xclk
1272 * @rdev: radeon_device pointer
1274 * Returns the reference clock used by the gfx engine
1277 u32 si_get_xclk(struct radeon_device *rdev)
1279 u32 reference_clock = rdev->clock.spll.reference_freq;
1282 tmp = RREG32(CG_CLKPIN_CNTL_2);
1283 if (tmp & MUX_TCLK_TO_XCLK)
1286 tmp = RREG32(CG_CLKPIN_CNTL);
1287 if (tmp & XTALIN_DIVIDE)
1288 return reference_clock / 4;
1290 return reference_clock;
1293 /* get temperature in millidegrees */
1294 int si_get_temp(struct radeon_device *rdev)
1297 int actual_temp = 0;
1299 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1305 actual_temp = temp & 0x1ff;
1307 actual_temp = (actual_temp * 1000);
1312 #define TAHITI_IO_MC_REGS_SIZE 36
1314 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1315 {0x0000006f, 0x03044000},
1316 {0x00000070, 0x0480c018},
1317 {0x00000071, 0x00000040},
1318 {0x00000072, 0x01000000},
1319 {0x00000074, 0x000000ff},
1320 {0x00000075, 0x00143400},
1321 {0x00000076, 0x08ec0800},
1322 {0x00000077, 0x040000cc},
1323 {0x00000079, 0x00000000},
1324 {0x0000007a, 0x21000409},
1325 {0x0000007c, 0x00000000},
1326 {0x0000007d, 0xe8000000},
1327 {0x0000007e, 0x044408a8},
1328 {0x0000007f, 0x00000003},
1329 {0x00000080, 0x00000000},
1330 {0x00000081, 0x01000000},
1331 {0x00000082, 0x02000000},
1332 {0x00000083, 0x00000000},
1333 {0x00000084, 0xe3f3e4f4},
1334 {0x00000085, 0x00052024},
1335 {0x00000087, 0x00000000},
1336 {0x00000088, 0x66036603},
1337 {0x00000089, 0x01000000},
1338 {0x0000008b, 0x1c0a0000},
1339 {0x0000008c, 0xff010000},
1340 {0x0000008e, 0xffffefff},
1341 {0x0000008f, 0xfff3efff},
1342 {0x00000090, 0xfff3efbf},
1343 {0x00000094, 0x00101101},
1344 {0x00000095, 0x00000fff},
1345 {0x00000096, 0x00116fff},
1346 {0x00000097, 0x60010000},
1347 {0x00000098, 0x10010000},
1348 {0x00000099, 0x00006000},
1349 {0x0000009a, 0x00001000},
1350 {0x0000009f, 0x00a77400}
1353 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1354 {0x0000006f, 0x03044000},
1355 {0x00000070, 0x0480c018},
1356 {0x00000071, 0x00000040},
1357 {0x00000072, 0x01000000},
1358 {0x00000074, 0x000000ff},
1359 {0x00000075, 0x00143400},
1360 {0x00000076, 0x08ec0800},
1361 {0x00000077, 0x040000cc},
1362 {0x00000079, 0x00000000},
1363 {0x0000007a, 0x21000409},
1364 {0x0000007c, 0x00000000},
1365 {0x0000007d, 0xe8000000},
1366 {0x0000007e, 0x044408a8},
1367 {0x0000007f, 0x00000003},
1368 {0x00000080, 0x00000000},
1369 {0x00000081, 0x01000000},
1370 {0x00000082, 0x02000000},
1371 {0x00000083, 0x00000000},
1372 {0x00000084, 0xe3f3e4f4},
1373 {0x00000085, 0x00052024},
1374 {0x00000087, 0x00000000},
1375 {0x00000088, 0x66036603},
1376 {0x00000089, 0x01000000},
1377 {0x0000008b, 0x1c0a0000},
1378 {0x0000008c, 0xff010000},
1379 {0x0000008e, 0xffffefff},
1380 {0x0000008f, 0xfff3efff},
1381 {0x00000090, 0xfff3efbf},
1382 {0x00000094, 0x00101101},
1383 {0x00000095, 0x00000fff},
1384 {0x00000096, 0x00116fff},
1385 {0x00000097, 0x60010000},
1386 {0x00000098, 0x10010000},
1387 {0x00000099, 0x00006000},
1388 {0x0000009a, 0x00001000},
1389 {0x0000009f, 0x00a47400}
1392 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1393 {0x0000006f, 0x03044000},
1394 {0x00000070, 0x0480c018},
1395 {0x00000071, 0x00000040},
1396 {0x00000072, 0x01000000},
1397 {0x00000074, 0x000000ff},
1398 {0x00000075, 0x00143400},
1399 {0x00000076, 0x08ec0800},
1400 {0x00000077, 0x040000cc},
1401 {0x00000079, 0x00000000},
1402 {0x0000007a, 0x21000409},
1403 {0x0000007c, 0x00000000},
1404 {0x0000007d, 0xe8000000},
1405 {0x0000007e, 0x044408a8},
1406 {0x0000007f, 0x00000003},
1407 {0x00000080, 0x00000000},
1408 {0x00000081, 0x01000000},
1409 {0x00000082, 0x02000000},
1410 {0x00000083, 0x00000000},
1411 {0x00000084, 0xe3f3e4f4},
1412 {0x00000085, 0x00052024},
1413 {0x00000087, 0x00000000},
1414 {0x00000088, 0x66036603},
1415 {0x00000089, 0x01000000},
1416 {0x0000008b, 0x1c0a0000},
1417 {0x0000008c, 0xff010000},
1418 {0x0000008e, 0xffffefff},
1419 {0x0000008f, 0xfff3efff},
1420 {0x00000090, 0xfff3efbf},
1421 {0x00000094, 0x00101101},
1422 {0x00000095, 0x00000fff},
1423 {0x00000096, 0x00116fff},
1424 {0x00000097, 0x60010000},
1425 {0x00000098, 0x10010000},
1426 {0x00000099, 0x00006000},
1427 {0x0000009a, 0x00001000},
1428 {0x0000009f, 0x00a37400}
1431 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1432 {0x0000006f, 0x03044000},
1433 {0x00000070, 0x0480c018},
1434 {0x00000071, 0x00000040},
1435 {0x00000072, 0x01000000},
1436 {0x00000074, 0x000000ff},
1437 {0x00000075, 0x00143400},
1438 {0x00000076, 0x08ec0800},
1439 {0x00000077, 0x040000cc},
1440 {0x00000079, 0x00000000},
1441 {0x0000007a, 0x21000409},
1442 {0x0000007c, 0x00000000},
1443 {0x0000007d, 0xe8000000},
1444 {0x0000007e, 0x044408a8},
1445 {0x0000007f, 0x00000003},
1446 {0x00000080, 0x00000000},
1447 {0x00000081, 0x01000000},
1448 {0x00000082, 0x02000000},
1449 {0x00000083, 0x00000000},
1450 {0x00000084, 0xe3f3e4f4},
1451 {0x00000085, 0x00052024},
1452 {0x00000087, 0x00000000},
1453 {0x00000088, 0x66036603},
1454 {0x00000089, 0x01000000},
1455 {0x0000008b, 0x1c0a0000},
1456 {0x0000008c, 0xff010000},
1457 {0x0000008e, 0xffffefff},
1458 {0x0000008f, 0xfff3efff},
1459 {0x00000090, 0xfff3efbf},
1460 {0x00000094, 0x00101101},
1461 {0x00000095, 0x00000fff},
1462 {0x00000096, 0x00116fff},
1463 {0x00000097, 0x60010000},
1464 {0x00000098, 0x10010000},
1465 {0x00000099, 0x00006000},
1466 {0x0000009a, 0x00001000},
1467 {0x0000009f, 0x00a17730}
1470 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1471 {0x0000006f, 0x03044000},
1472 {0x00000070, 0x0480c018},
1473 {0x00000071, 0x00000040},
1474 {0x00000072, 0x01000000},
1475 {0x00000074, 0x000000ff},
1476 {0x00000075, 0x00143400},
1477 {0x00000076, 0x08ec0800},
1478 {0x00000077, 0x040000cc},
1479 {0x00000079, 0x00000000},
1480 {0x0000007a, 0x21000409},
1481 {0x0000007c, 0x00000000},
1482 {0x0000007d, 0xe8000000},
1483 {0x0000007e, 0x044408a8},
1484 {0x0000007f, 0x00000003},
1485 {0x00000080, 0x00000000},
1486 {0x00000081, 0x01000000},
1487 {0x00000082, 0x02000000},
1488 {0x00000083, 0x00000000},
1489 {0x00000084, 0xe3f3e4f4},
1490 {0x00000085, 0x00052024},
1491 {0x00000087, 0x00000000},
1492 {0x00000088, 0x66036603},
1493 {0x00000089, 0x01000000},
1494 {0x0000008b, 0x1c0a0000},
1495 {0x0000008c, 0xff010000},
1496 {0x0000008e, 0xffffefff},
1497 {0x0000008f, 0xfff3efff},
1498 {0x00000090, 0xfff3efbf},
1499 {0x00000094, 0x00101101},
1500 {0x00000095, 0x00000fff},
1501 {0x00000096, 0x00116fff},
1502 {0x00000097, 0x60010000},
1503 {0x00000098, 0x10010000},
1504 {0x00000099, 0x00006000},
1505 {0x0000009a, 0x00001000},
1506 {0x0000009f, 0x00a07730}
1510 int si_mc_load_microcode(struct radeon_device *rdev)
1512 const __be32 *fw_data = NULL;
1513 const __le32 *new_fw_data = NULL;
1514 u32 running, blackout = 0;
1515 u32 *io_mc_regs = NULL;
1516 const __le32 *new_io_mc_regs = NULL;
1517 int i, regs_size, ucode_size;
1523 const struct mc_firmware_header_v1_0 *hdr =
1524 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1526 radeon_ucode_print_mc_hdr(&hdr->header);
1527 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1528 new_io_mc_regs = (const __le32 *)
1529 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1530 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1531 new_fw_data = (const __le32 *)
1532 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1534 ucode_size = rdev->mc_fw->size / 4;
1536 switch (rdev->family) {
1538 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1539 regs_size = TAHITI_IO_MC_REGS_SIZE;
1542 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1543 regs_size = TAHITI_IO_MC_REGS_SIZE;
1547 io_mc_regs = (u32 *)&verde_io_mc_regs;
1548 regs_size = TAHITI_IO_MC_REGS_SIZE;
1551 io_mc_regs = (u32 *)&oland_io_mc_regs;
1552 regs_size = TAHITI_IO_MC_REGS_SIZE;
1555 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1556 regs_size = TAHITI_IO_MC_REGS_SIZE;
1559 fw_data = (const __be32 *)rdev->mc_fw->data;
1562 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1566 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1567 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1570 /* reset the engine and set to writable */
1571 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1572 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1574 /* load mc io regs */
1575 for (i = 0; i < regs_size; i++) {
1577 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1578 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1580 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1581 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1584 /* load the MC ucode */
1585 for (i = 0; i < ucode_size; i++) {
1587 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1589 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1592 /* put the engine back into the active state */
1593 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1594 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1595 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1597 /* wait for training to complete */
1598 for (i = 0; i < rdev->usec_timeout; i++) {
1599 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1603 for (i = 0; i < rdev->usec_timeout; i++) {
1604 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1610 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1616 static int si_init_microcode(struct radeon_device *rdev)
1618 const char *chip_name;
1619 const char *new_chip_name;
1620 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1621 size_t smc_req_size, mc2_req_size;
1628 switch (rdev->family) {
1630 chip_name = "TAHITI";
1631 new_chip_name = "tahiti";
1632 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1633 me_req_size = SI_PM4_UCODE_SIZE * 4;
1634 ce_req_size = SI_CE_UCODE_SIZE * 4;
1635 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1636 mc_req_size = SI_MC_UCODE_SIZE * 4;
1637 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1638 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1641 chip_name = "PITCAIRN";
1642 new_chip_name = "pitcairn";
1643 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1644 me_req_size = SI_PM4_UCODE_SIZE * 4;
1645 ce_req_size = SI_CE_UCODE_SIZE * 4;
1646 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1647 mc_req_size = SI_MC_UCODE_SIZE * 4;
1648 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1649 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1652 chip_name = "VERDE";
1653 new_chip_name = "verde";
1654 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1655 me_req_size = SI_PM4_UCODE_SIZE * 4;
1656 ce_req_size = SI_CE_UCODE_SIZE * 4;
1657 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1658 mc_req_size = SI_MC_UCODE_SIZE * 4;
1659 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1660 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1663 chip_name = "OLAND";
1664 new_chip_name = "oland";
1665 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1666 me_req_size = SI_PM4_UCODE_SIZE * 4;
1667 ce_req_size = SI_CE_UCODE_SIZE * 4;
1668 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1669 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1670 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1673 chip_name = "HAINAN";
1674 new_chip_name = "hainan";
1675 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1676 me_req_size = SI_PM4_UCODE_SIZE * 4;
1677 ce_req_size = SI_CE_UCODE_SIZE * 4;
1678 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1679 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1680 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1685 DRM_INFO("Loading %s Microcode\n", new_chip_name);
1687 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1688 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1690 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1691 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1694 if (rdev->pfp_fw->size != pfp_req_size) {
1696 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1697 rdev->pfp_fw->size, fw_name);
1702 err = radeon_ucode_validate(rdev->pfp_fw);
1705 "si_cp: validation failed for firmware \"%s\"\n",
1713 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
1714 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1716 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1717 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1720 if (rdev->me_fw->size != me_req_size) {
1722 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1723 rdev->me_fw->size, fw_name);
1727 err = radeon_ucode_validate(rdev->me_fw);
1730 "si_cp: validation failed for firmware \"%s\"\n",
1738 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
1739 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1741 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1742 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1745 if (rdev->ce_fw->size != ce_req_size) {
1747 "si_cp: Bogus length %zu in firmware \"%s\"\n",
1748 rdev->ce_fw->size, fw_name);
1752 err = radeon_ucode_validate(rdev->ce_fw);
1755 "si_cp: validation failed for firmware \"%s\"\n",
1763 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
1764 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1766 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1767 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1770 if (rdev->rlc_fw->size != rlc_req_size) {
1772 "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1773 rdev->rlc_fw->size, fw_name);
1777 err = radeon_ucode_validate(rdev->rlc_fw);
1780 "si_cp: validation failed for firmware \"%s\"\n",
1788 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
1789 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1791 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1792 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1794 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1795 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1799 if ((rdev->mc_fw->size != mc_req_size) &&
1800 (rdev->mc_fw->size != mc2_req_size)) {
1802 "si_mc: Bogus length %zu in firmware \"%s\"\n",
1803 rdev->mc_fw->size, fw_name);
1806 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1808 err = radeon_ucode_validate(rdev->mc_fw);
1811 "si_cp: validation failed for firmware \"%s\"\n",
1819 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
1820 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1822 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1823 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1826 "smc: error loading firmware \"%s\"\n",
1828 release_firmware(rdev->smc_fw);
1829 rdev->smc_fw = NULL;
1831 } else if (rdev->smc_fw->size != smc_req_size) {
1833 "si_smc: Bogus length %zu in firmware \"%s\"\n",
1834 rdev->smc_fw->size, fw_name);
1838 err = radeon_ucode_validate(rdev->smc_fw);
1841 "si_cp: validation failed for firmware \"%s\"\n",
1850 rdev->new_fw = false;
1851 } else if (new_fw < 6) {
1852 printk(KERN_ERR "si_fw: mixing new and old firmware!\n");
1855 rdev->new_fw = true;
1861 "si_cp: Failed to load firmware \"%s\"\n",
1863 release_firmware(rdev->pfp_fw);
1864 rdev->pfp_fw = NULL;
1865 release_firmware(rdev->me_fw);
1867 release_firmware(rdev->ce_fw);
1869 release_firmware(rdev->rlc_fw);
1870 rdev->rlc_fw = NULL;
1871 release_firmware(rdev->mc_fw);
1873 release_firmware(rdev->smc_fw);
1874 rdev->smc_fw = NULL;
1879 /* watermark setup */
1880 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1881 struct radeon_crtc *radeon_crtc,
1882 struct drm_display_mode *mode,
1883 struct drm_display_mode *other_mode)
1885 u32 tmp, buffer_alloc, i;
1886 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1889 * There are 3 line buffers, each one shared by 2 display controllers.
1890 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1891 * the display controllers. The paritioning is done via one of four
1892 * preset allocations specified in bits 21:20:
1894 * 2 - whole lb, other crtc must be disabled
1896 /* this can get tricky if we have two large displays on a paired group
1897 * of crtcs. Ideally for multiple large displays we'd assign them to
1898 * non-linked crtcs for maximum line buffer allocation.
1900 if (radeon_crtc->base.enabled && mode) {
1905 tmp = 2; /* whole */
1913 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1914 DC_LB_MEMORY_CONFIG(tmp));
1916 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1917 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1918 for (i = 0; i < rdev->usec_timeout; i++) {
1919 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1920 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1925 if (radeon_crtc->base.enabled && mode) {
1935 /* controller not enabled, so no lb used */
1939 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1941 u32 tmp = RREG32(MC_SHARED_CHMAP);
1943 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1966 struct dce6_wm_params {
1967 u32 dram_channels; /* number of dram channels */
1968 u32 yclk; /* bandwidth per dram data pin in kHz */
1969 u32 sclk; /* engine clock in kHz */
1970 u32 disp_clk; /* display clock in kHz */
1971 u32 src_width; /* viewport width */
1972 u32 active_time; /* active display time in ns */
1973 u32 blank_time; /* blank time in ns */
1974 bool interlaced; /* mode is interlaced */
1975 fixed20_12 vsc; /* vertical scale ratio */
1976 u32 num_heads; /* number of active crtcs */
1977 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1978 u32 lb_size; /* line buffer allocated to pipe */
1979 u32 vtaps; /* vertical scaler taps */
1982 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1984 /* Calculate raw DRAM Bandwidth */
1985 fixed20_12 dram_efficiency; /* 0.7 */
1986 fixed20_12 yclk, dram_channels, bandwidth;
1989 a.full = dfixed_const(1000);
1990 yclk.full = dfixed_const(wm->yclk);
1991 yclk.full = dfixed_div(yclk, a);
1992 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1993 a.full = dfixed_const(10);
1994 dram_efficiency.full = dfixed_const(7);
1995 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1996 bandwidth.full = dfixed_mul(dram_channels, yclk);
1997 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1999 return dfixed_trunc(bandwidth);
2002 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2004 /* Calculate DRAM Bandwidth and the part allocated to display. */
2005 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2006 fixed20_12 yclk, dram_channels, bandwidth;
2009 a.full = dfixed_const(1000);
2010 yclk.full = dfixed_const(wm->yclk);
2011 yclk.full = dfixed_div(yclk, a);
2012 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2013 a.full = dfixed_const(10);
2014 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2015 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2016 bandwidth.full = dfixed_mul(dram_channels, yclk);
2017 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2019 return dfixed_trunc(bandwidth);
2022 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2024 /* Calculate the display Data return Bandwidth */
2025 fixed20_12 return_efficiency; /* 0.8 */
2026 fixed20_12 sclk, bandwidth;
2029 a.full = dfixed_const(1000);
2030 sclk.full = dfixed_const(wm->sclk);
2031 sclk.full = dfixed_div(sclk, a);
2032 a.full = dfixed_const(10);
2033 return_efficiency.full = dfixed_const(8);
2034 return_efficiency.full = dfixed_div(return_efficiency, a);
2035 a.full = dfixed_const(32);
2036 bandwidth.full = dfixed_mul(a, sclk);
2037 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2039 return dfixed_trunc(bandwidth);
2042 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2047 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2049 /* Calculate the DMIF Request Bandwidth */
2050 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2051 fixed20_12 disp_clk, sclk, bandwidth;
2052 fixed20_12 a, b1, b2;
2055 a.full = dfixed_const(1000);
2056 disp_clk.full = dfixed_const(wm->disp_clk);
2057 disp_clk.full = dfixed_div(disp_clk, a);
2058 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2059 b1.full = dfixed_mul(a, disp_clk);
2061 a.full = dfixed_const(1000);
2062 sclk.full = dfixed_const(wm->sclk);
2063 sclk.full = dfixed_div(sclk, a);
2064 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2065 b2.full = dfixed_mul(a, sclk);
2067 a.full = dfixed_const(10);
2068 disp_clk_request_efficiency.full = dfixed_const(8);
2069 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2071 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2073 a.full = dfixed_const(min_bandwidth);
2074 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2076 return dfixed_trunc(bandwidth);
2079 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2081 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2082 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2083 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2084 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2086 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2089 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2091 /* Calculate the display mode Average Bandwidth
2092 * DisplayMode should contain the source and destination dimensions,
2096 fixed20_12 line_time;
2097 fixed20_12 src_width;
2098 fixed20_12 bandwidth;
2101 a.full = dfixed_const(1000);
2102 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2103 line_time.full = dfixed_div(line_time, a);
2104 bpp.full = dfixed_const(wm->bytes_per_pixel);
2105 src_width.full = dfixed_const(wm->src_width);
2106 bandwidth.full = dfixed_mul(src_width, bpp);
2107 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2108 bandwidth.full = dfixed_div(bandwidth, line_time);
2110 return dfixed_trunc(bandwidth);
2113 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2115 /* First calcualte the latency in ns */
2116 u32 mc_latency = 2000; /* 2000 ns. */
2117 u32 available_bandwidth = dce6_available_bandwidth(wm);
2118 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2119 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2120 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2121 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2122 (wm->num_heads * cursor_line_pair_return_time);
2123 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2124 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2125 u32 tmp, dmif_size = 12288;
2128 if (wm->num_heads == 0)
2131 a.full = dfixed_const(2);
2132 b.full = dfixed_const(1);
2133 if ((wm->vsc.full > a.full) ||
2134 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2136 ((wm->vsc.full >= a.full) && wm->interlaced))
2137 max_src_lines_per_dst_line = 4;
2139 max_src_lines_per_dst_line = 2;
2141 a.full = dfixed_const(available_bandwidth);
2142 b.full = dfixed_const(wm->num_heads);
2143 a.full = dfixed_div(a, b);
2145 b.full = dfixed_const(mc_latency + 512);
2146 c.full = dfixed_const(wm->disp_clk);
2147 b.full = dfixed_div(b, c);
2149 c.full = dfixed_const(dmif_size);
2150 b.full = dfixed_div(c, b);
2152 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
2154 b.full = dfixed_const(1000);
2155 c.full = dfixed_const(wm->disp_clk);
2156 b.full = dfixed_div(c, b);
2157 c.full = dfixed_const(wm->bytes_per_pixel);
2158 b.full = dfixed_mul(b, c);
2160 lb_fill_bw = min(tmp, dfixed_trunc(b));
2162 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2163 b.full = dfixed_const(1000);
2164 c.full = dfixed_const(lb_fill_bw);
2165 b.full = dfixed_div(c, b);
2166 a.full = dfixed_div(a, b);
2167 line_fill_time = dfixed_trunc(a);
2169 if (line_fill_time < wm->active_time)
2172 return latency + (line_fill_time - wm->active_time);
2176 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2178 if (dce6_average_bandwidth(wm) <=
2179 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2185 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2187 if (dce6_average_bandwidth(wm) <=
2188 (dce6_available_bandwidth(wm) / wm->num_heads))
2194 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2196 u32 lb_partitions = wm->lb_size / wm->src_width;
2197 u32 line_time = wm->active_time + wm->blank_time;
2198 u32 latency_tolerant_lines;
2202 a.full = dfixed_const(1);
2203 if (wm->vsc.full > a.full)
2204 latency_tolerant_lines = 1;
2206 if (lb_partitions <= (wm->vtaps + 1))
2207 latency_tolerant_lines = 1;
2209 latency_tolerant_lines = 2;
2212 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2214 if (dce6_latency_watermark(wm) <= latency_hiding)
2220 static void dce6_program_watermarks(struct radeon_device *rdev,
2221 struct radeon_crtc *radeon_crtc,
2222 u32 lb_size, u32 num_heads)
2224 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2225 struct dce6_wm_params wm_low, wm_high;
2229 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2230 u32 priority_a_mark = 0, priority_b_mark = 0;
2231 u32 priority_a_cnt = PRIORITY_OFF;
2232 u32 priority_b_cnt = PRIORITY_OFF;
2233 u32 tmp, arb_control3;
2236 if (radeon_crtc->base.enabled && num_heads && mode) {
2237 pixel_period = 1000000 / (u32)mode->clock;
2238 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2242 if (rdev->family == CHIP_ARUBA)
2243 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2245 dram_channels = si_get_number_of_dram_channels(rdev);
2247 /* watermark for high clocks */
2248 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2250 radeon_dpm_get_mclk(rdev, false) * 10;
2252 radeon_dpm_get_sclk(rdev, false) * 10;
2254 wm_high.yclk = rdev->pm.current_mclk * 10;
2255 wm_high.sclk = rdev->pm.current_sclk * 10;
2258 wm_high.disp_clk = mode->clock;
2259 wm_high.src_width = mode->crtc_hdisplay;
2260 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2261 wm_high.blank_time = line_time - wm_high.active_time;
2262 wm_high.interlaced = false;
2263 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2264 wm_high.interlaced = true;
2265 wm_high.vsc = radeon_crtc->vsc;
2267 if (radeon_crtc->rmx_type != RMX_OFF)
2269 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2270 wm_high.lb_size = lb_size;
2271 wm_high.dram_channels = dram_channels;
2272 wm_high.num_heads = num_heads;
2274 /* watermark for low clocks */
2275 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2277 radeon_dpm_get_mclk(rdev, true) * 10;
2279 radeon_dpm_get_sclk(rdev, true) * 10;
2281 wm_low.yclk = rdev->pm.current_mclk * 10;
2282 wm_low.sclk = rdev->pm.current_sclk * 10;
2285 wm_low.disp_clk = mode->clock;
2286 wm_low.src_width = mode->crtc_hdisplay;
2287 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2288 wm_low.blank_time = line_time - wm_low.active_time;
2289 wm_low.interlaced = false;
2290 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2291 wm_low.interlaced = true;
2292 wm_low.vsc = radeon_crtc->vsc;
2294 if (radeon_crtc->rmx_type != RMX_OFF)
2296 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2297 wm_low.lb_size = lb_size;
2298 wm_low.dram_channels = dram_channels;
2299 wm_low.num_heads = num_heads;
2301 /* set for high clocks */
2302 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2303 /* set for low clocks */
2304 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2306 /* possibly force display priority to high */
2307 /* should really do this at mode validation time... */
2308 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2309 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2310 !dce6_check_latency_hiding(&wm_high) ||
2311 (rdev->disp_priority == 2)) {
2312 DRM_DEBUG_KMS("force priority to high\n");
2313 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2314 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2316 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2317 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2318 !dce6_check_latency_hiding(&wm_low) ||
2319 (rdev->disp_priority == 2)) {
2320 DRM_DEBUG_KMS("force priority to high\n");
2321 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2322 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2325 a.full = dfixed_const(1000);
2326 b.full = dfixed_const(mode->clock);
2327 b.full = dfixed_div(b, a);
2328 c.full = dfixed_const(latency_watermark_a);
2329 c.full = dfixed_mul(c, b);
2330 c.full = dfixed_mul(c, radeon_crtc->hsc);
2331 c.full = dfixed_div(c, a);
2332 a.full = dfixed_const(16);
2333 c.full = dfixed_div(c, a);
2334 priority_a_mark = dfixed_trunc(c);
2335 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2337 a.full = dfixed_const(1000);
2338 b.full = dfixed_const(mode->clock);
2339 b.full = dfixed_div(b, a);
2340 c.full = dfixed_const(latency_watermark_b);
2341 c.full = dfixed_mul(c, b);
2342 c.full = dfixed_mul(c, radeon_crtc->hsc);
2343 c.full = dfixed_div(c, a);
2344 a.full = dfixed_const(16);
2345 c.full = dfixed_div(c, a);
2346 priority_b_mark = dfixed_trunc(c);
2347 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2351 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2353 tmp &= ~LATENCY_WATERMARK_MASK(3);
2354 tmp |= LATENCY_WATERMARK_MASK(1);
2355 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2356 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2357 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2358 LATENCY_HIGH_WATERMARK(line_time)));
2360 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2361 tmp &= ~LATENCY_WATERMARK_MASK(3);
2362 tmp |= LATENCY_WATERMARK_MASK(2);
2363 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2364 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2365 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2366 LATENCY_HIGH_WATERMARK(line_time)));
2367 /* restore original selection */
2368 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2370 /* write the priority marks */
2371 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2372 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2374 /* save values for DPM */
2375 radeon_crtc->line_time = line_time;
2376 radeon_crtc->wm_high = latency_watermark_a;
2377 radeon_crtc->wm_low = latency_watermark_b;
2380 void dce6_bandwidth_update(struct radeon_device *rdev)
2382 struct drm_display_mode *mode0 = NULL;
2383 struct drm_display_mode *mode1 = NULL;
2384 u32 num_heads = 0, lb_size;
2387 radeon_update_display_priority(rdev);
2389 for (i = 0; i < rdev->num_crtc; i++) {
2390 if (rdev->mode_info.crtcs[i]->base.enabled)
2393 for (i = 0; i < rdev->num_crtc; i += 2) {
2394 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2395 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2396 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2397 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2398 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2399 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2406 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2408 const u32 num_tile_mode_states = 32;
2409 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2411 switch (rdev->config.si.mem_row_size_in_kb) {
2413 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2417 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2420 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2424 if ((rdev->family == CHIP_TAHITI) ||
2425 (rdev->family == CHIP_PITCAIRN)) {
2426 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2427 switch (reg_offset) {
2428 case 0: /* non-AA compressed depth or any compressed stencil */
2429 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2430 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2431 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2432 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2433 NUM_BANKS(ADDR_SURF_16_BANK) |
2434 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2435 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2436 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2438 case 1: /* 2xAA/4xAA compressed depth only */
2439 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2440 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2441 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2442 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2443 NUM_BANKS(ADDR_SURF_16_BANK) |
2444 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2445 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2446 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2448 case 2: /* 8xAA compressed depth only */
2449 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2450 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2451 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2452 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2453 NUM_BANKS(ADDR_SURF_16_BANK) |
2454 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2455 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2456 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2458 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2459 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2460 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2461 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2462 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2463 NUM_BANKS(ADDR_SURF_16_BANK) |
2464 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2465 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2466 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2468 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2469 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2470 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2471 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2472 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2473 NUM_BANKS(ADDR_SURF_16_BANK) |
2474 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2475 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2476 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2478 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2479 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2480 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2481 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2482 TILE_SPLIT(split_equal_to_row_size) |
2483 NUM_BANKS(ADDR_SURF_16_BANK) |
2484 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2486 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2488 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2489 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2490 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2491 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2492 TILE_SPLIT(split_equal_to_row_size) |
2493 NUM_BANKS(ADDR_SURF_16_BANK) |
2494 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2495 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2496 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2498 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2499 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2500 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2501 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2502 TILE_SPLIT(split_equal_to_row_size) |
2503 NUM_BANKS(ADDR_SURF_16_BANK) |
2504 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2505 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2506 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2508 case 8: /* 1D and 1D Array Surfaces */
2509 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2510 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2511 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2512 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2513 NUM_BANKS(ADDR_SURF_16_BANK) |
2514 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2515 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2516 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2518 case 9: /* Displayable maps. */
2519 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2520 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2521 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2522 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2523 NUM_BANKS(ADDR_SURF_16_BANK) |
2524 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2525 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2526 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2528 case 10: /* Display 8bpp. */
2529 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2530 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2531 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2532 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2533 NUM_BANKS(ADDR_SURF_16_BANK) |
2534 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2535 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2536 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2538 case 11: /* Display 16bpp. */
2539 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2540 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2542 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2543 NUM_BANKS(ADDR_SURF_16_BANK) |
2544 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2545 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2546 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2548 case 12: /* Display 32bpp. */
2549 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2550 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2551 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2552 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2553 NUM_BANKS(ADDR_SURF_16_BANK) |
2554 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2555 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2556 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2558 case 13: /* Thin. */
2559 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2560 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2561 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2562 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2563 NUM_BANKS(ADDR_SURF_16_BANK) |
2564 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2565 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2566 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2568 case 14: /* Thin 8 bpp. */
2569 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2570 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2571 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2572 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2573 NUM_BANKS(ADDR_SURF_16_BANK) |
2574 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2575 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2576 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2578 case 15: /* Thin 16 bpp. */
2579 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2580 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2581 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2582 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2583 NUM_BANKS(ADDR_SURF_16_BANK) |
2584 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2585 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2586 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2588 case 16: /* Thin 32 bpp. */
2589 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2590 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2591 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2592 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2593 NUM_BANKS(ADDR_SURF_16_BANK) |
2594 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2595 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2596 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2598 case 17: /* Thin 64 bpp. */
2599 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2600 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2601 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2602 TILE_SPLIT(split_equal_to_row_size) |
2603 NUM_BANKS(ADDR_SURF_16_BANK) |
2604 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2605 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2606 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2608 case 21: /* 8 bpp PRT. */
2609 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2610 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2611 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2612 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2613 NUM_BANKS(ADDR_SURF_16_BANK) |
2614 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2615 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2616 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2618 case 22: /* 16 bpp PRT */
2619 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2620 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2621 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2622 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2623 NUM_BANKS(ADDR_SURF_16_BANK) |
2624 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2625 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2626 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2628 case 23: /* 32 bpp PRT */
2629 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2630 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2631 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2632 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2633 NUM_BANKS(ADDR_SURF_16_BANK) |
2634 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2635 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2636 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2638 case 24: /* 64 bpp PRT */
2639 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2640 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2641 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2642 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2643 NUM_BANKS(ADDR_SURF_16_BANK) |
2644 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2645 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2646 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2648 case 25: /* 128 bpp PRT */
2649 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2650 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2651 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2652 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2653 NUM_BANKS(ADDR_SURF_8_BANK) |
2654 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2656 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2662 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2663 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2665 } else if ((rdev->family == CHIP_VERDE) ||
2666 (rdev->family == CHIP_OLAND) ||
2667 (rdev->family == CHIP_HAINAN)) {
2668 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2669 switch (reg_offset) {
2670 case 0: /* non-AA compressed depth or any compressed stencil */
2671 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2672 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2673 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2674 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2675 NUM_BANKS(ADDR_SURF_16_BANK) |
2676 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2677 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2678 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2680 case 1: /* 2xAA/4xAA compressed depth only */
2681 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2682 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2683 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2684 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2685 NUM_BANKS(ADDR_SURF_16_BANK) |
2686 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2687 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2688 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2690 case 2: /* 8xAA compressed depth only */
2691 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2692 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2693 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2694 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2695 NUM_BANKS(ADDR_SURF_16_BANK) |
2696 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2700 case 3: /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2701 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2702 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2703 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2704 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2705 NUM_BANKS(ADDR_SURF_16_BANK) |
2706 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2707 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2708 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2710 case 4: /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2711 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2712 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2713 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2714 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2715 NUM_BANKS(ADDR_SURF_16_BANK) |
2716 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2717 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2718 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2720 case 5: /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2721 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2722 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2723 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2724 TILE_SPLIT(split_equal_to_row_size) |
2725 NUM_BANKS(ADDR_SURF_16_BANK) |
2726 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2730 case 6: /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2731 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2732 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2733 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2734 TILE_SPLIT(split_equal_to_row_size) |
2735 NUM_BANKS(ADDR_SURF_16_BANK) |
2736 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2737 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2738 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2740 case 7: /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2741 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2742 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2743 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2744 TILE_SPLIT(split_equal_to_row_size) |
2745 NUM_BANKS(ADDR_SURF_16_BANK) |
2746 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2747 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2748 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2750 case 8: /* 1D and 1D Array Surfaces */
2751 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2752 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2753 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2754 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2755 NUM_BANKS(ADDR_SURF_16_BANK) |
2756 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2757 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2758 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2760 case 9: /* Displayable maps. */
2761 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2762 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2763 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2764 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2765 NUM_BANKS(ADDR_SURF_16_BANK) |
2766 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2767 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2768 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2770 case 10: /* Display 8bpp. */
2771 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2772 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2773 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2774 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2775 NUM_BANKS(ADDR_SURF_16_BANK) |
2776 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2777 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2778 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2780 case 11: /* Display 16bpp. */
2781 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2782 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2783 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2784 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2785 NUM_BANKS(ADDR_SURF_16_BANK) |
2786 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2787 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2788 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2790 case 12: /* Display 32bpp. */
2791 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2792 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2793 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2794 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2795 NUM_BANKS(ADDR_SURF_16_BANK) |
2796 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2797 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2798 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2800 case 13: /* Thin. */
2801 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2802 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2803 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2804 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2805 NUM_BANKS(ADDR_SURF_16_BANK) |
2806 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2807 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2808 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2810 case 14: /* Thin 8 bpp. */
2811 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2812 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2813 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2815 NUM_BANKS(ADDR_SURF_16_BANK) |
2816 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2817 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2818 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2820 case 15: /* Thin 16 bpp. */
2821 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2822 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2823 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2824 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2825 NUM_BANKS(ADDR_SURF_16_BANK) |
2826 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2827 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2828 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2830 case 16: /* Thin 32 bpp. */
2831 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2832 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2833 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2834 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2835 NUM_BANKS(ADDR_SURF_16_BANK) |
2836 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2837 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2838 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2840 case 17: /* Thin 64 bpp. */
2841 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2842 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2843 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2844 TILE_SPLIT(split_equal_to_row_size) |
2845 NUM_BANKS(ADDR_SURF_16_BANK) |
2846 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2847 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2848 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2850 case 21: /* 8 bpp PRT. */
2851 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2852 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2853 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2854 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2855 NUM_BANKS(ADDR_SURF_16_BANK) |
2856 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2857 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2858 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2860 case 22: /* 16 bpp PRT */
2861 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2862 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2863 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2864 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2865 NUM_BANKS(ADDR_SURF_16_BANK) |
2866 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2867 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2868 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2870 case 23: /* 32 bpp PRT */
2871 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2872 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2873 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2874 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2875 NUM_BANKS(ADDR_SURF_16_BANK) |
2876 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2877 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2878 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2880 case 24: /* 64 bpp PRT */
2881 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2882 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2883 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2884 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2885 NUM_BANKS(ADDR_SURF_16_BANK) |
2886 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2887 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2888 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2890 case 25: /* 128 bpp PRT */
2891 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2894 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2895 NUM_BANKS(ADDR_SURF_8_BANK) |
2896 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2897 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2898 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2904 rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2905 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2908 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2911 static void si_select_se_sh(struct radeon_device *rdev,
2912 u32 se_num, u32 sh_num)
2914 u32 data = INSTANCE_BROADCAST_WRITES;
2916 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2917 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2918 else if (se_num == 0xffffffff)
2919 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2920 else if (sh_num == 0xffffffff)
2921 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2923 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2924 WREG32(GRBM_GFX_INDEX, data);
2927 static u32 si_create_bitmask(u32 bit_width)
2931 for (i = 0; i < bit_width; i++) {
2938 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2942 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2944 data &= INACTIVE_CUS_MASK;
2947 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2949 data >>= INACTIVE_CUS_SHIFT;
2951 mask = si_create_bitmask(cu_per_sh);
2953 return ~data & mask;
2956 static void si_setup_spi(struct radeon_device *rdev,
2957 u32 se_num, u32 sh_per_se,
2961 u32 data, mask, active_cu;
2963 for (i = 0; i < se_num; i++) {
2964 for (j = 0; j < sh_per_se; j++) {
2965 si_select_se_sh(rdev, i, j);
2966 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2967 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2970 for (k = 0; k < 16; k++) {
2972 if (active_cu & mask) {
2974 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2980 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2983 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2984 u32 max_rb_num_per_se,
2989 data = RREG32(CC_RB_BACKEND_DISABLE);
2991 data &= BACKEND_DISABLE_MASK;
2994 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2996 data >>= BACKEND_DISABLE_SHIFT;
2998 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
3003 static void si_setup_rb(struct radeon_device *rdev,
3004 u32 se_num, u32 sh_per_se,
3005 u32 max_rb_num_per_se)
3009 u32 disabled_rbs = 0;
3010 u32 enabled_rbs = 0;
3012 for (i = 0; i < se_num; i++) {
3013 for (j = 0; j < sh_per_se; j++) {
3014 si_select_se_sh(rdev, i, j);
3015 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3016 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3019 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3022 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3023 if (!(disabled_rbs & mask))
3024 enabled_rbs |= mask;
3028 rdev->config.si.backend_enable_mask = enabled_rbs;
3030 for (i = 0; i < se_num; i++) {
3031 si_select_se_sh(rdev, i, 0xffffffff);
3033 for (j = 0; j < sh_per_se; j++) {
3034 switch (enabled_rbs & 3) {
3036 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3039 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3043 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3048 WREG32(PA_SC_RASTER_CONFIG, data);
3050 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3053 static void si_gpu_init(struct radeon_device *rdev)
3055 u32 gb_addr_config = 0;
3056 u32 mc_shared_chmap, mc_arb_ramcfg;
3058 u32 hdp_host_path_cntl;
3062 switch (rdev->family) {
3064 rdev->config.si.max_shader_engines = 2;
3065 rdev->config.si.max_tile_pipes = 12;
3066 rdev->config.si.max_cu_per_sh = 8;
3067 rdev->config.si.max_sh_per_se = 2;
3068 rdev->config.si.max_backends_per_se = 4;
3069 rdev->config.si.max_texture_channel_caches = 12;
3070 rdev->config.si.max_gprs = 256;
3071 rdev->config.si.max_gs_threads = 32;
3072 rdev->config.si.max_hw_contexts = 8;
3074 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3075 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3076 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3077 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3078 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3081 rdev->config.si.max_shader_engines = 2;
3082 rdev->config.si.max_tile_pipes = 8;
3083 rdev->config.si.max_cu_per_sh = 5;
3084 rdev->config.si.max_sh_per_se = 2;
3085 rdev->config.si.max_backends_per_se = 4;
3086 rdev->config.si.max_texture_channel_caches = 8;
3087 rdev->config.si.max_gprs = 256;
3088 rdev->config.si.max_gs_threads = 32;
3089 rdev->config.si.max_hw_contexts = 8;
3091 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3092 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3093 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3094 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3095 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3099 rdev->config.si.max_shader_engines = 1;
3100 rdev->config.si.max_tile_pipes = 4;
3101 rdev->config.si.max_cu_per_sh = 5;
3102 rdev->config.si.max_sh_per_se = 2;
3103 rdev->config.si.max_backends_per_se = 4;
3104 rdev->config.si.max_texture_channel_caches = 4;
3105 rdev->config.si.max_gprs = 256;
3106 rdev->config.si.max_gs_threads = 32;
3107 rdev->config.si.max_hw_contexts = 8;
3109 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3110 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3111 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3112 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3113 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3116 rdev->config.si.max_shader_engines = 1;
3117 rdev->config.si.max_tile_pipes = 4;
3118 rdev->config.si.max_cu_per_sh = 6;
3119 rdev->config.si.max_sh_per_se = 1;
3120 rdev->config.si.max_backends_per_se = 2;
3121 rdev->config.si.max_texture_channel_caches = 4;
3122 rdev->config.si.max_gprs = 256;
3123 rdev->config.si.max_gs_threads = 16;
3124 rdev->config.si.max_hw_contexts = 8;
3126 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3127 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3128 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3129 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3130 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3133 rdev->config.si.max_shader_engines = 1;
3134 rdev->config.si.max_tile_pipes = 4;
3135 rdev->config.si.max_cu_per_sh = 5;
3136 rdev->config.si.max_sh_per_se = 1;
3137 rdev->config.si.max_backends_per_se = 1;
3138 rdev->config.si.max_texture_channel_caches = 2;
3139 rdev->config.si.max_gprs = 256;
3140 rdev->config.si.max_gs_threads = 16;
3141 rdev->config.si.max_hw_contexts = 8;
3143 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3144 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3145 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3146 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3147 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3151 /* Initialize HDP */
3152 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3153 WREG32((0x2c14 + j), 0x00000000);
3154 WREG32((0x2c18 + j), 0x00000000);
3155 WREG32((0x2c1c + j), 0x00000000);
3156 WREG32((0x2c20 + j), 0x00000000);
3157 WREG32((0x2c24 + j), 0x00000000);
3160 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3162 evergreen_fix_pci_max_read_req_size(rdev);
3164 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3166 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3167 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3169 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3170 rdev->config.si.mem_max_burst_length_bytes = 256;
3171 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3172 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3173 if (rdev->config.si.mem_row_size_in_kb > 4)
3174 rdev->config.si.mem_row_size_in_kb = 4;
3175 /* XXX use MC settings? */
3176 rdev->config.si.shader_engine_tile_size = 32;
3177 rdev->config.si.num_gpus = 1;
3178 rdev->config.si.multi_gpu_tile_size = 64;
3180 /* fix up row size */
3181 gb_addr_config &= ~ROW_SIZE_MASK;
3182 switch (rdev->config.si.mem_row_size_in_kb) {
3185 gb_addr_config |= ROW_SIZE(0);
3188 gb_addr_config |= ROW_SIZE(1);
3191 gb_addr_config |= ROW_SIZE(2);
3195 /* setup tiling info dword. gb_addr_config is not adequate since it does
3196 * not have bank info, so create a custom tiling dword.
3197 * bits 3:0 num_pipes
3198 * bits 7:4 num_banks
3199 * bits 11:8 group_size
3200 * bits 15:12 row_size
3202 rdev->config.si.tile_config = 0;
3203 switch (rdev->config.si.num_tile_pipes) {
3205 rdev->config.si.tile_config |= (0 << 0);
3208 rdev->config.si.tile_config |= (1 << 0);
3211 rdev->config.si.tile_config |= (2 << 0);
3215 /* XXX what about 12? */
3216 rdev->config.si.tile_config |= (3 << 0);
3219 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3220 case 0: /* four banks */
3221 rdev->config.si.tile_config |= 0 << 4;
3223 case 1: /* eight banks */
3224 rdev->config.si.tile_config |= 1 << 4;
3226 case 2: /* sixteen banks */
3228 rdev->config.si.tile_config |= 2 << 4;
3231 rdev->config.si.tile_config |=
3232 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3233 rdev->config.si.tile_config |=
3234 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3236 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3237 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3238 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3239 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3240 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3241 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3242 if (rdev->has_uvd) {
3243 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3244 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3245 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3248 si_tiling_mode_table_init(rdev);
3250 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3251 rdev->config.si.max_sh_per_se,
3252 rdev->config.si.max_backends_per_se);
3254 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3255 rdev->config.si.max_sh_per_se,
3256 rdev->config.si.max_cu_per_sh);
3258 rdev->config.si.active_cus = 0;
3259 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3260 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3261 rdev->config.si.active_cus +=
3262 hweight32(si_get_cu_active_bitmap(rdev, i, j));
3266 /* set HW defaults for 3D engine */
3267 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3268 ROQ_IB2_START(0x2b)));
3269 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3271 sx_debug_1 = RREG32(SX_DEBUG_1);
3272 WREG32(SX_DEBUG_1, sx_debug_1);
3274 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3276 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3277 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3278 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3279 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3281 WREG32(VGT_NUM_INSTANCES, 1);
3283 WREG32(CP_PERFMON_CNTL, 0);
3285 WREG32(SQ_CONFIG, 0);
3287 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3288 FORCE_EOV_MAX_REZ_CNT(255)));
3290 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3291 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3293 WREG32(VGT_GS_VERTEX_REUSE, 16);
3294 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3296 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3297 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3298 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3299 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3300 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3301 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3302 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3303 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3305 tmp = RREG32(HDP_MISC_CNTL);
3306 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3307 WREG32(HDP_MISC_CNTL, tmp);
3309 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3310 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3312 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3318 * GPU scratch registers helpers function.
3320 static void si_scratch_init(struct radeon_device *rdev)
3324 rdev->scratch.num_reg = 7;
3325 rdev->scratch.reg_base = SCRATCH_REG0;
3326 for (i = 0; i < rdev->scratch.num_reg; i++) {
3327 rdev->scratch.free[i] = true;
3328 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3332 void si_fence_ring_emit(struct radeon_device *rdev,
3333 struct radeon_fence *fence)
3335 struct radeon_ring *ring = &rdev->ring[fence->ring];
3336 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3338 /* flush read cache over gart */
3339 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3340 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3341 radeon_ring_write(ring, 0);
3342 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3343 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3344 PACKET3_TC_ACTION_ENA |
3345 PACKET3_SH_KCACHE_ACTION_ENA |
3346 PACKET3_SH_ICACHE_ACTION_ENA);
3347 radeon_ring_write(ring, 0xFFFFFFFF);
3348 radeon_ring_write(ring, 0);
3349 radeon_ring_write(ring, 10); /* poll interval */
3350 /* EVENT_WRITE_EOP - flush caches, send int */
3351 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3352 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3353 radeon_ring_write(ring, lower_32_bits(addr));
3354 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3355 radeon_ring_write(ring, fence->seq);
3356 radeon_ring_write(ring, 0);
3362 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3364 struct radeon_ring *ring = &rdev->ring[ib->ring];
3367 if (ib->is_const_ib) {
3368 /* set switch buffer packet before const IB */
3369 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3370 radeon_ring_write(ring, 0);
3372 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3375 if (ring->rptr_save_reg) {
3376 next_rptr = ring->wptr + 3 + 4 + 8;
3377 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3378 radeon_ring_write(ring, ((ring->rptr_save_reg -
3379 PACKET3_SET_CONFIG_REG_START) >> 2));
3380 radeon_ring_write(ring, next_rptr);
3381 } else if (rdev->wb.enabled) {
3382 next_rptr = ring->wptr + 5 + 4 + 8;
3383 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3384 radeon_ring_write(ring, (1 << 8));
3385 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3386 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3387 radeon_ring_write(ring, next_rptr);
3390 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3393 radeon_ring_write(ring, header);
3394 radeon_ring_write(ring,
3398 (ib->gpu_addr & 0xFFFFFFFC));
3399 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3400 radeon_ring_write(ring, ib->length_dw |
3401 (ib->vm ? (ib->vm->id << 24) : 0));
3403 if (!ib->is_const_ib) {
3404 /* flush read cache over gart for this vmid */
3405 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3406 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3407 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
3408 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3409 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3410 PACKET3_TC_ACTION_ENA |
3411 PACKET3_SH_KCACHE_ACTION_ENA |
3412 PACKET3_SH_ICACHE_ACTION_ENA);
3413 radeon_ring_write(ring, 0xFFFFFFFF);
3414 radeon_ring_write(ring, 0);
3415 radeon_ring_write(ring, 10); /* poll interval */
3422 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3425 WREG32(CP_ME_CNTL, 0);
3427 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3428 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3429 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3430 WREG32(SCRATCH_UMSK, 0);
3431 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3432 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3433 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3438 static int si_cp_load_microcode(struct radeon_device *rdev)
3442 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3445 si_cp_enable(rdev, false);
3448 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3449 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3450 const struct gfx_firmware_header_v1_0 *ce_hdr =
3451 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3452 const struct gfx_firmware_header_v1_0 *me_hdr =
3453 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3454 const __le32 *fw_data;
3457 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3458 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3459 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3462 fw_data = (const __le32 *)
3463 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3464 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3465 WREG32(CP_PFP_UCODE_ADDR, 0);
3466 for (i = 0; i < fw_size; i++)
3467 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3468 WREG32(CP_PFP_UCODE_ADDR, 0);
3471 fw_data = (const __le32 *)
3472 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3473 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3474 WREG32(CP_CE_UCODE_ADDR, 0);
3475 for (i = 0; i < fw_size; i++)
3476 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3477 WREG32(CP_CE_UCODE_ADDR, 0);
3480 fw_data = (const __be32 *)
3481 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3482 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3483 WREG32(CP_ME_RAM_WADDR, 0);
3484 for (i = 0; i < fw_size; i++)
3485 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3486 WREG32(CP_ME_RAM_WADDR, 0);
3488 const __be32 *fw_data;
3491 fw_data = (const __be32 *)rdev->pfp_fw->data;
3492 WREG32(CP_PFP_UCODE_ADDR, 0);
3493 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3494 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3495 WREG32(CP_PFP_UCODE_ADDR, 0);
3498 fw_data = (const __be32 *)rdev->ce_fw->data;
3499 WREG32(CP_CE_UCODE_ADDR, 0);
3500 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3501 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3502 WREG32(CP_CE_UCODE_ADDR, 0);
3505 fw_data = (const __be32 *)rdev->me_fw->data;
3506 WREG32(CP_ME_RAM_WADDR, 0);
3507 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3508 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3509 WREG32(CP_ME_RAM_WADDR, 0);
3512 WREG32(CP_PFP_UCODE_ADDR, 0);
3513 WREG32(CP_CE_UCODE_ADDR, 0);
3514 WREG32(CP_ME_RAM_WADDR, 0);
3515 WREG32(CP_ME_RAM_RADDR, 0);
3519 static int si_cp_start(struct radeon_device *rdev)
3521 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3524 r = radeon_ring_lock(rdev, ring, 7 + 4);
3526 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3530 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3531 radeon_ring_write(ring, 0x1);
3532 radeon_ring_write(ring, 0x0);
3533 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3534 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3535 radeon_ring_write(ring, 0);
3536 radeon_ring_write(ring, 0);
3538 /* init the CE partitions */
3539 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3540 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3541 radeon_ring_write(ring, 0xc000);
3542 radeon_ring_write(ring, 0xe000);
3543 radeon_ring_unlock_commit(rdev, ring, false);
3545 si_cp_enable(rdev, true);
3547 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3549 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3553 /* setup clear context state */
3554 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3555 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3557 for (i = 0; i < si_default_size; i++)
3558 radeon_ring_write(ring, si_default_state[i]);
3560 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3561 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3563 /* set clear context state */
3564 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3565 radeon_ring_write(ring, 0);
3567 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3568 radeon_ring_write(ring, 0x00000316);
3569 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3570 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3572 radeon_ring_unlock_commit(rdev, ring, false);
3574 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3575 ring = &rdev->ring[i];
3576 r = radeon_ring_lock(rdev, ring, 2);
3578 /* clear the compute context state */
3579 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3580 radeon_ring_write(ring, 0);
3582 radeon_ring_unlock_commit(rdev, ring, false);
3588 static void si_cp_fini(struct radeon_device *rdev)
3590 struct radeon_ring *ring;
3591 si_cp_enable(rdev, false);
3593 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3594 radeon_ring_fini(rdev, ring);
3595 radeon_scratch_free(rdev, ring->rptr_save_reg);
3597 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3598 radeon_ring_fini(rdev, ring);
3599 radeon_scratch_free(rdev, ring->rptr_save_reg);
3601 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3602 radeon_ring_fini(rdev, ring);
3603 radeon_scratch_free(rdev, ring->rptr_save_reg);
3606 static int si_cp_resume(struct radeon_device *rdev)
3608 struct radeon_ring *ring;
3613 si_enable_gui_idle_interrupt(rdev, false);
3615 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3616 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3618 /* Set the write pointer delay */
3619 WREG32(CP_RB_WPTR_DELAY, 0);
3621 WREG32(CP_DEBUG, 0);
3622 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3624 /* ring 0 - compute and gfx */
3625 /* Set ring buffer size */
3626 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3627 rb_bufsz = order_base_2(ring->ring_size / 8);
3628 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3630 tmp |= BUF_SWAP_32BIT;
3632 WREG32(CP_RB0_CNTL, tmp);
3634 /* Initialize the ring buffer's read and write pointers */
3635 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3637 WREG32(CP_RB0_WPTR, ring->wptr);
3639 /* set the wb address whether it's enabled or not */
3640 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3641 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3643 if (rdev->wb.enabled)
3644 WREG32(SCRATCH_UMSK, 0xff);
3646 tmp |= RB_NO_UPDATE;
3647 WREG32(SCRATCH_UMSK, 0);
3651 WREG32(CP_RB0_CNTL, tmp);
3653 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3655 /* ring1 - compute only */
3656 /* Set ring buffer size */
3657 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3658 rb_bufsz = order_base_2(ring->ring_size / 8);
3659 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3661 tmp |= BUF_SWAP_32BIT;
3663 WREG32(CP_RB1_CNTL, tmp);
3665 /* Initialize the ring buffer's read and write pointers */
3666 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3668 WREG32(CP_RB1_WPTR, ring->wptr);
3670 /* set the wb address whether it's enabled or not */
3671 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3672 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3675 WREG32(CP_RB1_CNTL, tmp);
3677 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3679 /* ring2 - compute only */
3680 /* Set ring buffer size */
3681 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3682 rb_bufsz = order_base_2(ring->ring_size / 8);
3683 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3685 tmp |= BUF_SWAP_32BIT;
3687 WREG32(CP_RB2_CNTL, tmp);
3689 /* Initialize the ring buffer's read and write pointers */
3690 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3692 WREG32(CP_RB2_WPTR, ring->wptr);
3694 /* set the wb address whether it's enabled or not */
3695 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3696 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3699 WREG32(CP_RB2_CNTL, tmp);
3701 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3703 /* start the rings */
3705 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3706 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3707 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3708 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3710 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3711 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3712 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3715 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3717 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3719 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3721 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3724 si_enable_gui_idle_interrupt(rdev, true);
3726 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3727 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3732 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3738 tmp = RREG32(GRBM_STATUS);
3739 if (tmp & (PA_BUSY | SC_BUSY |
3740 BCI_BUSY | SX_BUSY |
3741 TA_BUSY | VGT_BUSY |
3743 GDS_BUSY | SPI_BUSY |
3744 IA_BUSY | IA_BUSY_NO_DMA))
3745 reset_mask |= RADEON_RESET_GFX;
3747 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3748 CP_BUSY | CP_COHERENCY_BUSY))
3749 reset_mask |= RADEON_RESET_CP;
3751 if (tmp & GRBM_EE_BUSY)
3752 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3755 tmp = RREG32(GRBM_STATUS2);
3756 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3757 reset_mask |= RADEON_RESET_RLC;
3759 /* DMA_STATUS_REG 0 */
3760 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3761 if (!(tmp & DMA_IDLE))
3762 reset_mask |= RADEON_RESET_DMA;
3764 /* DMA_STATUS_REG 1 */
3765 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3766 if (!(tmp & DMA_IDLE))
3767 reset_mask |= RADEON_RESET_DMA1;
3770 tmp = RREG32(SRBM_STATUS2);
3772 reset_mask |= RADEON_RESET_DMA;
3774 if (tmp & DMA1_BUSY)
3775 reset_mask |= RADEON_RESET_DMA1;
3778 tmp = RREG32(SRBM_STATUS);
3781 reset_mask |= RADEON_RESET_IH;
3784 reset_mask |= RADEON_RESET_SEM;
3786 if (tmp & GRBM_RQ_PENDING)
3787 reset_mask |= RADEON_RESET_GRBM;
3790 reset_mask |= RADEON_RESET_VMC;
3792 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3793 MCC_BUSY | MCD_BUSY))
3794 reset_mask |= RADEON_RESET_MC;
3796 if (evergreen_is_display_hung(rdev))
3797 reset_mask |= RADEON_RESET_DISPLAY;
3800 tmp = RREG32(VM_L2_STATUS);
3802 reset_mask |= RADEON_RESET_VMC;
3804 /* Skip MC reset as it's mostly likely not hung, just busy */
3805 if (reset_mask & RADEON_RESET_MC) {
3806 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3807 reset_mask &= ~RADEON_RESET_MC;
3813 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3815 struct evergreen_mc_save save;
3816 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3819 if (reset_mask == 0)
3822 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3824 evergreen_print_gpu_status_regs(rdev);
3825 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3826 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3827 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3828 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3837 /* Disable CP parsing/prefetching */
3838 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3840 if (reset_mask & RADEON_RESET_DMA) {
3842 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3843 tmp &= ~DMA_RB_ENABLE;
3844 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3846 if (reset_mask & RADEON_RESET_DMA1) {
3848 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3849 tmp &= ~DMA_RB_ENABLE;
3850 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3855 evergreen_mc_stop(rdev, &save);
3856 if (evergreen_mc_wait_for_idle(rdev)) {
3857 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3860 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3861 grbm_soft_reset = SOFT_RESET_CB |
3875 if (reset_mask & RADEON_RESET_CP) {
3876 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3878 srbm_soft_reset |= SOFT_RESET_GRBM;
3881 if (reset_mask & RADEON_RESET_DMA)
3882 srbm_soft_reset |= SOFT_RESET_DMA;
3884 if (reset_mask & RADEON_RESET_DMA1)
3885 srbm_soft_reset |= SOFT_RESET_DMA1;
3887 if (reset_mask & RADEON_RESET_DISPLAY)
3888 srbm_soft_reset |= SOFT_RESET_DC;
3890 if (reset_mask & RADEON_RESET_RLC)
3891 grbm_soft_reset |= SOFT_RESET_RLC;
3893 if (reset_mask & RADEON_RESET_SEM)
3894 srbm_soft_reset |= SOFT_RESET_SEM;
3896 if (reset_mask & RADEON_RESET_IH)
3897 srbm_soft_reset |= SOFT_RESET_IH;
3899 if (reset_mask & RADEON_RESET_GRBM)
3900 srbm_soft_reset |= SOFT_RESET_GRBM;
3902 if (reset_mask & RADEON_RESET_VMC)
3903 srbm_soft_reset |= SOFT_RESET_VMC;
3905 if (reset_mask & RADEON_RESET_MC)
3906 srbm_soft_reset |= SOFT_RESET_MC;
3908 if (grbm_soft_reset) {
3909 tmp = RREG32(GRBM_SOFT_RESET);
3910 tmp |= grbm_soft_reset;
3911 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3912 WREG32(GRBM_SOFT_RESET, tmp);
3913 tmp = RREG32(GRBM_SOFT_RESET);
3917 tmp &= ~grbm_soft_reset;
3918 WREG32(GRBM_SOFT_RESET, tmp);
3919 tmp = RREG32(GRBM_SOFT_RESET);
3922 if (srbm_soft_reset) {
3923 tmp = RREG32(SRBM_SOFT_RESET);
3924 tmp |= srbm_soft_reset;
3925 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3926 WREG32(SRBM_SOFT_RESET, tmp);
3927 tmp = RREG32(SRBM_SOFT_RESET);
3931 tmp &= ~srbm_soft_reset;
3932 WREG32(SRBM_SOFT_RESET, tmp);
3933 tmp = RREG32(SRBM_SOFT_RESET);
3936 /* Wait a little for things to settle down */
3939 evergreen_mc_resume(rdev, &save);
3942 evergreen_print_gpu_status_regs(rdev);
3945 static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3949 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3950 tmp |= SPLL_BYPASS_EN;
3951 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3953 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3954 tmp |= SPLL_CTLREQ_CHG;
3955 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3957 for (i = 0; i < rdev->usec_timeout; i++) {
3958 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3963 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3964 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3965 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3967 tmp = RREG32(MPLL_CNTL_MODE);
3968 tmp &= ~MPLL_MCLK_SEL;
3969 WREG32(MPLL_CNTL_MODE, tmp);
3972 static void si_spll_powerdown(struct radeon_device *rdev)
3976 tmp = RREG32(SPLL_CNTL_MODE);
3977 tmp |= SPLL_SW_DIR_CONTROL;
3978 WREG32(SPLL_CNTL_MODE, tmp);
3980 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3982 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3984 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3986 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3988 tmp = RREG32(SPLL_CNTL_MODE);
3989 tmp &= ~SPLL_SW_DIR_CONTROL;
3990 WREG32(SPLL_CNTL_MODE, tmp);
3993 static void si_gpu_pci_config_reset(struct radeon_device *rdev)
3995 struct evergreen_mc_save save;
3998 dev_info(rdev->dev, "GPU pci config reset\n");
4006 /* Disable CP parsing/prefetching */
4007 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4009 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
4010 tmp &= ~DMA_RB_ENABLE;
4011 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
4013 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
4014 tmp &= ~DMA_RB_ENABLE;
4015 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
4016 /* XXX other engines? */
4018 /* halt the rlc, disable cp internal ints */
4023 /* disable mem access */
4024 evergreen_mc_stop(rdev, &save);
4025 if (evergreen_mc_wait_for_idle(rdev)) {
4026 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4029 /* set mclk/sclk to bypass */
4030 si_set_clk_bypass_mode(rdev);
4031 /* powerdown spll */
4032 si_spll_powerdown(rdev);
4034 pci_clear_master(rdev->pdev);
4036 radeon_pci_config_reset(rdev);
4037 /* wait for asic to come out of reset */
4038 for (i = 0; i < rdev->usec_timeout; i++) {
4039 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4045 int si_asic_reset(struct radeon_device *rdev)
4049 reset_mask = si_gpu_check_soft_reset(rdev);
4052 r600_set_bios_scratch_engine_hung(rdev, true);
4054 /* try soft reset */
4055 si_gpu_soft_reset(rdev, reset_mask);
4057 reset_mask = si_gpu_check_soft_reset(rdev);
4059 /* try pci config reset */
4060 if (reset_mask && radeon_hard_reset)
4061 si_gpu_pci_config_reset(rdev);
4063 reset_mask = si_gpu_check_soft_reset(rdev);
4066 r600_set_bios_scratch_engine_hung(rdev, false);
4072 * si_gfx_is_lockup - Check if the GFX engine is locked up
4074 * @rdev: radeon_device pointer
4075 * @ring: radeon_ring structure holding ring information
4077 * Check if the GFX engine is locked up.
4078 * Returns true if the engine appears to be locked up, false if not.
4080 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4082 u32 reset_mask = si_gpu_check_soft_reset(rdev);
4084 if (!(reset_mask & (RADEON_RESET_GFX |
4085 RADEON_RESET_COMPUTE |
4086 RADEON_RESET_CP))) {
4087 radeon_ring_lockup_update(rdev, ring);
4090 return radeon_ring_test_lockup(rdev, ring);
4094 static void si_mc_program(struct radeon_device *rdev)
4096 struct evergreen_mc_save save;
4100 /* Initialize HDP */
4101 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4102 WREG32((0x2c14 + j), 0x00000000);
4103 WREG32((0x2c18 + j), 0x00000000);
4104 WREG32((0x2c1c + j), 0x00000000);
4105 WREG32((0x2c20 + j), 0x00000000);
4106 WREG32((0x2c24 + j), 0x00000000);
4108 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4110 evergreen_mc_stop(rdev, &save);
4111 if (radeon_mc_wait_for_idle(rdev)) {
4112 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4114 if (!ASIC_IS_NODCE(rdev))
4115 /* Lockout access through VGA aperture*/
4116 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4117 /* Update configuration */
4118 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4119 rdev->mc.vram_start >> 12);
4120 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4121 rdev->mc.vram_end >> 12);
4122 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4123 rdev->vram_scratch.gpu_addr >> 12);
4124 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4125 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4126 WREG32(MC_VM_FB_LOCATION, tmp);
4127 /* XXX double check these! */
4128 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4129 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4130 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4131 WREG32(MC_VM_AGP_BASE, 0);
4132 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4133 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4134 if (radeon_mc_wait_for_idle(rdev)) {
4135 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4137 evergreen_mc_resume(rdev, &save);
4138 if (!ASIC_IS_NODCE(rdev)) {
4139 /* we need to own VRAM, so turn off the VGA renderer here
4140 * to stop it overwriting our objects */
4141 rv515_vga_render_disable(rdev);
4145 void si_vram_gtt_location(struct radeon_device *rdev,
4146 struct radeon_mc *mc)
4148 if (mc->mc_vram_size > 0xFFC0000000ULL) {
4149 /* leave room for at least 1024M GTT */
4150 dev_warn(rdev->dev, "limiting VRAM\n");
4151 mc->real_vram_size = 0xFFC0000000ULL;
4152 mc->mc_vram_size = 0xFFC0000000ULL;
4154 radeon_vram_location(rdev, &rdev->mc, 0);
4155 rdev->mc.gtt_base_align = 0;
4156 radeon_gtt_location(rdev, mc);
4159 static int si_mc_init(struct radeon_device *rdev)
4162 int chansize, numchan;
4164 /* Get VRAM informations */
4165 rdev->mc.vram_is_ddr = true;
4166 tmp = RREG32(MC_ARB_RAMCFG);
4167 if (tmp & CHANSIZE_OVERRIDE) {
4169 } else if (tmp & CHANSIZE_MASK) {
4174 tmp = RREG32(MC_SHARED_CHMAP);
4175 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4205 rdev->mc.vram_width = numchan * chansize;
4206 /* Could aper size report 0 ? */
4207 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4208 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4209 /* size in MB on si */
4210 tmp = RREG32(CONFIG_MEMSIZE);
4211 /* some boards may have garbage in the upper 16 bits */
4212 if (tmp & 0xffff0000) {
4213 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4217 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4218 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4219 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4220 si_vram_gtt_location(rdev, &rdev->mc);
4221 radeon_update_bandwidth_info(rdev);
4229 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4231 /* flush hdp cache */
4232 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4234 /* bits 0-15 are the VM contexts0-15 */
4235 WREG32(VM_INVALIDATE_REQUEST, 1);
4238 static int si_pcie_gart_enable(struct radeon_device *rdev)
4242 if (rdev->gart.robj == NULL) {
4243 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4246 r = radeon_gart_table_vram_pin(rdev);
4249 /* Setup TLB control */
4250 WREG32(MC_VM_MX_L1_TLB_CNTL,
4253 ENABLE_L1_FRAGMENT_PROCESSING |
4254 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4255 ENABLE_ADVANCED_DRIVER_MODEL |
4256 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4257 /* Setup L2 cache */
4258 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4259 ENABLE_L2_FRAGMENT_PROCESSING |
4260 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4261 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4262 EFFECTIVE_L2_QUEUE_SIZE(7) |
4263 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4264 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4265 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4267 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
4268 /* setup context0 */
4269 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4270 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4271 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4272 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4273 (u32)(rdev->dummy_page.addr >> 12));
4274 WREG32(VM_CONTEXT0_CNTL2, 0);
4275 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4276 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4282 /* empty context1-15 */
4283 /* set vm size, must be a multiple of 4 */
4284 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4285 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
4286 /* Assign the pt base to something valid for now; the pts used for
4287 * the VMs are determined by the application and setup and assigned
4288 * on the fly in the vm part of radeon_gart.c
4290 for (i = 1; i < 16; i++) {
4292 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4293 rdev->gart.table_addr >> 12);
4295 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4296 rdev->gart.table_addr >> 12);
4299 /* enable context1-15 */
4300 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4301 (u32)(rdev->dummy_page.addr >> 12));
4302 WREG32(VM_CONTEXT1_CNTL2, 4);
4303 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4304 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
4305 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4306 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4307 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4308 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4309 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4310 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4311 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4312 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4313 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4314 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4315 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4316 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4318 si_pcie_gart_tlb_flush(rdev);
4319 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4320 (unsigned)(rdev->mc.gtt_size >> 20),
4321 (unsigned long long)rdev->gart.table_addr);
4322 rdev->gart.ready = true;
4326 static void si_pcie_gart_disable(struct radeon_device *rdev)
4328 /* Disable all tables */
4329 WREG32(VM_CONTEXT0_CNTL, 0);
4330 WREG32(VM_CONTEXT1_CNTL, 0);
4331 /* Setup TLB control */
4332 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4333 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4334 /* Setup L2 cache */
4335 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4336 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4337 EFFECTIVE_L2_QUEUE_SIZE(7) |
4338 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4339 WREG32(VM_L2_CNTL2, 0);
4340 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4341 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4342 radeon_gart_table_vram_unpin(rdev);
4345 static void si_pcie_gart_fini(struct radeon_device *rdev)
4347 si_pcie_gart_disable(rdev);
4348 radeon_gart_table_vram_free(rdev);
4349 radeon_gart_fini(rdev);
4353 static bool si_vm_reg_valid(u32 reg)
4355 /* context regs are fine */
4359 /* check config regs */
4361 case GRBM_GFX_INDEX:
4362 case CP_STRMOUT_CNTL:
4363 case VGT_VTX_VECT_EJECT_REG:
4364 case VGT_CACHE_INVALIDATION:
4365 case VGT_ESGS_RING_SIZE:
4366 case VGT_GSVS_RING_SIZE:
4367 case VGT_GS_VERTEX_REUSE:
4368 case VGT_PRIMITIVE_TYPE:
4369 case VGT_INDEX_TYPE:
4370 case VGT_NUM_INDICES:
4371 case VGT_NUM_INSTANCES:
4372 case VGT_TF_RING_SIZE:
4373 case VGT_HS_OFFCHIP_PARAM:
4374 case VGT_TF_MEMORY_BASE:
4376 case PA_SU_LINE_STIPPLE_VALUE:
4377 case PA_SC_LINE_STIPPLE_STATE:
4380 case SPI_STATIC_THREAD_MGMT_1:
4381 case SPI_STATIC_THREAD_MGMT_2:
4382 case SPI_STATIC_THREAD_MGMT_3:
4383 case SPI_PS_MAX_WAVE_ID:
4384 case SPI_CONFIG_CNTL:
4385 case SPI_CONFIG_CNTL_1:
4389 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4394 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4395 u32 *ib, struct radeon_cs_packet *pkt)
4397 switch (pkt->opcode) {
4399 case PACKET3_SET_BASE:
4400 case PACKET3_SET_CE_DE_COUNTERS:
4401 case PACKET3_LOAD_CONST_RAM:
4402 case PACKET3_WRITE_CONST_RAM:
4403 case PACKET3_WRITE_CONST_RAM_OFFSET:
4404 case PACKET3_DUMP_CONST_RAM:
4405 case PACKET3_INCREMENT_CE_COUNTER:
4406 case PACKET3_WAIT_ON_DE_COUNTER:
4407 case PACKET3_CE_WRITE:
4410 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4416 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4418 u32 start_reg, reg, i;
4419 u32 command = ib[idx + 4];
4420 u32 info = ib[idx + 1];
4421 u32 idx_value = ib[idx];
4422 if (command & PACKET3_CP_DMA_CMD_SAS) {
4423 /* src address space is register */
4424 if (((info & 0x60000000) >> 29) == 0) {
4425 start_reg = idx_value << 2;
4426 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4428 if (!si_vm_reg_valid(reg)) {
4429 DRM_ERROR("CP DMA Bad SRC register\n");
4433 for (i = 0; i < (command & 0x1fffff); i++) {
4434 reg = start_reg + (4 * i);
4435 if (!si_vm_reg_valid(reg)) {
4436 DRM_ERROR("CP DMA Bad SRC register\n");
4443 if (command & PACKET3_CP_DMA_CMD_DAS) {
4444 /* dst address space is register */
4445 if (((info & 0x00300000) >> 20) == 0) {
4446 start_reg = ib[idx + 2];
4447 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4449 if (!si_vm_reg_valid(reg)) {
4450 DRM_ERROR("CP DMA Bad DST register\n");
4454 for (i = 0; i < (command & 0x1fffff); i++) {
4455 reg = start_reg + (4 * i);
4456 if (!si_vm_reg_valid(reg)) {
4457 DRM_ERROR("CP DMA Bad DST register\n");
4467 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4468 u32 *ib, struct radeon_cs_packet *pkt)
4471 u32 idx = pkt->idx + 1;
4472 u32 idx_value = ib[idx];
4473 u32 start_reg, end_reg, reg, i;
4475 switch (pkt->opcode) {
4477 case PACKET3_SET_BASE:
4478 case PACKET3_CLEAR_STATE:
4479 case PACKET3_INDEX_BUFFER_SIZE:
4480 case PACKET3_DISPATCH_DIRECT:
4481 case PACKET3_DISPATCH_INDIRECT:
4482 case PACKET3_ALLOC_GDS:
4483 case PACKET3_WRITE_GDS_RAM:
4484 case PACKET3_ATOMIC_GDS:
4485 case PACKET3_ATOMIC:
4486 case PACKET3_OCCLUSION_QUERY:
4487 case PACKET3_SET_PREDICATION:
4488 case PACKET3_COND_EXEC:
4489 case PACKET3_PRED_EXEC:
4490 case PACKET3_DRAW_INDIRECT:
4491 case PACKET3_DRAW_INDEX_INDIRECT:
4492 case PACKET3_INDEX_BASE:
4493 case PACKET3_DRAW_INDEX_2:
4494 case PACKET3_CONTEXT_CONTROL:
4495 case PACKET3_INDEX_TYPE:
4496 case PACKET3_DRAW_INDIRECT_MULTI:
4497 case PACKET3_DRAW_INDEX_AUTO:
4498 case PACKET3_DRAW_INDEX_IMMD:
4499 case PACKET3_NUM_INSTANCES:
4500 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4501 case PACKET3_STRMOUT_BUFFER_UPDATE:
4502 case PACKET3_DRAW_INDEX_OFFSET_2:
4503 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4504 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4505 case PACKET3_MPEG_INDEX:
4506 case PACKET3_WAIT_REG_MEM:
4507 case PACKET3_MEM_WRITE:
4508 case PACKET3_PFP_SYNC_ME:
4509 case PACKET3_SURFACE_SYNC:
4510 case PACKET3_EVENT_WRITE:
4511 case PACKET3_EVENT_WRITE_EOP:
4512 case PACKET3_EVENT_WRITE_EOS:
4513 case PACKET3_SET_CONTEXT_REG:
4514 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4515 case PACKET3_SET_SH_REG:
4516 case PACKET3_SET_SH_REG_OFFSET:
4517 case PACKET3_INCREMENT_DE_COUNTER:
4518 case PACKET3_WAIT_ON_CE_COUNTER:
4519 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4520 case PACKET3_ME_WRITE:
4522 case PACKET3_COPY_DATA:
4523 if ((idx_value & 0xf00) == 0) {
4524 reg = ib[idx + 3] * 4;
4525 if (!si_vm_reg_valid(reg))
4529 case PACKET3_WRITE_DATA:
4530 if ((idx_value & 0xf00) == 0) {
4531 start_reg = ib[idx + 1] * 4;
4532 if (idx_value & 0x10000) {
4533 if (!si_vm_reg_valid(start_reg))
4536 for (i = 0; i < (pkt->count - 2); i++) {
4537 reg = start_reg + (4 * i);
4538 if (!si_vm_reg_valid(reg))
4544 case PACKET3_COND_WRITE:
4545 if (idx_value & 0x100) {
4546 reg = ib[idx + 5] * 4;
4547 if (!si_vm_reg_valid(reg))
4551 case PACKET3_COPY_DW:
4552 if (idx_value & 0x2) {
4553 reg = ib[idx + 3] * 4;
4554 if (!si_vm_reg_valid(reg))
4558 case PACKET3_SET_CONFIG_REG:
4559 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4560 end_reg = 4 * pkt->count + start_reg - 4;
4561 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4562 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4563 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4564 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4567 for (i = 0; i < pkt->count; i++) {
4568 reg = start_reg + (4 * i);
4569 if (!si_vm_reg_valid(reg))
4573 case PACKET3_CP_DMA:
4574 r = si_vm_packet3_cp_dma_check(ib, idx);
4579 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4585 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4586 u32 *ib, struct radeon_cs_packet *pkt)
4589 u32 idx = pkt->idx + 1;
4590 u32 idx_value = ib[idx];
4591 u32 start_reg, reg, i;
4593 switch (pkt->opcode) {
4595 case PACKET3_SET_BASE:
4596 case PACKET3_CLEAR_STATE:
4597 case PACKET3_DISPATCH_DIRECT:
4598 case PACKET3_DISPATCH_INDIRECT:
4599 case PACKET3_ALLOC_GDS:
4600 case PACKET3_WRITE_GDS_RAM:
4601 case PACKET3_ATOMIC_GDS:
4602 case PACKET3_ATOMIC:
4603 case PACKET3_OCCLUSION_QUERY:
4604 case PACKET3_SET_PREDICATION:
4605 case PACKET3_COND_EXEC:
4606 case PACKET3_PRED_EXEC:
4607 case PACKET3_CONTEXT_CONTROL:
4608 case PACKET3_STRMOUT_BUFFER_UPDATE:
4609 case PACKET3_WAIT_REG_MEM:
4610 case PACKET3_MEM_WRITE:
4611 case PACKET3_PFP_SYNC_ME:
4612 case PACKET3_SURFACE_SYNC:
4613 case PACKET3_EVENT_WRITE:
4614 case PACKET3_EVENT_WRITE_EOP:
4615 case PACKET3_EVENT_WRITE_EOS:
4616 case PACKET3_SET_CONTEXT_REG:
4617 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4618 case PACKET3_SET_SH_REG:
4619 case PACKET3_SET_SH_REG_OFFSET:
4620 case PACKET3_INCREMENT_DE_COUNTER:
4621 case PACKET3_WAIT_ON_CE_COUNTER:
4622 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4623 case PACKET3_ME_WRITE:
4625 case PACKET3_COPY_DATA:
4626 if ((idx_value & 0xf00) == 0) {
4627 reg = ib[idx + 3] * 4;
4628 if (!si_vm_reg_valid(reg))
4632 case PACKET3_WRITE_DATA:
4633 if ((idx_value & 0xf00) == 0) {
4634 start_reg = ib[idx + 1] * 4;
4635 if (idx_value & 0x10000) {
4636 if (!si_vm_reg_valid(start_reg))
4639 for (i = 0; i < (pkt->count - 2); i++) {
4640 reg = start_reg + (4 * i);
4641 if (!si_vm_reg_valid(reg))
4647 case PACKET3_COND_WRITE:
4648 if (idx_value & 0x100) {
4649 reg = ib[idx + 5] * 4;
4650 if (!si_vm_reg_valid(reg))
4654 case PACKET3_COPY_DW:
4655 if (idx_value & 0x2) {
4656 reg = ib[idx + 3] * 4;
4657 if (!si_vm_reg_valid(reg))
4661 case PACKET3_CP_DMA:
4662 r = si_vm_packet3_cp_dma_check(ib, idx);
4667 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4673 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4677 struct radeon_cs_packet pkt;
4681 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4682 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4685 case RADEON_PACKET_TYPE0:
4686 dev_err(rdev->dev, "Packet0 not allowed!\n");
4689 case RADEON_PACKET_TYPE2:
4692 case RADEON_PACKET_TYPE3:
4693 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4694 if (ib->is_const_ib)
4695 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4698 case RADEON_RING_TYPE_GFX_INDEX:
4699 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4701 case CAYMAN_RING_TYPE_CP1_INDEX:
4702 case CAYMAN_RING_TYPE_CP2_INDEX:
4703 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4706 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4711 idx += pkt.count + 2;
4714 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4720 } while (idx < ib->length_dw);
4728 int si_vm_init(struct radeon_device *rdev)
4731 rdev->vm_manager.nvm = 16;
4732 /* base offset of vram pages */
4733 rdev->vm_manager.vram_base_offset = 0;
4738 void si_vm_fini(struct radeon_device *rdev)
4743 * si_vm_decode_fault - print human readable fault info
4745 * @rdev: radeon_device pointer
4746 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4747 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4749 * Print human readable fault information (SI).
4751 static void si_vm_decode_fault(struct radeon_device *rdev,
4752 u32 status, u32 addr)
4754 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4755 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4756 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4759 if (rdev->family == CHIP_TAHITI) {
5000 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5001 protections, vmid, addr,
5002 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5006 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5008 struct radeon_ring *ring = &rdev->ring[ridx];
5013 /* write new base address */
5014 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5015 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5016 WRITE_DATA_DST_SEL(0)));
5019 radeon_ring_write(ring,
5020 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5022 radeon_ring_write(ring,
5023 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5025 radeon_ring_write(ring, 0);
5026 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5028 /* flush hdp cache */
5029 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5030 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5031 WRITE_DATA_DST_SEL(0)));
5032 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5033 radeon_ring_write(ring, 0);
5034 radeon_ring_write(ring, 0x1);
5036 /* bits 0-15 are the VM contexts0-15 */
5037 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5038 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5039 WRITE_DATA_DST_SEL(0)));
5040 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5041 radeon_ring_write(ring, 0);
5042 radeon_ring_write(ring, 1 << vm->id);
5044 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5045 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5046 radeon_ring_write(ring, 0x0);
5050 * Power and clock gating
5052 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5056 for (i = 0; i < rdev->usec_timeout; i++) {
5057 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5062 for (i = 0; i < rdev->usec_timeout; i++) {
5063 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5069 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5072 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5077 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5079 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5080 WREG32(CP_INT_CNTL_RING0, tmp);
5083 /* read a gfx register */
5084 tmp = RREG32(DB_DEPTH_INFO);
5086 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5087 for (i = 0; i < rdev->usec_timeout; i++) {
5088 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5095 static void si_set_uvd_dcm(struct radeon_device *rdev,
5100 tmp = RREG32(UVD_CGC_CTRL);
5101 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5102 tmp |= DCM | CG_DT(1) | CLK_OD(4);
5106 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5112 WREG32(UVD_CGC_CTRL, tmp);
5113 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5116 void si_init_uvd_internal_cg(struct radeon_device *rdev)
5118 bool hw_mode = true;
5121 si_set_uvd_dcm(rdev, false);
5123 u32 tmp = RREG32(UVD_CGC_CTRL);
5125 WREG32(UVD_CGC_CTRL, tmp);
5129 static u32 si_halt_rlc(struct radeon_device *rdev)
5133 orig = data = RREG32(RLC_CNTL);
5135 if (data & RLC_ENABLE) {
5136 data &= ~RLC_ENABLE;
5137 WREG32(RLC_CNTL, data);
5139 si_wait_for_rlc_serdes(rdev);
5145 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5149 tmp = RREG32(RLC_CNTL);
5151 WREG32(RLC_CNTL, rlc);
5154 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5158 orig = data = RREG32(DMA_PG);
5159 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
5160 data |= PG_CNTL_ENABLE;
5162 data &= ~PG_CNTL_ENABLE;
5164 WREG32(DMA_PG, data);
5167 static void si_init_dma_pg(struct radeon_device *rdev)
5171 WREG32(DMA_PGFSM_WRITE, 0x00002000);
5172 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5174 for (tmp = 0; tmp < 5; tmp++)
5175 WREG32(DMA_PGFSM_WRITE, 0);
5178 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5183 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5184 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5185 WREG32(RLC_TTOP_D, tmp);
5187 tmp = RREG32(RLC_PG_CNTL);
5188 tmp |= GFX_PG_ENABLE;
5189 WREG32(RLC_PG_CNTL, tmp);
5191 tmp = RREG32(RLC_AUTO_PG_CTRL);
5193 WREG32(RLC_AUTO_PG_CTRL, tmp);
5195 tmp = RREG32(RLC_AUTO_PG_CTRL);
5197 WREG32(RLC_AUTO_PG_CTRL, tmp);
5199 tmp = RREG32(DB_RENDER_CONTROL);
5203 static void si_init_gfx_cgpg(struct radeon_device *rdev)
5207 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5209 tmp = RREG32(RLC_PG_CNTL);
5211 WREG32(RLC_PG_CNTL, tmp);
5213 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5215 tmp = RREG32(RLC_AUTO_PG_CTRL);
5217 tmp &= ~GRBM_REG_SGIT_MASK;
5218 tmp |= GRBM_REG_SGIT(0x700);
5219 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5220 WREG32(RLC_AUTO_PG_CTRL, tmp);
5223 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5225 u32 mask = 0, tmp, tmp1;
5228 si_select_se_sh(rdev, se, sh);
5229 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5230 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5231 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5238 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5243 return (~tmp) & mask;
5246 static void si_init_ao_cu_mask(struct radeon_device *rdev)
5248 u32 i, j, k, active_cu_number = 0;
5249 u32 mask, counter, cu_bitmap;
5252 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5253 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5257 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5258 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5266 active_cu_number += counter;
5267 tmp |= (cu_bitmap << (i * 16 + j * 8));
5271 WREG32(RLC_PG_AO_CU_MASK, tmp);
5273 tmp = RREG32(RLC_MAX_PG_CU);
5274 tmp &= ~MAX_PU_CU_MASK;
5275 tmp |= MAX_PU_CU(active_cu_number);
5276 WREG32(RLC_MAX_PG_CU, tmp);
5279 static void si_enable_cgcg(struct radeon_device *rdev,
5282 u32 data, orig, tmp;
5284 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5286 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5287 si_enable_gui_idle_interrupt(rdev, true);
5289 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5291 tmp = si_halt_rlc(rdev);
5293 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5294 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5295 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5297 si_wait_for_rlc_serdes(rdev);
5299 si_update_rlc(rdev, tmp);
5301 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5303 data |= CGCG_EN | CGLS_EN;
5305 si_enable_gui_idle_interrupt(rdev, false);
5307 RREG32(CB_CGTT_SCLK_CTRL);
5308 RREG32(CB_CGTT_SCLK_CTRL);
5309 RREG32(CB_CGTT_SCLK_CTRL);
5310 RREG32(CB_CGTT_SCLK_CTRL);
5312 data &= ~(CGCG_EN | CGLS_EN);
5316 WREG32(RLC_CGCG_CGLS_CTRL, data);
5319 static void si_enable_mgcg(struct radeon_device *rdev,
5322 u32 data, orig, tmp = 0;
5324 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5325 orig = data = RREG32(CGTS_SM_CTRL_REG);
5328 WREG32(CGTS_SM_CTRL_REG, data);
5330 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5331 orig = data = RREG32(CP_MEM_SLP_CNTL);
5332 data |= CP_MEM_LS_EN;
5334 WREG32(CP_MEM_SLP_CNTL, data);
5337 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5340 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5342 tmp = si_halt_rlc(rdev);
5344 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5345 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5346 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5348 si_update_rlc(rdev, tmp);
5350 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5353 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5355 data = RREG32(CP_MEM_SLP_CNTL);
5356 if (data & CP_MEM_LS_EN) {
5357 data &= ~CP_MEM_LS_EN;
5358 WREG32(CP_MEM_SLP_CNTL, data);
5360 orig = data = RREG32(CGTS_SM_CTRL_REG);
5361 data |= LS_OVERRIDE | OVERRIDE;
5363 WREG32(CGTS_SM_CTRL_REG, data);
5365 tmp = si_halt_rlc(rdev);
5367 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5368 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5369 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5371 si_update_rlc(rdev, tmp);
5375 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5378 u32 orig, data, tmp;
5380 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5381 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5383 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5385 orig = data = RREG32(UVD_CGC_CTRL);
5388 WREG32(UVD_CGC_CTRL, data);
5390 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5391 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5393 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5395 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5397 orig = data = RREG32(UVD_CGC_CTRL);
5400 WREG32(UVD_CGC_CTRL, data);
5402 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5403 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5407 static const u32 mc_cg_registers[] =
5420 static void si_enable_mc_ls(struct radeon_device *rdev,
5426 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5427 orig = data = RREG32(mc_cg_registers[i]);
5428 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5429 data |= MC_LS_ENABLE;
5431 data &= ~MC_LS_ENABLE;
5433 WREG32(mc_cg_registers[i], data);
5437 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5443 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5444 orig = data = RREG32(mc_cg_registers[i]);
5445 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5446 data |= MC_CG_ENABLE;
5448 data &= ~MC_CG_ENABLE;
5450 WREG32(mc_cg_registers[i], data);
5454 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5457 u32 orig, data, offset;
5460 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5461 for (i = 0; i < 2; i++) {
5463 offset = DMA0_REGISTER_OFFSET;
5465 offset = DMA1_REGISTER_OFFSET;
5466 orig = data = RREG32(DMA_POWER_CNTL + offset);
5467 data &= ~MEM_POWER_OVERRIDE;
5469 WREG32(DMA_POWER_CNTL + offset, data);
5470 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5473 for (i = 0; i < 2; i++) {
5475 offset = DMA0_REGISTER_OFFSET;
5477 offset = DMA1_REGISTER_OFFSET;
5478 orig = data = RREG32(DMA_POWER_CNTL + offset);
5479 data |= MEM_POWER_OVERRIDE;
5481 WREG32(DMA_POWER_CNTL + offset, data);
5483 orig = data = RREG32(DMA_CLK_CTRL + offset);
5486 WREG32(DMA_CLK_CTRL + offset, data);
5491 static void si_enable_bif_mgls(struct radeon_device *rdev,
5496 orig = data = RREG32_PCIE(PCIE_CNTL2);
5498 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5499 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5500 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5502 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5503 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5506 WREG32_PCIE(PCIE_CNTL2, data);
5509 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5514 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5516 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5517 data &= ~CLOCK_GATING_DIS;
5519 data |= CLOCK_GATING_DIS;
5522 WREG32(HDP_HOST_PATH_CNTL, data);
5525 static void si_enable_hdp_ls(struct radeon_device *rdev,
5530 orig = data = RREG32(HDP_MEM_POWER_LS);
5532 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5533 data |= HDP_LS_ENABLE;
5535 data &= ~HDP_LS_ENABLE;
5538 WREG32(HDP_MEM_POWER_LS, data);
5541 static void si_update_cg(struct radeon_device *rdev,
5542 u32 block, bool enable)
5544 if (block & RADEON_CG_BLOCK_GFX) {
5545 si_enable_gui_idle_interrupt(rdev, false);
5546 /* order matters! */
5548 si_enable_mgcg(rdev, true);
5549 si_enable_cgcg(rdev, true);
5551 si_enable_cgcg(rdev, false);
5552 si_enable_mgcg(rdev, false);
5554 si_enable_gui_idle_interrupt(rdev, true);
5557 if (block & RADEON_CG_BLOCK_MC) {
5558 si_enable_mc_mgcg(rdev, enable);
5559 si_enable_mc_ls(rdev, enable);
5562 if (block & RADEON_CG_BLOCK_SDMA) {
5563 si_enable_dma_mgcg(rdev, enable);
5566 if (block & RADEON_CG_BLOCK_BIF) {
5567 si_enable_bif_mgls(rdev, enable);
5570 if (block & RADEON_CG_BLOCK_UVD) {
5571 if (rdev->has_uvd) {
5572 si_enable_uvd_mgcg(rdev, enable);
5576 if (block & RADEON_CG_BLOCK_HDP) {
5577 si_enable_hdp_mgcg(rdev, enable);
5578 si_enable_hdp_ls(rdev, enable);
5582 static void si_init_cg(struct radeon_device *rdev)
5584 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5585 RADEON_CG_BLOCK_MC |
5586 RADEON_CG_BLOCK_SDMA |
5587 RADEON_CG_BLOCK_BIF |
5588 RADEON_CG_BLOCK_HDP), true);
5589 if (rdev->has_uvd) {
5590 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5591 si_init_uvd_internal_cg(rdev);
5595 static void si_fini_cg(struct radeon_device *rdev)
5597 if (rdev->has_uvd) {
5598 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5600 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5601 RADEON_CG_BLOCK_MC |
5602 RADEON_CG_BLOCK_SDMA |
5603 RADEON_CG_BLOCK_BIF |
5604 RADEON_CG_BLOCK_HDP), false);
5607 u32 si_get_csb_size(struct radeon_device *rdev)
5610 const struct cs_section_def *sect = NULL;
5611 const struct cs_extent_def *ext = NULL;
5613 if (rdev->rlc.cs_data == NULL)
5616 /* begin clear state */
5618 /* context control state */
5621 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5622 for (ext = sect->section; ext->extent != NULL; ++ext) {
5623 if (sect->id == SECT_CONTEXT)
5624 count += 2 + ext->reg_count;
5629 /* pa_sc_raster_config */
5631 /* end clear state */
5639 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5642 const struct cs_section_def *sect = NULL;
5643 const struct cs_extent_def *ext = NULL;
5645 if (rdev->rlc.cs_data == NULL)
5650 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5651 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5653 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5654 buffer[count++] = cpu_to_le32(0x80000000);
5655 buffer[count++] = cpu_to_le32(0x80000000);
5657 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5658 for (ext = sect->section; ext->extent != NULL; ++ext) {
5659 if (sect->id == SECT_CONTEXT) {
5661 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5662 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5663 for (i = 0; i < ext->reg_count; i++)
5664 buffer[count++] = cpu_to_le32(ext->extent[i]);
5671 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5672 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5673 switch (rdev->family) {
5676 buffer[count++] = cpu_to_le32(0x2a00126a);
5679 buffer[count++] = cpu_to_le32(0x0000124a);
5682 buffer[count++] = cpu_to_le32(0x00000082);
5685 buffer[count++] = cpu_to_le32(0x00000000);
5688 buffer[count++] = cpu_to_le32(0x00000000);
5692 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5693 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5695 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5696 buffer[count++] = cpu_to_le32(0);
5699 static void si_init_pg(struct radeon_device *rdev)
5701 if (rdev->pg_flags) {
5702 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5703 si_init_dma_pg(rdev);
5705 si_init_ao_cu_mask(rdev);
5706 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5707 si_init_gfx_cgpg(rdev);
5709 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5710 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5712 si_enable_dma_pg(rdev, true);
5713 si_enable_gfx_cgpg(rdev, true);
5715 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5716 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5720 static void si_fini_pg(struct radeon_device *rdev)
5722 if (rdev->pg_flags) {
5723 si_enable_dma_pg(rdev, false);
5724 si_enable_gfx_cgpg(rdev, false);
5731 void si_rlc_reset(struct radeon_device *rdev)
5733 u32 tmp = RREG32(GRBM_SOFT_RESET);
5735 tmp |= SOFT_RESET_RLC;
5736 WREG32(GRBM_SOFT_RESET, tmp);
5738 tmp &= ~SOFT_RESET_RLC;
5739 WREG32(GRBM_SOFT_RESET, tmp);
5743 static void si_rlc_stop(struct radeon_device *rdev)
5745 WREG32(RLC_CNTL, 0);
5747 si_enable_gui_idle_interrupt(rdev, false);
5749 si_wait_for_rlc_serdes(rdev);
5752 static void si_rlc_start(struct radeon_device *rdev)
5754 WREG32(RLC_CNTL, RLC_ENABLE);
5756 si_enable_gui_idle_interrupt(rdev, true);
5761 static bool si_lbpw_supported(struct radeon_device *rdev)
5765 /* Enable LBPW only for DDR3 */
5766 tmp = RREG32(MC_SEQ_MISC0);
5767 if ((tmp & 0xF0000000) == 0xB0000000)
5772 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5776 tmp = RREG32(RLC_LB_CNTL);
5778 tmp |= LOAD_BALANCE_ENABLE;
5780 tmp &= ~LOAD_BALANCE_ENABLE;
5781 WREG32(RLC_LB_CNTL, tmp);
5784 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5785 WREG32(SPI_LB_CU_MASK, 0x00ff);
5789 static int si_rlc_resume(struct radeon_device *rdev)
5804 WREG32(RLC_RL_BASE, 0);
5805 WREG32(RLC_RL_SIZE, 0);
5806 WREG32(RLC_LB_CNTL, 0);
5807 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5808 WREG32(RLC_LB_CNTR_INIT, 0);
5809 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5811 WREG32(RLC_MC_CNTL, 0);
5812 WREG32(RLC_UCODE_CNTL, 0);
5815 const struct rlc_firmware_header_v1_0 *hdr =
5816 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5817 u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5818 const __le32 *fw_data = (const __le32 *)
5819 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5821 radeon_ucode_print_rlc_hdr(&hdr->header);
5823 for (i = 0; i < fw_size; i++) {
5824 WREG32(RLC_UCODE_ADDR, i);
5825 WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5828 const __be32 *fw_data =
5829 (const __be32 *)rdev->rlc_fw->data;
5830 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5831 WREG32(RLC_UCODE_ADDR, i);
5832 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5835 WREG32(RLC_UCODE_ADDR, 0);
5837 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5844 static void si_enable_interrupts(struct radeon_device *rdev)
5846 u32 ih_cntl = RREG32(IH_CNTL);
5847 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5849 ih_cntl |= ENABLE_INTR;
5850 ih_rb_cntl |= IH_RB_ENABLE;
5851 WREG32(IH_CNTL, ih_cntl);
5852 WREG32(IH_RB_CNTL, ih_rb_cntl);
5853 rdev->ih.enabled = true;
5856 static void si_disable_interrupts(struct radeon_device *rdev)
5858 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5859 u32 ih_cntl = RREG32(IH_CNTL);
5861 ih_rb_cntl &= ~IH_RB_ENABLE;
5862 ih_cntl &= ~ENABLE_INTR;
5863 WREG32(IH_RB_CNTL, ih_rb_cntl);
5864 WREG32(IH_CNTL, ih_cntl);
5865 /* set rptr, wptr to 0 */
5866 WREG32(IH_RB_RPTR, 0);
5867 WREG32(IH_RB_WPTR, 0);
5868 rdev->ih.enabled = false;
5872 static void si_disable_interrupt_state(struct radeon_device *rdev)
5876 tmp = RREG32(CP_INT_CNTL_RING0) &
5877 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5878 WREG32(CP_INT_CNTL_RING0, tmp);
5879 WREG32(CP_INT_CNTL_RING1, 0);
5880 WREG32(CP_INT_CNTL_RING2, 0);
5881 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5882 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5883 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5884 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5885 WREG32(GRBM_INT_CNTL, 0);
5886 if (rdev->num_crtc >= 2) {
5887 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5888 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5890 if (rdev->num_crtc >= 4) {
5891 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5892 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5894 if (rdev->num_crtc >= 6) {
5895 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5896 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5899 if (rdev->num_crtc >= 2) {
5900 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5901 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5903 if (rdev->num_crtc >= 4) {
5904 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5905 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5907 if (rdev->num_crtc >= 6) {
5908 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5909 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5912 if (!ASIC_IS_NODCE(rdev)) {
5913 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5915 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5916 WREG32(DC_HPD1_INT_CONTROL, tmp);
5917 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5918 WREG32(DC_HPD2_INT_CONTROL, tmp);
5919 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5920 WREG32(DC_HPD3_INT_CONTROL, tmp);
5921 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5922 WREG32(DC_HPD4_INT_CONTROL, tmp);
5923 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5924 WREG32(DC_HPD5_INT_CONTROL, tmp);
5925 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5926 WREG32(DC_HPD6_INT_CONTROL, tmp);
5930 static int si_irq_init(struct radeon_device *rdev)
5934 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5937 ret = r600_ih_ring_alloc(rdev);
5942 si_disable_interrupts(rdev);
5945 ret = si_rlc_resume(rdev);
5947 r600_ih_ring_fini(rdev);
5951 /* setup interrupt control */
5952 /* set dummy read address to ring address */
5953 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
5954 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5955 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5956 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5958 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5959 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5960 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5961 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5963 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5964 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5966 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5967 IH_WPTR_OVERFLOW_CLEAR |
5970 if (rdev->wb.enabled)
5971 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
5973 /* set the writeback address whether it's enabled or not */
5974 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
5975 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
5977 WREG32(IH_RB_CNTL, ih_rb_cntl);
5979 /* set rptr, wptr to 0 */
5980 WREG32(IH_RB_RPTR, 0);
5981 WREG32(IH_RB_WPTR, 0);
5983 /* Default settings for IH_CNTL (disabled at first) */
5984 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
5985 /* RPTR_REARM only works if msi's are enabled */
5986 if (rdev->msi_enabled)
5987 ih_cntl |= RPTR_REARM;
5988 WREG32(IH_CNTL, ih_cntl);
5990 /* force the active interrupt state to all disabled */
5991 si_disable_interrupt_state(rdev);
5993 pci_set_master(rdev->pdev);
5996 si_enable_interrupts(rdev);
6001 int si_irq_set(struct radeon_device *rdev)
6004 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
6005 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6006 u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
6007 u32 grbm_int_cntl = 0;
6008 u32 dma_cntl, dma_cntl1;
6009 u32 thermal_int = 0;
6011 if (!rdev->irq.installed) {
6012 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6015 /* don't enable anything if the ih is disabled */
6016 if (!rdev->ih.enabled) {
6017 si_disable_interrupts(rdev);
6018 /* force the active interrupt state to all disabled */
6019 si_disable_interrupt_state(rdev);
6023 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6024 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6026 if (!ASIC_IS_NODCE(rdev)) {
6027 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6028 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6029 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6030 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6031 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6032 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6035 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6036 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6038 thermal_int = RREG32(CG_THERMAL_INT) &
6039 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6041 /* enable CP interrupts on all rings */
6042 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6043 DRM_DEBUG("si_irq_set: sw int gfx\n");
6044 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6046 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6047 DRM_DEBUG("si_irq_set: sw int cp1\n");
6048 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6050 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6051 DRM_DEBUG("si_irq_set: sw int cp2\n");
6052 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6054 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6055 DRM_DEBUG("si_irq_set: sw int dma\n");
6056 dma_cntl |= TRAP_ENABLE;
6059 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6060 DRM_DEBUG("si_irq_set: sw int dma1\n");
6061 dma_cntl1 |= TRAP_ENABLE;
6063 if (rdev->irq.crtc_vblank_int[0] ||
6064 atomic_read(&rdev->irq.pflip[0])) {
6065 DRM_DEBUG("si_irq_set: vblank 0\n");
6066 crtc1 |= VBLANK_INT_MASK;
6068 if (rdev->irq.crtc_vblank_int[1] ||
6069 atomic_read(&rdev->irq.pflip[1])) {
6070 DRM_DEBUG("si_irq_set: vblank 1\n");
6071 crtc2 |= VBLANK_INT_MASK;
6073 if (rdev->irq.crtc_vblank_int[2] ||
6074 atomic_read(&rdev->irq.pflip[2])) {
6075 DRM_DEBUG("si_irq_set: vblank 2\n");
6076 crtc3 |= VBLANK_INT_MASK;
6078 if (rdev->irq.crtc_vblank_int[3] ||
6079 atomic_read(&rdev->irq.pflip[3])) {
6080 DRM_DEBUG("si_irq_set: vblank 3\n");
6081 crtc4 |= VBLANK_INT_MASK;
6083 if (rdev->irq.crtc_vblank_int[4] ||
6084 atomic_read(&rdev->irq.pflip[4])) {
6085 DRM_DEBUG("si_irq_set: vblank 4\n");
6086 crtc5 |= VBLANK_INT_MASK;
6088 if (rdev->irq.crtc_vblank_int[5] ||
6089 atomic_read(&rdev->irq.pflip[5])) {
6090 DRM_DEBUG("si_irq_set: vblank 5\n");
6091 crtc6 |= VBLANK_INT_MASK;
6093 if (rdev->irq.hpd[0]) {
6094 DRM_DEBUG("si_irq_set: hpd 1\n");
6095 hpd1 |= DC_HPDx_INT_EN;
6097 if (rdev->irq.hpd[1]) {
6098 DRM_DEBUG("si_irq_set: hpd 2\n");
6099 hpd2 |= DC_HPDx_INT_EN;
6101 if (rdev->irq.hpd[2]) {
6102 DRM_DEBUG("si_irq_set: hpd 3\n");
6103 hpd3 |= DC_HPDx_INT_EN;
6105 if (rdev->irq.hpd[3]) {
6106 DRM_DEBUG("si_irq_set: hpd 4\n");
6107 hpd4 |= DC_HPDx_INT_EN;
6109 if (rdev->irq.hpd[4]) {
6110 DRM_DEBUG("si_irq_set: hpd 5\n");
6111 hpd5 |= DC_HPDx_INT_EN;
6113 if (rdev->irq.hpd[5]) {
6114 DRM_DEBUG("si_irq_set: hpd 6\n");
6115 hpd6 |= DC_HPDx_INT_EN;
6118 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6119 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6120 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6122 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6123 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6125 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6127 if (rdev->irq.dpm_thermal) {
6128 DRM_DEBUG("dpm thermal\n");
6129 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6132 if (rdev->num_crtc >= 2) {
6133 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6134 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6136 if (rdev->num_crtc >= 4) {
6137 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6138 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6140 if (rdev->num_crtc >= 6) {
6141 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6142 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6145 if (rdev->num_crtc >= 2) {
6146 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
6147 GRPH_PFLIP_INT_MASK);
6148 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
6149 GRPH_PFLIP_INT_MASK);
6151 if (rdev->num_crtc >= 4) {
6152 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
6153 GRPH_PFLIP_INT_MASK);
6154 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
6155 GRPH_PFLIP_INT_MASK);
6157 if (rdev->num_crtc >= 6) {
6158 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
6159 GRPH_PFLIP_INT_MASK);
6160 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
6161 GRPH_PFLIP_INT_MASK);
6164 if (!ASIC_IS_NODCE(rdev)) {
6165 WREG32(DC_HPD1_INT_CONTROL, hpd1);
6166 WREG32(DC_HPD2_INT_CONTROL, hpd2);
6167 WREG32(DC_HPD3_INT_CONTROL, hpd3);
6168 WREG32(DC_HPD4_INT_CONTROL, hpd4);
6169 WREG32(DC_HPD5_INT_CONTROL, hpd5);
6170 WREG32(DC_HPD6_INT_CONTROL, hpd6);
6173 WREG32(CG_THERMAL_INT, thermal_int);
6178 static inline void si_irq_ack(struct radeon_device *rdev)
6182 if (ASIC_IS_NODCE(rdev))
6185 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6186 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6187 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6188 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6189 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6190 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6191 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
6192 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
6193 if (rdev->num_crtc >= 4) {
6194 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
6195 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
6197 if (rdev->num_crtc >= 6) {
6198 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
6199 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
6202 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
6203 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6204 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
6205 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6206 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
6207 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6208 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
6209 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6210 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6211 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6212 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6213 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6215 if (rdev->num_crtc >= 4) {
6216 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
6217 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6218 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
6219 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6220 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6221 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6222 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6223 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6224 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6225 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6226 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6227 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6230 if (rdev->num_crtc >= 6) {
6231 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
6232 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6233 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
6234 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6235 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6236 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6237 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6238 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6239 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6240 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6241 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6242 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6245 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6246 tmp = RREG32(DC_HPD1_INT_CONTROL);
6247 tmp |= DC_HPDx_INT_ACK;
6248 WREG32(DC_HPD1_INT_CONTROL, tmp);
6250 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6251 tmp = RREG32(DC_HPD2_INT_CONTROL);
6252 tmp |= DC_HPDx_INT_ACK;
6253 WREG32(DC_HPD2_INT_CONTROL, tmp);
6255 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6256 tmp = RREG32(DC_HPD3_INT_CONTROL);
6257 tmp |= DC_HPDx_INT_ACK;
6258 WREG32(DC_HPD3_INT_CONTROL, tmp);
6260 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6261 tmp = RREG32(DC_HPD4_INT_CONTROL);
6262 tmp |= DC_HPDx_INT_ACK;
6263 WREG32(DC_HPD4_INT_CONTROL, tmp);
6265 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6266 tmp = RREG32(DC_HPD5_INT_CONTROL);
6267 tmp |= DC_HPDx_INT_ACK;
6268 WREG32(DC_HPD5_INT_CONTROL, tmp);
6270 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6271 tmp = RREG32(DC_HPD5_INT_CONTROL);
6272 tmp |= DC_HPDx_INT_ACK;
6273 WREG32(DC_HPD6_INT_CONTROL, tmp);
6277 static void si_irq_disable(struct radeon_device *rdev)
6279 si_disable_interrupts(rdev);
6280 /* Wait and acknowledge irq */
6283 si_disable_interrupt_state(rdev);
6286 static void si_irq_suspend(struct radeon_device *rdev)
6288 si_irq_disable(rdev);
6292 static void si_irq_fini(struct radeon_device *rdev)
6294 si_irq_suspend(rdev);
6295 r600_ih_ring_fini(rdev);
6298 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6302 if (rdev->wb.enabled)
6303 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6305 wptr = RREG32(IH_RB_WPTR);
6307 if (wptr & RB_OVERFLOW) {
6308 /* When a ring buffer overflow happen start parsing interrupt
6309 * from the last not overwritten vector (wptr + 16). Hopefully
6310 * this should allow us to catchup.
6312 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
6313 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
6314 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6315 tmp = RREG32(IH_RB_CNTL);
6316 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6317 WREG32(IH_RB_CNTL, tmp);
6318 wptr &= ~RB_OVERFLOW;
6320 return (wptr & rdev->ih.ptr_mask);
6324 * Each IV ring entry is 128 bits:
6325 * [7:0] - interrupt source id
6327 * [59:32] - interrupt source data
6328 * [63:60] - reserved
6331 * [127:80] - reserved
6333 int si_irq_process(struct radeon_device *rdev)
6337 u32 src_id, src_data, ring_id;
6339 bool queue_hotplug = false;
6340 bool queue_thermal = false;
6343 if (!rdev->ih.enabled || rdev->shutdown)
6346 wptr = si_get_ih_wptr(rdev);
6349 /* is somebody else already processing irqs? */
6350 if (atomic_xchg(&rdev->ih.lock, 1))
6353 rptr = rdev->ih.rptr;
6354 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6356 /* Order reading of wptr vs. reading of IH ring data */
6359 /* display interrupts */
6362 while (rptr != wptr) {
6363 /* wptr/rptr are in bytes! */
6364 ring_index = rptr / 4;
6365 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6366 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6367 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6370 case 1: /* D1 vblank/vline */
6372 case 0: /* D1 vblank */
6373 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
6374 if (rdev->irq.crtc_vblank_int[0]) {
6375 drm_handle_vblank(rdev->ddev, 0);
6376 rdev->pm.vblank_sync = true;
6377 wake_up(&rdev->irq.vblank_queue);
6379 if (atomic_read(&rdev->irq.pflip[0]))
6380 radeon_crtc_handle_vblank(rdev, 0);
6381 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6382 DRM_DEBUG("IH: D1 vblank\n");
6385 case 1: /* D1 vline */
6386 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
6387 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6388 DRM_DEBUG("IH: D1 vline\n");
6392 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6396 case 2: /* D2 vblank/vline */
6398 case 0: /* D2 vblank */
6399 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
6400 if (rdev->irq.crtc_vblank_int[1]) {
6401 drm_handle_vblank(rdev->ddev, 1);
6402 rdev->pm.vblank_sync = true;
6403 wake_up(&rdev->irq.vblank_queue);
6405 if (atomic_read(&rdev->irq.pflip[1]))
6406 radeon_crtc_handle_vblank(rdev, 1);
6407 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6408 DRM_DEBUG("IH: D2 vblank\n");
6411 case 1: /* D2 vline */
6412 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
6413 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6414 DRM_DEBUG("IH: D2 vline\n");
6418 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6422 case 3: /* D3 vblank/vline */
6424 case 0: /* D3 vblank */
6425 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
6426 if (rdev->irq.crtc_vblank_int[2]) {
6427 drm_handle_vblank(rdev->ddev, 2);
6428 rdev->pm.vblank_sync = true;
6429 wake_up(&rdev->irq.vblank_queue);
6431 if (atomic_read(&rdev->irq.pflip[2]))
6432 radeon_crtc_handle_vblank(rdev, 2);
6433 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6434 DRM_DEBUG("IH: D3 vblank\n");
6437 case 1: /* D3 vline */
6438 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
6439 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6440 DRM_DEBUG("IH: D3 vline\n");
6444 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6448 case 4: /* D4 vblank/vline */
6450 case 0: /* D4 vblank */
6451 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
6452 if (rdev->irq.crtc_vblank_int[3]) {
6453 drm_handle_vblank(rdev->ddev, 3);
6454 rdev->pm.vblank_sync = true;
6455 wake_up(&rdev->irq.vblank_queue);
6457 if (atomic_read(&rdev->irq.pflip[3]))
6458 radeon_crtc_handle_vblank(rdev, 3);
6459 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6460 DRM_DEBUG("IH: D4 vblank\n");
6463 case 1: /* D4 vline */
6464 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
6465 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6466 DRM_DEBUG("IH: D4 vline\n");
6470 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6474 case 5: /* D5 vblank/vline */
6476 case 0: /* D5 vblank */
6477 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
6478 if (rdev->irq.crtc_vblank_int[4]) {
6479 drm_handle_vblank(rdev->ddev, 4);
6480 rdev->pm.vblank_sync = true;
6481 wake_up(&rdev->irq.vblank_queue);
6483 if (atomic_read(&rdev->irq.pflip[4]))
6484 radeon_crtc_handle_vblank(rdev, 4);
6485 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6486 DRM_DEBUG("IH: D5 vblank\n");
6489 case 1: /* D5 vline */
6490 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
6491 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6492 DRM_DEBUG("IH: D5 vline\n");
6496 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6500 case 6: /* D6 vblank/vline */
6502 case 0: /* D6 vblank */
6503 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
6504 if (rdev->irq.crtc_vblank_int[5]) {
6505 drm_handle_vblank(rdev->ddev, 5);
6506 rdev->pm.vblank_sync = true;
6507 wake_up(&rdev->irq.vblank_queue);
6509 if (atomic_read(&rdev->irq.pflip[5]))
6510 radeon_crtc_handle_vblank(rdev, 5);
6511 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6512 DRM_DEBUG("IH: D6 vblank\n");
6515 case 1: /* D6 vline */
6516 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
6517 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6518 DRM_DEBUG("IH: D6 vline\n");
6522 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6526 case 8: /* D1 page flip */
6527 case 10: /* D2 page flip */
6528 case 12: /* D3 page flip */
6529 case 14: /* D4 page flip */
6530 case 16: /* D5 page flip */
6531 case 18: /* D6 page flip */
6532 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6533 if (radeon_use_pflipirq > 0)
6534 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6536 case 42: /* HPD hotplug */
6539 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6540 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6541 queue_hotplug = true;
6542 DRM_DEBUG("IH: HPD1\n");
6546 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6547 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6548 queue_hotplug = true;
6549 DRM_DEBUG("IH: HPD2\n");
6553 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6554 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6555 queue_hotplug = true;
6556 DRM_DEBUG("IH: HPD3\n");
6560 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6561 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6562 queue_hotplug = true;
6563 DRM_DEBUG("IH: HPD4\n");
6567 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6568 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6569 queue_hotplug = true;
6570 DRM_DEBUG("IH: HPD5\n");
6574 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6575 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6576 queue_hotplug = true;
6577 DRM_DEBUG("IH: HPD6\n");
6581 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6586 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6587 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6591 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6592 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6593 /* reset addr and status */
6594 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6595 if (addr == 0x0 && status == 0x0)
6597 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6598 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6600 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6602 si_vm_decode_fault(rdev, status, addr);
6604 case 176: /* RINGID0 CP_INT */
6605 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6607 case 177: /* RINGID1 CP_INT */
6608 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6610 case 178: /* RINGID2 CP_INT */
6611 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6613 case 181: /* CP EOP event */
6614 DRM_DEBUG("IH: CP EOP\n");
6617 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6620 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6623 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6627 case 224: /* DMA trap event */
6628 DRM_DEBUG("IH: DMA trap\n");
6629 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6631 case 230: /* thermal low to high */
6632 DRM_DEBUG("IH: thermal low to high\n");
6633 rdev->pm.dpm.thermal.high_to_low = false;
6634 queue_thermal = true;
6636 case 231: /* thermal high to low */
6637 DRM_DEBUG("IH: thermal high to low\n");
6638 rdev->pm.dpm.thermal.high_to_low = true;
6639 queue_thermal = true;
6641 case 233: /* GUI IDLE */
6642 DRM_DEBUG("IH: GUI idle\n");
6644 case 244: /* DMA trap event */
6645 DRM_DEBUG("IH: DMA1 trap\n");
6646 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6649 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6653 /* wptr/rptr are in bytes! */
6655 rptr &= rdev->ih.ptr_mask;
6658 schedule_work(&rdev->hotplug_work);
6659 if (queue_thermal && rdev->pm.dpm_enabled)
6660 schedule_work(&rdev->pm.dpm.thermal.work);
6661 rdev->ih.rptr = rptr;
6662 WREG32(IH_RB_RPTR, rdev->ih.rptr);
6663 atomic_set(&rdev->ih.lock, 0);
6665 /* make sure wptr hasn't changed while processing */
6666 wptr = si_get_ih_wptr(rdev);
6674 * startup/shutdown callbacks
6676 static int si_startup(struct radeon_device *rdev)
6678 struct radeon_ring *ring;
6681 /* enable pcie gen2/3 link */
6682 si_pcie_gen3_enable(rdev);
6684 si_program_aspm(rdev);
6686 /* scratch needs to be initialized before MC */
6687 r = r600_vram_scratch_init(rdev);
6691 si_mc_program(rdev);
6693 if (!rdev->pm.dpm_enabled) {
6694 r = si_mc_load_microcode(rdev);
6696 DRM_ERROR("Failed to load MC firmware!\n");
6701 r = si_pcie_gart_enable(rdev);
6706 /* allocate rlc buffers */
6707 if (rdev->family == CHIP_VERDE) {
6708 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6709 rdev->rlc.reg_list_size =
6710 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6712 rdev->rlc.cs_data = si_cs_data;
6713 r = sumo_rlc_init(rdev);
6715 DRM_ERROR("Failed to init rlc BOs!\n");
6719 /* allocate wb buffer */
6720 r = radeon_wb_init(rdev);
6724 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6726 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6730 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6732 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6736 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6738 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6742 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6744 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6748 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6750 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6754 if (rdev->has_uvd) {
6755 r = uvd_v2_2_resume(rdev);
6757 r = radeon_fence_driver_start_ring(rdev,
6758 R600_RING_TYPE_UVD_INDEX);
6760 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6763 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6767 if (!rdev->irq.installed) {
6768 r = radeon_irq_kms_init(rdev);
6773 r = si_irq_init(rdev);
6775 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6776 radeon_irq_kms_fini(rdev);
6781 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6782 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6787 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6788 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6793 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6794 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6799 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6800 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6801 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6805 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6806 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6807 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6811 r = si_cp_load_microcode(rdev);
6814 r = si_cp_resume(rdev);
6818 r = cayman_dma_resume(rdev);
6822 if (rdev->has_uvd) {
6823 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6824 if (ring->ring_size) {
6825 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
6828 r = uvd_v1_0_init(rdev);
6830 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
6834 r = radeon_ib_pool_init(rdev);
6836 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6840 r = radeon_vm_manager_init(rdev);
6842 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6846 r = dce6_audio_init(rdev);
6853 int si_resume(struct radeon_device *rdev)
6857 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6858 * posting will perform necessary task to bring back GPU into good
6862 atom_asic_init(rdev->mode_info.atom_context);
6864 /* init golden registers */
6865 si_init_golden_registers(rdev);
6867 if (rdev->pm.pm_method == PM_METHOD_DPM)
6868 radeon_pm_resume(rdev);
6870 rdev->accel_working = true;
6871 r = si_startup(rdev);
6873 DRM_ERROR("si startup failed on resume\n");
6874 rdev->accel_working = false;
6882 int si_suspend(struct radeon_device *rdev)
6884 radeon_pm_suspend(rdev);
6885 dce6_audio_fini(rdev);
6886 radeon_vm_manager_fini(rdev);
6887 si_cp_enable(rdev, false);
6888 cayman_dma_stop(rdev);
6889 if (rdev->has_uvd) {
6890 uvd_v1_0_fini(rdev);
6891 radeon_uvd_suspend(rdev);
6895 si_irq_suspend(rdev);
6896 radeon_wb_disable(rdev);
6897 si_pcie_gart_disable(rdev);
6901 /* Plan is to move initialization in that function and use
6902 * helper function so that radeon_device_init pretty much
6903 * do nothing more than calling asic specific function. This
6904 * should also allow to remove a bunch of callback function
6907 int si_init(struct radeon_device *rdev)
6909 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6913 if (!radeon_get_bios(rdev)) {
6914 if (ASIC_IS_AVIVO(rdev))
6917 /* Must be an ATOMBIOS */
6918 if (!rdev->is_atom_bios) {
6919 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6922 r = radeon_atombios_init(rdev);
6926 /* Post card if necessary */
6927 if (!radeon_card_posted(rdev)) {
6929 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6932 DRM_INFO("GPU not posted. posting now...\n");
6933 atom_asic_init(rdev->mode_info.atom_context);
6935 /* init golden registers */
6936 si_init_golden_registers(rdev);
6937 /* Initialize scratch registers */
6938 si_scratch_init(rdev);
6939 /* Initialize surface registers */
6940 radeon_surface_init(rdev);
6941 /* Initialize clocks */
6942 radeon_get_clock_info(rdev->ddev);
6945 r = radeon_fence_driver_init(rdev);
6949 /* initialize memory controller */
6950 r = si_mc_init(rdev);
6953 /* Memory manager */
6954 r = radeon_bo_init(rdev);
6958 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6959 !rdev->rlc_fw || !rdev->mc_fw) {
6960 r = si_init_microcode(rdev);
6962 DRM_ERROR("Failed to load firmware!\n");
6967 /* Initialize power management */
6968 radeon_pm_init(rdev);
6970 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6971 ring->ring_obj = NULL;
6972 r600_ring_init(rdev, ring, 1024 * 1024);
6974 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6975 ring->ring_obj = NULL;
6976 r600_ring_init(rdev, ring, 1024 * 1024);
6978 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6979 ring->ring_obj = NULL;
6980 r600_ring_init(rdev, ring, 1024 * 1024);
6982 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6983 ring->ring_obj = NULL;
6984 r600_ring_init(rdev, ring, 64 * 1024);
6986 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6987 ring->ring_obj = NULL;
6988 r600_ring_init(rdev, ring, 64 * 1024);
6990 if (rdev->has_uvd) {
6991 r = radeon_uvd_init(rdev);
6993 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6994 ring->ring_obj = NULL;
6995 r600_ring_init(rdev, ring, 4096);
6999 rdev->ih.ring_obj = NULL;
7000 r600_ih_ring_init(rdev, 64 * 1024);
7002 r = r600_pcie_gart_init(rdev);
7006 rdev->accel_working = true;
7007 r = si_startup(rdev);
7009 dev_err(rdev->dev, "disabling GPU acceleration\n");
7011 cayman_dma_fini(rdev);
7013 sumo_rlc_fini(rdev);
7014 radeon_wb_fini(rdev);
7015 radeon_ib_pool_fini(rdev);
7016 radeon_vm_manager_fini(rdev);
7017 radeon_irq_kms_fini(rdev);
7018 si_pcie_gart_fini(rdev);
7019 rdev->accel_working = false;
7022 /* Don't start up if the MC ucode is missing.
7023 * The default clocks and voltages before the MC ucode
7024 * is loaded are not suffient for advanced operations.
7027 DRM_ERROR("radeon: MC ucode required for NI+.\n");
7034 void si_fini(struct radeon_device *rdev)
7036 radeon_pm_fini(rdev);
7038 cayman_dma_fini(rdev);
7042 sumo_rlc_fini(rdev);
7043 radeon_wb_fini(rdev);
7044 radeon_vm_manager_fini(rdev);
7045 radeon_ib_pool_fini(rdev);
7046 radeon_irq_kms_fini(rdev);
7047 if (rdev->has_uvd) {
7048 uvd_v1_0_fini(rdev);
7049 radeon_uvd_fini(rdev);
7051 si_pcie_gart_fini(rdev);
7052 r600_vram_scratch_fini(rdev);
7053 radeon_gem_fini(rdev);
7054 radeon_fence_driver_fini(rdev);
7055 radeon_bo_fini(rdev);
7056 radeon_atombios_fini(rdev);
7062 * si_get_gpu_clock_counter - return GPU clock counter snapshot
7064 * @rdev: radeon_device pointer
7066 * Fetches a GPU clock counter snapshot (SI).
7067 * Returns the 64 bit clock counter snapshot.
7069 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
7073 mutex_lock(&rdev->gpu_clock_mutex);
7074 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
7075 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
7076 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
7077 mutex_unlock(&rdev->gpu_clock_mutex);
7081 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
7083 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
7086 /* bypass vclk and dclk with bclk */
7087 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7088 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
7089 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7091 /* put PLL in bypass mode */
7092 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
7094 if (!vclk || !dclk) {
7095 /* keep the Bypass mode, put PLL to sleep */
7096 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
7100 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
7101 16384, 0x03FFFFFF, 0, 128, 5,
7102 &fb_div, &vclk_div, &dclk_div);
7106 /* set RESET_ANTI_MUX to 0 */
7107 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7109 /* set VCO_MODE to 1 */
7110 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7112 /* toggle UPLL_SLEEP to 1 then back to 0 */
7113 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
7114 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7116 /* deassert UPLL_RESET */
7117 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7121 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7125 /* assert UPLL_RESET again */
7126 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7128 /* disable spread spectrum. */
7129 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7131 /* set feedback divider */
7132 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
7134 /* set ref divider to 0 */
7135 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7137 if (fb_div < 307200)
7138 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7140 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7142 /* set PDIV_A and PDIV_B */
7143 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7144 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
7145 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7147 /* give the PLL some time to settle */
7150 /* deassert PLL_RESET */
7151 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7155 /* switch from bypass mode to normal mode */
7156 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7158 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7162 /* switch VCLK and DCLK selection */
7163 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7164 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7165 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7172 static void si_pcie_gen3_enable(struct radeon_device *rdev)
7174 struct pci_dev *root = rdev->pdev->bus->self;
7175 int bridge_pos, gpu_pos;
7176 u32 speed_cntl, mask, current_data_rate;
7180 if (radeon_pcie_gen2 == 0)
7183 if (rdev->flags & RADEON_IS_IGP)
7186 if (!(rdev->flags & RADEON_IS_PCIE))
7189 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
7193 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
7196 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7197 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7198 LC_CURRENT_DATA_RATE_SHIFT;
7199 if (mask & DRM_PCIE_SPEED_80) {
7200 if (current_data_rate == 2) {
7201 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7204 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7205 } else if (mask & DRM_PCIE_SPEED_50) {
7206 if (current_data_rate == 1) {
7207 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7210 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7213 bridge_pos = pci_pcie_cap(root);
7217 gpu_pos = pci_pcie_cap(rdev->pdev);
7221 if (mask & DRM_PCIE_SPEED_80) {
7222 /* re-try equalization if gen3 is not already enabled */
7223 if (current_data_rate != 2) {
7224 u16 bridge_cfg, gpu_cfg;
7225 u16 bridge_cfg2, gpu_cfg2;
7226 u32 max_lw, current_lw, tmp;
7228 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7229 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7231 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
7232 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7234 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
7235 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7237 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7238 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7239 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7241 if (current_lw < max_lw) {
7242 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7243 if (tmp & LC_RENEGOTIATION_SUPPORT) {
7244 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7245 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7246 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7247 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7251 for (i = 0; i < 10; i++) {
7253 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
7254 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7257 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7258 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7260 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7261 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7263 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7264 tmp |= LC_SET_QUIESCE;
7265 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7267 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7269 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7274 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7275 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7276 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7277 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7279 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7280 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7281 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7282 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7285 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7286 tmp16 &= ~((1 << 4) | (7 << 9));
7287 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7288 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7290 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7291 tmp16 &= ~((1 << 4) | (7 << 9));
7292 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7293 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7295 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7296 tmp &= ~LC_SET_QUIESCE;
7297 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7302 /* set the link speed */
7303 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7304 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7305 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7307 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7309 if (mask & DRM_PCIE_SPEED_80)
7310 tmp16 |= 3; /* gen3 */
7311 else if (mask & DRM_PCIE_SPEED_50)
7312 tmp16 |= 2; /* gen2 */
7314 tmp16 |= 1; /* gen1 */
7315 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7317 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7318 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7319 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7321 for (i = 0; i < rdev->usec_timeout; i++) {
7322 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7323 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7329 static void si_program_aspm(struct radeon_device *rdev)
7332 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7333 bool disable_clkreq = false;
7335 if (radeon_aspm == 0)
7338 if (!(rdev->flags & RADEON_IS_PCIE))
7341 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7342 data &= ~LC_XMIT_N_FTS_MASK;
7343 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7345 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7347 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7348 data |= LC_GO_TO_RECOVERY;
7350 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7352 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7353 data |= P_IGNORE_EDB_ERR;
7355 WREG32_PCIE(PCIE_P_CNTL, data);
7357 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7358 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7359 data |= LC_PMI_TO_L1_DIS;
7361 data |= LC_L0S_INACTIVITY(7);
7364 data |= LC_L1_INACTIVITY(7);
7365 data &= ~LC_PMI_TO_L1_DIS;
7367 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7369 if (!disable_plloff_in_l1) {
7370 bool clk_req_support;
7372 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7373 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7374 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7376 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7378 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7379 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7380 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7382 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7384 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7385 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7386 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7388 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7390 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7391 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7392 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7394 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7396 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7397 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7398 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7400 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7402 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7403 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7405 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7407 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7408 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7410 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7412 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7413 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7415 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7417 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7418 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7420 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7422 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7423 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7425 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7427 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7428 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7430 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7432 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7433 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7435 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7437 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7438 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7439 data |= LC_DYN_LANES_PWR_STATE(3);
7441 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7443 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7444 data &= ~LS2_EXIT_TIME_MASK;
7445 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7446 data |= LS2_EXIT_TIME(5);
7448 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7450 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7451 data &= ~LS2_EXIT_TIME_MASK;
7452 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7453 data |= LS2_EXIT_TIME(5);
7455 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7457 if (!disable_clkreq) {
7458 struct pci_dev *root = rdev->pdev->bus->self;
7461 clk_req_support = false;
7462 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7463 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7464 clk_req_support = true;
7466 clk_req_support = false;
7469 if (clk_req_support) {
7470 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7471 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7473 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7475 orig = data = RREG32(THM_CLK_CNTL);
7476 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7477 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7479 WREG32(THM_CLK_CNTL, data);
7481 orig = data = RREG32(MISC_CLK_CNTL);
7482 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7483 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7485 WREG32(MISC_CLK_CNTL, data);
7487 orig = data = RREG32(CG_CLKPIN_CNTL);
7488 data &= ~BCLK_AS_XCLK;
7490 WREG32(CG_CLKPIN_CNTL, data);
7492 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7493 data &= ~FORCE_BIF_REFCLK_EN;
7495 WREG32(CG_CLKPIN_CNTL_2, data);
7497 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7498 data &= ~MPLL_CLKOUT_SEL_MASK;
7499 data |= MPLL_CLKOUT_SEL(4);
7501 WREG32(MPLL_BYPASSCLK_SEL, data);
7503 orig = data = RREG32(SPLL_CNTL_MODE);
7504 data &= ~SPLL_REFCLK_SEL_MASK;
7506 WREG32(SPLL_CNTL_MODE, data);
7511 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7514 orig = data = RREG32_PCIE(PCIE_CNTL2);
7515 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7517 WREG32_PCIE(PCIE_CNTL2, data);
7520 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7521 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7522 data = RREG32_PCIE(PCIE_LC_STATUS1);
7523 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7524 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7525 data &= ~LC_L0S_INACTIVITY_MASK;
7527 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
projects / linux-2.6-block.git / blob