2 * Copyright 2012 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
45 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
46 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
47 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
48 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
49 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
50 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
51 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
52 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
53 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
54 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
55 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
56 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
57 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
58 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
59 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
60 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
61 MODULE_FIRMWARE("radeon/KABINI_me.bin");
62 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
63 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
64 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
65 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
67 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
68 extern void r600_ih_ring_fini(struct radeon_device *rdev);
69 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
70 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
71 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
72 extern void sumo_rlc_fini(struct radeon_device *rdev);
73 extern int sumo_rlc_init(struct radeon_device *rdev);
74 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
75 extern void si_rlc_reset(struct radeon_device *rdev);
76 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
77 extern int cik_sdma_resume(struct radeon_device *rdev);
78 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
79 extern void cik_sdma_fini(struct radeon_device *rdev);
80 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
81 static void cik_rlc_stop(struct radeon_device *rdev);
82 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
83 static void cik_program_aspm(struct radeon_device *rdev);
84 static void cik_init_pg(struct radeon_device *rdev);
85 static void cik_init_cg(struct radeon_device *rdev);
86 static void cik_fini_pg(struct radeon_device *rdev);
87 static void cik_fini_cg(struct radeon_device *rdev);
88 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
91 /* get temperature in millidegrees */
92 int ci_get_temp(struct radeon_device *rdev)
97 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
103 actual_temp = temp & 0x1ff;
105 actual_temp = actual_temp * 1000;
110 /* get temperature in millidegrees */
111 int kv_get_temp(struct radeon_device *rdev)
116 temp = RREG32_SMC(0xC0300E0C);
119 actual_temp = (temp / 8) - 49;
123 actual_temp = actual_temp * 1000;
129 * Indirect registers accessor
131 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
136 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
137 WREG32(PCIE_INDEX, reg);
138 (void)RREG32(PCIE_INDEX);
139 r = RREG32(PCIE_DATA);
140 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
144 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
148 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
149 WREG32(PCIE_INDEX, reg);
150 (void)RREG32(PCIE_INDEX);
151 WREG32(PCIE_DATA, v);
152 (void)RREG32(PCIE_DATA);
153 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
156 static const u32 spectre_rlc_save_restore_register_list[] =
158 (0x0e00 << 16) | (0xc12c >> 2),
160 (0x0e00 << 16) | (0xc140 >> 2),
162 (0x0e00 << 16) | (0xc150 >> 2),
164 (0x0e00 << 16) | (0xc15c >> 2),
166 (0x0e00 << 16) | (0xc168 >> 2),
168 (0x0e00 << 16) | (0xc170 >> 2),
170 (0x0e00 << 16) | (0xc178 >> 2),
172 (0x0e00 << 16) | (0xc204 >> 2),
174 (0x0e00 << 16) | (0xc2b4 >> 2),
176 (0x0e00 << 16) | (0xc2b8 >> 2),
178 (0x0e00 << 16) | (0xc2bc >> 2),
180 (0x0e00 << 16) | (0xc2c0 >> 2),
182 (0x0e00 << 16) | (0x8228 >> 2),
184 (0x0e00 << 16) | (0x829c >> 2),
186 (0x0e00 << 16) | (0x869c >> 2),
188 (0x0600 << 16) | (0x98f4 >> 2),
190 (0x0e00 << 16) | (0x98f8 >> 2),
192 (0x0e00 << 16) | (0x9900 >> 2),
194 (0x0e00 << 16) | (0xc260 >> 2),
196 (0x0e00 << 16) | (0x90e8 >> 2),
198 (0x0e00 << 16) | (0x3c000 >> 2),
200 (0x0e00 << 16) | (0x3c00c >> 2),
202 (0x0e00 << 16) | (0x8c1c >> 2),
204 (0x0e00 << 16) | (0x9700 >> 2),
206 (0x0e00 << 16) | (0xcd20 >> 2),
208 (0x4e00 << 16) | (0xcd20 >> 2),
210 (0x5e00 << 16) | (0xcd20 >> 2),
212 (0x6e00 << 16) | (0xcd20 >> 2),
214 (0x7e00 << 16) | (0xcd20 >> 2),
216 (0x8e00 << 16) | (0xcd20 >> 2),
218 (0x9e00 << 16) | (0xcd20 >> 2),
220 (0xae00 << 16) | (0xcd20 >> 2),
222 (0xbe00 << 16) | (0xcd20 >> 2),
224 (0x0e00 << 16) | (0x89bc >> 2),
226 (0x0e00 << 16) | (0x8900 >> 2),
229 (0x0e00 << 16) | (0xc130 >> 2),
231 (0x0e00 << 16) | (0xc134 >> 2),
233 (0x0e00 << 16) | (0xc1fc >> 2),
235 (0x0e00 << 16) | (0xc208 >> 2),
237 (0x0e00 << 16) | (0xc264 >> 2),
239 (0x0e00 << 16) | (0xc268 >> 2),
241 (0x0e00 << 16) | (0xc26c >> 2),
243 (0x0e00 << 16) | (0xc270 >> 2),
245 (0x0e00 << 16) | (0xc274 >> 2),
247 (0x0e00 << 16) | (0xc278 >> 2),
249 (0x0e00 << 16) | (0xc27c >> 2),
251 (0x0e00 << 16) | (0xc280 >> 2),
253 (0x0e00 << 16) | (0xc284 >> 2),
255 (0x0e00 << 16) | (0xc288 >> 2),
257 (0x0e00 << 16) | (0xc28c >> 2),
259 (0x0e00 << 16) | (0xc290 >> 2),
261 (0x0e00 << 16) | (0xc294 >> 2),
263 (0x0e00 << 16) | (0xc298 >> 2),
265 (0x0e00 << 16) | (0xc29c >> 2),
267 (0x0e00 << 16) | (0xc2a0 >> 2),
269 (0x0e00 << 16) | (0xc2a4 >> 2),
271 (0x0e00 << 16) | (0xc2a8 >> 2),
273 (0x0e00 << 16) | (0xc2ac >> 2),
275 (0x0e00 << 16) | (0xc2b0 >> 2),
277 (0x0e00 << 16) | (0x301d0 >> 2),
279 (0x0e00 << 16) | (0x30238 >> 2),
281 (0x0e00 << 16) | (0x30250 >> 2),
283 (0x0e00 << 16) | (0x30254 >> 2),
285 (0x0e00 << 16) | (0x30258 >> 2),
287 (0x0e00 << 16) | (0x3025c >> 2),
289 (0x4e00 << 16) | (0xc900 >> 2),
291 (0x5e00 << 16) | (0xc900 >> 2),
293 (0x6e00 << 16) | (0xc900 >> 2),
295 (0x7e00 << 16) | (0xc900 >> 2),
297 (0x8e00 << 16) | (0xc900 >> 2),
299 (0x9e00 << 16) | (0xc900 >> 2),
301 (0xae00 << 16) | (0xc900 >> 2),
303 (0xbe00 << 16) | (0xc900 >> 2),
305 (0x4e00 << 16) | (0xc904 >> 2),
307 (0x5e00 << 16) | (0xc904 >> 2),
309 (0x6e00 << 16) | (0xc904 >> 2),
311 (0x7e00 << 16) | (0xc904 >> 2),
313 (0x8e00 << 16) | (0xc904 >> 2),
315 (0x9e00 << 16) | (0xc904 >> 2),
317 (0xae00 << 16) | (0xc904 >> 2),
319 (0xbe00 << 16) | (0xc904 >> 2),
321 (0x4e00 << 16) | (0xc908 >> 2),
323 (0x5e00 << 16) | (0xc908 >> 2),
325 (0x6e00 << 16) | (0xc908 >> 2),
327 (0x7e00 << 16) | (0xc908 >> 2),
329 (0x8e00 << 16) | (0xc908 >> 2),
331 (0x9e00 << 16) | (0xc908 >> 2),
333 (0xae00 << 16) | (0xc908 >> 2),
335 (0xbe00 << 16) | (0xc908 >> 2),
337 (0x4e00 << 16) | (0xc90c >> 2),
339 (0x5e00 << 16) | (0xc90c >> 2),
341 (0x6e00 << 16) | (0xc90c >> 2),
343 (0x7e00 << 16) | (0xc90c >> 2),
345 (0x8e00 << 16) | (0xc90c >> 2),
347 (0x9e00 << 16) | (0xc90c >> 2),
349 (0xae00 << 16) | (0xc90c >> 2),
351 (0xbe00 << 16) | (0xc90c >> 2),
353 (0x4e00 << 16) | (0xc910 >> 2),
355 (0x5e00 << 16) | (0xc910 >> 2),
357 (0x6e00 << 16) | (0xc910 >> 2),
359 (0x7e00 << 16) | (0xc910 >> 2),
361 (0x8e00 << 16) | (0xc910 >> 2),
363 (0x9e00 << 16) | (0xc910 >> 2),
365 (0xae00 << 16) | (0xc910 >> 2),
367 (0xbe00 << 16) | (0xc910 >> 2),
369 (0x0e00 << 16) | (0xc99c >> 2),
371 (0x0e00 << 16) | (0x9834 >> 2),
373 (0x0000 << 16) | (0x30f00 >> 2),
375 (0x0001 << 16) | (0x30f00 >> 2),
377 (0x0000 << 16) | (0x30f04 >> 2),
379 (0x0001 << 16) | (0x30f04 >> 2),
381 (0x0000 << 16) | (0x30f08 >> 2),
383 (0x0001 << 16) | (0x30f08 >> 2),
385 (0x0000 << 16) | (0x30f0c >> 2),
387 (0x0001 << 16) | (0x30f0c >> 2),
389 (0x0600 << 16) | (0x9b7c >> 2),
391 (0x0e00 << 16) | (0x8a14 >> 2),
393 (0x0e00 << 16) | (0x8a18 >> 2),
395 (0x0600 << 16) | (0x30a00 >> 2),
397 (0x0e00 << 16) | (0x8bf0 >> 2),
399 (0x0e00 << 16) | (0x8bcc >> 2),
401 (0x0e00 << 16) | (0x8b24 >> 2),
403 (0x0e00 << 16) | (0x30a04 >> 2),
405 (0x0600 << 16) | (0x30a10 >> 2),
407 (0x0600 << 16) | (0x30a14 >> 2),
409 (0x0600 << 16) | (0x30a18 >> 2),
411 (0x0600 << 16) | (0x30a2c >> 2),
413 (0x0e00 << 16) | (0xc700 >> 2),
415 (0x0e00 << 16) | (0xc704 >> 2),
417 (0x0e00 << 16) | (0xc708 >> 2),
419 (0x0e00 << 16) | (0xc768 >> 2),
421 (0x0400 << 16) | (0xc770 >> 2),
423 (0x0400 << 16) | (0xc774 >> 2),
425 (0x0400 << 16) | (0xc778 >> 2),
427 (0x0400 << 16) | (0xc77c >> 2),
429 (0x0400 << 16) | (0xc780 >> 2),
431 (0x0400 << 16) | (0xc784 >> 2),
433 (0x0400 << 16) | (0xc788 >> 2),
435 (0x0400 << 16) | (0xc78c >> 2),
437 (0x0400 << 16) | (0xc798 >> 2),
439 (0x0400 << 16) | (0xc79c >> 2),
441 (0x0400 << 16) | (0xc7a0 >> 2),
443 (0x0400 << 16) | (0xc7a4 >> 2),
445 (0x0400 << 16) | (0xc7a8 >> 2),
447 (0x0400 << 16) | (0xc7ac >> 2),
449 (0x0400 << 16) | (0xc7b0 >> 2),
451 (0x0400 << 16) | (0xc7b4 >> 2),
453 (0x0e00 << 16) | (0x9100 >> 2),
455 (0x0e00 << 16) | (0x3c010 >> 2),
457 (0x0e00 << 16) | (0x92a8 >> 2),
459 (0x0e00 << 16) | (0x92ac >> 2),
461 (0x0e00 << 16) | (0x92b4 >> 2),
463 (0x0e00 << 16) | (0x92b8 >> 2),
465 (0x0e00 << 16) | (0x92bc >> 2),
467 (0x0e00 << 16) | (0x92c0 >> 2),
469 (0x0e00 << 16) | (0x92c4 >> 2),
471 (0x0e00 << 16) | (0x92c8 >> 2),
473 (0x0e00 << 16) | (0x92cc >> 2),
475 (0x0e00 << 16) | (0x92d0 >> 2),
477 (0x0e00 << 16) | (0x8c00 >> 2),
479 (0x0e00 << 16) | (0x8c04 >> 2),
481 (0x0e00 << 16) | (0x8c20 >> 2),
483 (0x0e00 << 16) | (0x8c38 >> 2),
485 (0x0e00 << 16) | (0x8c3c >> 2),
487 (0x0e00 << 16) | (0xae00 >> 2),
489 (0x0e00 << 16) | (0x9604 >> 2),
491 (0x0e00 << 16) | (0xac08 >> 2),
493 (0x0e00 << 16) | (0xac0c >> 2),
495 (0x0e00 << 16) | (0xac10 >> 2),
497 (0x0e00 << 16) | (0xac14 >> 2),
499 (0x0e00 << 16) | (0xac58 >> 2),
501 (0x0e00 << 16) | (0xac68 >> 2),
503 (0x0e00 << 16) | (0xac6c >> 2),
505 (0x0e00 << 16) | (0xac70 >> 2),
507 (0x0e00 << 16) | (0xac74 >> 2),
509 (0x0e00 << 16) | (0xac78 >> 2),
511 (0x0e00 << 16) | (0xac7c >> 2),
513 (0x0e00 << 16) | (0xac80 >> 2),
515 (0x0e00 << 16) | (0xac84 >> 2),
517 (0x0e00 << 16) | (0xac88 >> 2),
519 (0x0e00 << 16) | (0xac8c >> 2),
521 (0x0e00 << 16) | (0x970c >> 2),
523 (0x0e00 << 16) | (0x9714 >> 2),
525 (0x0e00 << 16) | (0x9718 >> 2),
527 (0x0e00 << 16) | (0x971c >> 2),
529 (0x0e00 << 16) | (0x31068 >> 2),
531 (0x4e00 << 16) | (0x31068 >> 2),
533 (0x5e00 << 16) | (0x31068 >> 2),
535 (0x6e00 << 16) | (0x31068 >> 2),
537 (0x7e00 << 16) | (0x31068 >> 2),
539 (0x8e00 << 16) | (0x31068 >> 2),
541 (0x9e00 << 16) | (0x31068 >> 2),
543 (0xae00 << 16) | (0x31068 >> 2),
545 (0xbe00 << 16) | (0x31068 >> 2),
547 (0x0e00 << 16) | (0xcd10 >> 2),
549 (0x0e00 << 16) | (0xcd14 >> 2),
551 (0x0e00 << 16) | (0x88b0 >> 2),
553 (0x0e00 << 16) | (0x88b4 >> 2),
555 (0x0e00 << 16) | (0x88b8 >> 2),
557 (0x0e00 << 16) | (0x88bc >> 2),
559 (0x0400 << 16) | (0x89c0 >> 2),
561 (0x0e00 << 16) | (0x88c4 >> 2),
563 (0x0e00 << 16) | (0x88c8 >> 2),
565 (0x0e00 << 16) | (0x88d0 >> 2),
567 (0x0e00 << 16) | (0x88d4 >> 2),
569 (0x0e00 << 16) | (0x88d8 >> 2),
571 (0x0e00 << 16) | (0x8980 >> 2),
573 (0x0e00 << 16) | (0x30938 >> 2),
575 (0x0e00 << 16) | (0x3093c >> 2),
577 (0x0e00 << 16) | (0x30940 >> 2),
579 (0x0e00 << 16) | (0x89a0 >> 2),
581 (0x0e00 << 16) | (0x30900 >> 2),
583 (0x0e00 << 16) | (0x30904 >> 2),
585 (0x0e00 << 16) | (0x89b4 >> 2),
587 (0x0e00 << 16) | (0x3c210 >> 2),
589 (0x0e00 << 16) | (0x3c214 >> 2),
591 (0x0e00 << 16) | (0x3c218 >> 2),
593 (0x0e00 << 16) | (0x8904 >> 2),
596 (0x0e00 << 16) | (0x8c28 >> 2),
597 (0x0e00 << 16) | (0x8c2c >> 2),
598 (0x0e00 << 16) | (0x8c30 >> 2),
599 (0x0e00 << 16) | (0x8c34 >> 2),
600 (0x0e00 << 16) | (0x9600 >> 2),
603 static const u32 kalindi_rlc_save_restore_register_list[] =
605 (0x0e00 << 16) | (0xc12c >> 2),
607 (0x0e00 << 16) | (0xc140 >> 2),
609 (0x0e00 << 16) | (0xc150 >> 2),
611 (0x0e00 << 16) | (0xc15c >> 2),
613 (0x0e00 << 16) | (0xc168 >> 2),
615 (0x0e00 << 16) | (0xc170 >> 2),
617 (0x0e00 << 16) | (0xc204 >> 2),
619 (0x0e00 << 16) | (0xc2b4 >> 2),
621 (0x0e00 << 16) | (0xc2b8 >> 2),
623 (0x0e00 << 16) | (0xc2bc >> 2),
625 (0x0e00 << 16) | (0xc2c0 >> 2),
627 (0x0e00 << 16) | (0x8228 >> 2),
629 (0x0e00 << 16) | (0x829c >> 2),
631 (0x0e00 << 16) | (0x869c >> 2),
633 (0x0600 << 16) | (0x98f4 >> 2),
635 (0x0e00 << 16) | (0x98f8 >> 2),
637 (0x0e00 << 16) | (0x9900 >> 2),
639 (0x0e00 << 16) | (0xc260 >> 2),
641 (0x0e00 << 16) | (0x90e8 >> 2),
643 (0x0e00 << 16) | (0x3c000 >> 2),
645 (0x0e00 << 16) | (0x3c00c >> 2),
647 (0x0e00 << 16) | (0x8c1c >> 2),
649 (0x0e00 << 16) | (0x9700 >> 2),
651 (0x0e00 << 16) | (0xcd20 >> 2),
653 (0x4e00 << 16) | (0xcd20 >> 2),
655 (0x5e00 << 16) | (0xcd20 >> 2),
657 (0x6e00 << 16) | (0xcd20 >> 2),
659 (0x7e00 << 16) | (0xcd20 >> 2),
661 (0x0e00 << 16) | (0x89bc >> 2),
663 (0x0e00 << 16) | (0x8900 >> 2),
666 (0x0e00 << 16) | (0xc130 >> 2),
668 (0x0e00 << 16) | (0xc134 >> 2),
670 (0x0e00 << 16) | (0xc1fc >> 2),
672 (0x0e00 << 16) | (0xc208 >> 2),
674 (0x0e00 << 16) | (0xc264 >> 2),
676 (0x0e00 << 16) | (0xc268 >> 2),
678 (0x0e00 << 16) | (0xc26c >> 2),
680 (0x0e00 << 16) | (0xc270 >> 2),
682 (0x0e00 << 16) | (0xc274 >> 2),
684 (0x0e00 << 16) | (0xc28c >> 2),
686 (0x0e00 << 16) | (0xc290 >> 2),
688 (0x0e00 << 16) | (0xc294 >> 2),
690 (0x0e00 << 16) | (0xc298 >> 2),
692 (0x0e00 << 16) | (0xc2a0 >> 2),
694 (0x0e00 << 16) | (0xc2a4 >> 2),
696 (0x0e00 << 16) | (0xc2a8 >> 2),
698 (0x0e00 << 16) | (0xc2ac >> 2),
700 (0x0e00 << 16) | (0x301d0 >> 2),
702 (0x0e00 << 16) | (0x30238 >> 2),
704 (0x0e00 << 16) | (0x30250 >> 2),
706 (0x0e00 << 16) | (0x30254 >> 2),
708 (0x0e00 << 16) | (0x30258 >> 2),
710 (0x0e00 << 16) | (0x3025c >> 2),
712 (0x4e00 << 16) | (0xc900 >> 2),
714 (0x5e00 << 16) | (0xc900 >> 2),
716 (0x6e00 << 16) | (0xc900 >> 2),
718 (0x7e00 << 16) | (0xc900 >> 2),
720 (0x4e00 << 16) | (0xc904 >> 2),
722 (0x5e00 << 16) | (0xc904 >> 2),
724 (0x6e00 << 16) | (0xc904 >> 2),
726 (0x7e00 << 16) | (0xc904 >> 2),
728 (0x4e00 << 16) | (0xc908 >> 2),
730 (0x5e00 << 16) | (0xc908 >> 2),
732 (0x6e00 << 16) | (0xc908 >> 2),
734 (0x7e00 << 16) | (0xc908 >> 2),
736 (0x4e00 << 16) | (0xc90c >> 2),
738 (0x5e00 << 16) | (0xc90c >> 2),
740 (0x6e00 << 16) | (0xc90c >> 2),
742 (0x7e00 << 16) | (0xc90c >> 2),
744 (0x4e00 << 16) | (0xc910 >> 2),
746 (0x5e00 << 16) | (0xc910 >> 2),
748 (0x6e00 << 16) | (0xc910 >> 2),
750 (0x7e00 << 16) | (0xc910 >> 2),
752 (0x0e00 << 16) | (0xc99c >> 2),
754 (0x0e00 << 16) | (0x9834 >> 2),
756 (0x0000 << 16) | (0x30f00 >> 2),
758 (0x0000 << 16) | (0x30f04 >> 2),
760 (0x0000 << 16) | (0x30f08 >> 2),
762 (0x0000 << 16) | (0x30f0c >> 2),
764 (0x0600 << 16) | (0x9b7c >> 2),
766 (0x0e00 << 16) | (0x8a14 >> 2),
768 (0x0e00 << 16) | (0x8a18 >> 2),
770 (0x0600 << 16) | (0x30a00 >> 2),
772 (0x0e00 << 16) | (0x8bf0 >> 2),
774 (0x0e00 << 16) | (0x8bcc >> 2),
776 (0x0e00 << 16) | (0x8b24 >> 2),
778 (0x0e00 << 16) | (0x30a04 >> 2),
780 (0x0600 << 16) | (0x30a10 >> 2),
782 (0x0600 << 16) | (0x30a14 >> 2),
784 (0x0600 << 16) | (0x30a18 >> 2),
786 (0x0600 << 16) | (0x30a2c >> 2),
788 (0x0e00 << 16) | (0xc700 >> 2),
790 (0x0e00 << 16) | (0xc704 >> 2),
792 (0x0e00 << 16) | (0xc708 >> 2),
794 (0x0e00 << 16) | (0xc768 >> 2),
796 (0x0400 << 16) | (0xc770 >> 2),
798 (0x0400 << 16) | (0xc774 >> 2),
800 (0x0400 << 16) | (0xc798 >> 2),
802 (0x0400 << 16) | (0xc79c >> 2),
804 (0x0e00 << 16) | (0x9100 >> 2),
806 (0x0e00 << 16) | (0x3c010 >> 2),
808 (0x0e00 << 16) | (0x8c00 >> 2),
810 (0x0e00 << 16) | (0x8c04 >> 2),
812 (0x0e00 << 16) | (0x8c20 >> 2),
814 (0x0e00 << 16) | (0x8c38 >> 2),
816 (0x0e00 << 16) | (0x8c3c >> 2),
818 (0x0e00 << 16) | (0xae00 >> 2),
820 (0x0e00 << 16) | (0x9604 >> 2),
822 (0x0e00 << 16) | (0xac08 >> 2),
824 (0x0e00 << 16) | (0xac0c >> 2),
826 (0x0e00 << 16) | (0xac10 >> 2),
828 (0x0e00 << 16) | (0xac14 >> 2),
830 (0x0e00 << 16) | (0xac58 >> 2),
832 (0x0e00 << 16) | (0xac68 >> 2),
834 (0x0e00 << 16) | (0xac6c >> 2),
836 (0x0e00 << 16) | (0xac70 >> 2),
838 (0x0e00 << 16) | (0xac74 >> 2),
840 (0x0e00 << 16) | (0xac78 >> 2),
842 (0x0e00 << 16) | (0xac7c >> 2),
844 (0x0e00 << 16) | (0xac80 >> 2),
846 (0x0e00 << 16) | (0xac84 >> 2),
848 (0x0e00 << 16) | (0xac88 >> 2),
850 (0x0e00 << 16) | (0xac8c >> 2),
852 (0x0e00 << 16) | (0x970c >> 2),
854 (0x0e00 << 16) | (0x9714 >> 2),
856 (0x0e00 << 16) | (0x9718 >> 2),
858 (0x0e00 << 16) | (0x971c >> 2),
860 (0x0e00 << 16) | (0x31068 >> 2),
862 (0x4e00 << 16) | (0x31068 >> 2),
864 (0x5e00 << 16) | (0x31068 >> 2),
866 (0x6e00 << 16) | (0x31068 >> 2),
868 (0x7e00 << 16) | (0x31068 >> 2),
870 (0x0e00 << 16) | (0xcd10 >> 2),
872 (0x0e00 << 16) | (0xcd14 >> 2),
874 (0x0e00 << 16) | (0x88b0 >> 2),
876 (0x0e00 << 16) | (0x88b4 >> 2),
878 (0x0e00 << 16) | (0x88b8 >> 2),
880 (0x0e00 << 16) | (0x88bc >> 2),
882 (0x0400 << 16) | (0x89c0 >> 2),
884 (0x0e00 << 16) | (0x88c4 >> 2),
886 (0x0e00 << 16) | (0x88c8 >> 2),
888 (0x0e00 << 16) | (0x88d0 >> 2),
890 (0x0e00 << 16) | (0x88d4 >> 2),
892 (0x0e00 << 16) | (0x88d8 >> 2),
894 (0x0e00 << 16) | (0x8980 >> 2),
896 (0x0e00 << 16) | (0x30938 >> 2),
898 (0x0e00 << 16) | (0x3093c >> 2),
900 (0x0e00 << 16) | (0x30940 >> 2),
902 (0x0e00 << 16) | (0x89a0 >> 2),
904 (0x0e00 << 16) | (0x30900 >> 2),
906 (0x0e00 << 16) | (0x30904 >> 2),
908 (0x0e00 << 16) | (0x89b4 >> 2),
910 (0x0e00 << 16) | (0x3e1fc >> 2),
912 (0x0e00 << 16) | (0x3c210 >> 2),
914 (0x0e00 << 16) | (0x3c214 >> 2),
916 (0x0e00 << 16) | (0x3c218 >> 2),
918 (0x0e00 << 16) | (0x8904 >> 2),
921 (0x0e00 << 16) | (0x8c28 >> 2),
922 (0x0e00 << 16) | (0x8c2c >> 2),
923 (0x0e00 << 16) | (0x8c30 >> 2),
924 (0x0e00 << 16) | (0x8c34 >> 2),
925 (0x0e00 << 16) | (0x9600 >> 2),
928 static const u32 bonaire_golden_spm_registers[] =
930 0x30800, 0xe0ffffff, 0xe0000000
933 static const u32 bonaire_golden_common_registers[] =
935 0xc770, 0xffffffff, 0x00000800,
936 0xc774, 0xffffffff, 0x00000800,
937 0xc798, 0xffffffff, 0x00007fbf,
938 0xc79c, 0xffffffff, 0x00007faf
941 static const u32 bonaire_golden_registers[] =
943 0x3354, 0x00000333, 0x00000333,
944 0x3350, 0x000c0fc0, 0x00040200,
945 0x9a10, 0x00010000, 0x00058208,
946 0x3c000, 0xffff1fff, 0x00140000,
947 0x3c200, 0xfdfc0fff, 0x00000100,
948 0x3c234, 0x40000000, 0x40000200,
949 0x9830, 0xffffffff, 0x00000000,
950 0x9834, 0xf00fffff, 0x00000400,
951 0x9838, 0x0002021c, 0x00020200,
952 0xc78, 0x00000080, 0x00000000,
953 0x5bb0, 0x000000f0, 0x00000070,
954 0x5bc0, 0xf0311fff, 0x80300000,
955 0x98f8, 0x73773777, 0x12010001,
956 0x350c, 0x00810000, 0x408af000,
957 0x7030, 0x31000111, 0x00000011,
958 0x2f48, 0x73773777, 0x12010001,
959 0x220c, 0x00007fb6, 0x0021a1b1,
960 0x2210, 0x00007fb6, 0x002021b1,
961 0x2180, 0x00007fb6, 0x00002191,
962 0x2218, 0x00007fb6, 0x002121b1,
963 0x221c, 0x00007fb6, 0x002021b1,
964 0x21dc, 0x00007fb6, 0x00002191,
965 0x21e0, 0x00007fb6, 0x00002191,
966 0x3628, 0x0000003f, 0x0000000a,
967 0x362c, 0x0000003f, 0x0000000a,
968 0x2ae4, 0x00073ffe, 0x000022a2,
969 0x240c, 0x000007ff, 0x00000000,
970 0x8a14, 0xf000003f, 0x00000007,
971 0x8bf0, 0x00002001, 0x00000001,
972 0x8b24, 0xffffffff, 0x00ffffff,
973 0x30a04, 0x0000ff0f, 0x00000000,
974 0x28a4c, 0x07ffffff, 0x06000000,
975 0x4d8, 0x00000fff, 0x00000100,
976 0x3e78, 0x00000001, 0x00000002,
977 0x9100, 0x03000000, 0x0362c688,
978 0x8c00, 0x000000ff, 0x00000001,
979 0xe40, 0x00001fff, 0x00001fff,
980 0x9060, 0x0000007f, 0x00000020,
981 0x9508, 0x00010000, 0x00010000,
982 0xac14, 0x000003ff, 0x000000f3,
983 0xac0c, 0xffffffff, 0x00001032
986 static const u32 bonaire_mgcg_cgcg_init[] =
988 0xc420, 0xffffffff, 0xfffffffc,
989 0x30800, 0xffffffff, 0xe0000000,
990 0x3c2a0, 0xffffffff, 0x00000100,
991 0x3c208, 0xffffffff, 0x00000100,
992 0x3c2c0, 0xffffffff, 0xc0000100,
993 0x3c2c8, 0xffffffff, 0xc0000100,
994 0x3c2c4, 0xffffffff, 0xc0000100,
995 0x55e4, 0xffffffff, 0x00600100,
996 0x3c280, 0xffffffff, 0x00000100,
997 0x3c214, 0xffffffff, 0x06000100,
998 0x3c220, 0xffffffff, 0x00000100,
999 0x3c218, 0xffffffff, 0x06000100,
1000 0x3c204, 0xffffffff, 0x00000100,
1001 0x3c2e0, 0xffffffff, 0x00000100,
1002 0x3c224, 0xffffffff, 0x00000100,
1003 0x3c200, 0xffffffff, 0x00000100,
1004 0x3c230, 0xffffffff, 0x00000100,
1005 0x3c234, 0xffffffff, 0x00000100,
1006 0x3c250, 0xffffffff, 0x00000100,
1007 0x3c254, 0xffffffff, 0x00000100,
1008 0x3c258, 0xffffffff, 0x00000100,
1009 0x3c25c, 0xffffffff, 0x00000100,
1010 0x3c260, 0xffffffff, 0x00000100,
1011 0x3c27c, 0xffffffff, 0x00000100,
1012 0x3c278, 0xffffffff, 0x00000100,
1013 0x3c210, 0xffffffff, 0x06000100,
1014 0x3c290, 0xffffffff, 0x00000100,
1015 0x3c274, 0xffffffff, 0x00000100,
1016 0x3c2b4, 0xffffffff, 0x00000100,
1017 0x3c2b0, 0xffffffff, 0x00000100,
1018 0x3c270, 0xffffffff, 0x00000100,
1019 0x30800, 0xffffffff, 0xe0000000,
1020 0x3c020, 0xffffffff, 0x00010000,
1021 0x3c024, 0xffffffff, 0x00030002,
1022 0x3c028, 0xffffffff, 0x00040007,
1023 0x3c02c, 0xffffffff, 0x00060005,
1024 0x3c030, 0xffffffff, 0x00090008,
1025 0x3c034, 0xffffffff, 0x00010000,
1026 0x3c038, 0xffffffff, 0x00030002,
1027 0x3c03c, 0xffffffff, 0x00040007,
1028 0x3c040, 0xffffffff, 0x00060005,
1029 0x3c044, 0xffffffff, 0x00090008,
1030 0x3c048, 0xffffffff, 0x00010000,
1031 0x3c04c, 0xffffffff, 0x00030002,
1032 0x3c050, 0xffffffff, 0x00040007,
1033 0x3c054, 0xffffffff, 0x00060005,
1034 0x3c058, 0xffffffff, 0x00090008,
1035 0x3c05c, 0xffffffff, 0x00010000,
1036 0x3c060, 0xffffffff, 0x00030002,
1037 0x3c064, 0xffffffff, 0x00040007,
1038 0x3c068, 0xffffffff, 0x00060005,
1039 0x3c06c, 0xffffffff, 0x00090008,
1040 0x3c070, 0xffffffff, 0x00010000,
1041 0x3c074, 0xffffffff, 0x00030002,
1042 0x3c078, 0xffffffff, 0x00040007,
1043 0x3c07c, 0xffffffff, 0x00060005,
1044 0x3c080, 0xffffffff, 0x00090008,
1045 0x3c084, 0xffffffff, 0x00010000,
1046 0x3c088, 0xffffffff, 0x00030002,
1047 0x3c08c, 0xffffffff, 0x00040007,
1048 0x3c090, 0xffffffff, 0x00060005,
1049 0x3c094, 0xffffffff, 0x00090008,
1050 0x3c098, 0xffffffff, 0x00010000,
1051 0x3c09c, 0xffffffff, 0x00030002,
1052 0x3c0a0, 0xffffffff, 0x00040007,
1053 0x3c0a4, 0xffffffff, 0x00060005,
1054 0x3c0a8, 0xffffffff, 0x00090008,
1055 0x3c000, 0xffffffff, 0x96e00200,
1056 0x8708, 0xffffffff, 0x00900100,
1057 0xc424, 0xffffffff, 0x0020003f,
1058 0x38, 0xffffffff, 0x0140001c,
1059 0x3c, 0x000f0000, 0x000f0000,
1060 0x220, 0xffffffff, 0xC060000C,
1061 0x224, 0xc0000fff, 0x00000100,
1062 0xf90, 0xffffffff, 0x00000100,
1063 0xf98, 0x00000101, 0x00000000,
1064 0x20a8, 0xffffffff, 0x00000104,
1065 0x55e4, 0xff000fff, 0x00000100,
1066 0x30cc, 0xc0000fff, 0x00000104,
1067 0xc1e4, 0x00000001, 0x00000001,
1068 0xd00c, 0xff000ff0, 0x00000100,
1069 0xd80c, 0xff000ff0, 0x00000100
1072 static const u32 spectre_golden_spm_registers[] =
1074 0x30800, 0xe0ffffff, 0xe0000000
1077 static const u32 spectre_golden_common_registers[] =
1079 0xc770, 0xffffffff, 0x00000800,
1080 0xc774, 0xffffffff, 0x00000800,
1081 0xc798, 0xffffffff, 0x00007fbf,
1082 0xc79c, 0xffffffff, 0x00007faf
1085 static const u32 spectre_golden_registers[] =
1087 0x3c000, 0xffff1fff, 0x96940200,
1088 0x3c00c, 0xffff0001, 0xff000000,
1089 0x3c200, 0xfffc0fff, 0x00000100,
1090 0x6ed8, 0x00010101, 0x00010000,
1091 0x9834, 0xf00fffff, 0x00000400,
1092 0x9838, 0xfffffffc, 0x00020200,
1093 0x5bb0, 0x000000f0, 0x00000070,
1094 0x5bc0, 0xf0311fff, 0x80300000,
1095 0x98f8, 0x73773777, 0x12010001,
1096 0x9b7c, 0x00ff0000, 0x00fc0000,
1097 0x2f48, 0x73773777, 0x12010001,
1098 0x8a14, 0xf000003f, 0x00000007,
1099 0x8b24, 0xffffffff, 0x00ffffff,
1100 0x28350, 0x3f3f3fff, 0x00000082,
1101 0x28354, 0x0000003f, 0x00000000,
1102 0x3e78, 0x00000001, 0x00000002,
1103 0x913c, 0xffff03df, 0x00000004,
1104 0xc768, 0x00000008, 0x00000008,
1105 0x8c00, 0x000008ff, 0x00000800,
1106 0x9508, 0x00010000, 0x00010000,
1107 0xac0c, 0xffffffff, 0x54763210,
1108 0x214f8, 0x01ff01ff, 0x00000002,
1109 0x21498, 0x007ff800, 0x00200000,
1110 0x2015c, 0xffffffff, 0x00000f40,
1111 0x30934, 0xffffffff, 0x00000001
1114 static const u32 spectre_mgcg_cgcg_init[] =
1116 0xc420, 0xffffffff, 0xfffffffc,
1117 0x30800, 0xffffffff, 0xe0000000,
1118 0x3c2a0, 0xffffffff, 0x00000100,
1119 0x3c208, 0xffffffff, 0x00000100,
1120 0x3c2c0, 0xffffffff, 0x00000100,
1121 0x3c2c8, 0xffffffff, 0x00000100,
1122 0x3c2c4, 0xffffffff, 0x00000100,
1123 0x55e4, 0xffffffff, 0x00600100,
1124 0x3c280, 0xffffffff, 0x00000100,
1125 0x3c214, 0xffffffff, 0x06000100,
1126 0x3c220, 0xffffffff, 0x00000100,
1127 0x3c218, 0xffffffff, 0x06000100,
1128 0x3c204, 0xffffffff, 0x00000100,
1129 0x3c2e0, 0xffffffff, 0x00000100,
1130 0x3c224, 0xffffffff, 0x00000100,
1131 0x3c200, 0xffffffff, 0x00000100,
1132 0x3c230, 0xffffffff, 0x00000100,
1133 0x3c234, 0xffffffff, 0x00000100,
1134 0x3c250, 0xffffffff, 0x00000100,
1135 0x3c254, 0xffffffff, 0x00000100,
1136 0x3c258, 0xffffffff, 0x00000100,
1137 0x3c25c, 0xffffffff, 0x00000100,
1138 0x3c260, 0xffffffff, 0x00000100,
1139 0x3c27c, 0xffffffff, 0x00000100,
1140 0x3c278, 0xffffffff, 0x00000100,
1141 0x3c210, 0xffffffff, 0x06000100,
1142 0x3c290, 0xffffffff, 0x00000100,
1143 0x3c274, 0xffffffff, 0x00000100,
1144 0x3c2b4, 0xffffffff, 0x00000100,
1145 0x3c2b0, 0xffffffff, 0x00000100,
1146 0x3c270, 0xffffffff, 0x00000100,
1147 0x30800, 0xffffffff, 0xe0000000,
1148 0x3c020, 0xffffffff, 0x00010000,
1149 0x3c024, 0xffffffff, 0x00030002,
1150 0x3c028, 0xffffffff, 0x00040007,
1151 0x3c02c, 0xffffffff, 0x00060005,
1152 0x3c030, 0xffffffff, 0x00090008,
1153 0x3c034, 0xffffffff, 0x00010000,
1154 0x3c038, 0xffffffff, 0x00030002,
1155 0x3c03c, 0xffffffff, 0x00040007,
1156 0x3c040, 0xffffffff, 0x00060005,
1157 0x3c044, 0xffffffff, 0x00090008,
1158 0x3c048, 0xffffffff, 0x00010000,
1159 0x3c04c, 0xffffffff, 0x00030002,
1160 0x3c050, 0xffffffff, 0x00040007,
1161 0x3c054, 0xffffffff, 0x00060005,
1162 0x3c058, 0xffffffff, 0x00090008,
1163 0x3c05c, 0xffffffff, 0x00010000,
1164 0x3c060, 0xffffffff, 0x00030002,
1165 0x3c064, 0xffffffff, 0x00040007,
1166 0x3c068, 0xffffffff, 0x00060005,
1167 0x3c06c, 0xffffffff, 0x00090008,
1168 0x3c070, 0xffffffff, 0x00010000,
1169 0x3c074, 0xffffffff, 0x00030002,
1170 0x3c078, 0xffffffff, 0x00040007,
1171 0x3c07c, 0xffffffff, 0x00060005,
1172 0x3c080, 0xffffffff, 0x00090008,
1173 0x3c084, 0xffffffff, 0x00010000,
1174 0x3c088, 0xffffffff, 0x00030002,
1175 0x3c08c, 0xffffffff, 0x00040007,
1176 0x3c090, 0xffffffff, 0x00060005,
1177 0x3c094, 0xffffffff, 0x00090008,
1178 0x3c098, 0xffffffff, 0x00010000,
1179 0x3c09c, 0xffffffff, 0x00030002,
1180 0x3c0a0, 0xffffffff, 0x00040007,
1181 0x3c0a4, 0xffffffff, 0x00060005,
1182 0x3c0a8, 0xffffffff, 0x00090008,
1183 0x3c0ac, 0xffffffff, 0x00010000,
1184 0x3c0b0, 0xffffffff, 0x00030002,
1185 0x3c0b4, 0xffffffff, 0x00040007,
1186 0x3c0b8, 0xffffffff, 0x00060005,
1187 0x3c0bc, 0xffffffff, 0x00090008,
1188 0x3c000, 0xffffffff, 0x96e00200,
1189 0x8708, 0xffffffff, 0x00900100,
1190 0xc424, 0xffffffff, 0x0020003f,
1191 0x38, 0xffffffff, 0x0140001c,
1192 0x3c, 0x000f0000, 0x000f0000,
1193 0x220, 0xffffffff, 0xC060000C,
1194 0x224, 0xc0000fff, 0x00000100,
1195 0xf90, 0xffffffff, 0x00000100,
1196 0xf98, 0x00000101, 0x00000000,
1197 0x20a8, 0xffffffff, 0x00000104,
1198 0x55e4, 0xff000fff, 0x00000100,
1199 0x30cc, 0xc0000fff, 0x00000104,
1200 0xc1e4, 0x00000001, 0x00000001,
1201 0xd00c, 0xff000ff0, 0x00000100,
1202 0xd80c, 0xff000ff0, 0x00000100
1205 static const u32 kalindi_golden_spm_registers[] =
1207 0x30800, 0xe0ffffff, 0xe0000000
1210 static const u32 kalindi_golden_common_registers[] =
1212 0xc770, 0xffffffff, 0x00000800,
1213 0xc774, 0xffffffff, 0x00000800,
1214 0xc798, 0xffffffff, 0x00007fbf,
1215 0xc79c, 0xffffffff, 0x00007faf
1218 static const u32 kalindi_golden_registers[] =
1220 0x3c000, 0xffffdfff, 0x6e944040,
1221 0x55e4, 0xff607fff, 0xfc000100,
1222 0x3c220, 0xff000fff, 0x00000100,
1223 0x3c224, 0xff000fff, 0x00000100,
1224 0x3c200, 0xfffc0fff, 0x00000100,
1225 0x6ed8, 0x00010101, 0x00010000,
1226 0x9830, 0xffffffff, 0x00000000,
1227 0x9834, 0xf00fffff, 0x00000400,
1228 0x5bb0, 0x000000f0, 0x00000070,
1229 0x5bc0, 0xf0311fff, 0x80300000,
1230 0x98f8, 0x73773777, 0x12010001,
1231 0x98fc, 0xffffffff, 0x00000010,
1232 0x9b7c, 0x00ff0000, 0x00fc0000,
1233 0x8030, 0x00001f0f, 0x0000100a,
1234 0x2f48, 0x73773777, 0x12010001,
1235 0x2408, 0x000fffff, 0x000c007f,
1236 0x8a14, 0xf000003f, 0x00000007,
1237 0x8b24, 0x3fff3fff, 0x00ffcfff,
1238 0x30a04, 0x0000ff0f, 0x00000000,
1239 0x28a4c, 0x07ffffff, 0x06000000,
1240 0x4d8, 0x00000fff, 0x00000100,
1241 0x3e78, 0x00000001, 0x00000002,
1242 0xc768, 0x00000008, 0x00000008,
1243 0x8c00, 0x000000ff, 0x00000003,
1244 0x214f8, 0x01ff01ff, 0x00000002,
1245 0x21498, 0x007ff800, 0x00200000,
1246 0x2015c, 0xffffffff, 0x00000f40,
1247 0x88c4, 0x001f3ae3, 0x00000082,
1248 0x88d4, 0x0000001f, 0x00000010,
1249 0x30934, 0xffffffff, 0x00000000
1252 static const u32 kalindi_mgcg_cgcg_init[] =
1254 0xc420, 0xffffffff, 0xfffffffc,
1255 0x30800, 0xffffffff, 0xe0000000,
1256 0x3c2a0, 0xffffffff, 0x00000100,
1257 0x3c208, 0xffffffff, 0x00000100,
1258 0x3c2c0, 0xffffffff, 0x00000100,
1259 0x3c2c8, 0xffffffff, 0x00000100,
1260 0x3c2c4, 0xffffffff, 0x00000100,
1261 0x55e4, 0xffffffff, 0x00600100,
1262 0x3c280, 0xffffffff, 0x00000100,
1263 0x3c214, 0xffffffff, 0x06000100,
1264 0x3c220, 0xffffffff, 0x00000100,
1265 0x3c218, 0xffffffff, 0x06000100,
1266 0x3c204, 0xffffffff, 0x00000100,
1267 0x3c2e0, 0xffffffff, 0x00000100,
1268 0x3c224, 0xffffffff, 0x00000100,
1269 0x3c200, 0xffffffff, 0x00000100,
1270 0x3c230, 0xffffffff, 0x00000100,
1271 0x3c234, 0xffffffff, 0x00000100,
1272 0x3c250, 0xffffffff, 0x00000100,
1273 0x3c254, 0xffffffff, 0x00000100,
1274 0x3c258, 0xffffffff, 0x00000100,
1275 0x3c25c, 0xffffffff, 0x00000100,
1276 0x3c260, 0xffffffff, 0x00000100,
1277 0x3c27c, 0xffffffff, 0x00000100,
1278 0x3c278, 0xffffffff, 0x00000100,
1279 0x3c210, 0xffffffff, 0x06000100,
1280 0x3c290, 0xffffffff, 0x00000100,
1281 0x3c274, 0xffffffff, 0x00000100,
1282 0x3c2b4, 0xffffffff, 0x00000100,
1283 0x3c2b0, 0xffffffff, 0x00000100,
1284 0x3c270, 0xffffffff, 0x00000100,
1285 0x30800, 0xffffffff, 0xe0000000,
1286 0x3c020, 0xffffffff, 0x00010000,
1287 0x3c024, 0xffffffff, 0x00030002,
1288 0x3c028, 0xffffffff, 0x00040007,
1289 0x3c02c, 0xffffffff, 0x00060005,
1290 0x3c030, 0xffffffff, 0x00090008,
1291 0x3c034, 0xffffffff, 0x00010000,
1292 0x3c038, 0xffffffff, 0x00030002,
1293 0x3c03c, 0xffffffff, 0x00040007,
1294 0x3c040, 0xffffffff, 0x00060005,
1295 0x3c044, 0xffffffff, 0x00090008,
1296 0x3c000, 0xffffffff, 0x96e00200,
1297 0x8708, 0xffffffff, 0x00900100,
1298 0xc424, 0xffffffff, 0x0020003f,
1299 0x38, 0xffffffff, 0x0140001c,
1300 0x3c, 0x000f0000, 0x000f0000,
1301 0x220, 0xffffffff, 0xC060000C,
1302 0x224, 0xc0000fff, 0x00000100,
1303 0x20a8, 0xffffffff, 0x00000104,
1304 0x55e4, 0xff000fff, 0x00000100,
1305 0x30cc, 0xc0000fff, 0x00000104,
1306 0xc1e4, 0x00000001, 0x00000001,
1307 0xd00c, 0xff000ff0, 0x00000100,
1308 0xd80c, 0xff000ff0, 0x00000100
1311 static const u32 hawaii_golden_spm_registers[] =
1313 0x30800, 0xe0ffffff, 0xe0000000
1316 static const u32 hawaii_golden_common_registers[] =
1318 0x30800, 0xffffffff, 0xe0000000,
1319 0x28350, 0xffffffff, 0x3a00161a,
1320 0x28354, 0xffffffff, 0x0000002e,
1321 0x9a10, 0xffffffff, 0x00018208,
1322 0x98f8, 0xffffffff, 0x12011003
1325 static const u32 hawaii_golden_registers[] =
1327 0x3354, 0x00000333, 0x00000333,
1328 0x9a10, 0x00010000, 0x00058208,
1329 0x9830, 0xffffffff, 0x00000000,
1330 0x9834, 0xf00fffff, 0x00000400,
1331 0x9838, 0x0002021c, 0x00020200,
1332 0xc78, 0x00000080, 0x00000000,
1333 0x5bb0, 0x000000f0, 0x00000070,
1334 0x5bc0, 0xf0311fff, 0x80300000,
1335 0x350c, 0x00810000, 0x408af000,
1336 0x7030, 0x31000111, 0x00000011,
1337 0x2f48, 0x73773777, 0x12010001,
1338 0x2120, 0x0000007f, 0x0000001b,
1339 0x21dc, 0x00007fb6, 0x00002191,
1340 0x3628, 0x0000003f, 0x0000000a,
1341 0x362c, 0x0000003f, 0x0000000a,
1342 0x2ae4, 0x00073ffe, 0x000022a2,
1343 0x240c, 0x000007ff, 0x00000000,
1344 0x8bf0, 0x00002001, 0x00000001,
1345 0x8b24, 0xffffffff, 0x00ffffff,
1346 0x30a04, 0x0000ff0f, 0x00000000,
1347 0x28a4c, 0x07ffffff, 0x06000000,
1348 0x3e78, 0x00000001, 0x00000002,
1349 0xc768, 0x00000008, 0x00000008,
1350 0xc770, 0x00000f00, 0x00000800,
1351 0xc774, 0x00000f00, 0x00000800,
1352 0xc798, 0x00ffffff, 0x00ff7fbf,
1353 0xc79c, 0x00ffffff, 0x00ff7faf,
1354 0x8c00, 0x000000ff, 0x00000800,
1355 0xe40, 0x00001fff, 0x00001fff,
1356 0x9060, 0x0000007f, 0x00000020,
1357 0x9508, 0x00010000, 0x00010000,
1358 0xae00, 0x00100000, 0x000ff07c,
1359 0xac14, 0x000003ff, 0x0000000f,
1360 0xac10, 0xffffffff, 0x7564fdec,
1361 0xac0c, 0xffffffff, 0x3120b9a8,
1362 0xac08, 0x20000000, 0x0f9c0000
1365 static const u32 hawaii_mgcg_cgcg_init[] =
1367 0xc420, 0xffffffff, 0xfffffffd,
1368 0x30800, 0xffffffff, 0xe0000000,
1369 0x3c2a0, 0xffffffff, 0x00000100,
1370 0x3c208, 0xffffffff, 0x00000100,
1371 0x3c2c0, 0xffffffff, 0x00000100,
1372 0x3c2c8, 0xffffffff, 0x00000100,
1373 0x3c2c4, 0xffffffff, 0x00000100,
1374 0x55e4, 0xffffffff, 0x00200100,
1375 0x3c280, 0xffffffff, 0x00000100,
1376 0x3c214, 0xffffffff, 0x06000100,
1377 0x3c220, 0xffffffff, 0x00000100,
1378 0x3c218, 0xffffffff, 0x06000100,
1379 0x3c204, 0xffffffff, 0x00000100,
1380 0x3c2e0, 0xffffffff, 0x00000100,
1381 0x3c224, 0xffffffff, 0x00000100,
1382 0x3c200, 0xffffffff, 0x00000100,
1383 0x3c230, 0xffffffff, 0x00000100,
1384 0x3c234, 0xffffffff, 0x00000100,
1385 0x3c250, 0xffffffff, 0x00000100,
1386 0x3c254, 0xffffffff, 0x00000100,
1387 0x3c258, 0xffffffff, 0x00000100,
1388 0x3c25c, 0xffffffff, 0x00000100,
1389 0x3c260, 0xffffffff, 0x00000100,
1390 0x3c27c, 0xffffffff, 0x00000100,
1391 0x3c278, 0xffffffff, 0x00000100,
1392 0x3c210, 0xffffffff, 0x06000100,
1393 0x3c290, 0xffffffff, 0x00000100,
1394 0x3c274, 0xffffffff, 0x00000100,
1395 0x3c2b4, 0xffffffff, 0x00000100,
1396 0x3c2b0, 0xffffffff, 0x00000100,
1397 0x3c270, 0xffffffff, 0x00000100,
1398 0x30800, 0xffffffff, 0xe0000000,
1399 0x3c020, 0xffffffff, 0x00010000,
1400 0x3c024, 0xffffffff, 0x00030002,
1401 0x3c028, 0xffffffff, 0x00040007,
1402 0x3c02c, 0xffffffff, 0x00060005,
1403 0x3c030, 0xffffffff, 0x00090008,
1404 0x3c034, 0xffffffff, 0x00010000,
1405 0x3c038, 0xffffffff, 0x00030002,
1406 0x3c03c, 0xffffffff, 0x00040007,
1407 0x3c040, 0xffffffff, 0x00060005,
1408 0x3c044, 0xffffffff, 0x00090008,
1409 0x3c048, 0xffffffff, 0x00010000,
1410 0x3c04c, 0xffffffff, 0x00030002,
1411 0x3c050, 0xffffffff, 0x00040007,
1412 0x3c054, 0xffffffff, 0x00060005,
1413 0x3c058, 0xffffffff, 0x00090008,
1414 0x3c05c, 0xffffffff, 0x00010000,
1415 0x3c060, 0xffffffff, 0x00030002,
1416 0x3c064, 0xffffffff, 0x00040007,
1417 0x3c068, 0xffffffff, 0x00060005,
1418 0x3c06c, 0xffffffff, 0x00090008,
1419 0x3c070, 0xffffffff, 0x00010000,
1420 0x3c074, 0xffffffff, 0x00030002,
1421 0x3c078, 0xffffffff, 0x00040007,
1422 0x3c07c, 0xffffffff, 0x00060005,
1423 0x3c080, 0xffffffff, 0x00090008,
1424 0x3c084, 0xffffffff, 0x00010000,
1425 0x3c088, 0xffffffff, 0x00030002,
1426 0x3c08c, 0xffffffff, 0x00040007,
1427 0x3c090, 0xffffffff, 0x00060005,
1428 0x3c094, 0xffffffff, 0x00090008,
1429 0x3c098, 0xffffffff, 0x00010000,
1430 0x3c09c, 0xffffffff, 0x00030002,
1431 0x3c0a0, 0xffffffff, 0x00040007,
1432 0x3c0a4, 0xffffffff, 0x00060005,
1433 0x3c0a8, 0xffffffff, 0x00090008,
1434 0x3c0ac, 0xffffffff, 0x00010000,
1435 0x3c0b0, 0xffffffff, 0x00030002,
1436 0x3c0b4, 0xffffffff, 0x00040007,
1437 0x3c0b8, 0xffffffff, 0x00060005,
1438 0x3c0bc, 0xffffffff, 0x00090008,
1439 0x3c0c0, 0xffffffff, 0x00010000,
1440 0x3c0c4, 0xffffffff, 0x00030002,
1441 0x3c0c8, 0xffffffff, 0x00040007,
1442 0x3c0cc, 0xffffffff, 0x00060005,
1443 0x3c0d0, 0xffffffff, 0x00090008,
1444 0x3c0d4, 0xffffffff, 0x00010000,
1445 0x3c0d8, 0xffffffff, 0x00030002,
1446 0x3c0dc, 0xffffffff, 0x00040007,
1447 0x3c0e0, 0xffffffff, 0x00060005,
1448 0x3c0e4, 0xffffffff, 0x00090008,
1449 0x3c0e8, 0xffffffff, 0x00010000,
1450 0x3c0ec, 0xffffffff, 0x00030002,
1451 0x3c0f0, 0xffffffff, 0x00040007,
1452 0x3c0f4, 0xffffffff, 0x00060005,
1453 0x3c0f8, 0xffffffff, 0x00090008,
1454 0xc318, 0xffffffff, 0x00020200,
1455 0x3350, 0xffffffff, 0x00000200,
1456 0x15c0, 0xffffffff, 0x00000400,
1457 0x55e8, 0xffffffff, 0x00000000,
1458 0x2f50, 0xffffffff, 0x00000902,
1459 0x3c000, 0xffffffff, 0x96940200,
1460 0x8708, 0xffffffff, 0x00900100,
1461 0xc424, 0xffffffff, 0x0020003f,
1462 0x38, 0xffffffff, 0x0140001c,
1463 0x3c, 0x000f0000, 0x000f0000,
1464 0x220, 0xffffffff, 0xc060000c,
1465 0x224, 0xc0000fff, 0x00000100,
1466 0xf90, 0xffffffff, 0x00000100,
1467 0xf98, 0x00000101, 0x00000000,
1468 0x20a8, 0xffffffff, 0x00000104,
1469 0x55e4, 0xff000fff, 0x00000100,
1470 0x30cc, 0xc0000fff, 0x00000104,
1471 0xc1e4, 0x00000001, 0x00000001,
1472 0xd00c, 0xff000ff0, 0x00000100,
1473 0xd80c, 0xff000ff0, 0x00000100
1476 static void cik_init_golden_registers(struct radeon_device *rdev)
1478 switch (rdev->family) {
1480 radeon_program_register_sequence(rdev,
1481 bonaire_mgcg_cgcg_init,
1482 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1483 radeon_program_register_sequence(rdev,
1484 bonaire_golden_registers,
1485 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1486 radeon_program_register_sequence(rdev,
1487 bonaire_golden_common_registers,
1488 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1489 radeon_program_register_sequence(rdev,
1490 bonaire_golden_spm_registers,
1491 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1494 radeon_program_register_sequence(rdev,
1495 kalindi_mgcg_cgcg_init,
1496 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1497 radeon_program_register_sequence(rdev,
1498 kalindi_golden_registers,
1499 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1500 radeon_program_register_sequence(rdev,
1501 kalindi_golden_common_registers,
1502 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1503 radeon_program_register_sequence(rdev,
1504 kalindi_golden_spm_registers,
1505 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1508 radeon_program_register_sequence(rdev,
1509 spectre_mgcg_cgcg_init,
1510 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1511 radeon_program_register_sequence(rdev,
1512 spectre_golden_registers,
1513 (const u32)ARRAY_SIZE(spectre_golden_registers));
1514 radeon_program_register_sequence(rdev,
1515 spectre_golden_common_registers,
1516 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1517 radeon_program_register_sequence(rdev,
1518 spectre_golden_spm_registers,
1519 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1522 radeon_program_register_sequence(rdev,
1523 hawaii_mgcg_cgcg_init,
1524 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1525 radeon_program_register_sequence(rdev,
1526 hawaii_golden_registers,
1527 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1528 radeon_program_register_sequence(rdev,
1529 hawaii_golden_common_registers,
1530 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1531 radeon_program_register_sequence(rdev,
1532 hawaii_golden_spm_registers,
1533 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1541 * cik_get_xclk - get the xclk
1543 * @rdev: radeon_device pointer
1545 * Returns the reference clock used by the gfx engine
1548 u32 cik_get_xclk(struct radeon_device *rdev)
1550 u32 reference_clock = rdev->clock.spll.reference_freq;
1552 if (rdev->flags & RADEON_IS_IGP) {
1553 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1554 return reference_clock / 2;
1556 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1557 return reference_clock / 4;
1559 return reference_clock;
1563 * cik_mm_rdoorbell - read a doorbell dword
1565 * @rdev: radeon_device pointer
1566 * @index: doorbell index
1568 * Returns the value in the doorbell aperture at the
1569 * requested doorbell index (CIK).
1571 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1573 if (index < rdev->doorbell.num_doorbells) {
1574 return readl(rdev->doorbell.ptr + index);
1576 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1582 * cik_mm_wdoorbell - write a doorbell dword
1584 * @rdev: radeon_device pointer
1585 * @index: doorbell index
1586 * @v: value to write
1588 * Writes @v to the doorbell aperture at the
1589 * requested doorbell index (CIK).
1591 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1593 if (index < rdev->doorbell.num_doorbells) {
1594 writel(v, rdev->doorbell.ptr + index);
1596 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1600 #define BONAIRE_IO_MC_REGS_SIZE 36
1602 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1604 {0x00000070, 0x04400000},
1605 {0x00000071, 0x80c01803},
1606 {0x00000072, 0x00004004},
1607 {0x00000073, 0x00000100},
1608 {0x00000074, 0x00ff0000},
1609 {0x00000075, 0x34000000},
1610 {0x00000076, 0x08000014},
1611 {0x00000077, 0x00cc08ec},
1612 {0x00000078, 0x00000400},
1613 {0x00000079, 0x00000000},
1614 {0x0000007a, 0x04090000},
1615 {0x0000007c, 0x00000000},
1616 {0x0000007e, 0x4408a8e8},
1617 {0x0000007f, 0x00000304},
1618 {0x00000080, 0x00000000},
1619 {0x00000082, 0x00000001},
1620 {0x00000083, 0x00000002},
1621 {0x00000084, 0xf3e4f400},
1622 {0x00000085, 0x052024e3},
1623 {0x00000087, 0x00000000},
1624 {0x00000088, 0x01000000},
1625 {0x0000008a, 0x1c0a0000},
1626 {0x0000008b, 0xff010000},
1627 {0x0000008d, 0xffffefff},
1628 {0x0000008e, 0xfff3efff},
1629 {0x0000008f, 0xfff3efbf},
1630 {0x00000092, 0xf7ffffff},
1631 {0x00000093, 0xffffff7f},
1632 {0x00000095, 0x00101101},
1633 {0x00000096, 0x00000fff},
1634 {0x00000097, 0x00116fff},
1635 {0x00000098, 0x60010000},
1636 {0x00000099, 0x10010000},
1637 {0x0000009a, 0x00006000},
1638 {0x0000009b, 0x00001000},
1639 {0x0000009f, 0x00b48000}
1642 #define HAWAII_IO_MC_REGS_SIZE 22
1644 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1646 {0x0000007d, 0x40000000},
1647 {0x0000007e, 0x40180304},
1648 {0x0000007f, 0x0000ff00},
1649 {0x00000081, 0x00000000},
1650 {0x00000083, 0x00000800},
1651 {0x00000086, 0x00000000},
1652 {0x00000087, 0x00000100},
1653 {0x00000088, 0x00020100},
1654 {0x00000089, 0x00000000},
1655 {0x0000008b, 0x00040000},
1656 {0x0000008c, 0x00000100},
1657 {0x0000008e, 0xff010000},
1658 {0x00000090, 0xffffefff},
1659 {0x00000091, 0xfff3efff},
1660 {0x00000092, 0xfff3efbf},
1661 {0x00000093, 0xf7ffffff},
1662 {0x00000094, 0xffffff7f},
1663 {0x00000095, 0x00000fff},
1664 {0x00000096, 0x00116fff},
1665 {0x00000097, 0x60010000},
1666 {0x00000098, 0x10010000},
1667 {0x0000009f, 0x00c79000}
1672 * cik_srbm_select - select specific register instances
1674 * @rdev: radeon_device pointer
1675 * @me: selected ME (micro engine)
1680 * Switches the currently active registers instances. Some
1681 * registers are instanced per VMID, others are instanced per
1682 * me/pipe/queue combination.
1684 static void cik_srbm_select(struct radeon_device *rdev,
1685 u32 me, u32 pipe, u32 queue, u32 vmid)
1687 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1690 QUEUEID(queue & 0x7));
1691 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1696 * ci_mc_load_microcode - load MC ucode into the hw
1698 * @rdev: radeon_device pointer
1700 * Load the GDDR MC ucode into the hw (CIK).
1701 * Returns 0 on success, error on failure.
1703 int ci_mc_load_microcode(struct radeon_device *rdev)
1705 const __be32 *fw_data;
1706 u32 running, blackout = 0;
1708 int i, regs_size, ucode_size;
1713 ucode_size = rdev->mc_fw->size / 4;
1715 switch (rdev->family) {
1717 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1718 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1721 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1722 regs_size = HAWAII_IO_MC_REGS_SIZE;
1728 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1732 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1733 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1736 /* reset the engine and set to writable */
1737 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1738 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1740 /* load mc io regs */
1741 for (i = 0; i < regs_size; i++) {
1742 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1743 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1745 /* load the MC ucode */
1746 fw_data = (const __be32 *)rdev->mc_fw->data;
1747 for (i = 0; i < ucode_size; i++)
1748 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1750 /* put the engine back into the active state */
1751 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1752 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1753 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1755 /* wait for training to complete */
1756 for (i = 0; i < rdev->usec_timeout; i++) {
1757 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1761 for (i = 0; i < rdev->usec_timeout; i++) {
1762 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1768 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1775 * cik_init_microcode - load ucode images from disk
1777 * @rdev: radeon_device pointer
1779 * Use the firmware interface to load the ucode images into
1780 * the driver (not loaded into hw).
1781 * Returns 0 on success, error on failure.
1783 static int cik_init_microcode(struct radeon_device *rdev)
1785 const char *chip_name;
1786 size_t pfp_req_size, me_req_size, ce_req_size,
1787 mec_req_size, rlc_req_size, mc_req_size = 0,
1788 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1794 switch (rdev->family) {
1796 chip_name = "BONAIRE";
1797 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1798 me_req_size = CIK_ME_UCODE_SIZE * 4;
1799 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1800 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1801 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1802 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1803 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1804 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1805 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1808 chip_name = "HAWAII";
1809 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1810 me_req_size = CIK_ME_UCODE_SIZE * 4;
1811 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1812 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1813 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1814 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1815 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1816 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1817 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1820 chip_name = "KAVERI";
1821 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1822 me_req_size = CIK_ME_UCODE_SIZE * 4;
1823 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1824 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1825 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1826 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1829 chip_name = "KABINI";
1830 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1831 me_req_size = CIK_ME_UCODE_SIZE * 4;
1832 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1833 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1834 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1835 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1840 DRM_INFO("Loading %s Microcode\n", chip_name);
1842 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1843 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1846 if (rdev->pfp_fw->size != pfp_req_size) {
1848 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1849 rdev->pfp_fw->size, fw_name);
1854 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1855 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1858 if (rdev->me_fw->size != me_req_size) {
1860 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1861 rdev->me_fw->size, fw_name);
1865 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1866 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1869 if (rdev->ce_fw->size != ce_req_size) {
1871 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1872 rdev->ce_fw->size, fw_name);
1876 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1877 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1880 if (rdev->mec_fw->size != mec_req_size) {
1882 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1883 rdev->mec_fw->size, fw_name);
1887 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1888 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1891 if (rdev->rlc_fw->size != rlc_req_size) {
1893 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1894 rdev->rlc_fw->size, fw_name);
1898 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1899 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1902 if (rdev->sdma_fw->size != sdma_req_size) {
1904 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1905 rdev->sdma_fw->size, fw_name);
1909 /* No SMC, MC ucode on APUs */
1910 if (!(rdev->flags & RADEON_IS_IGP)) {
1911 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1912 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1914 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1915 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1919 if ((rdev->mc_fw->size != mc_req_size) &&
1920 (rdev->mc_fw->size != mc2_req_size)){
1922 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1923 rdev->mc_fw->size, fw_name);
1926 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1928 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1929 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1932 "smc: error loading firmware \"%s\"\n",
1934 release_firmware(rdev->smc_fw);
1935 rdev->smc_fw = NULL;
1937 } else if (rdev->smc_fw->size != smc_req_size) {
1939 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1940 rdev->smc_fw->size, fw_name);
1949 "cik_cp: Failed to load firmware \"%s\"\n",
1951 release_firmware(rdev->pfp_fw);
1952 rdev->pfp_fw = NULL;
1953 release_firmware(rdev->me_fw);
1955 release_firmware(rdev->ce_fw);
1957 release_firmware(rdev->rlc_fw);
1958 rdev->rlc_fw = NULL;
1959 release_firmware(rdev->mc_fw);
1961 release_firmware(rdev->smc_fw);
1962 rdev->smc_fw = NULL;
1971 * cik_tiling_mode_table_init - init the hw tiling table
1973 * @rdev: radeon_device pointer
1975 * Starting with SI, the tiling setup is done globally in a
1976 * set of 32 tiling modes. Rather than selecting each set of
1977 * parameters per surface as on older asics, we just select
1978 * which index in the tiling table we want to use, and the
1979 * surface uses those parameters (CIK).
1981 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1983 const u32 num_tile_mode_states = 32;
1984 const u32 num_secondary_tile_mode_states = 16;
1985 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1986 u32 num_pipe_configs;
1987 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1988 rdev->config.cik.max_shader_engines;
1990 switch (rdev->config.cik.mem_row_size_in_kb) {
1992 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1996 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1999 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2003 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2004 if (num_pipe_configs > 8)
2005 num_pipe_configs = 16;
2007 if (num_pipe_configs == 16) {
2008 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2009 switch (reg_offset) {
2011 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2012 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2013 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2014 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2017 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2018 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2019 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2020 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2023 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2024 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2025 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2026 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2029 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2030 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2031 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2032 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2035 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2036 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2037 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2038 TILE_SPLIT(split_equal_to_row_size));
2041 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2042 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2043 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2046 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2047 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2048 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2049 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2052 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2053 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2054 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2055 TILE_SPLIT(split_equal_to_row_size));
2058 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2059 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2062 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2063 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2064 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2067 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2068 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2069 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2070 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2073 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2074 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2075 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2076 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2079 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2080 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2081 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2082 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2085 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2086 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2087 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2090 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2091 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2092 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2093 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2096 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2097 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2098 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2099 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2102 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2103 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2104 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2105 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2108 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2109 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2110 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2113 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2114 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2115 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2116 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2119 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2120 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2121 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2122 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2125 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2126 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2127 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2128 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2134 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2135 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2137 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2138 switch (reg_offset) {
2140 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2141 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2142 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2143 NUM_BANKS(ADDR_SURF_16_BANK));
2146 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2147 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2148 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2149 NUM_BANKS(ADDR_SURF_16_BANK));
2152 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2153 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2154 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2155 NUM_BANKS(ADDR_SURF_16_BANK));
2158 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2159 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2160 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2161 NUM_BANKS(ADDR_SURF_16_BANK));
2164 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2165 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2166 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2167 NUM_BANKS(ADDR_SURF_8_BANK));
2170 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2171 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2172 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2173 NUM_BANKS(ADDR_SURF_4_BANK));
2176 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2177 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2178 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2179 NUM_BANKS(ADDR_SURF_2_BANK));
2182 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2183 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2184 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2185 NUM_BANKS(ADDR_SURF_16_BANK));
2188 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2189 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2190 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2191 NUM_BANKS(ADDR_SURF_16_BANK));
2194 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2195 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2196 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2197 NUM_BANKS(ADDR_SURF_16_BANK));
2200 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2201 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2202 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2203 NUM_BANKS(ADDR_SURF_8_BANK));
2206 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2207 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2208 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2209 NUM_BANKS(ADDR_SURF_4_BANK));
2212 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2213 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2214 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2215 NUM_BANKS(ADDR_SURF_2_BANK));
2218 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2219 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2220 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2221 NUM_BANKS(ADDR_SURF_2_BANK));
2227 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2229 } else if (num_pipe_configs == 8) {
2230 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2231 switch (reg_offset) {
2233 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2234 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2235 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2236 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2239 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2240 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2241 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2242 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2245 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2246 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2247 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2248 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2251 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2252 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2253 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2254 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2257 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2258 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2259 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2260 TILE_SPLIT(split_equal_to_row_size));
2263 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2264 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2265 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2268 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2269 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2270 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2271 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2274 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2275 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2276 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2277 TILE_SPLIT(split_equal_to_row_size));
2280 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2281 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2284 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2285 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2286 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2289 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2290 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2291 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2292 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2295 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2296 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2297 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2298 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2301 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2302 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2303 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2304 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2307 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2308 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2312 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2313 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2314 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2315 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2318 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2319 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2320 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2321 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2324 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2325 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2326 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2327 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2330 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2331 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2332 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2335 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2336 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2337 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2338 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2341 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2342 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2343 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2344 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2347 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2348 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2349 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2350 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2356 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2357 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2359 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2360 switch (reg_offset) {
2362 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2363 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2364 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2365 NUM_BANKS(ADDR_SURF_16_BANK));
2368 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2369 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2370 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2371 NUM_BANKS(ADDR_SURF_16_BANK));
2374 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2375 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2376 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2377 NUM_BANKS(ADDR_SURF_16_BANK));
2380 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2381 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2382 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2383 NUM_BANKS(ADDR_SURF_16_BANK));
2386 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2387 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2388 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2389 NUM_BANKS(ADDR_SURF_8_BANK));
2392 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2393 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2394 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2395 NUM_BANKS(ADDR_SURF_4_BANK));
2398 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2399 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2400 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2401 NUM_BANKS(ADDR_SURF_2_BANK));
2404 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2405 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2406 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2407 NUM_BANKS(ADDR_SURF_16_BANK));
2410 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2411 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2412 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2413 NUM_BANKS(ADDR_SURF_16_BANK));
2416 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2417 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2418 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2419 NUM_BANKS(ADDR_SURF_16_BANK));
2422 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2423 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2424 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2425 NUM_BANKS(ADDR_SURF_16_BANK));
2428 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2429 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2430 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2431 NUM_BANKS(ADDR_SURF_8_BANK));
2434 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2435 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2436 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2437 NUM_BANKS(ADDR_SURF_4_BANK));
2440 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2441 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2442 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2443 NUM_BANKS(ADDR_SURF_2_BANK));
2449 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2450 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2452 } else if (num_pipe_configs == 4) {
2454 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2455 switch (reg_offset) {
2457 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2458 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2459 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2460 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2463 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2464 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2465 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2466 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2469 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2470 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2471 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2472 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2475 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2476 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2477 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2478 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2481 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2482 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2483 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2484 TILE_SPLIT(split_equal_to_row_size));
2487 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2488 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2489 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2492 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2493 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2494 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2495 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2498 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2499 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2500 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2501 TILE_SPLIT(split_equal_to_row_size));
2504 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2505 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2508 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2509 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2510 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2513 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2515 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2516 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2519 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2520 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2521 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2522 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2526 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2528 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2531 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2532 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2533 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2536 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2538 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2539 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2542 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2543 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2544 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2545 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2548 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2549 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2550 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2551 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2554 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2555 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2556 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2559 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2560 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2561 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2562 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2565 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2566 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2567 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2568 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2571 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2572 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2573 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2574 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2580 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2581 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2583 } else if (num_rbs < 4) {
2584 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2585 switch (reg_offset) {
2587 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2588 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2590 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2594 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2595 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2596 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2599 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2600 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2601 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2602 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2605 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2606 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2607 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2608 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2611 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2612 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2613 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2614 TILE_SPLIT(split_equal_to_row_size));
2617 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2618 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2619 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2622 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2623 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2624 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2625 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2628 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2629 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2630 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2631 TILE_SPLIT(split_equal_to_row_size));
2634 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2635 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2638 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2639 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2640 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2643 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2644 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2645 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2646 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2649 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2650 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2651 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2652 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2655 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2656 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2657 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2658 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2661 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2662 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2663 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2666 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2667 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2668 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2669 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2672 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2673 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2674 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2675 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2678 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2679 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2680 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2681 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2684 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2685 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2686 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2689 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2690 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2691 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2692 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2695 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2696 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2697 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2698 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2701 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2702 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2703 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2704 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2710 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2711 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2714 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2715 switch (reg_offset) {
2717 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2718 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2719 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2720 NUM_BANKS(ADDR_SURF_16_BANK));
2723 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2724 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2725 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2726 NUM_BANKS(ADDR_SURF_16_BANK));
2729 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2730 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2731 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2732 NUM_BANKS(ADDR_SURF_16_BANK));
2735 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2736 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2737 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2738 NUM_BANKS(ADDR_SURF_16_BANK));
2741 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2742 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2743 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2744 NUM_BANKS(ADDR_SURF_16_BANK));
2747 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2748 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2749 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2750 NUM_BANKS(ADDR_SURF_8_BANK));
2753 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2754 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2755 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2756 NUM_BANKS(ADDR_SURF_4_BANK));
2759 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2760 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2761 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2762 NUM_BANKS(ADDR_SURF_16_BANK));
2765 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2766 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2767 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2768 NUM_BANKS(ADDR_SURF_16_BANK));
2771 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2772 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2773 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2774 NUM_BANKS(ADDR_SURF_16_BANK));
2777 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2778 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2779 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2780 NUM_BANKS(ADDR_SURF_16_BANK));
2783 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2784 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2785 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2786 NUM_BANKS(ADDR_SURF_16_BANK));
2789 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2790 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2791 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2792 NUM_BANKS(ADDR_SURF_8_BANK));
2795 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2796 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2797 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2798 NUM_BANKS(ADDR_SURF_4_BANK));
2804 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2805 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2807 } else if (num_pipe_configs == 2) {
2808 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2809 switch (reg_offset) {
2811 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2812 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2813 PIPE_CONFIG(ADDR_SURF_P2) |
2814 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2817 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2818 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2819 PIPE_CONFIG(ADDR_SURF_P2) |
2820 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2823 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2824 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2825 PIPE_CONFIG(ADDR_SURF_P2) |
2826 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2829 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2830 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2831 PIPE_CONFIG(ADDR_SURF_P2) |
2832 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2835 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2836 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2837 PIPE_CONFIG(ADDR_SURF_P2) |
2838 TILE_SPLIT(split_equal_to_row_size));
2841 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2842 PIPE_CONFIG(ADDR_SURF_P2) |
2843 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2846 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2847 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2848 PIPE_CONFIG(ADDR_SURF_P2) |
2849 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2852 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2853 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2854 PIPE_CONFIG(ADDR_SURF_P2) |
2855 TILE_SPLIT(split_equal_to_row_size));
2858 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2859 PIPE_CONFIG(ADDR_SURF_P2);
2862 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2863 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2864 PIPE_CONFIG(ADDR_SURF_P2));
2867 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2868 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2869 PIPE_CONFIG(ADDR_SURF_P2) |
2870 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2873 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2874 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2875 PIPE_CONFIG(ADDR_SURF_P2) |
2876 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2879 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2880 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2881 PIPE_CONFIG(ADDR_SURF_P2) |
2882 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2885 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2886 PIPE_CONFIG(ADDR_SURF_P2) |
2887 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2890 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2891 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2892 PIPE_CONFIG(ADDR_SURF_P2) |
2893 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2896 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2897 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2898 PIPE_CONFIG(ADDR_SURF_P2) |
2899 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2902 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2903 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2904 PIPE_CONFIG(ADDR_SURF_P2) |
2905 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2908 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2909 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2910 PIPE_CONFIG(ADDR_SURF_P2));
2913 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2914 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2915 PIPE_CONFIG(ADDR_SURF_P2) |
2916 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2919 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2920 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2921 PIPE_CONFIG(ADDR_SURF_P2) |
2922 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2925 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2926 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2927 PIPE_CONFIG(ADDR_SURF_P2) |
2928 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2934 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2935 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2937 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2938 switch (reg_offset) {
2940 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2941 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2942 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2943 NUM_BANKS(ADDR_SURF_16_BANK));
2946 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2947 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2948 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2949 NUM_BANKS(ADDR_SURF_16_BANK));
2952 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2953 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2954 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2955 NUM_BANKS(ADDR_SURF_16_BANK));
2958 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2959 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2960 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2961 NUM_BANKS(ADDR_SURF_16_BANK));
2964 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2965 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2966 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2967 NUM_BANKS(ADDR_SURF_16_BANK));
2970 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2971 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2972 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2973 NUM_BANKS(ADDR_SURF_16_BANK));
2976 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2977 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2978 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2979 NUM_BANKS(ADDR_SURF_8_BANK));
2982 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2983 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2984 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2985 NUM_BANKS(ADDR_SURF_16_BANK));
2988 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2989 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2990 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2991 NUM_BANKS(ADDR_SURF_16_BANK));
2994 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2995 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2996 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2997 NUM_BANKS(ADDR_SURF_16_BANK));
3000 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3001 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3002 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3003 NUM_BANKS(ADDR_SURF_16_BANK));
3006 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3007 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3008 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3009 NUM_BANKS(ADDR_SURF_16_BANK));
3012 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3013 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3014 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3015 NUM_BANKS(ADDR_SURF_16_BANK));
3018 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3019 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3020 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3021 NUM_BANKS(ADDR_SURF_8_BANK));
3027 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3028 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3031 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3035 * cik_select_se_sh - select which SE, SH to address
3037 * @rdev: radeon_device pointer
3038 * @se_num: shader engine to address
3039 * @sh_num: sh block to address
3041 * Select which SE, SH combinations to address. Certain
3042 * registers are instanced per SE or SH. 0xffffffff means
3043 * broadcast to all SEs or SHs (CIK).
3045 static void cik_select_se_sh(struct radeon_device *rdev,
3046 u32 se_num, u32 sh_num)
3048 u32 data = INSTANCE_BROADCAST_WRITES;
3050 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3051 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3052 else if (se_num == 0xffffffff)
3053 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3054 else if (sh_num == 0xffffffff)
3055 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3057 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3058 WREG32(GRBM_GFX_INDEX, data);
3062 * cik_create_bitmask - create a bitmask
3064 * @bit_width: length of the mask
3066 * create a variable length bit mask (CIK).
3067 * Returns the bitmask.
3069 static u32 cik_create_bitmask(u32 bit_width)
3073 for (i = 0; i < bit_width; i++) {
3081 * cik_get_rb_disabled - computes the mask of disabled RBs
3083 * @rdev: radeon_device pointer
3084 * @max_rb_num: max RBs (render backends) for the asic
3085 * @se_num: number of SEs (shader engines) for the asic
3086 * @sh_per_se: number of SH blocks per SE for the asic
3088 * Calculates the bitmask of disabled RBs (CIK).
3089 * Returns the disabled RB bitmask.
3091 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3092 u32 max_rb_num_per_se,
3097 data = RREG32(CC_RB_BACKEND_DISABLE);
3099 data &= BACKEND_DISABLE_MASK;
3102 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3104 data >>= BACKEND_DISABLE_SHIFT;
3106 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3112 * cik_setup_rb - setup the RBs on the asic
3114 * @rdev: radeon_device pointer
3115 * @se_num: number of SEs (shader engines) for the asic
3116 * @sh_per_se: number of SH blocks per SE for the asic
3117 * @max_rb_num: max RBs (render backends) for the asic
3119 * Configures per-SE/SH RB registers (CIK).
3121 static void cik_setup_rb(struct radeon_device *rdev,
3122 u32 se_num, u32 sh_per_se,
3123 u32 max_rb_num_per_se)
3127 u32 disabled_rbs = 0;
3128 u32 enabled_rbs = 0;
3130 for (i = 0; i < se_num; i++) {
3131 for (j = 0; j < sh_per_se; j++) {
3132 cik_select_se_sh(rdev, i, j);
3133 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3134 if (rdev->family == CHIP_HAWAII)
3135 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3137 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3140 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3143 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3144 if (!(disabled_rbs & mask))
3145 enabled_rbs |= mask;
3149 rdev->config.cik.backend_enable_mask = enabled_rbs;
3151 for (i = 0; i < se_num; i++) {
3152 cik_select_se_sh(rdev, i, 0xffffffff);
3154 for (j = 0; j < sh_per_se; j++) {
3155 switch (enabled_rbs & 3) {
3158 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3160 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3163 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3166 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3170 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3175 WREG32(PA_SC_RASTER_CONFIG, data);
3177 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3181 * cik_gpu_init - setup the 3D engine
3183 * @rdev: radeon_device pointer
3185 * Configures the 3D engine and tiling configuration
3186 * registers so that the 3D engine is usable.
3188 static void cik_gpu_init(struct radeon_device *rdev)
3190 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3191 u32 mc_shared_chmap, mc_arb_ramcfg;
3192 u32 hdp_host_path_cntl;
3196 switch (rdev->family) {
3198 rdev->config.cik.max_shader_engines = 2;
3199 rdev->config.cik.max_tile_pipes = 4;
3200 rdev->config.cik.max_cu_per_sh = 7;
3201 rdev->config.cik.max_sh_per_se = 1;
3202 rdev->config.cik.max_backends_per_se = 2;
3203 rdev->config.cik.max_texture_channel_caches = 4;
3204 rdev->config.cik.max_gprs = 256;
3205 rdev->config.cik.max_gs_threads = 32;
3206 rdev->config.cik.max_hw_contexts = 8;
3208 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3209 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3210 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3211 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3212 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3215 rdev->config.cik.max_shader_engines = 4;
3216 rdev->config.cik.max_tile_pipes = 16;
3217 rdev->config.cik.max_cu_per_sh = 11;
3218 rdev->config.cik.max_sh_per_se = 1;
3219 rdev->config.cik.max_backends_per_se = 4;
3220 rdev->config.cik.max_texture_channel_caches = 16;
3221 rdev->config.cik.max_gprs = 256;
3222 rdev->config.cik.max_gs_threads = 32;
3223 rdev->config.cik.max_hw_contexts = 8;
3225 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3226 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3227 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3228 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3229 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3232 rdev->config.cik.max_shader_engines = 1;
3233 rdev->config.cik.max_tile_pipes = 4;
3234 if ((rdev->pdev->device == 0x1304) ||
3235 (rdev->pdev->device == 0x1305) ||
3236 (rdev->pdev->device == 0x130C) ||
3237 (rdev->pdev->device == 0x130F) ||
3238 (rdev->pdev->device == 0x1310) ||
3239 (rdev->pdev->device == 0x1311) ||
3240 (rdev->pdev->device == 0x131C)) {
3241 rdev->config.cik.max_cu_per_sh = 8;
3242 rdev->config.cik.max_backends_per_se = 2;
3243 } else if ((rdev->pdev->device == 0x1309) ||
3244 (rdev->pdev->device == 0x130A) ||
3245 (rdev->pdev->device == 0x130D) ||
3246 (rdev->pdev->device == 0x1313) ||
3247 (rdev->pdev->device == 0x131D)) {
3248 rdev->config.cik.max_cu_per_sh = 6;
3249 rdev->config.cik.max_backends_per_se = 2;
3250 } else if ((rdev->pdev->device == 0x1306) ||
3251 (rdev->pdev->device == 0x1307) ||
3252 (rdev->pdev->device == 0x130B) ||
3253 (rdev->pdev->device == 0x130E) ||
3254 (rdev->pdev->device == 0x1315) ||
3255 (rdev->pdev->device == 0x131B)) {
3256 rdev->config.cik.max_cu_per_sh = 4;
3257 rdev->config.cik.max_backends_per_se = 1;
3259 rdev->config.cik.max_cu_per_sh = 3;
3260 rdev->config.cik.max_backends_per_se = 1;
3262 rdev->config.cik.max_sh_per_se = 1;
3263 rdev->config.cik.max_texture_channel_caches = 4;
3264 rdev->config.cik.max_gprs = 256;
3265 rdev->config.cik.max_gs_threads = 16;
3266 rdev->config.cik.max_hw_contexts = 8;
3268 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3269 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3270 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3271 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3272 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3276 rdev->config.cik.max_shader_engines = 1;
3277 rdev->config.cik.max_tile_pipes = 2;
3278 rdev->config.cik.max_cu_per_sh = 2;
3279 rdev->config.cik.max_sh_per_se = 1;
3280 rdev->config.cik.max_backends_per_se = 1;
3281 rdev->config.cik.max_texture_channel_caches = 2;
3282 rdev->config.cik.max_gprs = 256;
3283 rdev->config.cik.max_gs_threads = 16;
3284 rdev->config.cik.max_hw_contexts = 8;
3286 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3287 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3288 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3289 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3290 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3294 /* Initialize HDP */
3295 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3296 WREG32((0x2c14 + j), 0x00000000);
3297 WREG32((0x2c18 + j), 0x00000000);
3298 WREG32((0x2c1c + j), 0x00000000);
3299 WREG32((0x2c20 + j), 0x00000000);
3300 WREG32((0x2c24 + j), 0x00000000);
3303 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3305 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3307 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3308 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3310 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3311 rdev->config.cik.mem_max_burst_length_bytes = 256;
3312 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3313 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3314 if (rdev->config.cik.mem_row_size_in_kb > 4)
3315 rdev->config.cik.mem_row_size_in_kb = 4;
3316 /* XXX use MC settings? */
3317 rdev->config.cik.shader_engine_tile_size = 32;
3318 rdev->config.cik.num_gpus = 1;
3319 rdev->config.cik.multi_gpu_tile_size = 64;
3321 /* fix up row size */
3322 gb_addr_config &= ~ROW_SIZE_MASK;
3323 switch (rdev->config.cik.mem_row_size_in_kb) {
3326 gb_addr_config |= ROW_SIZE(0);
3329 gb_addr_config |= ROW_SIZE(1);
3332 gb_addr_config |= ROW_SIZE(2);
3336 /* setup tiling info dword. gb_addr_config is not adequate since it does
3337 * not have bank info, so create a custom tiling dword.
3338 * bits 3:0 num_pipes
3339 * bits 7:4 num_banks
3340 * bits 11:8 group_size
3341 * bits 15:12 row_size
3343 rdev->config.cik.tile_config = 0;
3344 switch (rdev->config.cik.num_tile_pipes) {
3346 rdev->config.cik.tile_config |= (0 << 0);
3349 rdev->config.cik.tile_config |= (1 << 0);
3352 rdev->config.cik.tile_config |= (2 << 0);
3356 /* XXX what about 12? */
3357 rdev->config.cik.tile_config |= (3 << 0);
3360 rdev->config.cik.tile_config |=
3361 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3362 rdev->config.cik.tile_config |=
3363 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3364 rdev->config.cik.tile_config |=
3365 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3367 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3368 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3369 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3370 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3371 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3372 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3373 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3374 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3376 cik_tiling_mode_table_init(rdev);
3378 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3379 rdev->config.cik.max_sh_per_se,
3380 rdev->config.cik.max_backends_per_se);
3382 /* set HW defaults for 3D engine */
3383 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3385 WREG32(SX_DEBUG_1, 0x20);
3387 WREG32(TA_CNTL_AUX, 0x00010000);
3389 tmp = RREG32(SPI_CONFIG_CNTL);
3391 WREG32(SPI_CONFIG_CNTL, tmp);
3393 WREG32(SQ_CONFIG, 1);
3395 WREG32(DB_DEBUG, 0);
3397 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3399 WREG32(DB_DEBUG2, tmp);
3401 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3403 WREG32(DB_DEBUG3, tmp);
3405 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3407 WREG32(CB_HW_CONTROL, tmp);
3409 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3411 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3412 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3413 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3414 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3416 WREG32(VGT_NUM_INSTANCES, 1);
3418 WREG32(CP_PERFMON_CNTL, 0);
3420 WREG32(SQ_CONFIG, 0);
3422 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3423 FORCE_EOV_MAX_REZ_CNT(255)));
3425 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3426 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3428 WREG32(VGT_GS_VERTEX_REUSE, 16);
3429 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3431 tmp = RREG32(HDP_MISC_CNTL);
3432 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3433 WREG32(HDP_MISC_CNTL, tmp);
3435 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3436 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3438 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3439 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3445 * GPU scratch registers helpers function.
3448 * cik_scratch_init - setup driver info for CP scratch regs
3450 * @rdev: radeon_device pointer
3452 * Set up the number and offset of the CP scratch registers.
3453 * NOTE: use of CP scratch registers is a legacy inferface and
3454 * is not used by default on newer asics (r6xx+). On newer asics,
3455 * memory buffers are used for fences rather than scratch regs.
3457 static void cik_scratch_init(struct radeon_device *rdev)
3461 rdev->scratch.num_reg = 7;
3462 rdev->scratch.reg_base = SCRATCH_REG0;
3463 for (i = 0; i < rdev->scratch.num_reg; i++) {
3464 rdev->scratch.free[i] = true;
3465 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3470 * cik_ring_test - basic gfx ring test
3472 * @rdev: radeon_device pointer
3473 * @ring: radeon_ring structure holding ring information
3475 * Allocate a scratch register and write to it using the gfx ring (CIK).
3476 * Provides a basic gfx ring test to verify that the ring is working.
3477 * Used by cik_cp_gfx_resume();
3478 * Returns 0 on success, error on failure.
3480 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3487 r = radeon_scratch_get(rdev, &scratch);
3489 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3492 WREG32(scratch, 0xCAFEDEAD);
3493 r = radeon_ring_lock(rdev, ring, 3);
3495 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3496 radeon_scratch_free(rdev, scratch);
3499 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3500 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3501 radeon_ring_write(ring, 0xDEADBEEF);
3502 radeon_ring_unlock_commit(rdev, ring);
3504 for (i = 0; i < rdev->usec_timeout; i++) {
3505 tmp = RREG32(scratch);
3506 if (tmp == 0xDEADBEEF)
3510 if (i < rdev->usec_timeout) {
3511 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3513 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3514 ring->idx, scratch, tmp);
3517 radeon_scratch_free(rdev, scratch);
3522 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3524 * @rdev: radeon_device pointer
3525 * @ridx: radeon ring index
3527 * Emits an hdp flush on the cp.
3529 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3532 struct radeon_ring *ring = &rdev->ring[ridx];
3535 switch (ring->idx) {
3536 case CAYMAN_RING_TYPE_CP1_INDEX:
3537 case CAYMAN_RING_TYPE_CP2_INDEX:
3541 ref_and_mask = CP2 << ring->pipe;
3544 ref_and_mask = CP6 << ring->pipe;
3550 case RADEON_RING_TYPE_GFX_INDEX:
3555 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3556 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3557 WAIT_REG_MEM_FUNCTION(3) | /* == */
3558 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3559 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3560 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3561 radeon_ring_write(ring, ref_and_mask);
3562 radeon_ring_write(ring, ref_and_mask);
3563 radeon_ring_write(ring, 0x20); /* poll interval */
3567 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3569 * @rdev: radeon_device pointer
3570 * @fence: radeon fence object
3572 * Emits a fence sequnce number on the gfx ring and flushes
3575 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3576 struct radeon_fence *fence)
3578 struct radeon_ring *ring = &rdev->ring[fence->ring];
3579 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3581 /* EVENT_WRITE_EOP - flush caches, send int */
3582 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3583 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3585 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3587 radeon_ring_write(ring, addr & 0xfffffffc);
3588 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3589 radeon_ring_write(ring, fence->seq);
3590 radeon_ring_write(ring, 0);
3592 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3596 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3598 * @rdev: radeon_device pointer
3599 * @fence: radeon fence object
3601 * Emits a fence sequnce number on the compute ring and flushes
3604 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3605 struct radeon_fence *fence)
3607 struct radeon_ring *ring = &rdev->ring[fence->ring];
3608 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3610 /* RELEASE_MEM - flush caches, send int */
3611 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3612 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3614 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3616 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3617 radeon_ring_write(ring, addr & 0xfffffffc);
3618 radeon_ring_write(ring, upper_32_bits(addr));
3619 radeon_ring_write(ring, fence->seq);
3620 radeon_ring_write(ring, 0);
3622 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3625 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3626 struct radeon_ring *ring,
3627 struct radeon_semaphore *semaphore,
3630 uint64_t addr = semaphore->gpu_addr;
3631 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3633 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3634 radeon_ring_write(ring, addr & 0xffffffff);
3635 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3641 * cik_copy_cpdma - copy pages using the CP DMA engine
3643 * @rdev: radeon_device pointer
3644 * @src_offset: src GPU address
3645 * @dst_offset: dst GPU address
3646 * @num_gpu_pages: number of GPU pages to xfer
3647 * @fence: radeon fence object
3649 * Copy GPU paging using the CP DMA engine (CIK+).
3650 * Used by the radeon ttm implementation to move pages if
3651 * registered as the asic copy callback.
3653 int cik_copy_cpdma(struct radeon_device *rdev,
3654 uint64_t src_offset, uint64_t dst_offset,
3655 unsigned num_gpu_pages,
3656 struct radeon_fence **fence)
3658 struct radeon_semaphore *sem = NULL;
3659 int ring_index = rdev->asic->copy.blit_ring_index;
3660 struct radeon_ring *ring = &rdev->ring[ring_index];
3661 u32 size_in_bytes, cur_size_in_bytes, control;
3665 r = radeon_semaphore_create(rdev, &sem);
3667 DRM_ERROR("radeon: moving bo (%d).\n", r);
3671 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3672 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3673 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3675 DRM_ERROR("radeon: moving bo (%d).\n", r);
3676 radeon_semaphore_free(rdev, &sem, NULL);
3680 radeon_semaphore_sync_to(sem, *fence);
3681 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3683 for (i = 0; i < num_loops; i++) {
3684 cur_size_in_bytes = size_in_bytes;
3685 if (cur_size_in_bytes > 0x1fffff)
3686 cur_size_in_bytes = 0x1fffff;
3687 size_in_bytes -= cur_size_in_bytes;
3689 if (size_in_bytes == 0)
3690 control |= PACKET3_DMA_DATA_CP_SYNC;
3691 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3692 radeon_ring_write(ring, control);
3693 radeon_ring_write(ring, lower_32_bits(src_offset));
3694 radeon_ring_write(ring, upper_32_bits(src_offset));
3695 radeon_ring_write(ring, lower_32_bits(dst_offset));
3696 radeon_ring_write(ring, upper_32_bits(dst_offset));
3697 radeon_ring_write(ring, cur_size_in_bytes);
3698 src_offset += cur_size_in_bytes;
3699 dst_offset += cur_size_in_bytes;
3702 r = radeon_fence_emit(rdev, fence, ring->idx);
3704 radeon_ring_unlock_undo(rdev, ring);
3708 radeon_ring_unlock_commit(rdev, ring);
3709 radeon_semaphore_free(rdev, &sem, *fence);
3718 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3720 * @rdev: radeon_device pointer
3721 * @ib: radeon indirect buffer object
3723 * Emits an DE (drawing engine) or CE (constant engine) IB
3724 * on the gfx ring. IBs are usually generated by userspace
3725 * acceleration drivers and submitted to the kernel for
3726 * sheduling on the ring. This function schedules the IB
3727 * on the gfx ring for execution by the GPU.
3729 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3731 struct radeon_ring *ring = &rdev->ring[ib->ring];
3732 u32 header, control = INDIRECT_BUFFER_VALID;
3734 if (ib->is_const_ib) {
3735 /* set switch buffer packet before const IB */
3736 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3737 radeon_ring_write(ring, 0);
3739 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3742 if (ring->rptr_save_reg) {
3743 next_rptr = ring->wptr + 3 + 4;
3744 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3745 radeon_ring_write(ring, ((ring->rptr_save_reg -
3746 PACKET3_SET_UCONFIG_REG_START) >> 2));
3747 radeon_ring_write(ring, next_rptr);
3748 } else if (rdev->wb.enabled) {
3749 next_rptr = ring->wptr + 5 + 4;
3750 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3751 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3752 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3753 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3754 radeon_ring_write(ring, next_rptr);
3757 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3760 control |= ib->length_dw |
3761 (ib->vm ? (ib->vm->id << 24) : 0);
3763 radeon_ring_write(ring, header);
3764 radeon_ring_write(ring,
3768 (ib->gpu_addr & 0xFFFFFFFC));
3769 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3770 radeon_ring_write(ring, control);
3774 * cik_ib_test - basic gfx ring IB test
3776 * @rdev: radeon_device pointer
3777 * @ring: radeon_ring structure holding ring information
3779 * Allocate an IB and execute it on the gfx ring (CIK).
3780 * Provides a basic gfx ring test to verify that IBs are working.
3781 * Returns 0 on success, error on failure.
3783 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3785 struct radeon_ib ib;
3791 r = radeon_scratch_get(rdev, &scratch);
3793 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3796 WREG32(scratch, 0xCAFEDEAD);
3797 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3799 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3800 radeon_scratch_free(rdev, scratch);
3803 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3804 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3805 ib.ptr[2] = 0xDEADBEEF;
3807 r = radeon_ib_schedule(rdev, &ib, NULL);
3809 radeon_scratch_free(rdev, scratch);
3810 radeon_ib_free(rdev, &ib);
3811 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3814 r = radeon_fence_wait(ib.fence, false);
3816 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3817 radeon_scratch_free(rdev, scratch);
3818 radeon_ib_free(rdev, &ib);
3821 for (i = 0; i < rdev->usec_timeout; i++) {
3822 tmp = RREG32(scratch);
3823 if (tmp == 0xDEADBEEF)
3827 if (i < rdev->usec_timeout) {
3828 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3830 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3834 radeon_scratch_free(rdev, scratch);
3835 radeon_ib_free(rdev, &ib);
3841 * On CIK, gfx and compute now have independant command processors.
3844 * Gfx consists of a single ring and can process both gfx jobs and
3845 * compute jobs. The gfx CP consists of three microengines (ME):
3846 * PFP - Pre-Fetch Parser
3848 * CE - Constant Engine
3849 * The PFP and ME make up what is considered the Drawing Engine (DE).
3850 * The CE is an asynchronous engine used for updating buffer desciptors
3851 * used by the DE so that they can be loaded into cache in parallel
3852 * while the DE is processing state update packets.
3855 * The compute CP consists of two microengines (ME):
3856 * MEC1 - Compute MicroEngine 1
3857 * MEC2 - Compute MicroEngine 2
3858 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3859 * The queues are exposed to userspace and are programmed directly
3860 * by the compute runtime.
3863 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3865 * @rdev: radeon_device pointer
3866 * @enable: enable or disable the MEs
3868 * Halts or unhalts the gfx MEs.
3870 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3873 WREG32(CP_ME_CNTL, 0);
3875 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3876 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3877 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3878 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3884 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3886 * @rdev: radeon_device pointer
3888 * Loads the gfx PFP, ME, and CE ucode.
3889 * Returns 0 for success, -EINVAL if the ucode is not available.
3891 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3893 const __be32 *fw_data;
3896 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3899 cik_cp_gfx_enable(rdev, false);
3902 fw_data = (const __be32 *)rdev->pfp_fw->data;
3903 WREG32(CP_PFP_UCODE_ADDR, 0);
3904 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3905 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3906 WREG32(CP_PFP_UCODE_ADDR, 0);
3909 fw_data = (const __be32 *)rdev->ce_fw->data;
3910 WREG32(CP_CE_UCODE_ADDR, 0);
3911 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3912 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3913 WREG32(CP_CE_UCODE_ADDR, 0);
3916 fw_data = (const __be32 *)rdev->me_fw->data;
3917 WREG32(CP_ME_RAM_WADDR, 0);
3918 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3919 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3920 WREG32(CP_ME_RAM_WADDR, 0);
3922 WREG32(CP_PFP_UCODE_ADDR, 0);
3923 WREG32(CP_CE_UCODE_ADDR, 0);
3924 WREG32(CP_ME_RAM_WADDR, 0);
3925 WREG32(CP_ME_RAM_RADDR, 0);
3930 * cik_cp_gfx_start - start the gfx ring
3932 * @rdev: radeon_device pointer
3934 * Enables the ring and loads the clear state context and other
3935 * packets required to init the ring.
3936 * Returns 0 for success, error for failure.
3938 static int cik_cp_gfx_start(struct radeon_device *rdev)
3940 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3944 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3945 WREG32(CP_ENDIAN_SWAP, 0);
3946 WREG32(CP_DEVICE_ID, 1);
3948 cik_cp_gfx_enable(rdev, true);
3950 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3952 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3956 /* init the CE partitions. CE only used for gfx on CIK */
3957 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3958 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3959 radeon_ring_write(ring, 0xc000);
3960 radeon_ring_write(ring, 0xc000);
3962 /* setup clear context state */
3963 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3964 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3966 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3967 radeon_ring_write(ring, 0x80000000);
3968 radeon_ring_write(ring, 0x80000000);
3970 for (i = 0; i < cik_default_size; i++)
3971 radeon_ring_write(ring, cik_default_state[i]);
3973 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3974 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3976 /* set clear context state */
3977 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3978 radeon_ring_write(ring, 0);
3980 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3981 radeon_ring_write(ring, 0x00000316);
3982 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3983 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3985 radeon_ring_unlock_commit(rdev, ring);
3991 * cik_cp_gfx_fini - stop the gfx ring
3993 * @rdev: radeon_device pointer
3995 * Stop the gfx ring and tear down the driver ring
3998 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4000 cik_cp_gfx_enable(rdev, false);
4001 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4005 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4007 * @rdev: radeon_device pointer
4009 * Program the location and size of the gfx ring buffer
4010 * and test it to make sure it's working.
4011 * Returns 0 for success, error for failure.
4013 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4015 struct radeon_ring *ring;
4021 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4022 if (rdev->family != CHIP_HAWAII)
4023 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4025 /* Set the write pointer delay */
4026 WREG32(CP_RB_WPTR_DELAY, 0);
4028 /* set the RB to use vmid 0 */
4029 WREG32(CP_RB_VMID, 0);
4031 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4033 /* ring 0 - compute and gfx */
4034 /* Set ring buffer size */
4035 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4036 rb_bufsz = order_base_2(ring->ring_size / 8);
4037 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4039 tmp |= BUF_SWAP_32BIT;
4041 WREG32(CP_RB0_CNTL, tmp);
4043 /* Initialize the ring buffer's read and write pointers */
4044 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4046 WREG32(CP_RB0_WPTR, ring->wptr);
4048 /* set the wb address wether it's enabled or not */
4049 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4050 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4052 /* scratch register shadowing is no longer supported */
4053 WREG32(SCRATCH_UMSK, 0);
4055 if (!rdev->wb.enabled)
4056 tmp |= RB_NO_UPDATE;
4059 WREG32(CP_RB0_CNTL, tmp);
4061 rb_addr = ring->gpu_addr >> 8;
4062 WREG32(CP_RB0_BASE, rb_addr);
4063 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4065 /* start the ring */
4066 cik_cp_gfx_start(rdev);
4067 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4068 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4070 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4074 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4075 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4080 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4081 struct radeon_ring *ring)
4085 if (rdev->wb.enabled)
4086 rptr = rdev->wb.wb[ring->rptr_offs/4];
4088 rptr = RREG32(CP_RB0_RPTR);
4093 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4094 struct radeon_ring *ring)
4098 wptr = RREG32(CP_RB0_WPTR);
4103 void cik_gfx_set_wptr(struct radeon_device *rdev,
4104 struct radeon_ring *ring)
4106 WREG32(CP_RB0_WPTR, ring->wptr);
4107 (void)RREG32(CP_RB0_WPTR);
4110 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4111 struct radeon_ring *ring)
4115 if (rdev->wb.enabled) {
4116 rptr = rdev->wb.wb[ring->rptr_offs/4];
4118 mutex_lock(&rdev->srbm_mutex);
4119 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4120 rptr = RREG32(CP_HQD_PQ_RPTR);
4121 cik_srbm_select(rdev, 0, 0, 0, 0);
4122 mutex_unlock(&rdev->srbm_mutex);
4128 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4129 struct radeon_ring *ring)
4133 if (rdev->wb.enabled) {
4134 /* XXX check if swapping is necessary on BE */
4135 wptr = rdev->wb.wb[ring->wptr_offs/4];
4137 mutex_lock(&rdev->srbm_mutex);
4138 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4139 wptr = RREG32(CP_HQD_PQ_WPTR);
4140 cik_srbm_select(rdev, 0, 0, 0, 0);
4141 mutex_unlock(&rdev->srbm_mutex);
4147 void cik_compute_set_wptr(struct radeon_device *rdev,
4148 struct radeon_ring *ring)
4150 /* XXX check if swapping is necessary on BE */
4151 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4152 WDOORBELL32(ring->doorbell_index, ring->wptr);
4156 * cik_cp_compute_enable - enable/disable the compute CP MEs
4158 * @rdev: radeon_device pointer
4159 * @enable: enable or disable the MEs
4161 * Halts or unhalts the compute MEs.
4163 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4166 WREG32(CP_MEC_CNTL, 0);
4168 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4169 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4170 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4176 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4178 * @rdev: radeon_device pointer
4180 * Loads the compute MEC1&2 ucode.
4181 * Returns 0 for success, -EINVAL if the ucode is not available.
4183 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4185 const __be32 *fw_data;
4191 cik_cp_compute_enable(rdev, false);
4194 fw_data = (const __be32 *)rdev->mec_fw->data;
4195 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4196 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4197 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4198 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4200 if (rdev->family == CHIP_KAVERI) {
4202 fw_data = (const __be32 *)rdev->mec_fw->data;
4203 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4204 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4205 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4206 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4213 * cik_cp_compute_start - start the compute queues
4215 * @rdev: radeon_device pointer
4217 * Enable the compute queues.
4218 * Returns 0 for success, error for failure.
4220 static int cik_cp_compute_start(struct radeon_device *rdev)
4222 cik_cp_compute_enable(rdev, true);
4228 * cik_cp_compute_fini - stop the compute queues
4230 * @rdev: radeon_device pointer
4232 * Stop the compute queues and tear down the driver queue
4235 static void cik_cp_compute_fini(struct radeon_device *rdev)
4239 cik_cp_compute_enable(rdev, false);
4241 for (i = 0; i < 2; i++) {
4243 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4245 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4247 if (rdev->ring[idx].mqd_obj) {
4248 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4249 if (unlikely(r != 0))
4250 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4252 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4253 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4255 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4256 rdev->ring[idx].mqd_obj = NULL;
4261 static void cik_mec_fini(struct radeon_device *rdev)
4265 if (rdev->mec.hpd_eop_obj) {
4266 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4267 if (unlikely(r != 0))
4268 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4269 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4270 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4272 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4273 rdev->mec.hpd_eop_obj = NULL;
4277 #define MEC_HPD_SIZE 2048
4279 static int cik_mec_init(struct radeon_device *rdev)
4285 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4286 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4288 if (rdev->family == CHIP_KAVERI)
4289 rdev->mec.num_mec = 2;
4291 rdev->mec.num_mec = 1;
4292 rdev->mec.num_pipe = 4;
4293 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4295 if (rdev->mec.hpd_eop_obj == NULL) {
4296 r = radeon_bo_create(rdev,
4297 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4299 RADEON_GEM_DOMAIN_GTT, NULL,
4300 &rdev->mec.hpd_eop_obj);
4302 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4307 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4308 if (unlikely(r != 0)) {
4312 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4313 &rdev->mec.hpd_eop_gpu_addr);
4315 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4319 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4321 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4326 /* clear memory. Not sure if this is required or not */
4327 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4329 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4330 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4335 struct hqd_registers
4337 u32 cp_mqd_base_addr;
4338 u32 cp_mqd_base_addr_hi;
4341 u32 cp_hqd_persistent_state;
4342 u32 cp_hqd_pipe_priority;
4343 u32 cp_hqd_queue_priority;
4346 u32 cp_hqd_pq_base_hi;
4348 u32 cp_hqd_pq_rptr_report_addr;
4349 u32 cp_hqd_pq_rptr_report_addr_hi;
4350 u32 cp_hqd_pq_wptr_poll_addr;
4351 u32 cp_hqd_pq_wptr_poll_addr_hi;
4352 u32 cp_hqd_pq_doorbell_control;
4354 u32 cp_hqd_pq_control;
4355 u32 cp_hqd_ib_base_addr;
4356 u32 cp_hqd_ib_base_addr_hi;
4358 u32 cp_hqd_ib_control;
4359 u32 cp_hqd_iq_timer;
4361 u32 cp_hqd_dequeue_request;
4362 u32 cp_hqd_dma_offload;
4363 u32 cp_hqd_sema_cmd;
4364 u32 cp_hqd_msg_type;
4365 u32 cp_hqd_atomic0_preop_lo;
4366 u32 cp_hqd_atomic0_preop_hi;
4367 u32 cp_hqd_atomic1_preop_lo;
4368 u32 cp_hqd_atomic1_preop_hi;
4369 u32 cp_hqd_hq_scheduler0;
4370 u32 cp_hqd_hq_scheduler1;
4377 u32 dispatch_initiator;
4381 u32 pipeline_stat_enable;
4382 u32 perf_counter_enable;
4388 u32 resource_limits;
4389 u32 static_thread_mgmt01[2];
4391 u32 static_thread_mgmt23[2];
4393 u32 thread_trace_enable;
4396 u32 vgtcs_invoke_count[2];
4397 struct hqd_registers queue_state;
4399 u32 interrupt_queue[64];
4403 * cik_cp_compute_resume - setup the compute queue registers
4405 * @rdev: radeon_device pointer
4407 * Program the compute queues and test them to make sure they
4409 * Returns 0 for success, error for failure.
4411 static int cik_cp_compute_resume(struct radeon_device *rdev)
4415 bool use_doorbell = true;
4421 struct bonaire_mqd *mqd;
4423 r = cik_cp_compute_start(rdev);
4427 /* fix up chicken bits */
4428 tmp = RREG32(CP_CPF_DEBUG);
4430 WREG32(CP_CPF_DEBUG, tmp);
4432 /* init the pipes */
4433 mutex_lock(&rdev->srbm_mutex);
4434 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4435 int me = (i < 4) ? 1 : 2;
4436 int pipe = (i < 4) ? i : (i - 4);
4438 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4440 cik_srbm_select(rdev, me, pipe, 0, 0);
4442 /* write the EOP addr */
4443 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4444 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4446 /* set the VMID assigned */
4447 WREG32(CP_HPD_EOP_VMID, 0);
4449 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4450 tmp = RREG32(CP_HPD_EOP_CONTROL);
4451 tmp &= ~EOP_SIZE_MASK;
4452 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4453 WREG32(CP_HPD_EOP_CONTROL, tmp);
4455 cik_srbm_select(rdev, 0, 0, 0, 0);
4456 mutex_unlock(&rdev->srbm_mutex);
4458 /* init the queues. Just two for now. */
4459 for (i = 0; i < 2; i++) {
4461 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4463 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4465 if (rdev->ring[idx].mqd_obj == NULL) {
4466 r = radeon_bo_create(rdev,
4467 sizeof(struct bonaire_mqd),
4469 RADEON_GEM_DOMAIN_GTT, NULL,
4470 &rdev->ring[idx].mqd_obj);
4472 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4477 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4478 if (unlikely(r != 0)) {
4479 cik_cp_compute_fini(rdev);
4482 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4485 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4486 cik_cp_compute_fini(rdev);
4489 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4491 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4492 cik_cp_compute_fini(rdev);
4496 /* init the mqd struct */
4497 memset(buf, 0, sizeof(struct bonaire_mqd));
4499 mqd = (struct bonaire_mqd *)buf;
4500 mqd->header = 0xC0310800;
4501 mqd->static_thread_mgmt01[0] = 0xffffffff;
4502 mqd->static_thread_mgmt01[1] = 0xffffffff;
4503 mqd->static_thread_mgmt23[0] = 0xffffffff;
4504 mqd->static_thread_mgmt23[1] = 0xffffffff;
4506 mutex_lock(&rdev->srbm_mutex);
4507 cik_srbm_select(rdev, rdev->ring[idx].me,
4508 rdev->ring[idx].pipe,
4509 rdev->ring[idx].queue, 0);
4511 /* disable wptr polling */
4512 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4513 tmp &= ~WPTR_POLL_EN;
4514 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4516 /* enable doorbell? */
4517 mqd->queue_state.cp_hqd_pq_doorbell_control =
4518 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4520 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4522 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4523 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4524 mqd->queue_state.cp_hqd_pq_doorbell_control);
4526 /* disable the queue if it's active */
4527 mqd->queue_state.cp_hqd_dequeue_request = 0;
4528 mqd->queue_state.cp_hqd_pq_rptr = 0;
4529 mqd->queue_state.cp_hqd_pq_wptr= 0;
4530 if (RREG32(CP_HQD_ACTIVE) & 1) {
4531 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4532 for (i = 0; i < rdev->usec_timeout; i++) {
4533 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4537 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4538 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4539 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4542 /* set the pointer to the MQD */
4543 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4544 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4545 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4546 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4547 /* set MQD vmid to 0 */
4548 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4549 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4550 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4552 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4553 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4554 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4555 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4556 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4557 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4559 /* set up the HQD, this is similar to CP_RB0_CNTL */
4560 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4561 mqd->queue_state.cp_hqd_pq_control &=
4562 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4564 mqd->queue_state.cp_hqd_pq_control |=
4565 order_base_2(rdev->ring[idx].ring_size / 8);
4566 mqd->queue_state.cp_hqd_pq_control |=
4567 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4569 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4571 mqd->queue_state.cp_hqd_pq_control &=
4572 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4573 mqd->queue_state.cp_hqd_pq_control |=
4574 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4575 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4577 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4579 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4581 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4582 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4583 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4584 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4585 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4586 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4588 /* set the wb address wether it's enabled or not */
4590 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4592 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4593 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4594 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4595 upper_32_bits(wb_gpu_addr) & 0xffff;
4596 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4597 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4598 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4599 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4601 /* enable the doorbell if requested */
4603 mqd->queue_state.cp_hqd_pq_doorbell_control =
4604 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4605 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4606 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4607 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4608 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4609 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4610 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4613 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4615 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4616 mqd->queue_state.cp_hqd_pq_doorbell_control);
4618 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4619 rdev->ring[idx].wptr = 0;
4620 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4621 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4622 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4624 /* set the vmid for the queue */
4625 mqd->queue_state.cp_hqd_vmid = 0;
4626 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4628 /* activate the queue */
4629 mqd->queue_state.cp_hqd_active = 1;
4630 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4632 cik_srbm_select(rdev, 0, 0, 0, 0);
4633 mutex_unlock(&rdev->srbm_mutex);
4635 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4636 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4638 rdev->ring[idx].ready = true;
4639 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4641 rdev->ring[idx].ready = false;
4647 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4649 cik_cp_gfx_enable(rdev, enable);
4650 cik_cp_compute_enable(rdev, enable);
4653 static int cik_cp_load_microcode(struct radeon_device *rdev)
4657 r = cik_cp_gfx_load_microcode(rdev);
4660 r = cik_cp_compute_load_microcode(rdev);
4667 static void cik_cp_fini(struct radeon_device *rdev)
4669 cik_cp_gfx_fini(rdev);
4670 cik_cp_compute_fini(rdev);
4673 static int cik_cp_resume(struct radeon_device *rdev)
4677 cik_enable_gui_idle_interrupt(rdev, false);
4679 r = cik_cp_load_microcode(rdev);
4683 r = cik_cp_gfx_resume(rdev);
4686 r = cik_cp_compute_resume(rdev);
4690 cik_enable_gui_idle_interrupt(rdev, true);
4695 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4697 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4698 RREG32(GRBM_STATUS));
4699 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4700 RREG32(GRBM_STATUS2));
4701 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4702 RREG32(GRBM_STATUS_SE0));
4703 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4704 RREG32(GRBM_STATUS_SE1));
4705 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4706 RREG32(GRBM_STATUS_SE2));
4707 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4708 RREG32(GRBM_STATUS_SE3));
4709 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4710 RREG32(SRBM_STATUS));
4711 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4712 RREG32(SRBM_STATUS2));
4713 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4714 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4715 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4716 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4717 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4718 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4719 RREG32(CP_STALLED_STAT1));
4720 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4721 RREG32(CP_STALLED_STAT2));
4722 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4723 RREG32(CP_STALLED_STAT3));
4724 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4725 RREG32(CP_CPF_BUSY_STAT));
4726 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4727 RREG32(CP_CPF_STALLED_STAT1));
4728 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4729 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4730 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4731 RREG32(CP_CPC_STALLED_STAT1));
4732 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4736 * cik_gpu_check_soft_reset - check which blocks are busy
4738 * @rdev: radeon_device pointer
4740 * Check which blocks are busy and return the relevant reset
4741 * mask to be used by cik_gpu_soft_reset().
4742 * Returns a mask of the blocks to be reset.
4744 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4750 tmp = RREG32(GRBM_STATUS);
4751 if (tmp & (PA_BUSY | SC_BUSY |
4752 BCI_BUSY | SX_BUSY |
4753 TA_BUSY | VGT_BUSY |
4755 GDS_BUSY | SPI_BUSY |
4756 IA_BUSY | IA_BUSY_NO_DMA))
4757 reset_mask |= RADEON_RESET_GFX;
4759 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4760 reset_mask |= RADEON_RESET_CP;
4763 tmp = RREG32(GRBM_STATUS2);
4765 reset_mask |= RADEON_RESET_RLC;
4767 /* SDMA0_STATUS_REG */
4768 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4769 if (!(tmp & SDMA_IDLE))
4770 reset_mask |= RADEON_RESET_DMA;
4772 /* SDMA1_STATUS_REG */
4773 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4774 if (!(tmp & SDMA_IDLE))
4775 reset_mask |= RADEON_RESET_DMA1;
4778 tmp = RREG32(SRBM_STATUS2);
4779 if (tmp & SDMA_BUSY)
4780 reset_mask |= RADEON_RESET_DMA;
4782 if (tmp & SDMA1_BUSY)
4783 reset_mask |= RADEON_RESET_DMA1;
4786 tmp = RREG32(SRBM_STATUS);
4789 reset_mask |= RADEON_RESET_IH;
4792 reset_mask |= RADEON_RESET_SEM;
4794 if (tmp & GRBM_RQ_PENDING)
4795 reset_mask |= RADEON_RESET_GRBM;
4798 reset_mask |= RADEON_RESET_VMC;
4800 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4801 MCC_BUSY | MCD_BUSY))
4802 reset_mask |= RADEON_RESET_MC;
4804 if (evergreen_is_display_hung(rdev))
4805 reset_mask |= RADEON_RESET_DISPLAY;
4807 /* Skip MC reset as it's mostly likely not hung, just busy */
4808 if (reset_mask & RADEON_RESET_MC) {
4809 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4810 reset_mask &= ~RADEON_RESET_MC;
4817 * cik_gpu_soft_reset - soft reset GPU
4819 * @rdev: radeon_device pointer
4820 * @reset_mask: mask of which blocks to reset
4822 * Soft reset the blocks specified in @reset_mask.
4824 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4826 struct evergreen_mc_save save;
4827 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4830 if (reset_mask == 0)
4833 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4835 cik_print_gpu_status_regs(rdev);
4836 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4837 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4838 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4839 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4848 /* Disable GFX parsing/prefetching */
4849 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4851 /* Disable MEC parsing/prefetching */
4852 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4854 if (reset_mask & RADEON_RESET_DMA) {
4856 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4858 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4860 if (reset_mask & RADEON_RESET_DMA1) {
4862 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4864 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4867 evergreen_mc_stop(rdev, &save);
4868 if (evergreen_mc_wait_for_idle(rdev)) {
4869 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4872 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4873 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4875 if (reset_mask & RADEON_RESET_CP) {
4876 grbm_soft_reset |= SOFT_RESET_CP;
4878 srbm_soft_reset |= SOFT_RESET_GRBM;
4881 if (reset_mask & RADEON_RESET_DMA)
4882 srbm_soft_reset |= SOFT_RESET_SDMA;
4884 if (reset_mask & RADEON_RESET_DMA1)
4885 srbm_soft_reset |= SOFT_RESET_SDMA1;
4887 if (reset_mask & RADEON_RESET_DISPLAY)
4888 srbm_soft_reset |= SOFT_RESET_DC;
4890 if (reset_mask & RADEON_RESET_RLC)
4891 grbm_soft_reset |= SOFT_RESET_RLC;
4893 if (reset_mask & RADEON_RESET_SEM)
4894 srbm_soft_reset |= SOFT_RESET_SEM;
4896 if (reset_mask & RADEON_RESET_IH)
4897 srbm_soft_reset |= SOFT_RESET_IH;
4899 if (reset_mask & RADEON_RESET_GRBM)
4900 srbm_soft_reset |= SOFT_RESET_GRBM;
4902 if (reset_mask & RADEON_RESET_VMC)
4903 srbm_soft_reset |= SOFT_RESET_VMC;
4905 if (!(rdev->flags & RADEON_IS_IGP)) {
4906 if (reset_mask & RADEON_RESET_MC)
4907 srbm_soft_reset |= SOFT_RESET_MC;
4910 if (grbm_soft_reset) {
4911 tmp = RREG32(GRBM_SOFT_RESET);
4912 tmp |= grbm_soft_reset;
4913 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4914 WREG32(GRBM_SOFT_RESET, tmp);
4915 tmp = RREG32(GRBM_SOFT_RESET);
4919 tmp &= ~grbm_soft_reset;
4920 WREG32(GRBM_SOFT_RESET, tmp);
4921 tmp = RREG32(GRBM_SOFT_RESET);
4924 if (srbm_soft_reset) {
4925 tmp = RREG32(SRBM_SOFT_RESET);
4926 tmp |= srbm_soft_reset;
4927 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4928 WREG32(SRBM_SOFT_RESET, tmp);
4929 tmp = RREG32(SRBM_SOFT_RESET);
4933 tmp &= ~srbm_soft_reset;
4934 WREG32(SRBM_SOFT_RESET, tmp);
4935 tmp = RREG32(SRBM_SOFT_RESET);
4938 /* Wait a little for things to settle down */
4941 evergreen_mc_resume(rdev, &save);
4944 cik_print_gpu_status_regs(rdev);
4947 struct kv_reset_save_regs {
4948 u32 gmcon_reng_execute;
4953 static void kv_save_regs_for_reset(struct radeon_device *rdev,
4954 struct kv_reset_save_regs *save)
4956 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
4957 save->gmcon_misc = RREG32(GMCON_MISC);
4958 save->gmcon_misc3 = RREG32(GMCON_MISC3);
4960 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
4961 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
4962 STCTRL_STUTTER_EN));
4965 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
4966 struct kv_reset_save_regs *save)
4970 WREG32(GMCON_PGFSM_WRITE, 0);
4971 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
4973 for (i = 0; i < 5; i++)
4974 WREG32(GMCON_PGFSM_WRITE, 0);
4976 WREG32(GMCON_PGFSM_WRITE, 0);
4977 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
4979 for (i = 0; i < 5; i++)
4980 WREG32(GMCON_PGFSM_WRITE, 0);
4982 WREG32(GMCON_PGFSM_WRITE, 0x210000);
4983 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
4985 for (i = 0; i < 5; i++)
4986 WREG32(GMCON_PGFSM_WRITE, 0);
4988 WREG32(GMCON_PGFSM_WRITE, 0x21003);
4989 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
4991 for (i = 0; i < 5; i++)
4992 WREG32(GMCON_PGFSM_WRITE, 0);
4994 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
4995 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
4997 for (i = 0; i < 5; i++)
4998 WREG32(GMCON_PGFSM_WRITE, 0);
5000 WREG32(GMCON_PGFSM_WRITE, 0);
5001 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5003 for (i = 0; i < 5; i++)
5004 WREG32(GMCON_PGFSM_WRITE, 0);
5006 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5007 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5009 for (i = 0; i < 5; i++)
5010 WREG32(GMCON_PGFSM_WRITE, 0);
5012 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5013 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5015 for (i = 0; i < 5; i++)
5016 WREG32(GMCON_PGFSM_WRITE, 0);
5018 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5019 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5021 for (i = 0; i < 5; i++)
5022 WREG32(GMCON_PGFSM_WRITE, 0);
5024 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5025 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5027 for (i = 0; i < 5; i++)
5028 WREG32(GMCON_PGFSM_WRITE, 0);
5030 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5031 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5033 WREG32(GMCON_MISC3, save->gmcon_misc3);
5034 WREG32(GMCON_MISC, save->gmcon_misc);
5035 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5038 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5040 struct evergreen_mc_save save;
5041 struct kv_reset_save_regs kv_save = { 0 };
5044 dev_info(rdev->dev, "GPU pci config reset\n");
5052 /* Disable GFX parsing/prefetching */
5053 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5055 /* Disable MEC parsing/prefetching */
5056 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5059 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5061 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5063 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5065 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5066 /* XXX other engines? */
5068 /* halt the rlc, disable cp internal ints */
5073 /* disable mem access */
5074 evergreen_mc_stop(rdev, &save);
5075 if (evergreen_mc_wait_for_idle(rdev)) {
5076 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5079 if (rdev->flags & RADEON_IS_IGP)
5080 kv_save_regs_for_reset(rdev, &kv_save);
5083 pci_clear_master(rdev->pdev);
5085 radeon_pci_config_reset(rdev);
5089 /* wait for asic to come out of reset */
5090 for (i = 0; i < rdev->usec_timeout; i++) {
5091 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5096 /* does asic init need to be run first??? */
5097 if (rdev->flags & RADEON_IS_IGP)
5098 kv_restore_regs_for_reset(rdev, &kv_save);
5102 * cik_asic_reset - soft reset GPU
5104 * @rdev: radeon_device pointer
5106 * Look up which blocks are hung and attempt
5108 * Returns 0 for success.
5110 int cik_asic_reset(struct radeon_device *rdev)
5114 reset_mask = cik_gpu_check_soft_reset(rdev);
5117 r600_set_bios_scratch_engine_hung(rdev, true);
5119 /* try soft reset */
5120 cik_gpu_soft_reset(rdev, reset_mask);
5122 reset_mask = cik_gpu_check_soft_reset(rdev);
5124 /* try pci config reset */
5125 if (reset_mask && radeon_hard_reset)
5126 cik_gpu_pci_config_reset(rdev);
5128 reset_mask = cik_gpu_check_soft_reset(rdev);
5131 r600_set_bios_scratch_engine_hung(rdev, false);
5137 * cik_gfx_is_lockup - check if the 3D engine is locked up
5139 * @rdev: radeon_device pointer
5140 * @ring: radeon_ring structure holding ring information
5142 * Check if the 3D engine is locked up (CIK).
5143 * Returns true if the engine is locked, false if not.
5145 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5147 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5149 if (!(reset_mask & (RADEON_RESET_GFX |
5150 RADEON_RESET_COMPUTE |
5151 RADEON_RESET_CP))) {
5152 radeon_ring_lockup_update(rdev, ring);
5155 return radeon_ring_test_lockup(rdev, ring);
5160 * cik_mc_program - program the GPU memory controller
5162 * @rdev: radeon_device pointer
5164 * Set the location of vram, gart, and AGP in the GPU's
5165 * physical address space (CIK).
5167 static void cik_mc_program(struct radeon_device *rdev)
5169 struct evergreen_mc_save save;
5173 /* Initialize HDP */
5174 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5175 WREG32((0x2c14 + j), 0x00000000);
5176 WREG32((0x2c18 + j), 0x00000000);
5177 WREG32((0x2c1c + j), 0x00000000);
5178 WREG32((0x2c20 + j), 0x00000000);
5179 WREG32((0x2c24 + j), 0x00000000);
5181 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5183 evergreen_mc_stop(rdev, &save);
5184 if (radeon_mc_wait_for_idle(rdev)) {
5185 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5187 /* Lockout access through VGA aperture*/
5188 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5189 /* Update configuration */
5190 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5191 rdev->mc.vram_start >> 12);
5192 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5193 rdev->mc.vram_end >> 12);
5194 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5195 rdev->vram_scratch.gpu_addr >> 12);
5196 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5197 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5198 WREG32(MC_VM_FB_LOCATION, tmp);
5199 /* XXX double check these! */
5200 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5201 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5202 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5203 WREG32(MC_VM_AGP_BASE, 0);
5204 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5205 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5206 if (radeon_mc_wait_for_idle(rdev)) {
5207 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5209 evergreen_mc_resume(rdev, &save);
5210 /* we need to own VRAM, so turn off the VGA renderer here
5211 * to stop it overwriting our objects */
5212 rv515_vga_render_disable(rdev);
5216 * cik_mc_init - initialize the memory controller driver params
5218 * @rdev: radeon_device pointer
5220 * Look up the amount of vram, vram width, and decide how to place
5221 * vram and gart within the GPU's physical address space (CIK).
5222 * Returns 0 for success.
5224 static int cik_mc_init(struct radeon_device *rdev)
5227 int chansize, numchan;
5229 /* Get VRAM informations */
5230 rdev->mc.vram_is_ddr = true;
5231 tmp = RREG32(MC_ARB_RAMCFG);
5232 if (tmp & CHANSIZE_MASK) {
5237 tmp = RREG32(MC_SHARED_CHMAP);
5238 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5268 rdev->mc.vram_width = numchan * chansize;
5269 /* Could aper size report 0 ? */
5270 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5271 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5272 /* size in MB on si */
5273 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5274 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5275 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5276 si_vram_gtt_location(rdev, &rdev->mc);
5277 radeon_update_bandwidth_info(rdev);
5284 * VMID 0 is the physical GPU addresses as used by the kernel.
5285 * VMIDs 1-15 are used for userspace clients and are handled
5286 * by the radeon vm/hsa code.
5289 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5291 * @rdev: radeon_device pointer
5293 * Flush the TLB for the VMID 0 page table (CIK).
5295 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5297 /* flush hdp cache */
5298 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5300 /* bits 0-15 are the VM contexts0-15 */
5301 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5305 * cik_pcie_gart_enable - gart enable
5307 * @rdev: radeon_device pointer
5309 * This sets up the TLBs, programs the page tables for VMID0,
5310 * sets up the hw for VMIDs 1-15 which are allocated on
5311 * demand, and sets up the global locations for the LDS, GDS,
5312 * and GPUVM for FSA64 clients (CIK).
5313 * Returns 0 for success, errors for failure.
5315 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5319 if (rdev->gart.robj == NULL) {
5320 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5323 r = radeon_gart_table_vram_pin(rdev);
5326 radeon_gart_restore(rdev);
5327 /* Setup TLB control */
5328 WREG32(MC_VM_MX_L1_TLB_CNTL,
5331 ENABLE_L1_FRAGMENT_PROCESSING |
5332 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5333 ENABLE_ADVANCED_DRIVER_MODEL |
5334 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5335 /* Setup L2 cache */
5336 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5337 ENABLE_L2_FRAGMENT_PROCESSING |
5338 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5339 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5340 EFFECTIVE_L2_QUEUE_SIZE(7) |
5341 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5342 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5343 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5345 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5346 /* setup context0 */
5347 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5348 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5349 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5350 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5351 (u32)(rdev->dummy_page.addr >> 12));
5352 WREG32(VM_CONTEXT0_CNTL2, 0);
5353 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5354 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5360 /* empty context1-15 */
5361 /* FIXME start with 4G, once using 2 level pt switch to full
5364 /* set vm size, must be a multiple of 4 */
5365 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5366 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5367 for (i = 1; i < 16; i++) {
5369 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5370 rdev->gart.table_addr >> 12);
5372 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5373 rdev->gart.table_addr >> 12);
5376 /* enable context1-15 */
5377 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5378 (u32)(rdev->dummy_page.addr >> 12));
5379 WREG32(VM_CONTEXT1_CNTL2, 4);
5380 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5381 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5382 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5383 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5384 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5385 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5386 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5387 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5388 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5389 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5390 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5391 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5392 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5394 if (rdev->family == CHIP_KAVERI) {
5395 u32 tmp = RREG32(CHUB_CONTROL);
5397 WREG32(CHUB_CONTROL, tmp);
5400 /* XXX SH_MEM regs */
5401 /* where to put LDS, scratch, GPUVM in FSA64 space */
5402 mutex_lock(&rdev->srbm_mutex);
5403 for (i = 0; i < 16; i++) {
5404 cik_srbm_select(rdev, 0, 0, 0, i);
5405 /* CP and shaders */
5406 WREG32(SH_MEM_CONFIG, 0);
5407 WREG32(SH_MEM_APE1_BASE, 1);
5408 WREG32(SH_MEM_APE1_LIMIT, 0);
5409 WREG32(SH_MEM_BASES, 0);
5411 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5412 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5413 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5414 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5415 /* XXX SDMA RLC - todo */
5417 cik_srbm_select(rdev, 0, 0, 0, 0);
5418 mutex_unlock(&rdev->srbm_mutex);
5420 cik_pcie_gart_tlb_flush(rdev);
5421 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5422 (unsigned)(rdev->mc.gtt_size >> 20),
5423 (unsigned long long)rdev->gart.table_addr);
5424 rdev->gart.ready = true;
5429 * cik_pcie_gart_disable - gart disable
5431 * @rdev: radeon_device pointer
5433 * This disables all VM page table (CIK).
5435 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5437 /* Disable all tables */
5438 WREG32(VM_CONTEXT0_CNTL, 0);
5439 WREG32(VM_CONTEXT1_CNTL, 0);
5440 /* Setup TLB control */
5441 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5442 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5443 /* Setup L2 cache */
5445 ENABLE_L2_FRAGMENT_PROCESSING |
5446 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5447 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5448 EFFECTIVE_L2_QUEUE_SIZE(7) |
5449 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5450 WREG32(VM_L2_CNTL2, 0);
5451 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5452 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5453 radeon_gart_table_vram_unpin(rdev);
5457 * cik_pcie_gart_fini - vm fini callback
5459 * @rdev: radeon_device pointer
5461 * Tears down the driver GART/VM setup (CIK).
5463 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5465 cik_pcie_gart_disable(rdev);
5466 radeon_gart_table_vram_free(rdev);
5467 radeon_gart_fini(rdev);
5472 * cik_ib_parse - vm ib_parse callback
5474 * @rdev: radeon_device pointer
5475 * @ib: indirect buffer pointer
5477 * CIK uses hw IB checking so this is a nop (CIK).
5479 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5486 * VMID 0 is the physical GPU addresses as used by the kernel.
5487 * VMIDs 1-15 are used for userspace clients and are handled
5488 * by the radeon vm/hsa code.
5491 * cik_vm_init - cik vm init callback
5493 * @rdev: radeon_device pointer
5495 * Inits cik specific vm parameters (number of VMs, base of vram for
5496 * VMIDs 1-15) (CIK).
5497 * Returns 0 for success.
5499 int cik_vm_init(struct radeon_device *rdev)
5502 rdev->vm_manager.nvm = 16;
5503 /* base offset of vram pages */
5504 if (rdev->flags & RADEON_IS_IGP) {
5505 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5507 rdev->vm_manager.vram_base_offset = tmp;
5509 rdev->vm_manager.vram_base_offset = 0;
5515 * cik_vm_fini - cik vm fini callback
5517 * @rdev: radeon_device pointer
5519 * Tear down any asic specific VM setup (CIK).
5521 void cik_vm_fini(struct radeon_device *rdev)
5526 * cik_vm_decode_fault - print human readable fault info
5528 * @rdev: radeon_device pointer
5529 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5530 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5532 * Print human readable fault information (CIK).
5534 static void cik_vm_decode_fault(struct radeon_device *rdev,
5535 u32 status, u32 addr, u32 mc_client)
5538 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5539 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5540 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5541 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5543 if (rdev->family == CHIP_HAWAII)
5544 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5546 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5548 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5549 protections, vmid, addr,
5550 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5551 block, mc_client, mc_id);
5555 * cik_vm_flush - cik vm flush using the CP
5557 * @rdev: radeon_device pointer
5559 * Update the page table base and flush the VM TLB
5560 * using the CP (CIK).
5562 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5564 struct radeon_ring *ring = &rdev->ring[ridx];
5569 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5570 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5571 WRITE_DATA_DST_SEL(0)));
5573 radeon_ring_write(ring,
5574 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5576 radeon_ring_write(ring,
5577 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5579 radeon_ring_write(ring, 0);
5580 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5582 /* update SH_MEM_* regs */
5583 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5584 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5585 WRITE_DATA_DST_SEL(0)));
5586 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5587 radeon_ring_write(ring, 0);
5588 radeon_ring_write(ring, VMID(vm->id));
5590 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5591 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5592 WRITE_DATA_DST_SEL(0)));
5593 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5594 radeon_ring_write(ring, 0);
5596 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5597 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5598 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5599 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5601 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5602 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5603 WRITE_DATA_DST_SEL(0)));
5604 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5605 radeon_ring_write(ring, 0);
5606 radeon_ring_write(ring, VMID(0));
5609 cik_hdp_flush_cp_ring_emit(rdev, ridx);
5611 /* bits 0-15 are the VM contexts0-15 */
5612 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5613 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5614 WRITE_DATA_DST_SEL(0)));
5615 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5616 radeon_ring_write(ring, 0);
5617 radeon_ring_write(ring, 1 << vm->id);
5619 /* compute doesn't have PFP */
5620 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5621 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5622 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5623 radeon_ring_write(ring, 0x0);
5629 * The RLC is a multi-purpose microengine that handles a
5630 * variety of functions, the most important of which is
5631 * the interrupt controller.
5633 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5636 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5639 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5641 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5642 WREG32(CP_INT_CNTL_RING0, tmp);
5645 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5649 tmp = RREG32(RLC_LB_CNTL);
5651 tmp |= LOAD_BALANCE_ENABLE;
5653 tmp &= ~LOAD_BALANCE_ENABLE;
5654 WREG32(RLC_LB_CNTL, tmp);
5657 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5662 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5663 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5664 cik_select_se_sh(rdev, i, j);
5665 for (k = 0; k < rdev->usec_timeout; k++) {
5666 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5672 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5674 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5675 for (k = 0; k < rdev->usec_timeout; k++) {
5676 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5682 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5686 tmp = RREG32(RLC_CNTL);
5688 WREG32(RLC_CNTL, rlc);
5691 static u32 cik_halt_rlc(struct radeon_device *rdev)
5695 orig = data = RREG32(RLC_CNTL);
5697 if (data & RLC_ENABLE) {
5700 data &= ~RLC_ENABLE;
5701 WREG32(RLC_CNTL, data);
5703 for (i = 0; i < rdev->usec_timeout; i++) {
5704 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5709 cik_wait_for_rlc_serdes(rdev);
5715 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5719 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5720 WREG32(RLC_GPR_REG2, tmp);
5722 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5723 for (i = 0; i < rdev->usec_timeout; i++) {
5724 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5729 for (i = 0; i < rdev->usec_timeout; i++) {
5730 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5736 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5740 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5741 WREG32(RLC_GPR_REG2, tmp);
5745 * cik_rlc_stop - stop the RLC ME
5747 * @rdev: radeon_device pointer
5749 * Halt the RLC ME (MicroEngine) (CIK).
5751 static void cik_rlc_stop(struct radeon_device *rdev)
5753 WREG32(RLC_CNTL, 0);
5755 cik_enable_gui_idle_interrupt(rdev, false);
5757 cik_wait_for_rlc_serdes(rdev);
5761 * cik_rlc_start - start the RLC ME
5763 * @rdev: radeon_device pointer
5765 * Unhalt the RLC ME (MicroEngine) (CIK).
5767 static void cik_rlc_start(struct radeon_device *rdev)
5769 WREG32(RLC_CNTL, RLC_ENABLE);
5771 cik_enable_gui_idle_interrupt(rdev, true);
5777 * cik_rlc_resume - setup the RLC hw
5779 * @rdev: radeon_device pointer
5781 * Initialize the RLC registers, load the ucode,
5782 * and start the RLC (CIK).
5783 * Returns 0 for success, -EINVAL if the ucode is not available.
5785 static int cik_rlc_resume(struct radeon_device *rdev)
5788 const __be32 *fw_data;
5793 switch (rdev->family) {
5797 size = BONAIRE_RLC_UCODE_SIZE;
5800 size = KV_RLC_UCODE_SIZE;
5803 size = KB_RLC_UCODE_SIZE;
5810 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5811 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5819 WREG32(RLC_LB_CNTR_INIT, 0);
5820 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5822 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5823 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5824 WREG32(RLC_LB_PARAMS, 0x00600408);
5825 WREG32(RLC_LB_CNTL, 0x80000004);
5827 WREG32(RLC_MC_CNTL, 0);
5828 WREG32(RLC_UCODE_CNTL, 0);
5830 fw_data = (const __be32 *)rdev->rlc_fw->data;
5831 WREG32(RLC_GPM_UCODE_ADDR, 0);
5832 for (i = 0; i < size; i++)
5833 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5834 WREG32(RLC_GPM_UCODE_ADDR, 0);
5836 /* XXX - find out what chips support lbpw */
5837 cik_enable_lbpw(rdev, false);
5839 if (rdev->family == CHIP_BONAIRE)
5840 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5842 cik_rlc_start(rdev);
5847 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5849 u32 data, orig, tmp, tmp2;
5851 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5853 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5854 cik_enable_gui_idle_interrupt(rdev, true);
5856 tmp = cik_halt_rlc(rdev);
5858 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5859 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5860 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5861 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5862 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5864 cik_update_rlc(rdev, tmp);
5866 data |= CGCG_EN | CGLS_EN;
5868 cik_enable_gui_idle_interrupt(rdev, false);
5870 RREG32(CB_CGTT_SCLK_CTRL);
5871 RREG32(CB_CGTT_SCLK_CTRL);
5872 RREG32(CB_CGTT_SCLK_CTRL);
5873 RREG32(CB_CGTT_SCLK_CTRL);
5875 data &= ~(CGCG_EN | CGLS_EN);
5879 WREG32(RLC_CGCG_CGLS_CTRL, data);
5883 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5885 u32 data, orig, tmp = 0;
5887 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5888 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5889 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5890 orig = data = RREG32(CP_MEM_SLP_CNTL);
5891 data |= CP_MEM_LS_EN;
5893 WREG32(CP_MEM_SLP_CNTL, data);
5897 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5900 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5902 tmp = cik_halt_rlc(rdev);
5904 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5905 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5906 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5907 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5908 WREG32(RLC_SERDES_WR_CTRL, data);
5910 cik_update_rlc(rdev, tmp);
5912 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5913 orig = data = RREG32(CGTS_SM_CTRL_REG);
5914 data &= ~SM_MODE_MASK;
5915 data |= SM_MODE(0x2);
5916 data |= SM_MODE_ENABLE;
5917 data &= ~CGTS_OVERRIDE;
5918 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5919 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5920 data &= ~CGTS_LS_OVERRIDE;
5921 data &= ~ON_MONITOR_ADD_MASK;
5922 data |= ON_MONITOR_ADD_EN;
5923 data |= ON_MONITOR_ADD(0x96);
5925 WREG32(CGTS_SM_CTRL_REG, data);
5928 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5931 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5933 data = RREG32(RLC_MEM_SLP_CNTL);
5934 if (data & RLC_MEM_LS_EN) {
5935 data &= ~RLC_MEM_LS_EN;
5936 WREG32(RLC_MEM_SLP_CNTL, data);
5939 data = RREG32(CP_MEM_SLP_CNTL);
5940 if (data & CP_MEM_LS_EN) {
5941 data &= ~CP_MEM_LS_EN;
5942 WREG32(CP_MEM_SLP_CNTL, data);
5945 orig = data = RREG32(CGTS_SM_CTRL_REG);
5946 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5948 WREG32(CGTS_SM_CTRL_REG, data);
5950 tmp = cik_halt_rlc(rdev);
5952 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5953 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5954 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5955 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5956 WREG32(RLC_SERDES_WR_CTRL, data);
5958 cik_update_rlc(rdev, tmp);
5962 static const u32 mc_cg_registers[] =
5975 static void cik_enable_mc_ls(struct radeon_device *rdev,
5981 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5982 orig = data = RREG32(mc_cg_registers[i]);
5983 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5984 data |= MC_LS_ENABLE;
5986 data &= ~MC_LS_ENABLE;
5988 WREG32(mc_cg_registers[i], data);
5992 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5998 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5999 orig = data = RREG32(mc_cg_registers[i]);
6000 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6001 data |= MC_CG_ENABLE;
6003 data &= ~MC_CG_ENABLE;
6005 WREG32(mc_cg_registers[i], data);
6009 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6014 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6015 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6016 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6018 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6021 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6023 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6026 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6030 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6035 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6036 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6039 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6041 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6044 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6046 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6049 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6051 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6054 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6058 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6063 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6064 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6066 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6068 orig = data = RREG32(UVD_CGC_CTRL);
6071 WREG32(UVD_CGC_CTRL, data);
6073 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6075 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6077 orig = data = RREG32(UVD_CGC_CTRL);
6080 WREG32(UVD_CGC_CTRL, data);
6084 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6089 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6091 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6092 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6093 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6095 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6096 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6099 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6102 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6107 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6109 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6110 data &= ~CLOCK_GATING_DIS;
6112 data |= CLOCK_GATING_DIS;
6115 WREG32(HDP_HOST_PATH_CNTL, data);
6118 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6123 orig = data = RREG32(HDP_MEM_POWER_LS);
6125 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6126 data |= HDP_LS_ENABLE;
6128 data &= ~HDP_LS_ENABLE;
6131 WREG32(HDP_MEM_POWER_LS, data);
6134 void cik_update_cg(struct radeon_device *rdev,
6135 u32 block, bool enable)
6138 if (block & RADEON_CG_BLOCK_GFX) {
6139 cik_enable_gui_idle_interrupt(rdev, false);
6140 /* order matters! */
6142 cik_enable_mgcg(rdev, true);
6143 cik_enable_cgcg(rdev, true);
6145 cik_enable_cgcg(rdev, false);
6146 cik_enable_mgcg(rdev, false);
6148 cik_enable_gui_idle_interrupt(rdev, true);
6151 if (block & RADEON_CG_BLOCK_MC) {
6152 if (!(rdev->flags & RADEON_IS_IGP)) {
6153 cik_enable_mc_mgcg(rdev, enable);
6154 cik_enable_mc_ls(rdev, enable);
6158 if (block & RADEON_CG_BLOCK_SDMA) {
6159 cik_enable_sdma_mgcg(rdev, enable);
6160 cik_enable_sdma_mgls(rdev, enable);
6163 if (block & RADEON_CG_BLOCK_BIF) {
6164 cik_enable_bif_mgls(rdev, enable);
6167 if (block & RADEON_CG_BLOCK_UVD) {
6169 cik_enable_uvd_mgcg(rdev, enable);
6172 if (block & RADEON_CG_BLOCK_HDP) {
6173 cik_enable_hdp_mgcg(rdev, enable);
6174 cik_enable_hdp_ls(rdev, enable);
6177 if (block & RADEON_CG_BLOCK_VCE) {
6178 vce_v2_0_enable_mgcg(rdev, enable);
6182 static void cik_init_cg(struct radeon_device *rdev)
6185 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6188 si_init_uvd_internal_cg(rdev);
6190 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6191 RADEON_CG_BLOCK_SDMA |
6192 RADEON_CG_BLOCK_BIF |
6193 RADEON_CG_BLOCK_UVD |
6194 RADEON_CG_BLOCK_HDP), true);
6197 static void cik_fini_cg(struct radeon_device *rdev)
6199 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6200 RADEON_CG_BLOCK_SDMA |
6201 RADEON_CG_BLOCK_BIF |
6202 RADEON_CG_BLOCK_UVD |
6203 RADEON_CG_BLOCK_HDP), false);
6205 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6208 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6213 orig = data = RREG32(RLC_PG_CNTL);
6214 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6215 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6217 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6219 WREG32(RLC_PG_CNTL, data);
6222 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6227 orig = data = RREG32(RLC_PG_CNTL);
6228 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6229 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6231 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6233 WREG32(RLC_PG_CNTL, data);
6236 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6240 orig = data = RREG32(RLC_PG_CNTL);
6241 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6242 data &= ~DISABLE_CP_PG;
6244 data |= DISABLE_CP_PG;
6246 WREG32(RLC_PG_CNTL, data);
6249 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6253 orig = data = RREG32(RLC_PG_CNTL);
6254 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6255 data &= ~DISABLE_GDS_PG;
6257 data |= DISABLE_GDS_PG;
6259 WREG32(RLC_PG_CNTL, data);
6262 #define CP_ME_TABLE_SIZE 96
6263 #define CP_ME_TABLE_OFFSET 2048
6264 #define CP_MEC_TABLE_OFFSET 4096
6266 void cik_init_cp_pg_table(struct radeon_device *rdev)
6268 const __be32 *fw_data;
6269 volatile u32 *dst_ptr;
6270 int me, i, max_me = 4;
6274 if (rdev->family == CHIP_KAVERI)
6277 if (rdev->rlc.cp_table_ptr == NULL)
6280 /* write the cp table buffer */
6281 dst_ptr = rdev->rlc.cp_table_ptr;
6282 for (me = 0; me < max_me; me++) {
6284 fw_data = (const __be32 *)rdev->ce_fw->data;
6285 table_offset = CP_ME_TABLE_OFFSET;
6286 } else if (me == 1) {
6287 fw_data = (const __be32 *)rdev->pfp_fw->data;
6288 table_offset = CP_ME_TABLE_OFFSET;
6289 } else if (me == 2) {
6290 fw_data = (const __be32 *)rdev->me_fw->data;
6291 table_offset = CP_ME_TABLE_OFFSET;
6293 fw_data = (const __be32 *)rdev->mec_fw->data;
6294 table_offset = CP_MEC_TABLE_OFFSET;
6297 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6298 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6300 bo_offset += CP_ME_TABLE_SIZE;
6304 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6309 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6310 orig = data = RREG32(RLC_PG_CNTL);
6311 data |= GFX_PG_ENABLE;
6313 WREG32(RLC_PG_CNTL, data);
6315 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6318 WREG32(RLC_AUTO_PG_CTRL, data);
6320 orig = data = RREG32(RLC_PG_CNTL);
6321 data &= ~GFX_PG_ENABLE;
6323 WREG32(RLC_PG_CNTL, data);
6325 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6326 data &= ~AUTO_PG_EN;
6328 WREG32(RLC_AUTO_PG_CTRL, data);
6330 data = RREG32(DB_RENDER_CONTROL);
6334 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6336 u32 mask = 0, tmp, tmp1;
6339 cik_select_se_sh(rdev, se, sh);
6340 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6341 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6342 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6349 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6354 return (~tmp) & mask;
6357 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6359 u32 i, j, k, active_cu_number = 0;
6360 u32 mask, counter, cu_bitmap;
6363 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6364 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6368 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6369 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6377 active_cu_number += counter;
6378 tmp |= (cu_bitmap << (i * 16 + j * 8));
6382 WREG32(RLC_PG_AO_CU_MASK, tmp);
6384 tmp = RREG32(RLC_MAX_PG_CU);
6385 tmp &= ~MAX_PU_CU_MASK;
6386 tmp |= MAX_PU_CU(active_cu_number);
6387 WREG32(RLC_MAX_PG_CU, tmp);
6390 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6395 orig = data = RREG32(RLC_PG_CNTL);
6396 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6397 data |= STATIC_PER_CU_PG_ENABLE;
6399 data &= ~STATIC_PER_CU_PG_ENABLE;
6401 WREG32(RLC_PG_CNTL, data);
6404 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6409 orig = data = RREG32(RLC_PG_CNTL);
6410 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6411 data |= DYN_PER_CU_PG_ENABLE;
6413 data &= ~DYN_PER_CU_PG_ENABLE;
6415 WREG32(RLC_PG_CNTL, data);
6418 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6419 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6421 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6426 if (rdev->rlc.cs_data) {
6427 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6428 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6429 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6430 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6432 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6433 for (i = 0; i < 3; i++)
6434 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6436 if (rdev->rlc.reg_list) {
6437 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6438 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6439 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6442 orig = data = RREG32(RLC_PG_CNTL);
6445 WREG32(RLC_PG_CNTL, data);
6447 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6448 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6450 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6451 data &= ~IDLE_POLL_COUNT_MASK;
6452 data |= IDLE_POLL_COUNT(0x60);
6453 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6456 WREG32(RLC_PG_DELAY, data);
6458 data = RREG32(RLC_PG_DELAY_2);
6461 WREG32(RLC_PG_DELAY_2, data);
6463 data = RREG32(RLC_AUTO_PG_CTRL);
6464 data &= ~GRBM_REG_SGIT_MASK;
6465 data |= GRBM_REG_SGIT(0x700);
6466 WREG32(RLC_AUTO_PG_CTRL, data);
6470 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6472 cik_enable_gfx_cgpg(rdev, enable);
6473 cik_enable_gfx_static_mgpg(rdev, enable);
6474 cik_enable_gfx_dynamic_mgpg(rdev, enable);
6477 u32 cik_get_csb_size(struct radeon_device *rdev)
6480 const struct cs_section_def *sect = NULL;
6481 const struct cs_extent_def *ext = NULL;
6483 if (rdev->rlc.cs_data == NULL)
6486 /* begin clear state */
6488 /* context control state */
6491 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6492 for (ext = sect->section; ext->extent != NULL; ++ext) {
6493 if (sect->id == SECT_CONTEXT)
6494 count += 2 + ext->reg_count;
6499 /* pa_sc_raster_config/pa_sc_raster_config1 */
6501 /* end clear state */
6509 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6512 const struct cs_section_def *sect = NULL;
6513 const struct cs_extent_def *ext = NULL;
6515 if (rdev->rlc.cs_data == NULL)
6520 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6521 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6523 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6524 buffer[count++] = cpu_to_le32(0x80000000);
6525 buffer[count++] = cpu_to_le32(0x80000000);
6527 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6528 for (ext = sect->section; ext->extent != NULL; ++ext) {
6529 if (sect->id == SECT_CONTEXT) {
6531 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6532 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6533 for (i = 0; i < ext->reg_count; i++)
6534 buffer[count++] = cpu_to_le32(ext->extent[i]);
6541 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6542 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6543 switch (rdev->family) {
6545 buffer[count++] = cpu_to_le32(0x16000012);
6546 buffer[count++] = cpu_to_le32(0x00000000);
6549 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6550 buffer[count++] = cpu_to_le32(0x00000000);
6553 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6554 buffer[count++] = cpu_to_le32(0x00000000);
6557 buffer[count++] = cpu_to_le32(0x3a00161a);
6558 buffer[count++] = cpu_to_le32(0x0000002e);
6561 buffer[count++] = cpu_to_le32(0x00000000);
6562 buffer[count++] = cpu_to_le32(0x00000000);
6566 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6567 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6569 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6570 buffer[count++] = cpu_to_le32(0);
6573 static void cik_init_pg(struct radeon_device *rdev)
6575 if (rdev->pg_flags) {
6576 cik_enable_sck_slowdown_on_pu(rdev, true);
6577 cik_enable_sck_slowdown_on_pd(rdev, true);
6578 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6579 cik_init_gfx_cgpg(rdev);
6580 cik_enable_cp_pg(rdev, true);
6581 cik_enable_gds_pg(rdev, true);
6583 cik_init_ao_cu_mask(rdev);
6584 cik_update_gfx_pg(rdev, true);
6588 static void cik_fini_pg(struct radeon_device *rdev)
6590 if (rdev->pg_flags) {
6591 cik_update_gfx_pg(rdev, false);
6592 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6593 cik_enable_cp_pg(rdev, false);
6594 cik_enable_gds_pg(rdev, false);
6601 * Starting with r6xx, interrupts are handled via a ring buffer.
6602 * Ring buffers are areas of GPU accessible memory that the GPU
6603 * writes interrupt vectors into and the host reads vectors out of.
6604 * There is a rptr (read pointer) that determines where the
6605 * host is currently reading, and a wptr (write pointer)
6606 * which determines where the GPU has written. When the
6607 * pointers are equal, the ring is idle. When the GPU
6608 * writes vectors to the ring buffer, it increments the
6609 * wptr. When there is an interrupt, the host then starts
6610 * fetching commands and processing them until the pointers are
6611 * equal again at which point it updates the rptr.
6615 * cik_enable_interrupts - Enable the interrupt ring buffer
6617 * @rdev: radeon_device pointer
6619 * Enable the interrupt ring buffer (CIK).
6621 static void cik_enable_interrupts(struct radeon_device *rdev)
6623 u32 ih_cntl = RREG32(IH_CNTL);
6624 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6626 ih_cntl |= ENABLE_INTR;
6627 ih_rb_cntl |= IH_RB_ENABLE;
6628 WREG32(IH_CNTL, ih_cntl);
6629 WREG32(IH_RB_CNTL, ih_rb_cntl);
6630 rdev->ih.enabled = true;
6634 * cik_disable_interrupts - Disable the interrupt ring buffer
6636 * @rdev: radeon_device pointer
6638 * Disable the interrupt ring buffer (CIK).
6640 static void cik_disable_interrupts(struct radeon_device *rdev)
6642 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6643 u32 ih_cntl = RREG32(IH_CNTL);
6645 ih_rb_cntl &= ~IH_RB_ENABLE;
6646 ih_cntl &= ~ENABLE_INTR;
6647 WREG32(IH_RB_CNTL, ih_rb_cntl);
6648 WREG32(IH_CNTL, ih_cntl);
6649 /* set rptr, wptr to 0 */
6650 WREG32(IH_RB_RPTR, 0);
6651 WREG32(IH_RB_WPTR, 0);
6652 rdev->ih.enabled = false;
6657 * cik_disable_interrupt_state - Disable all interrupt sources
6659 * @rdev: radeon_device pointer
6661 * Clear all interrupt enable bits used by the driver (CIK).
6663 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6668 tmp = RREG32(CP_INT_CNTL_RING0) &
6669 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6670 WREG32(CP_INT_CNTL_RING0, tmp);
6672 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6673 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6674 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6675 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6676 /* compute queues */
6677 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6678 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6679 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6680 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6681 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6682 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6683 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6684 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6686 WREG32(GRBM_INT_CNTL, 0);
6687 /* vline/vblank, etc. */
6688 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6689 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6690 if (rdev->num_crtc >= 4) {
6691 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6692 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6694 if (rdev->num_crtc >= 6) {
6695 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6696 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6700 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6702 /* digital hotplug */
6703 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6704 WREG32(DC_HPD1_INT_CONTROL, tmp);
6705 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6706 WREG32(DC_HPD2_INT_CONTROL, tmp);
6707 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6708 WREG32(DC_HPD3_INT_CONTROL, tmp);
6709 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6710 WREG32(DC_HPD4_INT_CONTROL, tmp);
6711 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6712 WREG32(DC_HPD5_INT_CONTROL, tmp);
6713 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6714 WREG32(DC_HPD6_INT_CONTROL, tmp);
6719 * cik_irq_init - init and enable the interrupt ring
6721 * @rdev: radeon_device pointer
6723 * Allocate a ring buffer for the interrupt controller,
6724 * enable the RLC, disable interrupts, enable the IH
6725 * ring buffer and enable it (CIK).
6726 * Called at device load and reume.
6727 * Returns 0 for success, errors for failure.
6729 static int cik_irq_init(struct radeon_device *rdev)
6733 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6736 ret = r600_ih_ring_alloc(rdev);
6741 cik_disable_interrupts(rdev);
6744 ret = cik_rlc_resume(rdev);
6746 r600_ih_ring_fini(rdev);
6750 /* setup interrupt control */
6751 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6752 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6753 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6754 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6755 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6757 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6758 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6759 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6760 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6762 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6763 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6765 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6766 IH_WPTR_OVERFLOW_CLEAR |
6769 if (rdev->wb.enabled)
6770 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6772 /* set the writeback address whether it's enabled or not */
6773 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6774 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6776 WREG32(IH_RB_CNTL, ih_rb_cntl);
6778 /* set rptr, wptr to 0 */
6779 WREG32(IH_RB_RPTR, 0);
6780 WREG32(IH_RB_WPTR, 0);
6782 /* Default settings for IH_CNTL (disabled at first) */
6783 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6784 /* RPTR_REARM only works if msi's are enabled */
6785 if (rdev->msi_enabled)
6786 ih_cntl |= RPTR_REARM;
6787 WREG32(IH_CNTL, ih_cntl);
6789 /* force the active interrupt state to all disabled */
6790 cik_disable_interrupt_state(rdev);
6792 pci_set_master(rdev->pdev);
6795 cik_enable_interrupts(rdev);
6801 * cik_irq_set - enable/disable interrupt sources
6803 * @rdev: radeon_device pointer
6805 * Enable interrupt sources on the GPU (vblanks, hpd,
6807 * Returns 0 for success, errors for failure.
6809 int cik_irq_set(struct radeon_device *rdev)
6812 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6813 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6814 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6815 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6816 u32 grbm_int_cntl = 0;
6817 u32 dma_cntl, dma_cntl1;
6820 if (!rdev->irq.installed) {
6821 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6824 /* don't enable anything if the ih is disabled */
6825 if (!rdev->ih.enabled) {
6826 cik_disable_interrupts(rdev);
6827 /* force the active interrupt state to all disabled */
6828 cik_disable_interrupt_state(rdev);
6832 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6833 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6834 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6836 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6837 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6838 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6839 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6840 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6841 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6843 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6844 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6846 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6847 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6848 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6849 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6850 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6851 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6852 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6853 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6855 if (rdev->flags & RADEON_IS_IGP)
6856 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6857 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6859 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6860 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6862 /* enable CP interrupts on all rings */
6863 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6864 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6865 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6867 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6868 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6869 DRM_DEBUG("si_irq_set: sw int cp1\n");
6870 if (ring->me == 1) {
6871 switch (ring->pipe) {
6873 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6876 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6879 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6882 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6885 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6888 } else if (ring->me == 2) {
6889 switch (ring->pipe) {
6891 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6894 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6897 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6900 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6903 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6907 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6910 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6911 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6912 DRM_DEBUG("si_irq_set: sw int cp2\n");
6913 if (ring->me == 1) {
6914 switch (ring->pipe) {
6916 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6919 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6922 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6925 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6928 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6931 } else if (ring->me == 2) {
6932 switch (ring->pipe) {
6934 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6937 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6940 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6943 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6946 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6950 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6954 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6955 DRM_DEBUG("cik_irq_set: sw int dma\n");
6956 dma_cntl |= TRAP_ENABLE;
6959 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6960 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6961 dma_cntl1 |= TRAP_ENABLE;
6964 if (rdev->irq.crtc_vblank_int[0] ||
6965 atomic_read(&rdev->irq.pflip[0])) {
6966 DRM_DEBUG("cik_irq_set: vblank 0\n");
6967 crtc1 |= VBLANK_INTERRUPT_MASK;
6969 if (rdev->irq.crtc_vblank_int[1] ||
6970 atomic_read(&rdev->irq.pflip[1])) {
6971 DRM_DEBUG("cik_irq_set: vblank 1\n");
6972 crtc2 |= VBLANK_INTERRUPT_MASK;
6974 if (rdev->irq.crtc_vblank_int[2] ||
6975 atomic_read(&rdev->irq.pflip[2])) {
6976 DRM_DEBUG("cik_irq_set: vblank 2\n");
6977 crtc3 |= VBLANK_INTERRUPT_MASK;
6979 if (rdev->irq.crtc_vblank_int[3] ||
6980 atomic_read(&rdev->irq.pflip[3])) {
6981 DRM_DEBUG("cik_irq_set: vblank 3\n");
6982 crtc4 |= VBLANK_INTERRUPT_MASK;
6984 if (rdev->irq.crtc_vblank_int[4] ||
6985 atomic_read(&rdev->irq.pflip[4])) {
6986 DRM_DEBUG("cik_irq_set: vblank 4\n");
6987 crtc5 |= VBLANK_INTERRUPT_MASK;
6989 if (rdev->irq.crtc_vblank_int[5] ||
6990 atomic_read(&rdev->irq.pflip[5])) {
6991 DRM_DEBUG("cik_irq_set: vblank 5\n");
6992 crtc6 |= VBLANK_INTERRUPT_MASK;
6994 if (rdev->irq.hpd[0]) {
6995 DRM_DEBUG("cik_irq_set: hpd 1\n");
6996 hpd1 |= DC_HPDx_INT_EN;
6998 if (rdev->irq.hpd[1]) {
6999 DRM_DEBUG("cik_irq_set: hpd 2\n");
7000 hpd2 |= DC_HPDx_INT_EN;
7002 if (rdev->irq.hpd[2]) {
7003 DRM_DEBUG("cik_irq_set: hpd 3\n");
7004 hpd3 |= DC_HPDx_INT_EN;
7006 if (rdev->irq.hpd[3]) {
7007 DRM_DEBUG("cik_irq_set: hpd 4\n");
7008 hpd4 |= DC_HPDx_INT_EN;
7010 if (rdev->irq.hpd[4]) {
7011 DRM_DEBUG("cik_irq_set: hpd 5\n");
7012 hpd5 |= DC_HPDx_INT_EN;
7014 if (rdev->irq.hpd[5]) {
7015 DRM_DEBUG("cik_irq_set: hpd 6\n");
7016 hpd6 |= DC_HPDx_INT_EN;
7019 if (rdev->irq.dpm_thermal) {
7020 DRM_DEBUG("dpm thermal\n");
7021 if (rdev->flags & RADEON_IS_IGP)
7022 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7024 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7027 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7029 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7030 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7032 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7033 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7034 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7035 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7036 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7037 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7038 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7039 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7041 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7043 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7044 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7045 if (rdev->num_crtc >= 4) {
7046 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7047 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7049 if (rdev->num_crtc >= 6) {
7050 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7051 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7054 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7055 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7056 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7057 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7058 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7059 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7061 if (rdev->flags & RADEON_IS_IGP)
7062 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7064 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7070 * cik_irq_ack - ack interrupt sources
7072 * @rdev: radeon_device pointer
7074 * Ack interrupt sources on the GPU (vblanks, hpd,
7075 * etc.) (CIK). Certain interrupts sources are sw
7076 * generated and do not require an explicit ack.
7078 static inline void cik_irq_ack(struct radeon_device *rdev)
7082 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7083 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7084 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7085 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7086 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7087 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7088 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7090 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7091 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7092 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7093 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7094 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7095 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7096 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7097 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7099 if (rdev->num_crtc >= 4) {
7100 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7101 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7102 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7103 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7104 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7105 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7106 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7107 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7110 if (rdev->num_crtc >= 6) {
7111 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7112 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7113 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7114 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7115 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7116 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7117 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7118 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7121 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7122 tmp = RREG32(DC_HPD1_INT_CONTROL);
7123 tmp |= DC_HPDx_INT_ACK;
7124 WREG32(DC_HPD1_INT_CONTROL, tmp);
7126 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7127 tmp = RREG32(DC_HPD2_INT_CONTROL);
7128 tmp |= DC_HPDx_INT_ACK;
7129 WREG32(DC_HPD2_INT_CONTROL, tmp);
7131 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7132 tmp = RREG32(DC_HPD3_INT_CONTROL);
7133 tmp |= DC_HPDx_INT_ACK;
7134 WREG32(DC_HPD3_INT_CONTROL, tmp);
7136 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7137 tmp = RREG32(DC_HPD4_INT_CONTROL);
7138 tmp |= DC_HPDx_INT_ACK;
7139 WREG32(DC_HPD4_INT_CONTROL, tmp);
7141 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7142 tmp = RREG32(DC_HPD5_INT_CONTROL);
7143 tmp |= DC_HPDx_INT_ACK;
7144 WREG32(DC_HPD5_INT_CONTROL, tmp);
7146 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7147 tmp = RREG32(DC_HPD5_INT_CONTROL);
7148 tmp |= DC_HPDx_INT_ACK;
7149 WREG32(DC_HPD6_INT_CONTROL, tmp);
7154 * cik_irq_disable - disable interrupts
7156 * @rdev: radeon_device pointer
7158 * Disable interrupts on the hw (CIK).
7160 static void cik_irq_disable(struct radeon_device *rdev)
7162 cik_disable_interrupts(rdev);
7163 /* Wait and acknowledge irq */
7166 cik_disable_interrupt_state(rdev);
7170 * cik_irq_disable - disable interrupts for suspend
7172 * @rdev: radeon_device pointer
7174 * Disable interrupts and stop the RLC (CIK).
7177 static void cik_irq_suspend(struct radeon_device *rdev)
7179 cik_irq_disable(rdev);
7184 * cik_irq_fini - tear down interrupt support
7186 * @rdev: radeon_device pointer
7188 * Disable interrupts on the hw and free the IH ring
7190 * Used for driver unload.
7192 static void cik_irq_fini(struct radeon_device *rdev)
7194 cik_irq_suspend(rdev);
7195 r600_ih_ring_fini(rdev);
7199 * cik_get_ih_wptr - get the IH ring buffer wptr
7201 * @rdev: radeon_device pointer
7203 * Get the IH ring buffer wptr from either the register
7204 * or the writeback memory buffer (CIK). Also check for
7205 * ring buffer overflow and deal with it.
7206 * Used by cik_irq_process().
7207 * Returns the value of the wptr.
7209 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7213 if (rdev->wb.enabled)
7214 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7216 wptr = RREG32(IH_RB_WPTR);
7218 if (wptr & RB_OVERFLOW) {
7219 /* When a ring buffer overflow happen start parsing interrupt
7220 * from the last not overwritten vector (wptr + 16). Hopefully
7221 * this should allow us to catchup.
7223 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7224 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7225 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7226 tmp = RREG32(IH_RB_CNTL);
7227 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7228 WREG32(IH_RB_CNTL, tmp);
7230 return (wptr & rdev->ih.ptr_mask);
7234 * Each IV ring entry is 128 bits:
7235 * [7:0] - interrupt source id
7237 * [59:32] - interrupt source data
7238 * [63:60] - reserved
7241 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7242 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7243 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7244 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7245 * PIPE_ID - ME0 0=3D
7246 * - ME1&2 compute dispatcher (4 pipes each)
7248 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7249 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7250 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7253 * [127:96] - reserved
7256 * cik_irq_process - interrupt handler
7258 * @rdev: radeon_device pointer
7260 * Interrupt hander (CIK). Walk the IH ring,
7261 * ack interrupts and schedule work to handle
7263 * Returns irq process return code.
7265 int cik_irq_process(struct radeon_device *rdev)
7267 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7268 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7271 u32 src_id, src_data, ring_id;
7272 u8 me_id, pipe_id, queue_id;
7274 bool queue_hotplug = false;
7275 bool queue_reset = false;
7276 u32 addr, status, mc_client;
7277 bool queue_thermal = false;
7279 if (!rdev->ih.enabled || rdev->shutdown)
7282 wptr = cik_get_ih_wptr(rdev);
7285 /* is somebody else already processing irqs? */
7286 if (atomic_xchg(&rdev->ih.lock, 1))
7289 rptr = rdev->ih.rptr;
7290 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7292 /* Order reading of wptr vs. reading of IH ring data */
7295 /* display interrupts */
7298 while (rptr != wptr) {
7299 /* wptr/rptr are in bytes! */
7300 ring_index = rptr / 4;
7301 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7302 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7303 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7306 case 1: /* D1 vblank/vline */
7308 case 0: /* D1 vblank */
7309 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7310 if (rdev->irq.crtc_vblank_int[0]) {
7311 drm_handle_vblank(rdev->ddev, 0);
7312 rdev->pm.vblank_sync = true;
7313 wake_up(&rdev->irq.vblank_queue);
7315 if (atomic_read(&rdev->irq.pflip[0]))
7316 radeon_crtc_handle_flip(rdev, 0);
7317 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7318 DRM_DEBUG("IH: D1 vblank\n");
7321 case 1: /* D1 vline */
7322 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7323 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7324 DRM_DEBUG("IH: D1 vline\n");
7328 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7332 case 2: /* D2 vblank/vline */
7334 case 0: /* D2 vblank */
7335 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7336 if (rdev->irq.crtc_vblank_int[1]) {
7337 drm_handle_vblank(rdev->ddev, 1);
7338 rdev->pm.vblank_sync = true;
7339 wake_up(&rdev->irq.vblank_queue);
7341 if (atomic_read(&rdev->irq.pflip[1]))
7342 radeon_crtc_handle_flip(rdev, 1);
7343 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7344 DRM_DEBUG("IH: D2 vblank\n");
7347 case 1: /* D2 vline */
7348 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7349 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7350 DRM_DEBUG("IH: D2 vline\n");
7354 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7358 case 3: /* D3 vblank/vline */
7360 case 0: /* D3 vblank */
7361 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7362 if (rdev->irq.crtc_vblank_int[2]) {
7363 drm_handle_vblank(rdev->ddev, 2);
7364 rdev->pm.vblank_sync = true;
7365 wake_up(&rdev->irq.vblank_queue);
7367 if (atomic_read(&rdev->irq.pflip[2]))
7368 radeon_crtc_handle_flip(rdev, 2);
7369 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7370 DRM_DEBUG("IH: D3 vblank\n");
7373 case 1: /* D3 vline */
7374 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7375 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7376 DRM_DEBUG("IH: D3 vline\n");
7380 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7384 case 4: /* D4 vblank/vline */
7386 case 0: /* D4 vblank */
7387 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7388 if (rdev->irq.crtc_vblank_int[3]) {
7389 drm_handle_vblank(rdev->ddev, 3);
7390 rdev->pm.vblank_sync = true;
7391 wake_up(&rdev->irq.vblank_queue);
7393 if (atomic_read(&rdev->irq.pflip[3]))
7394 radeon_crtc_handle_flip(rdev, 3);
7395 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7396 DRM_DEBUG("IH: D4 vblank\n");
7399 case 1: /* D4 vline */
7400 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7401 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7402 DRM_DEBUG("IH: D4 vline\n");
7406 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7410 case 5: /* D5 vblank/vline */
7412 case 0: /* D5 vblank */
7413 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7414 if (rdev->irq.crtc_vblank_int[4]) {
7415 drm_handle_vblank(rdev->ddev, 4);
7416 rdev->pm.vblank_sync = true;
7417 wake_up(&rdev->irq.vblank_queue);
7419 if (atomic_read(&rdev->irq.pflip[4]))
7420 radeon_crtc_handle_flip(rdev, 4);
7421 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7422 DRM_DEBUG("IH: D5 vblank\n");
7425 case 1: /* D5 vline */
7426 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7427 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7428 DRM_DEBUG("IH: D5 vline\n");
7432 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7436 case 6: /* D6 vblank/vline */
7438 case 0: /* D6 vblank */
7439 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7440 if (rdev->irq.crtc_vblank_int[5]) {
7441 drm_handle_vblank(rdev->ddev, 5);
7442 rdev->pm.vblank_sync = true;
7443 wake_up(&rdev->irq.vblank_queue);
7445 if (atomic_read(&rdev->irq.pflip[5]))
7446 radeon_crtc_handle_flip(rdev, 5);
7447 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7448 DRM_DEBUG("IH: D6 vblank\n");
7451 case 1: /* D6 vline */
7452 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7453 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7454 DRM_DEBUG("IH: D6 vline\n");
7458 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7462 case 42: /* HPD hotplug */
7465 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7466 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7467 queue_hotplug = true;
7468 DRM_DEBUG("IH: HPD1\n");
7472 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7473 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7474 queue_hotplug = true;
7475 DRM_DEBUG("IH: HPD2\n");
7479 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7480 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7481 queue_hotplug = true;
7482 DRM_DEBUG("IH: HPD3\n");
7486 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7487 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7488 queue_hotplug = true;
7489 DRM_DEBUG("IH: HPD4\n");
7493 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7494 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7495 queue_hotplug = true;
7496 DRM_DEBUG("IH: HPD5\n");
7500 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7501 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7502 queue_hotplug = true;
7503 DRM_DEBUG("IH: HPD6\n");
7507 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7512 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7513 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7517 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7518 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7519 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7520 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7521 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
7523 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7525 cik_vm_decode_fault(rdev, status, addr, mc_client);
7526 /* reset addr and status */
7527 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7530 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7533 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7536 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7539 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7543 case 176: /* GFX RB CP_INT */
7544 case 177: /* GFX IB CP_INT */
7545 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7547 case 181: /* CP EOP event */
7548 DRM_DEBUG("IH: CP EOP\n");
7549 /* XXX check the bitfield order! */
7550 me_id = (ring_id & 0x60) >> 5;
7551 pipe_id = (ring_id & 0x18) >> 3;
7552 queue_id = (ring_id & 0x7) >> 0;
7555 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7559 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7560 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7561 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7562 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7566 case 184: /* CP Privileged reg access */
7567 DRM_ERROR("Illegal register access in command stream\n");
7568 /* XXX check the bitfield order! */
7569 me_id = (ring_id & 0x60) >> 5;
7570 pipe_id = (ring_id & 0x18) >> 3;
7571 queue_id = (ring_id & 0x7) >> 0;
7574 /* This results in a full GPU reset, but all we need to do is soft
7575 * reset the CP for gfx
7589 case 185: /* CP Privileged inst */
7590 DRM_ERROR("Illegal instruction in command stream\n");
7591 /* XXX check the bitfield order! */
7592 me_id = (ring_id & 0x60) >> 5;
7593 pipe_id = (ring_id & 0x18) >> 3;
7594 queue_id = (ring_id & 0x7) >> 0;
7597 /* This results in a full GPU reset, but all we need to do is soft
7598 * reset the CP for gfx
7612 case 224: /* SDMA trap event */
7613 /* XXX check the bitfield order! */
7614 me_id = (ring_id & 0x3) >> 0;
7615 queue_id = (ring_id & 0xc) >> 2;
7616 DRM_DEBUG("IH: SDMA trap\n");
7621 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7634 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7646 case 230: /* thermal low to high */
7647 DRM_DEBUG("IH: thermal low to high\n");
7648 rdev->pm.dpm.thermal.high_to_low = false;
7649 queue_thermal = true;
7651 case 231: /* thermal high to low */
7652 DRM_DEBUG("IH: thermal high to low\n");
7653 rdev->pm.dpm.thermal.high_to_low = true;
7654 queue_thermal = true;
7656 case 233: /* GUI IDLE */
7657 DRM_DEBUG("IH: GUI idle\n");
7659 case 241: /* SDMA Privileged inst */
7660 case 247: /* SDMA Privileged inst */
7661 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7662 /* XXX check the bitfield order! */
7663 me_id = (ring_id & 0x3) >> 0;
7664 queue_id = (ring_id & 0xc) >> 2;
7699 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7703 /* wptr/rptr are in bytes! */
7705 rptr &= rdev->ih.ptr_mask;
7708 schedule_work(&rdev->hotplug_work);
7710 schedule_work(&rdev->reset_work);
7712 schedule_work(&rdev->pm.dpm.thermal.work);
7713 rdev->ih.rptr = rptr;
7714 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7715 atomic_set(&rdev->ih.lock, 0);
7717 /* make sure wptr hasn't changed while processing */
7718 wptr = cik_get_ih_wptr(rdev);
7726 * startup/shutdown callbacks
7729 * cik_startup - program the asic to a functional state
7731 * @rdev: radeon_device pointer
7733 * Programs the asic to a functional state (CIK).
7734 * Called by cik_init() and cik_resume().
7735 * Returns 0 for success, error for failure.
7737 static int cik_startup(struct radeon_device *rdev)
7739 struct radeon_ring *ring;
7742 /* enable pcie gen2/3 link */
7743 cik_pcie_gen3_enable(rdev);
7745 cik_program_aspm(rdev);
7747 /* scratch needs to be initialized before MC */
7748 r = r600_vram_scratch_init(rdev);
7752 cik_mc_program(rdev);
7754 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7755 r = ci_mc_load_microcode(rdev);
7757 DRM_ERROR("Failed to load MC firmware!\n");
7762 r = cik_pcie_gart_enable(rdev);
7767 /* allocate rlc buffers */
7768 if (rdev->flags & RADEON_IS_IGP) {
7769 if (rdev->family == CHIP_KAVERI) {
7770 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7771 rdev->rlc.reg_list_size =
7772 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7774 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7775 rdev->rlc.reg_list_size =
7776 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7779 rdev->rlc.cs_data = ci_cs_data;
7780 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7781 r = sumo_rlc_init(rdev);
7783 DRM_ERROR("Failed to init rlc BOs!\n");
7787 /* allocate wb buffer */
7788 r = radeon_wb_init(rdev);
7792 /* allocate mec buffers */
7793 r = cik_mec_init(rdev);
7795 DRM_ERROR("Failed to init MEC BOs!\n");
7799 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7801 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7805 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7807 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7811 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7813 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7817 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7819 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7823 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7825 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7829 r = radeon_uvd_resume(rdev);
7831 r = uvd_v4_2_resume(rdev);
7833 r = radeon_fence_driver_start_ring(rdev,
7834 R600_RING_TYPE_UVD_INDEX);
7836 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7840 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7842 r = radeon_vce_resume(rdev);
7844 r = vce_v2_0_resume(rdev);
7846 r = radeon_fence_driver_start_ring(rdev,
7847 TN_RING_TYPE_VCE1_INDEX);
7849 r = radeon_fence_driver_start_ring(rdev,
7850 TN_RING_TYPE_VCE2_INDEX);
7853 dev_err(rdev->dev, "VCE init error (%d).\n", r);
7854 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
7855 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
7859 if (!rdev->irq.installed) {
7860 r = radeon_irq_kms_init(rdev);
7865 r = cik_irq_init(rdev);
7867 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7868 radeon_irq_kms_fini(rdev);
7873 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7874 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7875 PACKET3(PACKET3_NOP, 0x3FFF));
7879 /* set up the compute queues */
7880 /* type-2 packets are deprecated on MEC, use type-3 instead */
7881 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7882 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7883 PACKET3(PACKET3_NOP, 0x3FFF));
7886 ring->me = 1; /* first MEC */
7887 ring->pipe = 0; /* first pipe */
7888 ring->queue = 0; /* first queue */
7889 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7891 /* type-2 packets are deprecated on MEC, use type-3 instead */
7892 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7893 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7894 PACKET3(PACKET3_NOP, 0x3FFF));
7897 /* dGPU only have 1 MEC */
7898 ring->me = 1; /* first MEC */
7899 ring->pipe = 0; /* first pipe */
7900 ring->queue = 1; /* second queue */
7901 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7903 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7904 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7905 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7909 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7910 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7911 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7915 r = cik_cp_resume(rdev);
7919 r = cik_sdma_resume(rdev);
7923 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7924 if (ring->ring_size) {
7925 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7928 r = uvd_v1_0_init(rdev);
7930 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7935 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
7936 if (ring->ring_size)
7937 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7940 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
7941 if (ring->ring_size)
7942 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7946 r = vce_v1_0_init(rdev);
7947 else if (r != -ENOENT)
7948 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
7950 r = radeon_ib_pool_init(rdev);
7952 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7956 r = radeon_vm_manager_init(rdev);
7958 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7962 r = dce6_audio_init(rdev);
7970 * cik_resume - resume the asic to a functional state
7972 * @rdev: radeon_device pointer
7974 * Programs the asic to a functional state (CIK).
7976 * Returns 0 for success, error for failure.
7978 int cik_resume(struct radeon_device *rdev)
7983 atom_asic_init(rdev->mode_info.atom_context);
7985 /* init golden registers */
7986 cik_init_golden_registers(rdev);
7988 if (rdev->pm.pm_method == PM_METHOD_DPM)
7989 radeon_pm_resume(rdev);
7991 rdev->accel_working = true;
7992 r = cik_startup(rdev);
7994 DRM_ERROR("cik startup failed on resume\n");
7995 rdev->accel_working = false;
8004 * cik_suspend - suspend the asic
8006 * @rdev: radeon_device pointer
8008 * Bring the chip into a state suitable for suspend (CIK).
8009 * Called at suspend.
8010 * Returns 0 for success.
8012 int cik_suspend(struct radeon_device *rdev)
8014 radeon_pm_suspend(rdev);
8015 dce6_audio_fini(rdev);
8016 radeon_vm_manager_fini(rdev);
8017 cik_cp_enable(rdev, false);
8018 cik_sdma_enable(rdev, false);
8019 uvd_v1_0_fini(rdev);
8020 radeon_uvd_suspend(rdev);
8021 radeon_vce_suspend(rdev);
8024 cik_irq_suspend(rdev);
8025 radeon_wb_disable(rdev);
8026 cik_pcie_gart_disable(rdev);
8030 /* Plan is to move initialization in that function and use
8031 * helper function so that radeon_device_init pretty much
8032 * do nothing more than calling asic specific function. This
8033 * should also allow to remove a bunch of callback function
8037 * cik_init - asic specific driver and hw init
8039 * @rdev: radeon_device pointer
8041 * Setup asic specific driver variables and program the hw
8042 * to a functional state (CIK).
8043 * Called at driver startup.
8044 * Returns 0 for success, errors for failure.
8046 int cik_init(struct radeon_device *rdev)
8048 struct radeon_ring *ring;
8052 if (!radeon_get_bios(rdev)) {
8053 if (ASIC_IS_AVIVO(rdev))
8056 /* Must be an ATOMBIOS */
8057 if (!rdev->is_atom_bios) {
8058 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8061 r = radeon_atombios_init(rdev);
8065 /* Post card if necessary */
8066 if (!radeon_card_posted(rdev)) {
8068 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8071 DRM_INFO("GPU not posted. posting now...\n");
8072 atom_asic_init(rdev->mode_info.atom_context);
8074 /* init golden registers */
8075 cik_init_golden_registers(rdev);
8076 /* Initialize scratch registers */
8077 cik_scratch_init(rdev);
8078 /* Initialize surface registers */
8079 radeon_surface_init(rdev);
8080 /* Initialize clocks */
8081 radeon_get_clock_info(rdev->ddev);
8084 r = radeon_fence_driver_init(rdev);
8088 /* initialize memory controller */
8089 r = cik_mc_init(rdev);
8092 /* Memory manager */
8093 r = radeon_bo_init(rdev);
8097 if (rdev->flags & RADEON_IS_IGP) {
8098 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8099 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8100 r = cik_init_microcode(rdev);
8102 DRM_ERROR("Failed to load firmware!\n");
8107 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8108 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8110 r = cik_init_microcode(rdev);
8112 DRM_ERROR("Failed to load firmware!\n");
8118 /* Initialize power management */
8119 radeon_pm_init(rdev);
8121 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8122 ring->ring_obj = NULL;
8123 r600_ring_init(rdev, ring, 1024 * 1024);
8125 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8126 ring->ring_obj = NULL;
8127 r600_ring_init(rdev, ring, 1024 * 1024);
8128 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8132 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8133 ring->ring_obj = NULL;
8134 r600_ring_init(rdev, ring, 1024 * 1024);
8135 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8139 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8140 ring->ring_obj = NULL;
8141 r600_ring_init(rdev, ring, 256 * 1024);
8143 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8144 ring->ring_obj = NULL;
8145 r600_ring_init(rdev, ring, 256 * 1024);
8147 r = radeon_uvd_init(rdev);
8149 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8150 ring->ring_obj = NULL;
8151 r600_ring_init(rdev, ring, 4096);
8154 r = radeon_vce_init(rdev);
8156 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8157 ring->ring_obj = NULL;
8158 r600_ring_init(rdev, ring, 4096);
8160 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8161 ring->ring_obj = NULL;
8162 r600_ring_init(rdev, ring, 4096);
8165 rdev->ih.ring_obj = NULL;
8166 r600_ih_ring_init(rdev, 64 * 1024);
8168 r = r600_pcie_gart_init(rdev);
8172 rdev->accel_working = true;
8173 r = cik_startup(rdev);
8175 dev_err(rdev->dev, "disabling GPU acceleration\n");
8177 cik_sdma_fini(rdev);
8179 sumo_rlc_fini(rdev);
8181 radeon_wb_fini(rdev);
8182 radeon_ib_pool_fini(rdev);
8183 radeon_vm_manager_fini(rdev);
8184 radeon_irq_kms_fini(rdev);
8185 cik_pcie_gart_fini(rdev);
8186 rdev->accel_working = false;
8189 /* Don't start up if the MC ucode is missing.
8190 * The default clocks and voltages before the MC ucode
8191 * is loaded are not suffient for advanced operations.
8193 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8194 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8202 * cik_fini - asic specific driver and hw fini
8204 * @rdev: radeon_device pointer
8206 * Tear down the asic specific driver variables and program the hw
8207 * to an idle state (CIK).
8208 * Called at driver unload.
8210 void cik_fini(struct radeon_device *rdev)
8212 radeon_pm_fini(rdev);
8214 cik_sdma_fini(rdev);
8218 sumo_rlc_fini(rdev);
8220 radeon_wb_fini(rdev);
8221 radeon_vm_manager_fini(rdev);
8222 radeon_ib_pool_fini(rdev);
8223 radeon_irq_kms_fini(rdev);
8224 uvd_v1_0_fini(rdev);
8225 radeon_uvd_fini(rdev);
8226 radeon_vce_fini(rdev);
8227 cik_pcie_gart_fini(rdev);
8228 r600_vram_scratch_fini(rdev);
8229 radeon_gem_fini(rdev);
8230 radeon_fence_driver_fini(rdev);
8231 radeon_bo_fini(rdev);
8232 radeon_atombios_fini(rdev);
8237 void dce8_program_fmt(struct drm_encoder *encoder)
8239 struct drm_device *dev = encoder->dev;
8240 struct radeon_device *rdev = dev->dev_private;
8241 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8242 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8243 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8246 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8249 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8250 bpc = radeon_get_monitor_bpc(connector);
8251 dither = radeon_connector->dither;
8254 /* LVDS/eDP FMT is set up by atom */
8255 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8258 /* not needed for analog */
8259 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8260 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8268 if (dither == RADEON_FMT_DITHER_ENABLE)
8269 /* XXX sort out optimal dither settings */
8270 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8271 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8273 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8276 if (dither == RADEON_FMT_DITHER_ENABLE)
8277 /* XXX sort out optimal dither settings */
8278 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8279 FMT_RGB_RANDOM_ENABLE |
8280 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8282 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8285 if (dither == RADEON_FMT_DITHER_ENABLE)
8286 /* XXX sort out optimal dither settings */
8287 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8288 FMT_RGB_RANDOM_ENABLE |
8289 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8291 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8298 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8301 /* display watermark setup */
8303 * dce8_line_buffer_adjust - Set up the line buffer
8305 * @rdev: radeon_device pointer
8306 * @radeon_crtc: the selected display controller
8307 * @mode: the current display mode on the selected display
8310 * Setup up the line buffer allocation for
8311 * the selected display controller (CIK).
8312 * Returns the line buffer size in pixels.
8314 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8315 struct radeon_crtc *radeon_crtc,
8316 struct drm_display_mode *mode)
8318 u32 tmp, buffer_alloc, i;
8319 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8322 * There are 6 line buffers, one for each display controllers.
8323 * There are 3 partitions per LB. Select the number of partitions
8324 * to enable based on the display width. For display widths larger
8325 * than 4096, you need use to use 2 display controllers and combine
8326 * them using the stereo blender.
8328 if (radeon_crtc->base.enabled && mode) {
8329 if (mode->crtc_hdisplay < 1920) {
8332 } else if (mode->crtc_hdisplay < 2560) {
8335 } else if (mode->crtc_hdisplay < 4096) {
8337 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8339 DRM_DEBUG_KMS("Mode too big for LB!\n");
8341 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8348 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8349 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8351 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8352 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8353 for (i = 0; i < rdev->usec_timeout; i++) {
8354 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8355 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8360 if (radeon_crtc->base.enabled && mode) {
8372 /* controller not enabled, so no lb used */
8377 * cik_get_number_of_dram_channels - get the number of dram channels
8379 * @rdev: radeon_device pointer
8381 * Look up the number of video ram channels (CIK).
8382 * Used for display watermark bandwidth calculations
8383 * Returns the number of dram channels
8385 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8387 u32 tmp = RREG32(MC_SHARED_CHMAP);
8389 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8412 struct dce8_wm_params {
8413 u32 dram_channels; /* number of dram channels */
8414 u32 yclk; /* bandwidth per dram data pin in kHz */
8415 u32 sclk; /* engine clock in kHz */
8416 u32 disp_clk; /* display clock in kHz */
8417 u32 src_width; /* viewport width */
8418 u32 active_time; /* active display time in ns */
8419 u32 blank_time; /* blank time in ns */
8420 bool interlaced; /* mode is interlaced */
8421 fixed20_12 vsc; /* vertical scale ratio */
8422 u32 num_heads; /* number of active crtcs */
8423 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8424 u32 lb_size; /* line buffer allocated to pipe */
8425 u32 vtaps; /* vertical scaler taps */
8429 * dce8_dram_bandwidth - get the dram bandwidth
8431 * @wm: watermark calculation data
8433 * Calculate the raw dram bandwidth (CIK).
8434 * Used for display watermark bandwidth calculations
8435 * Returns the dram bandwidth in MBytes/s
8437 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8439 /* Calculate raw DRAM Bandwidth */
8440 fixed20_12 dram_efficiency; /* 0.7 */
8441 fixed20_12 yclk, dram_channels, bandwidth;
8444 a.full = dfixed_const(1000);
8445 yclk.full = dfixed_const(wm->yclk);
8446 yclk.full = dfixed_div(yclk, a);
8447 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8448 a.full = dfixed_const(10);
8449 dram_efficiency.full = dfixed_const(7);
8450 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8451 bandwidth.full = dfixed_mul(dram_channels, yclk);
8452 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8454 return dfixed_trunc(bandwidth);
8458 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8460 * @wm: watermark calculation data
8462 * Calculate the dram bandwidth used for display (CIK).
8463 * Used for display watermark bandwidth calculations
8464 * Returns the dram bandwidth for display in MBytes/s
8466 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8468 /* Calculate DRAM Bandwidth and the part allocated to display. */
8469 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8470 fixed20_12 yclk, dram_channels, bandwidth;
8473 a.full = dfixed_const(1000);
8474 yclk.full = dfixed_const(wm->yclk);
8475 yclk.full = dfixed_div(yclk, a);
8476 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8477 a.full = dfixed_const(10);
8478 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8479 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8480 bandwidth.full = dfixed_mul(dram_channels, yclk);
8481 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8483 return dfixed_trunc(bandwidth);
8487 * dce8_data_return_bandwidth - get the data return bandwidth
8489 * @wm: watermark calculation data
8491 * Calculate the data return bandwidth used for display (CIK).
8492 * Used for display watermark bandwidth calculations
8493 * Returns the data return bandwidth in MBytes/s
8495 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8497 /* Calculate the display Data return Bandwidth */
8498 fixed20_12 return_efficiency; /* 0.8 */
8499 fixed20_12 sclk, bandwidth;
8502 a.full = dfixed_const(1000);
8503 sclk.full = dfixed_const(wm->sclk);
8504 sclk.full = dfixed_div(sclk, a);
8505 a.full = dfixed_const(10);
8506 return_efficiency.full = dfixed_const(8);
8507 return_efficiency.full = dfixed_div(return_efficiency, a);
8508 a.full = dfixed_const(32);
8509 bandwidth.full = dfixed_mul(a, sclk);
8510 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8512 return dfixed_trunc(bandwidth);
8516 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8518 * @wm: watermark calculation data
8520 * Calculate the dmif bandwidth used for display (CIK).
8521 * Used for display watermark bandwidth calculations
8522 * Returns the dmif bandwidth in MBytes/s
8524 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8526 /* Calculate the DMIF Request Bandwidth */
8527 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8528 fixed20_12 disp_clk, bandwidth;
8531 a.full = dfixed_const(1000);
8532 disp_clk.full = dfixed_const(wm->disp_clk);
8533 disp_clk.full = dfixed_div(disp_clk, a);
8534 a.full = dfixed_const(32);
8535 b.full = dfixed_mul(a, disp_clk);
8537 a.full = dfixed_const(10);
8538 disp_clk_request_efficiency.full = dfixed_const(8);
8539 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8541 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8543 return dfixed_trunc(bandwidth);
8547 * dce8_available_bandwidth - get the min available bandwidth
8549 * @wm: watermark calculation data
8551 * Calculate the min available bandwidth used for display (CIK).
8552 * Used for display watermark bandwidth calculations
8553 * Returns the min available bandwidth in MBytes/s
8555 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8557 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8558 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8559 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8560 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8562 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8566 * dce8_average_bandwidth - get the average available bandwidth
8568 * @wm: watermark calculation data
8570 * Calculate the average available bandwidth used for display (CIK).
8571 * Used for display watermark bandwidth calculations
8572 * Returns the average available bandwidth in MBytes/s
8574 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8576 /* Calculate the display mode Average Bandwidth
8577 * DisplayMode should contain the source and destination dimensions,
8581 fixed20_12 line_time;
8582 fixed20_12 src_width;
8583 fixed20_12 bandwidth;
8586 a.full = dfixed_const(1000);
8587 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8588 line_time.full = dfixed_div(line_time, a);
8589 bpp.full = dfixed_const(wm->bytes_per_pixel);
8590 src_width.full = dfixed_const(wm->src_width);
8591 bandwidth.full = dfixed_mul(src_width, bpp);
8592 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8593 bandwidth.full = dfixed_div(bandwidth, line_time);
8595 return dfixed_trunc(bandwidth);
8599 * dce8_latency_watermark - get the latency watermark
8601 * @wm: watermark calculation data
8603 * Calculate the latency watermark (CIK).
8604 * Used for display watermark bandwidth calculations
8605 * Returns the latency watermark in ns
8607 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8609 /* First calculate the latency in ns */
8610 u32 mc_latency = 2000; /* 2000 ns. */
8611 u32 available_bandwidth = dce8_available_bandwidth(wm);
8612 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8613 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8614 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8615 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8616 (wm->num_heads * cursor_line_pair_return_time);
8617 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8618 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8619 u32 tmp, dmif_size = 12288;
8622 if (wm->num_heads == 0)
8625 a.full = dfixed_const(2);
8626 b.full = dfixed_const(1);
8627 if ((wm->vsc.full > a.full) ||
8628 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8630 ((wm->vsc.full >= a.full) && wm->interlaced))
8631 max_src_lines_per_dst_line = 4;
8633 max_src_lines_per_dst_line = 2;
8635 a.full = dfixed_const(available_bandwidth);
8636 b.full = dfixed_const(wm->num_heads);
8637 a.full = dfixed_div(a, b);
8639 b.full = dfixed_const(mc_latency + 512);
8640 c.full = dfixed_const(wm->disp_clk);
8641 b.full = dfixed_div(b, c);
8643 c.full = dfixed_const(dmif_size);
8644 b.full = dfixed_div(c, b);
8646 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8648 b.full = dfixed_const(1000);
8649 c.full = dfixed_const(wm->disp_clk);
8650 b.full = dfixed_div(c, b);
8651 c.full = dfixed_const(wm->bytes_per_pixel);
8652 b.full = dfixed_mul(b, c);
8654 lb_fill_bw = min(tmp, dfixed_trunc(b));
8656 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8657 b.full = dfixed_const(1000);
8658 c.full = dfixed_const(lb_fill_bw);
8659 b.full = dfixed_div(c, b);
8660 a.full = dfixed_div(a, b);
8661 line_fill_time = dfixed_trunc(a);
8663 if (line_fill_time < wm->active_time)
8666 return latency + (line_fill_time - wm->active_time);
8671 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8672 * average and available dram bandwidth
8674 * @wm: watermark calculation data
8676 * Check if the display average bandwidth fits in the display
8677 * dram bandwidth (CIK).
8678 * Used for display watermark bandwidth calculations
8679 * Returns true if the display fits, false if not.
8681 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8683 if (dce8_average_bandwidth(wm) <=
8684 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8691 * dce8_average_bandwidth_vs_available_bandwidth - check
8692 * average and available bandwidth
8694 * @wm: watermark calculation data
8696 * Check if the display average bandwidth fits in the display
8697 * available bandwidth (CIK).
8698 * Used for display watermark bandwidth calculations
8699 * Returns true if the display fits, false if not.
8701 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8703 if (dce8_average_bandwidth(wm) <=
8704 (dce8_available_bandwidth(wm) / wm->num_heads))
8711 * dce8_check_latency_hiding - check latency hiding
8713 * @wm: watermark calculation data
8715 * Check latency hiding (CIK).
8716 * Used for display watermark bandwidth calculations
8717 * Returns true if the display fits, false if not.
8719 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8721 u32 lb_partitions = wm->lb_size / wm->src_width;
8722 u32 line_time = wm->active_time + wm->blank_time;
8723 u32 latency_tolerant_lines;
8727 a.full = dfixed_const(1);
8728 if (wm->vsc.full > a.full)
8729 latency_tolerant_lines = 1;
8731 if (lb_partitions <= (wm->vtaps + 1))
8732 latency_tolerant_lines = 1;
8734 latency_tolerant_lines = 2;
8737 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8739 if (dce8_latency_watermark(wm) <= latency_hiding)
8746 * dce8_program_watermarks - program display watermarks
8748 * @rdev: radeon_device pointer
8749 * @radeon_crtc: the selected display controller
8750 * @lb_size: line buffer size
8751 * @num_heads: number of display controllers in use
8753 * Calculate and program the display watermarks for the
8754 * selected display controller (CIK).
8756 static void dce8_program_watermarks(struct radeon_device *rdev,
8757 struct radeon_crtc *radeon_crtc,
8758 u32 lb_size, u32 num_heads)
8760 struct drm_display_mode *mode = &radeon_crtc->base.mode;
8761 struct dce8_wm_params wm_low, wm_high;
8764 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8767 if (radeon_crtc->base.enabled && num_heads && mode) {
8768 pixel_period = 1000000 / (u32)mode->clock;
8769 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8771 /* watermark for high clocks */
8772 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8773 rdev->pm.dpm_enabled) {
8775 radeon_dpm_get_mclk(rdev, false) * 10;
8777 radeon_dpm_get_sclk(rdev, false) * 10;
8779 wm_high.yclk = rdev->pm.current_mclk * 10;
8780 wm_high.sclk = rdev->pm.current_sclk * 10;
8783 wm_high.disp_clk = mode->clock;
8784 wm_high.src_width = mode->crtc_hdisplay;
8785 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8786 wm_high.blank_time = line_time - wm_high.active_time;
8787 wm_high.interlaced = false;
8788 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8789 wm_high.interlaced = true;
8790 wm_high.vsc = radeon_crtc->vsc;
8792 if (radeon_crtc->rmx_type != RMX_OFF)
8794 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8795 wm_high.lb_size = lb_size;
8796 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8797 wm_high.num_heads = num_heads;
8799 /* set for high clocks */
8800 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8802 /* possibly force display priority to high */
8803 /* should really do this at mode validation time... */
8804 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8805 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8806 !dce8_check_latency_hiding(&wm_high) ||
8807 (rdev->disp_priority == 2)) {
8808 DRM_DEBUG_KMS("force priority to high\n");
8811 /* watermark for low clocks */
8812 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8813 rdev->pm.dpm_enabled) {
8815 radeon_dpm_get_mclk(rdev, true) * 10;
8817 radeon_dpm_get_sclk(rdev, true) * 10;
8819 wm_low.yclk = rdev->pm.current_mclk * 10;
8820 wm_low.sclk = rdev->pm.current_sclk * 10;
8823 wm_low.disp_clk = mode->clock;
8824 wm_low.src_width = mode->crtc_hdisplay;
8825 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8826 wm_low.blank_time = line_time - wm_low.active_time;
8827 wm_low.interlaced = false;
8828 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8829 wm_low.interlaced = true;
8830 wm_low.vsc = radeon_crtc->vsc;
8832 if (radeon_crtc->rmx_type != RMX_OFF)
8834 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8835 wm_low.lb_size = lb_size;
8836 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8837 wm_low.num_heads = num_heads;
8839 /* set for low clocks */
8840 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8842 /* possibly force display priority to high */
8843 /* should really do this at mode validation time... */
8844 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8845 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8846 !dce8_check_latency_hiding(&wm_low) ||
8847 (rdev->disp_priority == 2)) {
8848 DRM_DEBUG_KMS("force priority to high\n");
8853 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8855 tmp &= ~LATENCY_WATERMARK_MASK(3);
8856 tmp |= LATENCY_WATERMARK_MASK(1);
8857 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8858 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8859 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8860 LATENCY_HIGH_WATERMARK(line_time)));
8862 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8863 tmp &= ~LATENCY_WATERMARK_MASK(3);
8864 tmp |= LATENCY_WATERMARK_MASK(2);
8865 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8866 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8867 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8868 LATENCY_HIGH_WATERMARK(line_time)));
8869 /* restore original selection */
8870 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
8872 /* save values for DPM */
8873 radeon_crtc->line_time = line_time;
8874 radeon_crtc->wm_high = latency_watermark_a;
8875 radeon_crtc->wm_low = latency_watermark_b;
8879 * dce8_bandwidth_update - program display watermarks
8881 * @rdev: radeon_device pointer
8883 * Calculate and program the display watermarks and line
8884 * buffer allocation (CIK).
8886 void dce8_bandwidth_update(struct radeon_device *rdev)
8888 struct drm_display_mode *mode = NULL;
8889 u32 num_heads = 0, lb_size;
8892 radeon_update_display_priority(rdev);
8894 for (i = 0; i < rdev->num_crtc; i++) {
8895 if (rdev->mode_info.crtcs[i]->base.enabled)
8898 for (i = 0; i < rdev->num_crtc; i++) {
8899 mode = &rdev->mode_info.crtcs[i]->base.mode;
8900 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8901 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8906 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8908 * @rdev: radeon_device pointer
8910 * Fetches a GPU clock counter snapshot (SI).
8911 * Returns the 64 bit clock counter snapshot.
8913 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8917 mutex_lock(&rdev->gpu_clock_mutex);
8918 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8919 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8920 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8921 mutex_unlock(&rdev->gpu_clock_mutex);
8925 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8926 u32 cntl_reg, u32 status_reg)
8929 struct atom_clock_dividers dividers;
8932 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8933 clock, false, ÷rs);
8937 tmp = RREG32_SMC(cntl_reg);
8938 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8939 tmp |= dividers.post_divider;
8940 WREG32_SMC(cntl_reg, tmp);
8942 for (i = 0; i < 100; i++) {
8943 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8953 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8957 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8961 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8965 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
8968 struct atom_clock_dividers dividers;
8971 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8972 ecclk, false, ÷rs);
8976 for (i = 0; i < 100; i++) {
8977 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
8984 tmp = RREG32_SMC(CG_ECLK_CNTL);
8985 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
8986 tmp |= dividers.post_divider;
8987 WREG32_SMC(CG_ECLK_CNTL, tmp);
8989 for (i = 0; i < 100; i++) {
8990 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9000 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9002 struct pci_dev *root = rdev->pdev->bus->self;
9003 int bridge_pos, gpu_pos;
9004 u32 speed_cntl, mask, current_data_rate;
9008 if (radeon_pcie_gen2 == 0)
9011 if (rdev->flags & RADEON_IS_IGP)
9014 if (!(rdev->flags & RADEON_IS_PCIE))
9017 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9021 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9024 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9025 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9026 LC_CURRENT_DATA_RATE_SHIFT;
9027 if (mask & DRM_PCIE_SPEED_80) {
9028 if (current_data_rate == 2) {
9029 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9032 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9033 } else if (mask & DRM_PCIE_SPEED_50) {
9034 if (current_data_rate == 1) {
9035 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9038 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9041 bridge_pos = pci_pcie_cap(root);
9045 gpu_pos = pci_pcie_cap(rdev->pdev);
9049 if (mask & DRM_PCIE_SPEED_80) {
9050 /* re-try equalization if gen3 is not already enabled */
9051 if (current_data_rate != 2) {
9052 u16 bridge_cfg, gpu_cfg;
9053 u16 bridge_cfg2, gpu_cfg2;
9054 u32 max_lw, current_lw, tmp;
9056 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9057 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9059 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9060 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9062 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9063 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9065 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9066 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9067 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9069 if (current_lw < max_lw) {
9070 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9071 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9072 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9073 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9074 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9075 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9079 for (i = 0; i < 10; i++) {
9081 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9082 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9085 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9086 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9088 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9089 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9091 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9092 tmp |= LC_SET_QUIESCE;
9093 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9095 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9097 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9102 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9103 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9104 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9105 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9107 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9108 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9109 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9110 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9113 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9114 tmp16 &= ~((1 << 4) | (7 << 9));
9115 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9116 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9118 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9119 tmp16 &= ~((1 << 4) | (7 << 9));
9120 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9121 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9123 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9124 tmp &= ~LC_SET_QUIESCE;
9125 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9130 /* set the link speed */
9131 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9132 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9133 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9135 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9137 if (mask & DRM_PCIE_SPEED_80)
9138 tmp16 |= 3; /* gen3 */
9139 else if (mask & DRM_PCIE_SPEED_50)
9140 tmp16 |= 2; /* gen2 */
9142 tmp16 |= 1; /* gen1 */
9143 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9145 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9146 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9147 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9149 for (i = 0; i < rdev->usec_timeout; i++) {
9150 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9151 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9157 static void cik_program_aspm(struct radeon_device *rdev)
9160 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9161 bool disable_clkreq = false;
9163 if (radeon_aspm == 0)
9166 /* XXX double check IGPs */
9167 if (rdev->flags & RADEON_IS_IGP)
9170 if (!(rdev->flags & RADEON_IS_PCIE))
9173 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9174 data &= ~LC_XMIT_N_FTS_MASK;
9175 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9177 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9179 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9180 data |= LC_GO_TO_RECOVERY;
9182 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9184 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9185 data |= P_IGNORE_EDB_ERR;
9187 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9189 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9190 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9191 data |= LC_PMI_TO_L1_DIS;
9193 data |= LC_L0S_INACTIVITY(7);
9196 data |= LC_L1_INACTIVITY(7);
9197 data &= ~LC_PMI_TO_L1_DIS;
9199 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9201 if (!disable_plloff_in_l1) {
9202 bool clk_req_support;
9204 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9205 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9206 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9208 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9210 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9211 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9212 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9214 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9216 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9217 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9218 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9220 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9222 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9223 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9224 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9226 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9228 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9229 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9230 data |= LC_DYN_LANES_PWR_STATE(3);
9232 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9234 if (!disable_clkreq) {
9235 struct pci_dev *root = rdev->pdev->bus->self;
9238 clk_req_support = false;
9239 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9240 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9241 clk_req_support = true;
9243 clk_req_support = false;
9246 if (clk_req_support) {
9247 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9248 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9250 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9252 orig = data = RREG32_SMC(THM_CLK_CNTL);
9253 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9254 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9256 WREG32_SMC(THM_CLK_CNTL, data);
9258 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9259 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9260 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9262 WREG32_SMC(MISC_CLK_CTRL, data);
9264 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9265 data &= ~BCLK_AS_XCLK;
9267 WREG32_SMC(CG_CLKPIN_CNTL, data);
9269 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9270 data &= ~FORCE_BIF_REFCLK_EN;
9272 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9274 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9275 data &= ~MPLL_CLKOUT_SEL_MASK;
9276 data |= MPLL_CLKOUT_SEL(4);
9278 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9283 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9286 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9287 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9289 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9292 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9293 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9294 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9295 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9296 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9297 data &= ~LC_L0S_INACTIVITY_MASK;
9299 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);