2 * Copyright 2012 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
45 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
46 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
47 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
48 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
49 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
50 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
51 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
52 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
53 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
54 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
55 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
56 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
57 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
58 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
59 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
60 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
61 MODULE_FIRMWARE("radeon/KABINI_me.bin");
62 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
63 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
64 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
65 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
66 MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
67 MODULE_FIRMWARE("radeon/MULLINS_me.bin");
68 MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
69 MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
70 MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
71 MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
73 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
74 extern void r600_ih_ring_fini(struct radeon_device *rdev);
75 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
76 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
77 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
78 extern void sumo_rlc_fini(struct radeon_device *rdev);
79 extern int sumo_rlc_init(struct radeon_device *rdev);
80 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
81 extern void si_rlc_reset(struct radeon_device *rdev);
82 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
83 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
84 extern int cik_sdma_resume(struct radeon_device *rdev);
85 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
86 extern void cik_sdma_fini(struct radeon_device *rdev);
87 extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
88 static void cik_rlc_stop(struct radeon_device *rdev);
89 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
90 static void cik_program_aspm(struct radeon_device *rdev);
91 static void cik_init_pg(struct radeon_device *rdev);
92 static void cik_init_cg(struct radeon_device *rdev);
93 static void cik_fini_pg(struct radeon_device *rdev);
94 static void cik_fini_cg(struct radeon_device *rdev);
95 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
98 /* get temperature in millidegrees */
99 int ci_get_temp(struct radeon_device *rdev)
104 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
110 actual_temp = temp & 0x1ff;
112 actual_temp = actual_temp * 1000;
117 /* get temperature in millidegrees */
118 int kv_get_temp(struct radeon_device *rdev)
123 temp = RREG32_SMC(0xC0300E0C);
126 actual_temp = (temp / 8) - 49;
130 actual_temp = actual_temp * 1000;
136 * Indirect registers accessor
138 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
143 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
144 WREG32(PCIE_INDEX, reg);
145 (void)RREG32(PCIE_INDEX);
146 r = RREG32(PCIE_DATA);
147 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
151 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
155 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
156 WREG32(PCIE_INDEX, reg);
157 (void)RREG32(PCIE_INDEX);
158 WREG32(PCIE_DATA, v);
159 (void)RREG32(PCIE_DATA);
160 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
163 static const u32 spectre_rlc_save_restore_register_list[] =
165 (0x0e00 << 16) | (0xc12c >> 2),
167 (0x0e00 << 16) | (0xc140 >> 2),
169 (0x0e00 << 16) | (0xc150 >> 2),
171 (0x0e00 << 16) | (0xc15c >> 2),
173 (0x0e00 << 16) | (0xc168 >> 2),
175 (0x0e00 << 16) | (0xc170 >> 2),
177 (0x0e00 << 16) | (0xc178 >> 2),
179 (0x0e00 << 16) | (0xc204 >> 2),
181 (0x0e00 << 16) | (0xc2b4 >> 2),
183 (0x0e00 << 16) | (0xc2b8 >> 2),
185 (0x0e00 << 16) | (0xc2bc >> 2),
187 (0x0e00 << 16) | (0xc2c0 >> 2),
189 (0x0e00 << 16) | (0x8228 >> 2),
191 (0x0e00 << 16) | (0x829c >> 2),
193 (0x0e00 << 16) | (0x869c >> 2),
195 (0x0600 << 16) | (0x98f4 >> 2),
197 (0x0e00 << 16) | (0x98f8 >> 2),
199 (0x0e00 << 16) | (0x9900 >> 2),
201 (0x0e00 << 16) | (0xc260 >> 2),
203 (0x0e00 << 16) | (0x90e8 >> 2),
205 (0x0e00 << 16) | (0x3c000 >> 2),
207 (0x0e00 << 16) | (0x3c00c >> 2),
209 (0x0e00 << 16) | (0x8c1c >> 2),
211 (0x0e00 << 16) | (0x9700 >> 2),
213 (0x0e00 << 16) | (0xcd20 >> 2),
215 (0x4e00 << 16) | (0xcd20 >> 2),
217 (0x5e00 << 16) | (0xcd20 >> 2),
219 (0x6e00 << 16) | (0xcd20 >> 2),
221 (0x7e00 << 16) | (0xcd20 >> 2),
223 (0x8e00 << 16) | (0xcd20 >> 2),
225 (0x9e00 << 16) | (0xcd20 >> 2),
227 (0xae00 << 16) | (0xcd20 >> 2),
229 (0xbe00 << 16) | (0xcd20 >> 2),
231 (0x0e00 << 16) | (0x89bc >> 2),
233 (0x0e00 << 16) | (0x8900 >> 2),
236 (0x0e00 << 16) | (0xc130 >> 2),
238 (0x0e00 << 16) | (0xc134 >> 2),
240 (0x0e00 << 16) | (0xc1fc >> 2),
242 (0x0e00 << 16) | (0xc208 >> 2),
244 (0x0e00 << 16) | (0xc264 >> 2),
246 (0x0e00 << 16) | (0xc268 >> 2),
248 (0x0e00 << 16) | (0xc26c >> 2),
250 (0x0e00 << 16) | (0xc270 >> 2),
252 (0x0e00 << 16) | (0xc274 >> 2),
254 (0x0e00 << 16) | (0xc278 >> 2),
256 (0x0e00 << 16) | (0xc27c >> 2),
258 (0x0e00 << 16) | (0xc280 >> 2),
260 (0x0e00 << 16) | (0xc284 >> 2),
262 (0x0e00 << 16) | (0xc288 >> 2),
264 (0x0e00 << 16) | (0xc28c >> 2),
266 (0x0e00 << 16) | (0xc290 >> 2),
268 (0x0e00 << 16) | (0xc294 >> 2),
270 (0x0e00 << 16) | (0xc298 >> 2),
272 (0x0e00 << 16) | (0xc29c >> 2),
274 (0x0e00 << 16) | (0xc2a0 >> 2),
276 (0x0e00 << 16) | (0xc2a4 >> 2),
278 (0x0e00 << 16) | (0xc2a8 >> 2),
280 (0x0e00 << 16) | (0xc2ac >> 2),
282 (0x0e00 << 16) | (0xc2b0 >> 2),
284 (0x0e00 << 16) | (0x301d0 >> 2),
286 (0x0e00 << 16) | (0x30238 >> 2),
288 (0x0e00 << 16) | (0x30250 >> 2),
290 (0x0e00 << 16) | (0x30254 >> 2),
292 (0x0e00 << 16) | (0x30258 >> 2),
294 (0x0e00 << 16) | (0x3025c >> 2),
296 (0x4e00 << 16) | (0xc900 >> 2),
298 (0x5e00 << 16) | (0xc900 >> 2),
300 (0x6e00 << 16) | (0xc900 >> 2),
302 (0x7e00 << 16) | (0xc900 >> 2),
304 (0x8e00 << 16) | (0xc900 >> 2),
306 (0x9e00 << 16) | (0xc900 >> 2),
308 (0xae00 << 16) | (0xc900 >> 2),
310 (0xbe00 << 16) | (0xc900 >> 2),
312 (0x4e00 << 16) | (0xc904 >> 2),
314 (0x5e00 << 16) | (0xc904 >> 2),
316 (0x6e00 << 16) | (0xc904 >> 2),
318 (0x7e00 << 16) | (0xc904 >> 2),
320 (0x8e00 << 16) | (0xc904 >> 2),
322 (0x9e00 << 16) | (0xc904 >> 2),
324 (0xae00 << 16) | (0xc904 >> 2),
326 (0xbe00 << 16) | (0xc904 >> 2),
328 (0x4e00 << 16) | (0xc908 >> 2),
330 (0x5e00 << 16) | (0xc908 >> 2),
332 (0x6e00 << 16) | (0xc908 >> 2),
334 (0x7e00 << 16) | (0xc908 >> 2),
336 (0x8e00 << 16) | (0xc908 >> 2),
338 (0x9e00 << 16) | (0xc908 >> 2),
340 (0xae00 << 16) | (0xc908 >> 2),
342 (0xbe00 << 16) | (0xc908 >> 2),
344 (0x4e00 << 16) | (0xc90c >> 2),
346 (0x5e00 << 16) | (0xc90c >> 2),
348 (0x6e00 << 16) | (0xc90c >> 2),
350 (0x7e00 << 16) | (0xc90c >> 2),
352 (0x8e00 << 16) | (0xc90c >> 2),
354 (0x9e00 << 16) | (0xc90c >> 2),
356 (0xae00 << 16) | (0xc90c >> 2),
358 (0xbe00 << 16) | (0xc90c >> 2),
360 (0x4e00 << 16) | (0xc910 >> 2),
362 (0x5e00 << 16) | (0xc910 >> 2),
364 (0x6e00 << 16) | (0xc910 >> 2),
366 (0x7e00 << 16) | (0xc910 >> 2),
368 (0x8e00 << 16) | (0xc910 >> 2),
370 (0x9e00 << 16) | (0xc910 >> 2),
372 (0xae00 << 16) | (0xc910 >> 2),
374 (0xbe00 << 16) | (0xc910 >> 2),
376 (0x0e00 << 16) | (0xc99c >> 2),
378 (0x0e00 << 16) | (0x9834 >> 2),
380 (0x0000 << 16) | (0x30f00 >> 2),
382 (0x0001 << 16) | (0x30f00 >> 2),
384 (0x0000 << 16) | (0x30f04 >> 2),
386 (0x0001 << 16) | (0x30f04 >> 2),
388 (0x0000 << 16) | (0x30f08 >> 2),
390 (0x0001 << 16) | (0x30f08 >> 2),
392 (0x0000 << 16) | (0x30f0c >> 2),
394 (0x0001 << 16) | (0x30f0c >> 2),
396 (0x0600 << 16) | (0x9b7c >> 2),
398 (0x0e00 << 16) | (0x8a14 >> 2),
400 (0x0e00 << 16) | (0x8a18 >> 2),
402 (0x0600 << 16) | (0x30a00 >> 2),
404 (0x0e00 << 16) | (0x8bf0 >> 2),
406 (0x0e00 << 16) | (0x8bcc >> 2),
408 (0x0e00 << 16) | (0x8b24 >> 2),
410 (0x0e00 << 16) | (0x30a04 >> 2),
412 (0x0600 << 16) | (0x30a10 >> 2),
414 (0x0600 << 16) | (0x30a14 >> 2),
416 (0x0600 << 16) | (0x30a18 >> 2),
418 (0x0600 << 16) | (0x30a2c >> 2),
420 (0x0e00 << 16) | (0xc700 >> 2),
422 (0x0e00 << 16) | (0xc704 >> 2),
424 (0x0e00 << 16) | (0xc708 >> 2),
426 (0x0e00 << 16) | (0xc768 >> 2),
428 (0x0400 << 16) | (0xc770 >> 2),
430 (0x0400 << 16) | (0xc774 >> 2),
432 (0x0400 << 16) | (0xc778 >> 2),
434 (0x0400 << 16) | (0xc77c >> 2),
436 (0x0400 << 16) | (0xc780 >> 2),
438 (0x0400 << 16) | (0xc784 >> 2),
440 (0x0400 << 16) | (0xc788 >> 2),
442 (0x0400 << 16) | (0xc78c >> 2),
444 (0x0400 << 16) | (0xc798 >> 2),
446 (0x0400 << 16) | (0xc79c >> 2),
448 (0x0400 << 16) | (0xc7a0 >> 2),
450 (0x0400 << 16) | (0xc7a4 >> 2),
452 (0x0400 << 16) | (0xc7a8 >> 2),
454 (0x0400 << 16) | (0xc7ac >> 2),
456 (0x0400 << 16) | (0xc7b0 >> 2),
458 (0x0400 << 16) | (0xc7b4 >> 2),
460 (0x0e00 << 16) | (0x9100 >> 2),
462 (0x0e00 << 16) | (0x3c010 >> 2),
464 (0x0e00 << 16) | (0x92a8 >> 2),
466 (0x0e00 << 16) | (0x92ac >> 2),
468 (0x0e00 << 16) | (0x92b4 >> 2),
470 (0x0e00 << 16) | (0x92b8 >> 2),
472 (0x0e00 << 16) | (0x92bc >> 2),
474 (0x0e00 << 16) | (0x92c0 >> 2),
476 (0x0e00 << 16) | (0x92c4 >> 2),
478 (0x0e00 << 16) | (0x92c8 >> 2),
480 (0x0e00 << 16) | (0x92cc >> 2),
482 (0x0e00 << 16) | (0x92d0 >> 2),
484 (0x0e00 << 16) | (0x8c00 >> 2),
486 (0x0e00 << 16) | (0x8c04 >> 2),
488 (0x0e00 << 16) | (0x8c20 >> 2),
490 (0x0e00 << 16) | (0x8c38 >> 2),
492 (0x0e00 << 16) | (0x8c3c >> 2),
494 (0x0e00 << 16) | (0xae00 >> 2),
496 (0x0e00 << 16) | (0x9604 >> 2),
498 (0x0e00 << 16) | (0xac08 >> 2),
500 (0x0e00 << 16) | (0xac0c >> 2),
502 (0x0e00 << 16) | (0xac10 >> 2),
504 (0x0e00 << 16) | (0xac14 >> 2),
506 (0x0e00 << 16) | (0xac58 >> 2),
508 (0x0e00 << 16) | (0xac68 >> 2),
510 (0x0e00 << 16) | (0xac6c >> 2),
512 (0x0e00 << 16) | (0xac70 >> 2),
514 (0x0e00 << 16) | (0xac74 >> 2),
516 (0x0e00 << 16) | (0xac78 >> 2),
518 (0x0e00 << 16) | (0xac7c >> 2),
520 (0x0e00 << 16) | (0xac80 >> 2),
522 (0x0e00 << 16) | (0xac84 >> 2),
524 (0x0e00 << 16) | (0xac88 >> 2),
526 (0x0e00 << 16) | (0xac8c >> 2),
528 (0x0e00 << 16) | (0x970c >> 2),
530 (0x0e00 << 16) | (0x9714 >> 2),
532 (0x0e00 << 16) | (0x9718 >> 2),
534 (0x0e00 << 16) | (0x971c >> 2),
536 (0x0e00 << 16) | (0x31068 >> 2),
538 (0x4e00 << 16) | (0x31068 >> 2),
540 (0x5e00 << 16) | (0x31068 >> 2),
542 (0x6e00 << 16) | (0x31068 >> 2),
544 (0x7e00 << 16) | (0x31068 >> 2),
546 (0x8e00 << 16) | (0x31068 >> 2),
548 (0x9e00 << 16) | (0x31068 >> 2),
550 (0xae00 << 16) | (0x31068 >> 2),
552 (0xbe00 << 16) | (0x31068 >> 2),
554 (0x0e00 << 16) | (0xcd10 >> 2),
556 (0x0e00 << 16) | (0xcd14 >> 2),
558 (0x0e00 << 16) | (0x88b0 >> 2),
560 (0x0e00 << 16) | (0x88b4 >> 2),
562 (0x0e00 << 16) | (0x88b8 >> 2),
564 (0x0e00 << 16) | (0x88bc >> 2),
566 (0x0400 << 16) | (0x89c0 >> 2),
568 (0x0e00 << 16) | (0x88c4 >> 2),
570 (0x0e00 << 16) | (0x88c8 >> 2),
572 (0x0e00 << 16) | (0x88d0 >> 2),
574 (0x0e00 << 16) | (0x88d4 >> 2),
576 (0x0e00 << 16) | (0x88d8 >> 2),
578 (0x0e00 << 16) | (0x8980 >> 2),
580 (0x0e00 << 16) | (0x30938 >> 2),
582 (0x0e00 << 16) | (0x3093c >> 2),
584 (0x0e00 << 16) | (0x30940 >> 2),
586 (0x0e00 << 16) | (0x89a0 >> 2),
588 (0x0e00 << 16) | (0x30900 >> 2),
590 (0x0e00 << 16) | (0x30904 >> 2),
592 (0x0e00 << 16) | (0x89b4 >> 2),
594 (0x0e00 << 16) | (0x3c210 >> 2),
596 (0x0e00 << 16) | (0x3c214 >> 2),
598 (0x0e00 << 16) | (0x3c218 >> 2),
600 (0x0e00 << 16) | (0x8904 >> 2),
603 (0x0e00 << 16) | (0x8c28 >> 2),
604 (0x0e00 << 16) | (0x8c2c >> 2),
605 (0x0e00 << 16) | (0x8c30 >> 2),
606 (0x0e00 << 16) | (0x8c34 >> 2),
607 (0x0e00 << 16) | (0x9600 >> 2),
610 static const u32 kalindi_rlc_save_restore_register_list[] =
612 (0x0e00 << 16) | (0xc12c >> 2),
614 (0x0e00 << 16) | (0xc140 >> 2),
616 (0x0e00 << 16) | (0xc150 >> 2),
618 (0x0e00 << 16) | (0xc15c >> 2),
620 (0x0e00 << 16) | (0xc168 >> 2),
622 (0x0e00 << 16) | (0xc170 >> 2),
624 (0x0e00 << 16) | (0xc204 >> 2),
626 (0x0e00 << 16) | (0xc2b4 >> 2),
628 (0x0e00 << 16) | (0xc2b8 >> 2),
630 (0x0e00 << 16) | (0xc2bc >> 2),
632 (0x0e00 << 16) | (0xc2c0 >> 2),
634 (0x0e00 << 16) | (0x8228 >> 2),
636 (0x0e00 << 16) | (0x829c >> 2),
638 (0x0e00 << 16) | (0x869c >> 2),
640 (0x0600 << 16) | (0x98f4 >> 2),
642 (0x0e00 << 16) | (0x98f8 >> 2),
644 (0x0e00 << 16) | (0x9900 >> 2),
646 (0x0e00 << 16) | (0xc260 >> 2),
648 (0x0e00 << 16) | (0x90e8 >> 2),
650 (0x0e00 << 16) | (0x3c000 >> 2),
652 (0x0e00 << 16) | (0x3c00c >> 2),
654 (0x0e00 << 16) | (0x8c1c >> 2),
656 (0x0e00 << 16) | (0x9700 >> 2),
658 (0x0e00 << 16) | (0xcd20 >> 2),
660 (0x4e00 << 16) | (0xcd20 >> 2),
662 (0x5e00 << 16) | (0xcd20 >> 2),
664 (0x6e00 << 16) | (0xcd20 >> 2),
666 (0x7e00 << 16) | (0xcd20 >> 2),
668 (0x0e00 << 16) | (0x89bc >> 2),
670 (0x0e00 << 16) | (0x8900 >> 2),
673 (0x0e00 << 16) | (0xc130 >> 2),
675 (0x0e00 << 16) | (0xc134 >> 2),
677 (0x0e00 << 16) | (0xc1fc >> 2),
679 (0x0e00 << 16) | (0xc208 >> 2),
681 (0x0e00 << 16) | (0xc264 >> 2),
683 (0x0e00 << 16) | (0xc268 >> 2),
685 (0x0e00 << 16) | (0xc26c >> 2),
687 (0x0e00 << 16) | (0xc270 >> 2),
689 (0x0e00 << 16) | (0xc274 >> 2),
691 (0x0e00 << 16) | (0xc28c >> 2),
693 (0x0e00 << 16) | (0xc290 >> 2),
695 (0x0e00 << 16) | (0xc294 >> 2),
697 (0x0e00 << 16) | (0xc298 >> 2),
699 (0x0e00 << 16) | (0xc2a0 >> 2),
701 (0x0e00 << 16) | (0xc2a4 >> 2),
703 (0x0e00 << 16) | (0xc2a8 >> 2),
705 (0x0e00 << 16) | (0xc2ac >> 2),
707 (0x0e00 << 16) | (0x301d0 >> 2),
709 (0x0e00 << 16) | (0x30238 >> 2),
711 (0x0e00 << 16) | (0x30250 >> 2),
713 (0x0e00 << 16) | (0x30254 >> 2),
715 (0x0e00 << 16) | (0x30258 >> 2),
717 (0x0e00 << 16) | (0x3025c >> 2),
719 (0x4e00 << 16) | (0xc900 >> 2),
721 (0x5e00 << 16) | (0xc900 >> 2),
723 (0x6e00 << 16) | (0xc900 >> 2),
725 (0x7e00 << 16) | (0xc900 >> 2),
727 (0x4e00 << 16) | (0xc904 >> 2),
729 (0x5e00 << 16) | (0xc904 >> 2),
731 (0x6e00 << 16) | (0xc904 >> 2),
733 (0x7e00 << 16) | (0xc904 >> 2),
735 (0x4e00 << 16) | (0xc908 >> 2),
737 (0x5e00 << 16) | (0xc908 >> 2),
739 (0x6e00 << 16) | (0xc908 >> 2),
741 (0x7e00 << 16) | (0xc908 >> 2),
743 (0x4e00 << 16) | (0xc90c >> 2),
745 (0x5e00 << 16) | (0xc90c >> 2),
747 (0x6e00 << 16) | (0xc90c >> 2),
749 (0x7e00 << 16) | (0xc90c >> 2),
751 (0x4e00 << 16) | (0xc910 >> 2),
753 (0x5e00 << 16) | (0xc910 >> 2),
755 (0x6e00 << 16) | (0xc910 >> 2),
757 (0x7e00 << 16) | (0xc910 >> 2),
759 (0x0e00 << 16) | (0xc99c >> 2),
761 (0x0e00 << 16) | (0x9834 >> 2),
763 (0x0000 << 16) | (0x30f00 >> 2),
765 (0x0000 << 16) | (0x30f04 >> 2),
767 (0x0000 << 16) | (0x30f08 >> 2),
769 (0x0000 << 16) | (0x30f0c >> 2),
771 (0x0600 << 16) | (0x9b7c >> 2),
773 (0x0e00 << 16) | (0x8a14 >> 2),
775 (0x0e00 << 16) | (0x8a18 >> 2),
777 (0x0600 << 16) | (0x30a00 >> 2),
779 (0x0e00 << 16) | (0x8bf0 >> 2),
781 (0x0e00 << 16) | (0x8bcc >> 2),
783 (0x0e00 << 16) | (0x8b24 >> 2),
785 (0x0e00 << 16) | (0x30a04 >> 2),
787 (0x0600 << 16) | (0x30a10 >> 2),
789 (0x0600 << 16) | (0x30a14 >> 2),
791 (0x0600 << 16) | (0x30a18 >> 2),
793 (0x0600 << 16) | (0x30a2c >> 2),
795 (0x0e00 << 16) | (0xc700 >> 2),
797 (0x0e00 << 16) | (0xc704 >> 2),
799 (0x0e00 << 16) | (0xc708 >> 2),
801 (0x0e00 << 16) | (0xc768 >> 2),
803 (0x0400 << 16) | (0xc770 >> 2),
805 (0x0400 << 16) | (0xc774 >> 2),
807 (0x0400 << 16) | (0xc798 >> 2),
809 (0x0400 << 16) | (0xc79c >> 2),
811 (0x0e00 << 16) | (0x9100 >> 2),
813 (0x0e00 << 16) | (0x3c010 >> 2),
815 (0x0e00 << 16) | (0x8c00 >> 2),
817 (0x0e00 << 16) | (0x8c04 >> 2),
819 (0x0e00 << 16) | (0x8c20 >> 2),
821 (0x0e00 << 16) | (0x8c38 >> 2),
823 (0x0e00 << 16) | (0x8c3c >> 2),
825 (0x0e00 << 16) | (0xae00 >> 2),
827 (0x0e00 << 16) | (0x9604 >> 2),
829 (0x0e00 << 16) | (0xac08 >> 2),
831 (0x0e00 << 16) | (0xac0c >> 2),
833 (0x0e00 << 16) | (0xac10 >> 2),
835 (0x0e00 << 16) | (0xac14 >> 2),
837 (0x0e00 << 16) | (0xac58 >> 2),
839 (0x0e00 << 16) | (0xac68 >> 2),
841 (0x0e00 << 16) | (0xac6c >> 2),
843 (0x0e00 << 16) | (0xac70 >> 2),
845 (0x0e00 << 16) | (0xac74 >> 2),
847 (0x0e00 << 16) | (0xac78 >> 2),
849 (0x0e00 << 16) | (0xac7c >> 2),
851 (0x0e00 << 16) | (0xac80 >> 2),
853 (0x0e00 << 16) | (0xac84 >> 2),
855 (0x0e00 << 16) | (0xac88 >> 2),
857 (0x0e00 << 16) | (0xac8c >> 2),
859 (0x0e00 << 16) | (0x970c >> 2),
861 (0x0e00 << 16) | (0x9714 >> 2),
863 (0x0e00 << 16) | (0x9718 >> 2),
865 (0x0e00 << 16) | (0x971c >> 2),
867 (0x0e00 << 16) | (0x31068 >> 2),
869 (0x4e00 << 16) | (0x31068 >> 2),
871 (0x5e00 << 16) | (0x31068 >> 2),
873 (0x6e00 << 16) | (0x31068 >> 2),
875 (0x7e00 << 16) | (0x31068 >> 2),
877 (0x0e00 << 16) | (0xcd10 >> 2),
879 (0x0e00 << 16) | (0xcd14 >> 2),
881 (0x0e00 << 16) | (0x88b0 >> 2),
883 (0x0e00 << 16) | (0x88b4 >> 2),
885 (0x0e00 << 16) | (0x88b8 >> 2),
887 (0x0e00 << 16) | (0x88bc >> 2),
889 (0x0400 << 16) | (0x89c0 >> 2),
891 (0x0e00 << 16) | (0x88c4 >> 2),
893 (0x0e00 << 16) | (0x88c8 >> 2),
895 (0x0e00 << 16) | (0x88d0 >> 2),
897 (0x0e00 << 16) | (0x88d4 >> 2),
899 (0x0e00 << 16) | (0x88d8 >> 2),
901 (0x0e00 << 16) | (0x8980 >> 2),
903 (0x0e00 << 16) | (0x30938 >> 2),
905 (0x0e00 << 16) | (0x3093c >> 2),
907 (0x0e00 << 16) | (0x30940 >> 2),
909 (0x0e00 << 16) | (0x89a0 >> 2),
911 (0x0e00 << 16) | (0x30900 >> 2),
913 (0x0e00 << 16) | (0x30904 >> 2),
915 (0x0e00 << 16) | (0x89b4 >> 2),
917 (0x0e00 << 16) | (0x3e1fc >> 2),
919 (0x0e00 << 16) | (0x3c210 >> 2),
921 (0x0e00 << 16) | (0x3c214 >> 2),
923 (0x0e00 << 16) | (0x3c218 >> 2),
925 (0x0e00 << 16) | (0x8904 >> 2),
928 (0x0e00 << 16) | (0x8c28 >> 2),
929 (0x0e00 << 16) | (0x8c2c >> 2),
930 (0x0e00 << 16) | (0x8c30 >> 2),
931 (0x0e00 << 16) | (0x8c34 >> 2),
932 (0x0e00 << 16) | (0x9600 >> 2),
935 static const u32 bonaire_golden_spm_registers[] =
937 0x30800, 0xe0ffffff, 0xe0000000
940 static const u32 bonaire_golden_common_registers[] =
942 0xc770, 0xffffffff, 0x00000800,
943 0xc774, 0xffffffff, 0x00000800,
944 0xc798, 0xffffffff, 0x00007fbf,
945 0xc79c, 0xffffffff, 0x00007faf
948 static const u32 bonaire_golden_registers[] =
950 0x3354, 0x00000333, 0x00000333,
951 0x3350, 0x000c0fc0, 0x00040200,
952 0x9a10, 0x00010000, 0x00058208,
953 0x3c000, 0xffff1fff, 0x00140000,
954 0x3c200, 0xfdfc0fff, 0x00000100,
955 0x3c234, 0x40000000, 0x40000200,
956 0x9830, 0xffffffff, 0x00000000,
957 0x9834, 0xf00fffff, 0x00000400,
958 0x9838, 0x0002021c, 0x00020200,
959 0xc78, 0x00000080, 0x00000000,
960 0x5bb0, 0x000000f0, 0x00000070,
961 0x5bc0, 0xf0311fff, 0x80300000,
962 0x98f8, 0x73773777, 0x12010001,
963 0x350c, 0x00810000, 0x408af000,
964 0x7030, 0x31000111, 0x00000011,
965 0x2f48, 0x73773777, 0x12010001,
966 0x220c, 0x00007fb6, 0x0021a1b1,
967 0x2210, 0x00007fb6, 0x002021b1,
968 0x2180, 0x00007fb6, 0x00002191,
969 0x2218, 0x00007fb6, 0x002121b1,
970 0x221c, 0x00007fb6, 0x002021b1,
971 0x21dc, 0x00007fb6, 0x00002191,
972 0x21e0, 0x00007fb6, 0x00002191,
973 0x3628, 0x0000003f, 0x0000000a,
974 0x362c, 0x0000003f, 0x0000000a,
975 0x2ae4, 0x00073ffe, 0x000022a2,
976 0x240c, 0x000007ff, 0x00000000,
977 0x8a14, 0xf000003f, 0x00000007,
978 0x8bf0, 0x00002001, 0x00000001,
979 0x8b24, 0xffffffff, 0x00ffffff,
980 0x30a04, 0x0000ff0f, 0x00000000,
981 0x28a4c, 0x07ffffff, 0x06000000,
982 0x4d8, 0x00000fff, 0x00000100,
983 0x3e78, 0x00000001, 0x00000002,
984 0x9100, 0x03000000, 0x0362c688,
985 0x8c00, 0x000000ff, 0x00000001,
986 0xe40, 0x00001fff, 0x00001fff,
987 0x9060, 0x0000007f, 0x00000020,
988 0x9508, 0x00010000, 0x00010000,
989 0xac14, 0x000003ff, 0x000000f3,
990 0xac0c, 0xffffffff, 0x00001032
993 static const u32 bonaire_mgcg_cgcg_init[] =
995 0xc420, 0xffffffff, 0xfffffffc,
996 0x30800, 0xffffffff, 0xe0000000,
997 0x3c2a0, 0xffffffff, 0x00000100,
998 0x3c208, 0xffffffff, 0x00000100,
999 0x3c2c0, 0xffffffff, 0xc0000100,
1000 0x3c2c8, 0xffffffff, 0xc0000100,
1001 0x3c2c4, 0xffffffff, 0xc0000100,
1002 0x55e4, 0xffffffff, 0x00600100,
1003 0x3c280, 0xffffffff, 0x00000100,
1004 0x3c214, 0xffffffff, 0x06000100,
1005 0x3c220, 0xffffffff, 0x00000100,
1006 0x3c218, 0xffffffff, 0x06000100,
1007 0x3c204, 0xffffffff, 0x00000100,
1008 0x3c2e0, 0xffffffff, 0x00000100,
1009 0x3c224, 0xffffffff, 0x00000100,
1010 0x3c200, 0xffffffff, 0x00000100,
1011 0x3c230, 0xffffffff, 0x00000100,
1012 0x3c234, 0xffffffff, 0x00000100,
1013 0x3c250, 0xffffffff, 0x00000100,
1014 0x3c254, 0xffffffff, 0x00000100,
1015 0x3c258, 0xffffffff, 0x00000100,
1016 0x3c25c, 0xffffffff, 0x00000100,
1017 0x3c260, 0xffffffff, 0x00000100,
1018 0x3c27c, 0xffffffff, 0x00000100,
1019 0x3c278, 0xffffffff, 0x00000100,
1020 0x3c210, 0xffffffff, 0x06000100,
1021 0x3c290, 0xffffffff, 0x00000100,
1022 0x3c274, 0xffffffff, 0x00000100,
1023 0x3c2b4, 0xffffffff, 0x00000100,
1024 0x3c2b0, 0xffffffff, 0x00000100,
1025 0x3c270, 0xffffffff, 0x00000100,
1026 0x30800, 0xffffffff, 0xe0000000,
1027 0x3c020, 0xffffffff, 0x00010000,
1028 0x3c024, 0xffffffff, 0x00030002,
1029 0x3c028, 0xffffffff, 0x00040007,
1030 0x3c02c, 0xffffffff, 0x00060005,
1031 0x3c030, 0xffffffff, 0x00090008,
1032 0x3c034, 0xffffffff, 0x00010000,
1033 0x3c038, 0xffffffff, 0x00030002,
1034 0x3c03c, 0xffffffff, 0x00040007,
1035 0x3c040, 0xffffffff, 0x00060005,
1036 0x3c044, 0xffffffff, 0x00090008,
1037 0x3c048, 0xffffffff, 0x00010000,
1038 0x3c04c, 0xffffffff, 0x00030002,
1039 0x3c050, 0xffffffff, 0x00040007,
1040 0x3c054, 0xffffffff, 0x00060005,
1041 0x3c058, 0xffffffff, 0x00090008,
1042 0x3c05c, 0xffffffff, 0x00010000,
1043 0x3c060, 0xffffffff, 0x00030002,
1044 0x3c064, 0xffffffff, 0x00040007,
1045 0x3c068, 0xffffffff, 0x00060005,
1046 0x3c06c, 0xffffffff, 0x00090008,
1047 0x3c070, 0xffffffff, 0x00010000,
1048 0x3c074, 0xffffffff, 0x00030002,
1049 0x3c078, 0xffffffff, 0x00040007,
1050 0x3c07c, 0xffffffff, 0x00060005,
1051 0x3c080, 0xffffffff, 0x00090008,
1052 0x3c084, 0xffffffff, 0x00010000,
1053 0x3c088, 0xffffffff, 0x00030002,
1054 0x3c08c, 0xffffffff, 0x00040007,
1055 0x3c090, 0xffffffff, 0x00060005,
1056 0x3c094, 0xffffffff, 0x00090008,
1057 0x3c098, 0xffffffff, 0x00010000,
1058 0x3c09c, 0xffffffff, 0x00030002,
1059 0x3c0a0, 0xffffffff, 0x00040007,
1060 0x3c0a4, 0xffffffff, 0x00060005,
1061 0x3c0a8, 0xffffffff, 0x00090008,
1062 0x3c000, 0xffffffff, 0x96e00200,
1063 0x8708, 0xffffffff, 0x00900100,
1064 0xc424, 0xffffffff, 0x0020003f,
1065 0x38, 0xffffffff, 0x0140001c,
1066 0x3c, 0x000f0000, 0x000f0000,
1067 0x220, 0xffffffff, 0xC060000C,
1068 0x224, 0xc0000fff, 0x00000100,
1069 0xf90, 0xffffffff, 0x00000100,
1070 0xf98, 0x00000101, 0x00000000,
1071 0x20a8, 0xffffffff, 0x00000104,
1072 0x55e4, 0xff000fff, 0x00000100,
1073 0x30cc, 0xc0000fff, 0x00000104,
1074 0xc1e4, 0x00000001, 0x00000001,
1075 0xd00c, 0xff000ff0, 0x00000100,
1076 0xd80c, 0xff000ff0, 0x00000100
1079 static const u32 spectre_golden_spm_registers[] =
1081 0x30800, 0xe0ffffff, 0xe0000000
1084 static const u32 spectre_golden_common_registers[] =
1086 0xc770, 0xffffffff, 0x00000800,
1087 0xc774, 0xffffffff, 0x00000800,
1088 0xc798, 0xffffffff, 0x00007fbf,
1089 0xc79c, 0xffffffff, 0x00007faf
1092 static const u32 spectre_golden_registers[] =
1094 0x3c000, 0xffff1fff, 0x96940200,
1095 0x3c00c, 0xffff0001, 0xff000000,
1096 0x3c200, 0xfffc0fff, 0x00000100,
1097 0x6ed8, 0x00010101, 0x00010000,
1098 0x9834, 0xf00fffff, 0x00000400,
1099 0x9838, 0xfffffffc, 0x00020200,
1100 0x5bb0, 0x000000f0, 0x00000070,
1101 0x5bc0, 0xf0311fff, 0x80300000,
1102 0x98f8, 0x73773777, 0x12010001,
1103 0x9b7c, 0x00ff0000, 0x00fc0000,
1104 0x2f48, 0x73773777, 0x12010001,
1105 0x8a14, 0xf000003f, 0x00000007,
1106 0x8b24, 0xffffffff, 0x00ffffff,
1107 0x28350, 0x3f3f3fff, 0x00000082,
1108 0x28354, 0x0000003f, 0x00000000,
1109 0x3e78, 0x00000001, 0x00000002,
1110 0x913c, 0xffff03df, 0x00000004,
1111 0xc768, 0x00000008, 0x00000008,
1112 0x8c00, 0x000008ff, 0x00000800,
1113 0x9508, 0x00010000, 0x00010000,
1114 0xac0c, 0xffffffff, 0x54763210,
1115 0x214f8, 0x01ff01ff, 0x00000002,
1116 0x21498, 0x007ff800, 0x00200000,
1117 0x2015c, 0xffffffff, 0x00000f40,
1118 0x30934, 0xffffffff, 0x00000001
1121 static const u32 spectre_mgcg_cgcg_init[] =
1123 0xc420, 0xffffffff, 0xfffffffc,
1124 0x30800, 0xffffffff, 0xe0000000,
1125 0x3c2a0, 0xffffffff, 0x00000100,
1126 0x3c208, 0xffffffff, 0x00000100,
1127 0x3c2c0, 0xffffffff, 0x00000100,
1128 0x3c2c8, 0xffffffff, 0x00000100,
1129 0x3c2c4, 0xffffffff, 0x00000100,
1130 0x55e4, 0xffffffff, 0x00600100,
1131 0x3c280, 0xffffffff, 0x00000100,
1132 0x3c214, 0xffffffff, 0x06000100,
1133 0x3c220, 0xffffffff, 0x00000100,
1134 0x3c218, 0xffffffff, 0x06000100,
1135 0x3c204, 0xffffffff, 0x00000100,
1136 0x3c2e0, 0xffffffff, 0x00000100,
1137 0x3c224, 0xffffffff, 0x00000100,
1138 0x3c200, 0xffffffff, 0x00000100,
1139 0x3c230, 0xffffffff, 0x00000100,
1140 0x3c234, 0xffffffff, 0x00000100,
1141 0x3c250, 0xffffffff, 0x00000100,
1142 0x3c254, 0xffffffff, 0x00000100,
1143 0x3c258, 0xffffffff, 0x00000100,
1144 0x3c25c, 0xffffffff, 0x00000100,
1145 0x3c260, 0xffffffff, 0x00000100,
1146 0x3c27c, 0xffffffff, 0x00000100,
1147 0x3c278, 0xffffffff, 0x00000100,
1148 0x3c210, 0xffffffff, 0x06000100,
1149 0x3c290, 0xffffffff, 0x00000100,
1150 0x3c274, 0xffffffff, 0x00000100,
1151 0x3c2b4, 0xffffffff, 0x00000100,
1152 0x3c2b0, 0xffffffff, 0x00000100,
1153 0x3c270, 0xffffffff, 0x00000100,
1154 0x30800, 0xffffffff, 0xe0000000,
1155 0x3c020, 0xffffffff, 0x00010000,
1156 0x3c024, 0xffffffff, 0x00030002,
1157 0x3c028, 0xffffffff, 0x00040007,
1158 0x3c02c, 0xffffffff, 0x00060005,
1159 0x3c030, 0xffffffff, 0x00090008,
1160 0x3c034, 0xffffffff, 0x00010000,
1161 0x3c038, 0xffffffff, 0x00030002,
1162 0x3c03c, 0xffffffff, 0x00040007,
1163 0x3c040, 0xffffffff, 0x00060005,
1164 0x3c044, 0xffffffff, 0x00090008,
1165 0x3c048, 0xffffffff, 0x00010000,
1166 0x3c04c, 0xffffffff, 0x00030002,
1167 0x3c050, 0xffffffff, 0x00040007,
1168 0x3c054, 0xffffffff, 0x00060005,
1169 0x3c058, 0xffffffff, 0x00090008,
1170 0x3c05c, 0xffffffff, 0x00010000,
1171 0x3c060, 0xffffffff, 0x00030002,
1172 0x3c064, 0xffffffff, 0x00040007,
1173 0x3c068, 0xffffffff, 0x00060005,
1174 0x3c06c, 0xffffffff, 0x00090008,
1175 0x3c070, 0xffffffff, 0x00010000,
1176 0x3c074, 0xffffffff, 0x00030002,
1177 0x3c078, 0xffffffff, 0x00040007,
1178 0x3c07c, 0xffffffff, 0x00060005,
1179 0x3c080, 0xffffffff, 0x00090008,
1180 0x3c084, 0xffffffff, 0x00010000,
1181 0x3c088, 0xffffffff, 0x00030002,
1182 0x3c08c, 0xffffffff, 0x00040007,
1183 0x3c090, 0xffffffff, 0x00060005,
1184 0x3c094, 0xffffffff, 0x00090008,
1185 0x3c098, 0xffffffff, 0x00010000,
1186 0x3c09c, 0xffffffff, 0x00030002,
1187 0x3c0a0, 0xffffffff, 0x00040007,
1188 0x3c0a4, 0xffffffff, 0x00060005,
1189 0x3c0a8, 0xffffffff, 0x00090008,
1190 0x3c0ac, 0xffffffff, 0x00010000,
1191 0x3c0b0, 0xffffffff, 0x00030002,
1192 0x3c0b4, 0xffffffff, 0x00040007,
1193 0x3c0b8, 0xffffffff, 0x00060005,
1194 0x3c0bc, 0xffffffff, 0x00090008,
1195 0x3c000, 0xffffffff, 0x96e00200,
1196 0x8708, 0xffffffff, 0x00900100,
1197 0xc424, 0xffffffff, 0x0020003f,
1198 0x38, 0xffffffff, 0x0140001c,
1199 0x3c, 0x000f0000, 0x000f0000,
1200 0x220, 0xffffffff, 0xC060000C,
1201 0x224, 0xc0000fff, 0x00000100,
1202 0xf90, 0xffffffff, 0x00000100,
1203 0xf98, 0x00000101, 0x00000000,
1204 0x20a8, 0xffffffff, 0x00000104,
1205 0x55e4, 0xff000fff, 0x00000100,
1206 0x30cc, 0xc0000fff, 0x00000104,
1207 0xc1e4, 0x00000001, 0x00000001,
1208 0xd00c, 0xff000ff0, 0x00000100,
1209 0xd80c, 0xff000ff0, 0x00000100
1212 static const u32 kalindi_golden_spm_registers[] =
1214 0x30800, 0xe0ffffff, 0xe0000000
1217 static const u32 kalindi_golden_common_registers[] =
1219 0xc770, 0xffffffff, 0x00000800,
1220 0xc774, 0xffffffff, 0x00000800,
1221 0xc798, 0xffffffff, 0x00007fbf,
1222 0xc79c, 0xffffffff, 0x00007faf
1225 static const u32 kalindi_golden_registers[] =
1227 0x3c000, 0xffffdfff, 0x6e944040,
1228 0x55e4, 0xff607fff, 0xfc000100,
1229 0x3c220, 0xff000fff, 0x00000100,
1230 0x3c224, 0xff000fff, 0x00000100,
1231 0x3c200, 0xfffc0fff, 0x00000100,
1232 0x6ed8, 0x00010101, 0x00010000,
1233 0x9830, 0xffffffff, 0x00000000,
1234 0x9834, 0xf00fffff, 0x00000400,
1235 0x5bb0, 0x000000f0, 0x00000070,
1236 0x5bc0, 0xf0311fff, 0x80300000,
1237 0x98f8, 0x73773777, 0x12010001,
1238 0x98fc, 0xffffffff, 0x00000010,
1239 0x9b7c, 0x00ff0000, 0x00fc0000,
1240 0x8030, 0x00001f0f, 0x0000100a,
1241 0x2f48, 0x73773777, 0x12010001,
1242 0x2408, 0x000fffff, 0x000c007f,
1243 0x8a14, 0xf000003f, 0x00000007,
1244 0x8b24, 0x3fff3fff, 0x00ffcfff,
1245 0x30a04, 0x0000ff0f, 0x00000000,
1246 0x28a4c, 0x07ffffff, 0x06000000,
1247 0x4d8, 0x00000fff, 0x00000100,
1248 0x3e78, 0x00000001, 0x00000002,
1249 0xc768, 0x00000008, 0x00000008,
1250 0x8c00, 0x000000ff, 0x00000003,
1251 0x214f8, 0x01ff01ff, 0x00000002,
1252 0x21498, 0x007ff800, 0x00200000,
1253 0x2015c, 0xffffffff, 0x00000f40,
1254 0x88c4, 0x001f3ae3, 0x00000082,
1255 0x88d4, 0x0000001f, 0x00000010,
1256 0x30934, 0xffffffff, 0x00000000
1259 static const u32 kalindi_mgcg_cgcg_init[] =
1261 0xc420, 0xffffffff, 0xfffffffc,
1262 0x30800, 0xffffffff, 0xe0000000,
1263 0x3c2a0, 0xffffffff, 0x00000100,
1264 0x3c208, 0xffffffff, 0x00000100,
1265 0x3c2c0, 0xffffffff, 0x00000100,
1266 0x3c2c8, 0xffffffff, 0x00000100,
1267 0x3c2c4, 0xffffffff, 0x00000100,
1268 0x55e4, 0xffffffff, 0x00600100,
1269 0x3c280, 0xffffffff, 0x00000100,
1270 0x3c214, 0xffffffff, 0x06000100,
1271 0x3c220, 0xffffffff, 0x00000100,
1272 0x3c218, 0xffffffff, 0x06000100,
1273 0x3c204, 0xffffffff, 0x00000100,
1274 0x3c2e0, 0xffffffff, 0x00000100,
1275 0x3c224, 0xffffffff, 0x00000100,
1276 0x3c200, 0xffffffff, 0x00000100,
1277 0x3c230, 0xffffffff, 0x00000100,
1278 0x3c234, 0xffffffff, 0x00000100,
1279 0x3c250, 0xffffffff, 0x00000100,
1280 0x3c254, 0xffffffff, 0x00000100,
1281 0x3c258, 0xffffffff, 0x00000100,
1282 0x3c25c, 0xffffffff, 0x00000100,
1283 0x3c260, 0xffffffff, 0x00000100,
1284 0x3c27c, 0xffffffff, 0x00000100,
1285 0x3c278, 0xffffffff, 0x00000100,
1286 0x3c210, 0xffffffff, 0x06000100,
1287 0x3c290, 0xffffffff, 0x00000100,
1288 0x3c274, 0xffffffff, 0x00000100,
1289 0x3c2b4, 0xffffffff, 0x00000100,
1290 0x3c2b0, 0xffffffff, 0x00000100,
1291 0x3c270, 0xffffffff, 0x00000100,
1292 0x30800, 0xffffffff, 0xe0000000,
1293 0x3c020, 0xffffffff, 0x00010000,
1294 0x3c024, 0xffffffff, 0x00030002,
1295 0x3c028, 0xffffffff, 0x00040007,
1296 0x3c02c, 0xffffffff, 0x00060005,
1297 0x3c030, 0xffffffff, 0x00090008,
1298 0x3c034, 0xffffffff, 0x00010000,
1299 0x3c038, 0xffffffff, 0x00030002,
1300 0x3c03c, 0xffffffff, 0x00040007,
1301 0x3c040, 0xffffffff, 0x00060005,
1302 0x3c044, 0xffffffff, 0x00090008,
1303 0x3c000, 0xffffffff, 0x96e00200,
1304 0x8708, 0xffffffff, 0x00900100,
1305 0xc424, 0xffffffff, 0x0020003f,
1306 0x38, 0xffffffff, 0x0140001c,
1307 0x3c, 0x000f0000, 0x000f0000,
1308 0x220, 0xffffffff, 0xC060000C,
1309 0x224, 0xc0000fff, 0x00000100,
1310 0x20a8, 0xffffffff, 0x00000104,
1311 0x55e4, 0xff000fff, 0x00000100,
1312 0x30cc, 0xc0000fff, 0x00000104,
1313 0xc1e4, 0x00000001, 0x00000001,
1314 0xd00c, 0xff000ff0, 0x00000100,
1315 0xd80c, 0xff000ff0, 0x00000100
1318 static const u32 hawaii_golden_spm_registers[] =
1320 0x30800, 0xe0ffffff, 0xe0000000
1323 static const u32 hawaii_golden_common_registers[] =
1325 0x30800, 0xffffffff, 0xe0000000,
1326 0x28350, 0xffffffff, 0x3a00161a,
1327 0x28354, 0xffffffff, 0x0000002e,
1328 0x9a10, 0xffffffff, 0x00018208,
1329 0x98f8, 0xffffffff, 0x12011003
1332 static const u32 hawaii_golden_registers[] =
1334 0x3354, 0x00000333, 0x00000333,
1335 0x9a10, 0x00010000, 0x00058208,
1336 0x9830, 0xffffffff, 0x00000000,
1337 0x9834, 0xf00fffff, 0x00000400,
1338 0x9838, 0x0002021c, 0x00020200,
1339 0xc78, 0x00000080, 0x00000000,
1340 0x5bb0, 0x000000f0, 0x00000070,
1341 0x5bc0, 0xf0311fff, 0x80300000,
1342 0x350c, 0x00810000, 0x408af000,
1343 0x7030, 0x31000111, 0x00000011,
1344 0x2f48, 0x73773777, 0x12010001,
1345 0x2120, 0x0000007f, 0x0000001b,
1346 0x21dc, 0x00007fb6, 0x00002191,
1347 0x3628, 0x0000003f, 0x0000000a,
1348 0x362c, 0x0000003f, 0x0000000a,
1349 0x2ae4, 0x00073ffe, 0x000022a2,
1350 0x240c, 0x000007ff, 0x00000000,
1351 0x8bf0, 0x00002001, 0x00000001,
1352 0x8b24, 0xffffffff, 0x00ffffff,
1353 0x30a04, 0x0000ff0f, 0x00000000,
1354 0x28a4c, 0x07ffffff, 0x06000000,
1355 0x3e78, 0x00000001, 0x00000002,
1356 0xc768, 0x00000008, 0x00000008,
1357 0xc770, 0x00000f00, 0x00000800,
1358 0xc774, 0x00000f00, 0x00000800,
1359 0xc798, 0x00ffffff, 0x00ff7fbf,
1360 0xc79c, 0x00ffffff, 0x00ff7faf,
1361 0x8c00, 0x000000ff, 0x00000800,
1362 0xe40, 0x00001fff, 0x00001fff,
1363 0x9060, 0x0000007f, 0x00000020,
1364 0x9508, 0x00010000, 0x00010000,
1365 0xae00, 0x00100000, 0x000ff07c,
1366 0xac14, 0x000003ff, 0x0000000f,
1367 0xac10, 0xffffffff, 0x7564fdec,
1368 0xac0c, 0xffffffff, 0x3120b9a8,
1369 0xac08, 0x20000000, 0x0f9c0000
1372 static const u32 hawaii_mgcg_cgcg_init[] =
1374 0xc420, 0xffffffff, 0xfffffffd,
1375 0x30800, 0xffffffff, 0xe0000000,
1376 0x3c2a0, 0xffffffff, 0x00000100,
1377 0x3c208, 0xffffffff, 0x00000100,
1378 0x3c2c0, 0xffffffff, 0x00000100,
1379 0x3c2c8, 0xffffffff, 0x00000100,
1380 0x3c2c4, 0xffffffff, 0x00000100,
1381 0x55e4, 0xffffffff, 0x00200100,
1382 0x3c280, 0xffffffff, 0x00000100,
1383 0x3c214, 0xffffffff, 0x06000100,
1384 0x3c220, 0xffffffff, 0x00000100,
1385 0x3c218, 0xffffffff, 0x06000100,
1386 0x3c204, 0xffffffff, 0x00000100,
1387 0x3c2e0, 0xffffffff, 0x00000100,
1388 0x3c224, 0xffffffff, 0x00000100,
1389 0x3c200, 0xffffffff, 0x00000100,
1390 0x3c230, 0xffffffff, 0x00000100,
1391 0x3c234, 0xffffffff, 0x00000100,
1392 0x3c250, 0xffffffff, 0x00000100,
1393 0x3c254, 0xffffffff, 0x00000100,
1394 0x3c258, 0xffffffff, 0x00000100,
1395 0x3c25c, 0xffffffff, 0x00000100,
1396 0x3c260, 0xffffffff, 0x00000100,
1397 0x3c27c, 0xffffffff, 0x00000100,
1398 0x3c278, 0xffffffff, 0x00000100,
1399 0x3c210, 0xffffffff, 0x06000100,
1400 0x3c290, 0xffffffff, 0x00000100,
1401 0x3c274, 0xffffffff, 0x00000100,
1402 0x3c2b4, 0xffffffff, 0x00000100,
1403 0x3c2b0, 0xffffffff, 0x00000100,
1404 0x3c270, 0xffffffff, 0x00000100,
1405 0x30800, 0xffffffff, 0xe0000000,
1406 0x3c020, 0xffffffff, 0x00010000,
1407 0x3c024, 0xffffffff, 0x00030002,
1408 0x3c028, 0xffffffff, 0x00040007,
1409 0x3c02c, 0xffffffff, 0x00060005,
1410 0x3c030, 0xffffffff, 0x00090008,
1411 0x3c034, 0xffffffff, 0x00010000,
1412 0x3c038, 0xffffffff, 0x00030002,
1413 0x3c03c, 0xffffffff, 0x00040007,
1414 0x3c040, 0xffffffff, 0x00060005,
1415 0x3c044, 0xffffffff, 0x00090008,
1416 0x3c048, 0xffffffff, 0x00010000,
1417 0x3c04c, 0xffffffff, 0x00030002,
1418 0x3c050, 0xffffffff, 0x00040007,
1419 0x3c054, 0xffffffff, 0x00060005,
1420 0x3c058, 0xffffffff, 0x00090008,
1421 0x3c05c, 0xffffffff, 0x00010000,
1422 0x3c060, 0xffffffff, 0x00030002,
1423 0x3c064, 0xffffffff, 0x00040007,
1424 0x3c068, 0xffffffff, 0x00060005,
1425 0x3c06c, 0xffffffff, 0x00090008,
1426 0x3c070, 0xffffffff, 0x00010000,
1427 0x3c074, 0xffffffff, 0x00030002,
1428 0x3c078, 0xffffffff, 0x00040007,
1429 0x3c07c, 0xffffffff, 0x00060005,
1430 0x3c080, 0xffffffff, 0x00090008,
1431 0x3c084, 0xffffffff, 0x00010000,
1432 0x3c088, 0xffffffff, 0x00030002,
1433 0x3c08c, 0xffffffff, 0x00040007,
1434 0x3c090, 0xffffffff, 0x00060005,
1435 0x3c094, 0xffffffff, 0x00090008,
1436 0x3c098, 0xffffffff, 0x00010000,
1437 0x3c09c, 0xffffffff, 0x00030002,
1438 0x3c0a0, 0xffffffff, 0x00040007,
1439 0x3c0a4, 0xffffffff, 0x00060005,
1440 0x3c0a8, 0xffffffff, 0x00090008,
1441 0x3c0ac, 0xffffffff, 0x00010000,
1442 0x3c0b0, 0xffffffff, 0x00030002,
1443 0x3c0b4, 0xffffffff, 0x00040007,
1444 0x3c0b8, 0xffffffff, 0x00060005,
1445 0x3c0bc, 0xffffffff, 0x00090008,
1446 0x3c0c0, 0xffffffff, 0x00010000,
1447 0x3c0c4, 0xffffffff, 0x00030002,
1448 0x3c0c8, 0xffffffff, 0x00040007,
1449 0x3c0cc, 0xffffffff, 0x00060005,
1450 0x3c0d0, 0xffffffff, 0x00090008,
1451 0x3c0d4, 0xffffffff, 0x00010000,
1452 0x3c0d8, 0xffffffff, 0x00030002,
1453 0x3c0dc, 0xffffffff, 0x00040007,
1454 0x3c0e0, 0xffffffff, 0x00060005,
1455 0x3c0e4, 0xffffffff, 0x00090008,
1456 0x3c0e8, 0xffffffff, 0x00010000,
1457 0x3c0ec, 0xffffffff, 0x00030002,
1458 0x3c0f0, 0xffffffff, 0x00040007,
1459 0x3c0f4, 0xffffffff, 0x00060005,
1460 0x3c0f8, 0xffffffff, 0x00090008,
1461 0xc318, 0xffffffff, 0x00020200,
1462 0x3350, 0xffffffff, 0x00000200,
1463 0x15c0, 0xffffffff, 0x00000400,
1464 0x55e8, 0xffffffff, 0x00000000,
1465 0x2f50, 0xffffffff, 0x00000902,
1466 0x3c000, 0xffffffff, 0x96940200,
1467 0x8708, 0xffffffff, 0x00900100,
1468 0xc424, 0xffffffff, 0x0020003f,
1469 0x38, 0xffffffff, 0x0140001c,
1470 0x3c, 0x000f0000, 0x000f0000,
1471 0x220, 0xffffffff, 0xc060000c,
1472 0x224, 0xc0000fff, 0x00000100,
1473 0xf90, 0xffffffff, 0x00000100,
1474 0xf98, 0x00000101, 0x00000000,
1475 0x20a8, 0xffffffff, 0x00000104,
1476 0x55e4, 0xff000fff, 0x00000100,
1477 0x30cc, 0xc0000fff, 0x00000104,
1478 0xc1e4, 0x00000001, 0x00000001,
1479 0xd00c, 0xff000ff0, 0x00000100,
1480 0xd80c, 0xff000ff0, 0x00000100
1483 static const u32 godavari_golden_registers[] =
1485 0x55e4, 0xff607fff, 0xfc000100,
1486 0x6ed8, 0x00010101, 0x00010000,
1487 0x9830, 0xffffffff, 0x00000000,
1488 0x98302, 0xf00fffff, 0x00000400,
1489 0x6130, 0xffffffff, 0x00010000,
1490 0x5bb0, 0x000000f0, 0x00000070,
1491 0x5bc0, 0xf0311fff, 0x80300000,
1492 0x98f8, 0x73773777, 0x12010001,
1493 0x98fc, 0xffffffff, 0x00000010,
1494 0x8030, 0x00001f0f, 0x0000100a,
1495 0x2f48, 0x73773777, 0x12010001,
1496 0x2408, 0x000fffff, 0x000c007f,
1497 0x8a14, 0xf000003f, 0x00000007,
1498 0x8b24, 0xffffffff, 0x00ff0fff,
1499 0x30a04, 0x0000ff0f, 0x00000000,
1500 0x28a4c, 0x07ffffff, 0x06000000,
1501 0x4d8, 0x00000fff, 0x00000100,
1502 0xd014, 0x00010000, 0x00810001,
1503 0xd814, 0x00010000, 0x00810001,
1504 0x3e78, 0x00000001, 0x00000002,
1505 0xc768, 0x00000008, 0x00000008,
1506 0xc770, 0x00000f00, 0x00000800,
1507 0xc774, 0x00000f00, 0x00000800,
1508 0xc798, 0x00ffffff, 0x00ff7fbf,
1509 0xc79c, 0x00ffffff, 0x00ff7faf,
1510 0x8c00, 0x000000ff, 0x00000001,
1511 0x214f8, 0x01ff01ff, 0x00000002,
1512 0x21498, 0x007ff800, 0x00200000,
1513 0x2015c, 0xffffffff, 0x00000f40,
1514 0x88c4, 0x001f3ae3, 0x00000082,
1515 0x88d4, 0x0000001f, 0x00000010,
1516 0x30934, 0xffffffff, 0x00000000
1520 static void cik_init_golden_registers(struct radeon_device *rdev)
1522 switch (rdev->family) {
1524 radeon_program_register_sequence(rdev,
1525 bonaire_mgcg_cgcg_init,
1526 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1527 radeon_program_register_sequence(rdev,
1528 bonaire_golden_registers,
1529 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1530 radeon_program_register_sequence(rdev,
1531 bonaire_golden_common_registers,
1532 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1533 radeon_program_register_sequence(rdev,
1534 bonaire_golden_spm_registers,
1535 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1538 radeon_program_register_sequence(rdev,
1539 kalindi_mgcg_cgcg_init,
1540 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1541 radeon_program_register_sequence(rdev,
1542 kalindi_golden_registers,
1543 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1544 radeon_program_register_sequence(rdev,
1545 kalindi_golden_common_registers,
1546 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1547 radeon_program_register_sequence(rdev,
1548 kalindi_golden_spm_registers,
1549 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1552 radeon_program_register_sequence(rdev,
1553 kalindi_mgcg_cgcg_init,
1554 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1555 radeon_program_register_sequence(rdev,
1556 godavari_golden_registers,
1557 (const u32)ARRAY_SIZE(godavari_golden_registers));
1558 radeon_program_register_sequence(rdev,
1559 kalindi_golden_common_registers,
1560 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1561 radeon_program_register_sequence(rdev,
1562 kalindi_golden_spm_registers,
1563 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1566 radeon_program_register_sequence(rdev,
1567 spectre_mgcg_cgcg_init,
1568 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1569 radeon_program_register_sequence(rdev,
1570 spectre_golden_registers,
1571 (const u32)ARRAY_SIZE(spectre_golden_registers));
1572 radeon_program_register_sequence(rdev,
1573 spectre_golden_common_registers,
1574 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1575 radeon_program_register_sequence(rdev,
1576 spectre_golden_spm_registers,
1577 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1580 radeon_program_register_sequence(rdev,
1581 hawaii_mgcg_cgcg_init,
1582 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1583 radeon_program_register_sequence(rdev,
1584 hawaii_golden_registers,
1585 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1586 radeon_program_register_sequence(rdev,
1587 hawaii_golden_common_registers,
1588 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1589 radeon_program_register_sequence(rdev,
1590 hawaii_golden_spm_registers,
1591 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1599 * cik_get_xclk - get the xclk
1601 * @rdev: radeon_device pointer
1603 * Returns the reference clock used by the gfx engine
1606 u32 cik_get_xclk(struct radeon_device *rdev)
1608 u32 reference_clock = rdev->clock.spll.reference_freq;
1610 if (rdev->flags & RADEON_IS_IGP) {
1611 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1612 return reference_clock / 2;
1614 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1615 return reference_clock / 4;
1617 return reference_clock;
1621 * cik_mm_rdoorbell - read a doorbell dword
1623 * @rdev: radeon_device pointer
1624 * @index: doorbell index
1626 * Returns the value in the doorbell aperture at the
1627 * requested doorbell index (CIK).
1629 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1631 if (index < rdev->doorbell.num_doorbells) {
1632 return readl(rdev->doorbell.ptr + index);
1634 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1640 * cik_mm_wdoorbell - write a doorbell dword
1642 * @rdev: radeon_device pointer
1643 * @index: doorbell index
1644 * @v: value to write
1646 * Writes @v to the doorbell aperture at the
1647 * requested doorbell index (CIK).
1649 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1651 if (index < rdev->doorbell.num_doorbells) {
1652 writel(v, rdev->doorbell.ptr + index);
1654 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1658 #define BONAIRE_IO_MC_REGS_SIZE 36
1660 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1662 {0x00000070, 0x04400000},
1663 {0x00000071, 0x80c01803},
1664 {0x00000072, 0x00004004},
1665 {0x00000073, 0x00000100},
1666 {0x00000074, 0x00ff0000},
1667 {0x00000075, 0x34000000},
1668 {0x00000076, 0x08000014},
1669 {0x00000077, 0x00cc08ec},
1670 {0x00000078, 0x00000400},
1671 {0x00000079, 0x00000000},
1672 {0x0000007a, 0x04090000},
1673 {0x0000007c, 0x00000000},
1674 {0x0000007e, 0x4408a8e8},
1675 {0x0000007f, 0x00000304},
1676 {0x00000080, 0x00000000},
1677 {0x00000082, 0x00000001},
1678 {0x00000083, 0x00000002},
1679 {0x00000084, 0xf3e4f400},
1680 {0x00000085, 0x052024e3},
1681 {0x00000087, 0x00000000},
1682 {0x00000088, 0x01000000},
1683 {0x0000008a, 0x1c0a0000},
1684 {0x0000008b, 0xff010000},
1685 {0x0000008d, 0xffffefff},
1686 {0x0000008e, 0xfff3efff},
1687 {0x0000008f, 0xfff3efbf},
1688 {0x00000092, 0xf7ffffff},
1689 {0x00000093, 0xffffff7f},
1690 {0x00000095, 0x00101101},
1691 {0x00000096, 0x00000fff},
1692 {0x00000097, 0x00116fff},
1693 {0x00000098, 0x60010000},
1694 {0x00000099, 0x10010000},
1695 {0x0000009a, 0x00006000},
1696 {0x0000009b, 0x00001000},
1697 {0x0000009f, 0x00b48000}
1700 #define HAWAII_IO_MC_REGS_SIZE 22
1702 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1704 {0x0000007d, 0x40000000},
1705 {0x0000007e, 0x40180304},
1706 {0x0000007f, 0x0000ff00},
1707 {0x00000081, 0x00000000},
1708 {0x00000083, 0x00000800},
1709 {0x00000086, 0x00000000},
1710 {0x00000087, 0x00000100},
1711 {0x00000088, 0x00020100},
1712 {0x00000089, 0x00000000},
1713 {0x0000008b, 0x00040000},
1714 {0x0000008c, 0x00000100},
1715 {0x0000008e, 0xff010000},
1716 {0x00000090, 0xffffefff},
1717 {0x00000091, 0xfff3efff},
1718 {0x00000092, 0xfff3efbf},
1719 {0x00000093, 0xf7ffffff},
1720 {0x00000094, 0xffffff7f},
1721 {0x00000095, 0x00000fff},
1722 {0x00000096, 0x00116fff},
1723 {0x00000097, 0x60010000},
1724 {0x00000098, 0x10010000},
1725 {0x0000009f, 0x00c79000}
1730 * cik_srbm_select - select specific register instances
1732 * @rdev: radeon_device pointer
1733 * @me: selected ME (micro engine)
1738 * Switches the currently active registers instances. Some
1739 * registers are instanced per VMID, others are instanced per
1740 * me/pipe/queue combination.
1742 static void cik_srbm_select(struct radeon_device *rdev,
1743 u32 me, u32 pipe, u32 queue, u32 vmid)
1745 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1748 QUEUEID(queue & 0x7));
1749 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1754 * ci_mc_load_microcode - load MC ucode into the hw
1756 * @rdev: radeon_device pointer
1758 * Load the GDDR MC ucode into the hw (CIK).
1759 * Returns 0 on success, error on failure.
1761 int ci_mc_load_microcode(struct radeon_device *rdev)
1763 const __be32 *fw_data;
1764 u32 running, blackout = 0;
1766 int i, regs_size, ucode_size;
1771 ucode_size = rdev->mc_fw->size / 4;
1773 switch (rdev->family) {
1775 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1776 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1779 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1780 regs_size = HAWAII_IO_MC_REGS_SIZE;
1786 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1790 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1791 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1794 /* reset the engine and set to writable */
1795 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1796 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1798 /* load mc io regs */
1799 for (i = 0; i < regs_size; i++) {
1800 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1801 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1803 /* load the MC ucode */
1804 fw_data = (const __be32 *)rdev->mc_fw->data;
1805 for (i = 0; i < ucode_size; i++)
1806 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1808 /* put the engine back into the active state */
1809 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1810 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1811 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1813 /* wait for training to complete */
1814 for (i = 0; i < rdev->usec_timeout; i++) {
1815 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1819 for (i = 0; i < rdev->usec_timeout; i++) {
1820 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1826 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1833 * cik_init_microcode - load ucode images from disk
1835 * @rdev: radeon_device pointer
1837 * Use the firmware interface to load the ucode images into
1838 * the driver (not loaded into hw).
1839 * Returns 0 on success, error on failure.
1841 static int cik_init_microcode(struct radeon_device *rdev)
1843 const char *chip_name;
1844 size_t pfp_req_size, me_req_size, ce_req_size,
1845 mec_req_size, rlc_req_size, mc_req_size = 0,
1846 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1852 switch (rdev->family) {
1854 chip_name = "BONAIRE";
1855 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1856 me_req_size = CIK_ME_UCODE_SIZE * 4;
1857 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1858 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1859 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1860 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1861 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1862 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1863 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1866 chip_name = "HAWAII";
1867 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1868 me_req_size = CIK_ME_UCODE_SIZE * 4;
1869 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1870 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1871 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1872 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1873 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1874 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1875 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1878 chip_name = "KAVERI";
1879 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1880 me_req_size = CIK_ME_UCODE_SIZE * 4;
1881 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1882 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1883 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1884 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1887 chip_name = "KABINI";
1888 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1889 me_req_size = CIK_ME_UCODE_SIZE * 4;
1890 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1891 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1892 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1893 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1896 chip_name = "MULLINS";
1897 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1898 me_req_size = CIK_ME_UCODE_SIZE * 4;
1899 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1900 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1901 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1902 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1907 DRM_INFO("Loading %s Microcode\n", chip_name);
1909 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1910 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1913 if (rdev->pfp_fw->size != pfp_req_size) {
1915 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1916 rdev->pfp_fw->size, fw_name);
1921 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1922 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1925 if (rdev->me_fw->size != me_req_size) {
1927 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1928 rdev->me_fw->size, fw_name);
1932 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1933 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1936 if (rdev->ce_fw->size != ce_req_size) {
1938 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1939 rdev->ce_fw->size, fw_name);
1943 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1944 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1947 if (rdev->mec_fw->size != mec_req_size) {
1949 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1950 rdev->mec_fw->size, fw_name);
1954 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1955 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1958 if (rdev->rlc_fw->size != rlc_req_size) {
1960 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1961 rdev->rlc_fw->size, fw_name);
1965 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1966 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1969 if (rdev->sdma_fw->size != sdma_req_size) {
1971 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1972 rdev->sdma_fw->size, fw_name);
1976 /* No SMC, MC ucode on APUs */
1977 if (!(rdev->flags & RADEON_IS_IGP)) {
1978 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1979 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1981 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1982 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1986 if ((rdev->mc_fw->size != mc_req_size) &&
1987 (rdev->mc_fw->size != mc2_req_size)){
1989 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1990 rdev->mc_fw->size, fw_name);
1993 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1995 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1996 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1999 "smc: error loading firmware \"%s\"\n",
2001 release_firmware(rdev->smc_fw);
2002 rdev->smc_fw = NULL;
2004 } else if (rdev->smc_fw->size != smc_req_size) {
2006 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2007 rdev->smc_fw->size, fw_name);
2016 "cik_cp: Failed to load firmware \"%s\"\n",
2018 release_firmware(rdev->pfp_fw);
2019 rdev->pfp_fw = NULL;
2020 release_firmware(rdev->me_fw);
2022 release_firmware(rdev->ce_fw);
2024 release_firmware(rdev->rlc_fw);
2025 rdev->rlc_fw = NULL;
2026 release_firmware(rdev->mc_fw);
2028 release_firmware(rdev->smc_fw);
2029 rdev->smc_fw = NULL;
2038 * cik_tiling_mode_table_init - init the hw tiling table
2040 * @rdev: radeon_device pointer
2042 * Starting with SI, the tiling setup is done globally in a
2043 * set of 32 tiling modes. Rather than selecting each set of
2044 * parameters per surface as on older asics, we just select
2045 * which index in the tiling table we want to use, and the
2046 * surface uses those parameters (CIK).
2048 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2050 const u32 num_tile_mode_states = 32;
2051 const u32 num_secondary_tile_mode_states = 16;
2052 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2053 u32 num_pipe_configs;
2054 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2055 rdev->config.cik.max_shader_engines;
2057 switch (rdev->config.cik.mem_row_size_in_kb) {
2059 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2063 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2066 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2070 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2071 if (num_pipe_configs > 8)
2072 num_pipe_configs = 16;
2074 if (num_pipe_configs == 16) {
2075 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2076 switch (reg_offset) {
2078 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2079 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2080 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2081 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2084 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2085 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2086 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2087 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2090 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2091 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2092 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2093 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2096 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2097 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2098 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2099 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2102 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2103 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2104 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2105 TILE_SPLIT(split_equal_to_row_size));
2108 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2109 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2110 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2113 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2114 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2115 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2116 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2119 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2120 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2121 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2122 TILE_SPLIT(split_equal_to_row_size));
2125 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2126 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2129 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2130 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2131 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2134 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2135 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2136 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2137 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2140 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2141 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2142 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2143 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2146 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2147 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2148 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2149 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2152 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2153 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2154 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2157 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2158 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2159 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2160 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2163 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2164 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2165 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2166 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2169 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2170 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2171 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2172 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2175 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2176 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2177 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2180 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2181 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2182 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2183 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2186 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2187 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2188 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2189 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2192 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2193 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2194 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2195 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2201 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2202 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2204 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2205 switch (reg_offset) {
2207 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2208 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2209 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2210 NUM_BANKS(ADDR_SURF_16_BANK));
2213 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2214 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2215 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2216 NUM_BANKS(ADDR_SURF_16_BANK));
2219 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2220 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2221 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2222 NUM_BANKS(ADDR_SURF_16_BANK));
2225 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2226 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2227 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2228 NUM_BANKS(ADDR_SURF_16_BANK));
2231 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2232 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2233 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2234 NUM_BANKS(ADDR_SURF_8_BANK));
2237 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2238 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2239 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2240 NUM_BANKS(ADDR_SURF_4_BANK));
2243 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2244 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2245 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2246 NUM_BANKS(ADDR_SURF_2_BANK));
2249 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2250 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2251 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2252 NUM_BANKS(ADDR_SURF_16_BANK));
2255 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2256 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2257 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2258 NUM_BANKS(ADDR_SURF_16_BANK));
2261 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2262 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2263 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2264 NUM_BANKS(ADDR_SURF_16_BANK));
2267 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2268 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2269 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2270 NUM_BANKS(ADDR_SURF_8_BANK));
2273 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2274 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2275 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2276 NUM_BANKS(ADDR_SURF_4_BANK));
2279 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2280 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2281 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2282 NUM_BANKS(ADDR_SURF_2_BANK));
2285 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2286 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2287 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2288 NUM_BANKS(ADDR_SURF_2_BANK));
2294 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2296 } else if (num_pipe_configs == 8) {
2297 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2298 switch (reg_offset) {
2300 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2301 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2302 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2303 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2306 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2307 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2308 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2309 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2312 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2313 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2314 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2315 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2318 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2319 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2320 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2321 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2324 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2325 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2326 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2327 TILE_SPLIT(split_equal_to_row_size));
2330 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2331 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2332 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2335 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2336 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2337 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2338 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2341 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2342 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2343 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2344 TILE_SPLIT(split_equal_to_row_size));
2347 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2348 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2351 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2352 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2353 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2356 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2357 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2358 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2359 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2362 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2363 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2364 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2365 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2368 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2369 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2370 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2371 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2374 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2375 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2376 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2379 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2380 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2381 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2382 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2385 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2386 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2387 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2388 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2391 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2392 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2393 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2394 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2397 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2398 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2399 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2402 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2403 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2404 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2405 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2408 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2409 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2410 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2411 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2414 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2415 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2416 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2417 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2423 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2424 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2426 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2427 switch (reg_offset) {
2429 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2430 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2431 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2432 NUM_BANKS(ADDR_SURF_16_BANK));
2435 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2436 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2437 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2438 NUM_BANKS(ADDR_SURF_16_BANK));
2441 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2442 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2443 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2444 NUM_BANKS(ADDR_SURF_16_BANK));
2447 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2448 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2449 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2450 NUM_BANKS(ADDR_SURF_16_BANK));
2453 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2454 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2455 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2456 NUM_BANKS(ADDR_SURF_8_BANK));
2459 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2460 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2461 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2462 NUM_BANKS(ADDR_SURF_4_BANK));
2465 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2466 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2467 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2468 NUM_BANKS(ADDR_SURF_2_BANK));
2471 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2472 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2473 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2474 NUM_BANKS(ADDR_SURF_16_BANK));
2477 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2478 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2479 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2480 NUM_BANKS(ADDR_SURF_16_BANK));
2483 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2484 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2485 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2486 NUM_BANKS(ADDR_SURF_16_BANK));
2489 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2490 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2491 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2492 NUM_BANKS(ADDR_SURF_16_BANK));
2495 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2496 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2497 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2498 NUM_BANKS(ADDR_SURF_8_BANK));
2501 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2502 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2503 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2504 NUM_BANKS(ADDR_SURF_4_BANK));
2507 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2508 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2509 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2510 NUM_BANKS(ADDR_SURF_2_BANK));
2516 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2517 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2519 } else if (num_pipe_configs == 4) {
2521 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2522 switch (reg_offset) {
2524 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2525 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2526 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2527 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2530 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2531 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2532 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2533 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2536 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2538 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2539 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2542 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2543 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2544 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2545 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2548 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2549 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2550 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2551 TILE_SPLIT(split_equal_to_row_size));
2554 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2555 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2556 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2559 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2560 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2561 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2562 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2565 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2566 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2567 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2568 TILE_SPLIT(split_equal_to_row_size));
2571 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2572 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2575 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2576 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2577 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2580 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2581 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2582 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2583 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2586 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2587 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2588 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2589 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2592 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2593 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2594 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2595 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2598 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2599 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2600 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2603 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2604 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2605 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2606 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2609 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2610 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2611 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2612 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2615 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2616 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2617 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2618 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2621 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2622 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2623 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2626 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2627 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2628 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2629 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2632 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2633 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2634 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2635 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2638 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2639 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2640 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2641 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2647 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2648 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2650 } else if (num_rbs < 4) {
2651 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2652 switch (reg_offset) {
2654 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2655 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2656 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2657 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2660 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2661 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2662 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2663 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2666 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2667 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2668 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2669 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2672 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2673 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2674 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2675 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2678 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2679 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2680 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2681 TILE_SPLIT(split_equal_to_row_size));
2684 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2685 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2686 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2689 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2690 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2691 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2692 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2695 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2696 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2697 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2698 TILE_SPLIT(split_equal_to_row_size));
2701 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2702 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2705 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2706 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2707 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2710 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2711 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2712 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2713 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2716 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2717 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2718 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2719 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2722 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2723 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2724 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2725 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2728 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2729 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2730 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2733 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2734 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2735 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2736 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2739 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2740 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2741 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2742 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2745 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2746 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2747 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2748 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2751 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2752 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2753 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2756 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2757 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2758 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2759 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2762 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2763 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2764 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2765 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2768 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2769 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2770 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2771 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2777 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2778 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2781 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2782 switch (reg_offset) {
2784 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2785 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2786 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2787 NUM_BANKS(ADDR_SURF_16_BANK));
2790 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2791 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2792 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2793 NUM_BANKS(ADDR_SURF_16_BANK));
2796 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2797 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2798 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2799 NUM_BANKS(ADDR_SURF_16_BANK));
2802 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2803 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2804 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2805 NUM_BANKS(ADDR_SURF_16_BANK));
2808 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2809 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2810 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2811 NUM_BANKS(ADDR_SURF_16_BANK));
2814 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2815 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2816 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2817 NUM_BANKS(ADDR_SURF_8_BANK));
2820 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2821 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2822 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2823 NUM_BANKS(ADDR_SURF_4_BANK));
2826 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2827 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2828 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2829 NUM_BANKS(ADDR_SURF_16_BANK));
2832 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2833 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2834 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2835 NUM_BANKS(ADDR_SURF_16_BANK));
2838 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2839 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2840 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2841 NUM_BANKS(ADDR_SURF_16_BANK));
2844 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2845 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2846 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2847 NUM_BANKS(ADDR_SURF_16_BANK));
2850 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2851 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2852 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2853 NUM_BANKS(ADDR_SURF_16_BANK));
2856 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2857 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2858 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2859 NUM_BANKS(ADDR_SURF_8_BANK));
2862 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2863 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2864 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2865 NUM_BANKS(ADDR_SURF_4_BANK));
2871 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2872 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2874 } else if (num_pipe_configs == 2) {
2875 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2876 switch (reg_offset) {
2878 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2879 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2880 PIPE_CONFIG(ADDR_SURF_P2) |
2881 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2884 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2885 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2886 PIPE_CONFIG(ADDR_SURF_P2) |
2887 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2890 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2891 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2892 PIPE_CONFIG(ADDR_SURF_P2) |
2893 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2896 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2897 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2898 PIPE_CONFIG(ADDR_SURF_P2) |
2899 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2902 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2903 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2904 PIPE_CONFIG(ADDR_SURF_P2) |
2905 TILE_SPLIT(split_equal_to_row_size));
2908 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2909 PIPE_CONFIG(ADDR_SURF_P2) |
2910 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2913 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2914 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2915 PIPE_CONFIG(ADDR_SURF_P2) |
2916 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2919 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2920 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2921 PIPE_CONFIG(ADDR_SURF_P2) |
2922 TILE_SPLIT(split_equal_to_row_size));
2925 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2926 PIPE_CONFIG(ADDR_SURF_P2);
2929 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2930 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2931 PIPE_CONFIG(ADDR_SURF_P2));
2934 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2935 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2936 PIPE_CONFIG(ADDR_SURF_P2) |
2937 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2940 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2941 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2942 PIPE_CONFIG(ADDR_SURF_P2) |
2943 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2946 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2947 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2948 PIPE_CONFIG(ADDR_SURF_P2) |
2949 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2952 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2953 PIPE_CONFIG(ADDR_SURF_P2) |
2954 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2957 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2958 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2959 PIPE_CONFIG(ADDR_SURF_P2) |
2960 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2963 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2964 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2965 PIPE_CONFIG(ADDR_SURF_P2) |
2966 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2969 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2970 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2971 PIPE_CONFIG(ADDR_SURF_P2) |
2972 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2975 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2976 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2977 PIPE_CONFIG(ADDR_SURF_P2));
2980 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2981 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2982 PIPE_CONFIG(ADDR_SURF_P2) |
2983 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2986 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2987 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2988 PIPE_CONFIG(ADDR_SURF_P2) |
2989 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2992 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2993 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2994 PIPE_CONFIG(ADDR_SURF_P2) |
2995 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3001 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
3002 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3004 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3005 switch (reg_offset) {
3007 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3008 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3009 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3010 NUM_BANKS(ADDR_SURF_16_BANK));
3013 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3014 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3015 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3016 NUM_BANKS(ADDR_SURF_16_BANK));
3019 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3020 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3021 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3022 NUM_BANKS(ADDR_SURF_16_BANK));
3025 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3026 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3027 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3028 NUM_BANKS(ADDR_SURF_16_BANK));
3031 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3032 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3033 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3034 NUM_BANKS(ADDR_SURF_16_BANK));
3037 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3038 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3039 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3040 NUM_BANKS(ADDR_SURF_16_BANK));
3043 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3044 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3045 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3046 NUM_BANKS(ADDR_SURF_8_BANK));
3049 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3050 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3051 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3052 NUM_BANKS(ADDR_SURF_16_BANK));
3055 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3056 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3057 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3058 NUM_BANKS(ADDR_SURF_16_BANK));
3061 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3062 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3063 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3064 NUM_BANKS(ADDR_SURF_16_BANK));
3067 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3068 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3069 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3070 NUM_BANKS(ADDR_SURF_16_BANK));
3073 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3074 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3075 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3076 NUM_BANKS(ADDR_SURF_16_BANK));
3079 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3080 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3081 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3082 NUM_BANKS(ADDR_SURF_16_BANK));
3085 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3086 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3087 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3088 NUM_BANKS(ADDR_SURF_8_BANK));
3094 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3095 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3098 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3102 * cik_select_se_sh - select which SE, SH to address
3104 * @rdev: radeon_device pointer
3105 * @se_num: shader engine to address
3106 * @sh_num: sh block to address
3108 * Select which SE, SH combinations to address. Certain
3109 * registers are instanced per SE or SH. 0xffffffff means
3110 * broadcast to all SEs or SHs (CIK).
3112 static void cik_select_se_sh(struct radeon_device *rdev,
3113 u32 se_num, u32 sh_num)
3115 u32 data = INSTANCE_BROADCAST_WRITES;
3117 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3118 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3119 else if (se_num == 0xffffffff)
3120 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3121 else if (sh_num == 0xffffffff)
3122 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3124 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3125 WREG32(GRBM_GFX_INDEX, data);
3129 * cik_create_bitmask - create a bitmask
3131 * @bit_width: length of the mask
3133 * create a variable length bit mask (CIK).
3134 * Returns the bitmask.
3136 static u32 cik_create_bitmask(u32 bit_width)
3140 for (i = 0; i < bit_width; i++) {
3148 * cik_get_rb_disabled - computes the mask of disabled RBs
3150 * @rdev: radeon_device pointer
3151 * @max_rb_num: max RBs (render backends) for the asic
3152 * @se_num: number of SEs (shader engines) for the asic
3153 * @sh_per_se: number of SH blocks per SE for the asic
3155 * Calculates the bitmask of disabled RBs (CIK).
3156 * Returns the disabled RB bitmask.
3158 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3159 u32 max_rb_num_per_se,
3164 data = RREG32(CC_RB_BACKEND_DISABLE);
3166 data &= BACKEND_DISABLE_MASK;
3169 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3171 data >>= BACKEND_DISABLE_SHIFT;
3173 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3179 * cik_setup_rb - setup the RBs on the asic
3181 * @rdev: radeon_device pointer
3182 * @se_num: number of SEs (shader engines) for the asic
3183 * @sh_per_se: number of SH blocks per SE for the asic
3184 * @max_rb_num: max RBs (render backends) for the asic
3186 * Configures per-SE/SH RB registers (CIK).
3188 static void cik_setup_rb(struct radeon_device *rdev,
3189 u32 se_num, u32 sh_per_se,
3190 u32 max_rb_num_per_se)
3194 u32 disabled_rbs = 0;
3195 u32 enabled_rbs = 0;
3197 for (i = 0; i < se_num; i++) {
3198 for (j = 0; j < sh_per_se; j++) {
3199 cik_select_se_sh(rdev, i, j);
3200 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3201 if (rdev->family == CHIP_HAWAII)
3202 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3204 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3207 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3210 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3211 if (!(disabled_rbs & mask))
3212 enabled_rbs |= mask;
3216 rdev->config.cik.backend_enable_mask = enabled_rbs;
3218 for (i = 0; i < se_num; i++) {
3219 cik_select_se_sh(rdev, i, 0xffffffff);
3221 for (j = 0; j < sh_per_se; j++) {
3222 switch (enabled_rbs & 3) {
3225 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3227 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3230 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3233 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3237 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3242 WREG32(PA_SC_RASTER_CONFIG, data);
3244 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3248 * cik_gpu_init - setup the 3D engine
3250 * @rdev: radeon_device pointer
3252 * Configures the 3D engine and tiling configuration
3253 * registers so that the 3D engine is usable.
3255 static void cik_gpu_init(struct radeon_device *rdev)
3257 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3258 u32 mc_shared_chmap, mc_arb_ramcfg;
3259 u32 hdp_host_path_cntl;
3263 switch (rdev->family) {
3265 rdev->config.cik.max_shader_engines = 2;
3266 rdev->config.cik.max_tile_pipes = 4;
3267 rdev->config.cik.max_cu_per_sh = 7;
3268 rdev->config.cik.max_sh_per_se = 1;
3269 rdev->config.cik.max_backends_per_se = 2;
3270 rdev->config.cik.max_texture_channel_caches = 4;
3271 rdev->config.cik.max_gprs = 256;
3272 rdev->config.cik.max_gs_threads = 32;
3273 rdev->config.cik.max_hw_contexts = 8;
3275 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3276 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3277 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3278 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3279 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3282 rdev->config.cik.max_shader_engines = 4;
3283 rdev->config.cik.max_tile_pipes = 16;
3284 rdev->config.cik.max_cu_per_sh = 11;
3285 rdev->config.cik.max_sh_per_se = 1;
3286 rdev->config.cik.max_backends_per_se = 4;
3287 rdev->config.cik.max_texture_channel_caches = 16;
3288 rdev->config.cik.max_gprs = 256;
3289 rdev->config.cik.max_gs_threads = 32;
3290 rdev->config.cik.max_hw_contexts = 8;
3292 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3293 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3294 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3295 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3296 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3299 rdev->config.cik.max_shader_engines = 1;
3300 rdev->config.cik.max_tile_pipes = 4;
3301 if ((rdev->pdev->device == 0x1304) ||
3302 (rdev->pdev->device == 0x1305) ||
3303 (rdev->pdev->device == 0x130C) ||
3304 (rdev->pdev->device == 0x130F) ||
3305 (rdev->pdev->device == 0x1310) ||
3306 (rdev->pdev->device == 0x1311) ||
3307 (rdev->pdev->device == 0x131C)) {
3308 rdev->config.cik.max_cu_per_sh = 8;
3309 rdev->config.cik.max_backends_per_se = 2;
3310 } else if ((rdev->pdev->device == 0x1309) ||
3311 (rdev->pdev->device == 0x130A) ||
3312 (rdev->pdev->device == 0x130D) ||
3313 (rdev->pdev->device == 0x1313) ||
3314 (rdev->pdev->device == 0x131D)) {
3315 rdev->config.cik.max_cu_per_sh = 6;
3316 rdev->config.cik.max_backends_per_se = 2;
3317 } else if ((rdev->pdev->device == 0x1306) ||
3318 (rdev->pdev->device == 0x1307) ||
3319 (rdev->pdev->device == 0x130B) ||
3320 (rdev->pdev->device == 0x130E) ||
3321 (rdev->pdev->device == 0x1315) ||
3322 (rdev->pdev->device == 0x131B)) {
3323 rdev->config.cik.max_cu_per_sh = 4;
3324 rdev->config.cik.max_backends_per_se = 1;
3326 rdev->config.cik.max_cu_per_sh = 3;
3327 rdev->config.cik.max_backends_per_se = 1;
3329 rdev->config.cik.max_sh_per_se = 1;
3330 rdev->config.cik.max_texture_channel_caches = 4;
3331 rdev->config.cik.max_gprs = 256;
3332 rdev->config.cik.max_gs_threads = 16;
3333 rdev->config.cik.max_hw_contexts = 8;
3335 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3336 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3337 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3338 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3339 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3344 rdev->config.cik.max_shader_engines = 1;
3345 rdev->config.cik.max_tile_pipes = 2;
3346 rdev->config.cik.max_cu_per_sh = 2;
3347 rdev->config.cik.max_sh_per_se = 1;
3348 rdev->config.cik.max_backends_per_se = 1;
3349 rdev->config.cik.max_texture_channel_caches = 2;
3350 rdev->config.cik.max_gprs = 256;
3351 rdev->config.cik.max_gs_threads = 16;
3352 rdev->config.cik.max_hw_contexts = 8;
3354 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3355 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3356 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3357 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3358 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3362 /* Initialize HDP */
3363 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3364 WREG32((0x2c14 + j), 0x00000000);
3365 WREG32((0x2c18 + j), 0x00000000);
3366 WREG32((0x2c1c + j), 0x00000000);
3367 WREG32((0x2c20 + j), 0x00000000);
3368 WREG32((0x2c24 + j), 0x00000000);
3371 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3373 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3375 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3376 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3378 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3379 rdev->config.cik.mem_max_burst_length_bytes = 256;
3380 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3381 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3382 if (rdev->config.cik.mem_row_size_in_kb > 4)
3383 rdev->config.cik.mem_row_size_in_kb = 4;
3384 /* XXX use MC settings? */
3385 rdev->config.cik.shader_engine_tile_size = 32;
3386 rdev->config.cik.num_gpus = 1;
3387 rdev->config.cik.multi_gpu_tile_size = 64;
3389 /* fix up row size */
3390 gb_addr_config &= ~ROW_SIZE_MASK;
3391 switch (rdev->config.cik.mem_row_size_in_kb) {
3394 gb_addr_config |= ROW_SIZE(0);
3397 gb_addr_config |= ROW_SIZE(1);
3400 gb_addr_config |= ROW_SIZE(2);
3404 /* setup tiling info dword. gb_addr_config is not adequate since it does
3405 * not have bank info, so create a custom tiling dword.
3406 * bits 3:0 num_pipes
3407 * bits 7:4 num_banks
3408 * bits 11:8 group_size
3409 * bits 15:12 row_size
3411 rdev->config.cik.tile_config = 0;
3412 switch (rdev->config.cik.num_tile_pipes) {
3414 rdev->config.cik.tile_config |= (0 << 0);
3417 rdev->config.cik.tile_config |= (1 << 0);
3420 rdev->config.cik.tile_config |= (2 << 0);
3424 /* XXX what about 12? */
3425 rdev->config.cik.tile_config |= (3 << 0);
3428 rdev->config.cik.tile_config |=
3429 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3430 rdev->config.cik.tile_config |=
3431 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3432 rdev->config.cik.tile_config |=
3433 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3435 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3436 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3437 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3438 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3439 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3440 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3441 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3442 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3444 cik_tiling_mode_table_init(rdev);
3446 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3447 rdev->config.cik.max_sh_per_se,
3448 rdev->config.cik.max_backends_per_se);
3450 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3451 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
3452 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k++) {
3453 rdev->config.cik.active_cus +=
3454 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
3459 /* set HW defaults for 3D engine */
3460 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3462 WREG32(SX_DEBUG_1, 0x20);
3464 WREG32(TA_CNTL_AUX, 0x00010000);
3466 tmp = RREG32(SPI_CONFIG_CNTL);
3468 WREG32(SPI_CONFIG_CNTL, tmp);
3470 WREG32(SQ_CONFIG, 1);
3472 WREG32(DB_DEBUG, 0);
3474 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3476 WREG32(DB_DEBUG2, tmp);
3478 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3480 WREG32(DB_DEBUG3, tmp);
3482 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3484 WREG32(CB_HW_CONTROL, tmp);
3486 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3488 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3489 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3490 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3491 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3493 WREG32(VGT_NUM_INSTANCES, 1);
3495 WREG32(CP_PERFMON_CNTL, 0);
3497 WREG32(SQ_CONFIG, 0);
3499 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3500 FORCE_EOV_MAX_REZ_CNT(255)));
3502 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3503 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3505 WREG32(VGT_GS_VERTEX_REUSE, 16);
3506 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3508 tmp = RREG32(HDP_MISC_CNTL);
3509 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3510 WREG32(HDP_MISC_CNTL, tmp);
3512 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3513 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3515 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3516 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3522 * GPU scratch registers helpers function.
3525 * cik_scratch_init - setup driver info for CP scratch regs
3527 * @rdev: radeon_device pointer
3529 * Set up the number and offset of the CP scratch registers.
3530 * NOTE: use of CP scratch registers is a legacy inferface and
3531 * is not used by default on newer asics (r6xx+). On newer asics,
3532 * memory buffers are used for fences rather than scratch regs.
3534 static void cik_scratch_init(struct radeon_device *rdev)
3538 rdev->scratch.num_reg = 7;
3539 rdev->scratch.reg_base = SCRATCH_REG0;
3540 for (i = 0; i < rdev->scratch.num_reg; i++) {
3541 rdev->scratch.free[i] = true;
3542 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3547 * cik_ring_test - basic gfx ring test
3549 * @rdev: radeon_device pointer
3550 * @ring: radeon_ring structure holding ring information
3552 * Allocate a scratch register and write to it using the gfx ring (CIK).
3553 * Provides a basic gfx ring test to verify that the ring is working.
3554 * Used by cik_cp_gfx_resume();
3555 * Returns 0 on success, error on failure.
3557 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3564 r = radeon_scratch_get(rdev, &scratch);
3566 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3569 WREG32(scratch, 0xCAFEDEAD);
3570 r = radeon_ring_lock(rdev, ring, 3);
3572 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3573 radeon_scratch_free(rdev, scratch);
3576 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3577 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3578 radeon_ring_write(ring, 0xDEADBEEF);
3579 radeon_ring_unlock_commit(rdev, ring);
3581 for (i = 0; i < rdev->usec_timeout; i++) {
3582 tmp = RREG32(scratch);
3583 if (tmp == 0xDEADBEEF)
3587 if (i < rdev->usec_timeout) {
3588 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3590 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3591 ring->idx, scratch, tmp);
3594 radeon_scratch_free(rdev, scratch);
3599 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3601 * @rdev: radeon_device pointer
3602 * @ridx: radeon ring index
3604 * Emits an hdp flush on the cp.
3606 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3609 struct radeon_ring *ring = &rdev->ring[ridx];
3612 switch (ring->idx) {
3613 case CAYMAN_RING_TYPE_CP1_INDEX:
3614 case CAYMAN_RING_TYPE_CP2_INDEX:
3618 ref_and_mask = CP2 << ring->pipe;
3621 ref_and_mask = CP6 << ring->pipe;
3627 case RADEON_RING_TYPE_GFX_INDEX:
3632 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3633 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3634 WAIT_REG_MEM_FUNCTION(3) | /* == */
3635 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3636 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3637 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3638 radeon_ring_write(ring, ref_and_mask);
3639 radeon_ring_write(ring, ref_and_mask);
3640 radeon_ring_write(ring, 0x20); /* poll interval */
3644 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3646 * @rdev: radeon_device pointer
3647 * @fence: radeon fence object
3649 * Emits a fence sequnce number on the gfx ring and flushes
3652 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3653 struct radeon_fence *fence)
3655 struct radeon_ring *ring = &rdev->ring[fence->ring];
3656 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3658 /* EVENT_WRITE_EOP - flush caches, send int */
3659 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3660 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3662 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3664 radeon_ring_write(ring, addr & 0xfffffffc);
3665 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3666 radeon_ring_write(ring, fence->seq);
3667 radeon_ring_write(ring, 0);
3669 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3673 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3675 * @rdev: radeon_device pointer
3676 * @fence: radeon fence object
3678 * Emits a fence sequnce number on the compute ring and flushes
3681 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3682 struct radeon_fence *fence)
3684 struct radeon_ring *ring = &rdev->ring[fence->ring];
3685 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3687 /* RELEASE_MEM - flush caches, send int */
3688 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3689 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3691 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3693 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3694 radeon_ring_write(ring, addr & 0xfffffffc);
3695 radeon_ring_write(ring, upper_32_bits(addr));
3696 radeon_ring_write(ring, fence->seq);
3697 radeon_ring_write(ring, 0);
3699 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3702 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3703 struct radeon_ring *ring,
3704 struct radeon_semaphore *semaphore,
3707 uint64_t addr = semaphore->gpu_addr;
3708 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3710 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3711 radeon_ring_write(ring, lower_32_bits(addr));
3712 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3718 * cik_copy_cpdma - copy pages using the CP DMA engine
3720 * @rdev: radeon_device pointer
3721 * @src_offset: src GPU address
3722 * @dst_offset: dst GPU address
3723 * @num_gpu_pages: number of GPU pages to xfer
3724 * @fence: radeon fence object
3726 * Copy GPU paging using the CP DMA engine (CIK+).
3727 * Used by the radeon ttm implementation to move pages if
3728 * registered as the asic copy callback.
3730 int cik_copy_cpdma(struct radeon_device *rdev,
3731 uint64_t src_offset, uint64_t dst_offset,
3732 unsigned num_gpu_pages,
3733 struct radeon_fence **fence)
3735 struct radeon_semaphore *sem = NULL;
3736 int ring_index = rdev->asic->copy.blit_ring_index;
3737 struct radeon_ring *ring = &rdev->ring[ring_index];
3738 u32 size_in_bytes, cur_size_in_bytes, control;
3742 r = radeon_semaphore_create(rdev, &sem);
3744 DRM_ERROR("radeon: moving bo (%d).\n", r);
3748 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3749 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3750 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3752 DRM_ERROR("radeon: moving bo (%d).\n", r);
3753 radeon_semaphore_free(rdev, &sem, NULL);
3757 radeon_semaphore_sync_to(sem, *fence);
3758 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3760 for (i = 0; i < num_loops; i++) {
3761 cur_size_in_bytes = size_in_bytes;
3762 if (cur_size_in_bytes > 0x1fffff)
3763 cur_size_in_bytes = 0x1fffff;
3764 size_in_bytes -= cur_size_in_bytes;
3766 if (size_in_bytes == 0)
3767 control |= PACKET3_DMA_DATA_CP_SYNC;
3768 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3769 radeon_ring_write(ring, control);
3770 radeon_ring_write(ring, lower_32_bits(src_offset));
3771 radeon_ring_write(ring, upper_32_bits(src_offset));
3772 radeon_ring_write(ring, lower_32_bits(dst_offset));
3773 radeon_ring_write(ring, upper_32_bits(dst_offset));
3774 radeon_ring_write(ring, cur_size_in_bytes);
3775 src_offset += cur_size_in_bytes;
3776 dst_offset += cur_size_in_bytes;
3779 r = radeon_fence_emit(rdev, fence, ring->idx);
3781 radeon_ring_unlock_undo(rdev, ring);
3782 radeon_semaphore_free(rdev, &sem, NULL);
3786 radeon_ring_unlock_commit(rdev, ring);
3787 radeon_semaphore_free(rdev, &sem, *fence);
3796 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3798 * @rdev: radeon_device pointer
3799 * @ib: radeon indirect buffer object
3801 * Emits an DE (drawing engine) or CE (constant engine) IB
3802 * on the gfx ring. IBs are usually generated by userspace
3803 * acceleration drivers and submitted to the kernel for
3804 * sheduling on the ring. This function schedules the IB
3805 * on the gfx ring for execution by the GPU.
3807 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3809 struct radeon_ring *ring = &rdev->ring[ib->ring];
3810 u32 header, control = INDIRECT_BUFFER_VALID;
3812 if (ib->is_const_ib) {
3813 /* set switch buffer packet before const IB */
3814 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3815 radeon_ring_write(ring, 0);
3817 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3820 if (ring->rptr_save_reg) {
3821 next_rptr = ring->wptr + 3 + 4;
3822 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3823 radeon_ring_write(ring, ((ring->rptr_save_reg -
3824 PACKET3_SET_UCONFIG_REG_START) >> 2));
3825 radeon_ring_write(ring, next_rptr);
3826 } else if (rdev->wb.enabled) {
3827 next_rptr = ring->wptr + 5 + 4;
3828 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3829 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3830 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3831 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3832 radeon_ring_write(ring, next_rptr);
3835 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3838 control |= ib->length_dw |
3839 (ib->vm ? (ib->vm->id << 24) : 0);
3841 radeon_ring_write(ring, header);
3842 radeon_ring_write(ring,
3846 (ib->gpu_addr & 0xFFFFFFFC));
3847 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3848 radeon_ring_write(ring, control);
3852 * cik_ib_test - basic gfx ring IB test
3854 * @rdev: radeon_device pointer
3855 * @ring: radeon_ring structure holding ring information
3857 * Allocate an IB and execute it on the gfx ring (CIK).
3858 * Provides a basic gfx ring test to verify that IBs are working.
3859 * Returns 0 on success, error on failure.
3861 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3863 struct radeon_ib ib;
3869 r = radeon_scratch_get(rdev, &scratch);
3871 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3874 WREG32(scratch, 0xCAFEDEAD);
3875 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3877 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3878 radeon_scratch_free(rdev, scratch);
3881 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3882 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3883 ib.ptr[2] = 0xDEADBEEF;
3885 r = radeon_ib_schedule(rdev, &ib, NULL);
3887 radeon_scratch_free(rdev, scratch);
3888 radeon_ib_free(rdev, &ib);
3889 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3892 r = radeon_fence_wait(ib.fence, false);
3894 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3895 radeon_scratch_free(rdev, scratch);
3896 radeon_ib_free(rdev, &ib);
3899 for (i = 0; i < rdev->usec_timeout; i++) {
3900 tmp = RREG32(scratch);
3901 if (tmp == 0xDEADBEEF)
3905 if (i < rdev->usec_timeout) {
3906 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3908 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3912 radeon_scratch_free(rdev, scratch);
3913 radeon_ib_free(rdev, &ib);
3919 * On CIK, gfx and compute now have independant command processors.
3922 * Gfx consists of a single ring and can process both gfx jobs and
3923 * compute jobs. The gfx CP consists of three microengines (ME):
3924 * PFP - Pre-Fetch Parser
3926 * CE - Constant Engine
3927 * The PFP and ME make up what is considered the Drawing Engine (DE).
3928 * The CE is an asynchronous engine used for updating buffer desciptors
3929 * used by the DE so that they can be loaded into cache in parallel
3930 * while the DE is processing state update packets.
3933 * The compute CP consists of two microengines (ME):
3934 * MEC1 - Compute MicroEngine 1
3935 * MEC2 - Compute MicroEngine 2
3936 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3937 * The queues are exposed to userspace and are programmed directly
3938 * by the compute runtime.
3941 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3943 * @rdev: radeon_device pointer
3944 * @enable: enable or disable the MEs
3946 * Halts or unhalts the gfx MEs.
3948 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3951 WREG32(CP_ME_CNTL, 0);
3953 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3954 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3955 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3956 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3962 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3964 * @rdev: radeon_device pointer
3966 * Loads the gfx PFP, ME, and CE ucode.
3967 * Returns 0 for success, -EINVAL if the ucode is not available.
3969 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3971 const __be32 *fw_data;
3974 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3977 cik_cp_gfx_enable(rdev, false);
3980 fw_data = (const __be32 *)rdev->pfp_fw->data;
3981 WREG32(CP_PFP_UCODE_ADDR, 0);
3982 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3983 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3984 WREG32(CP_PFP_UCODE_ADDR, 0);
3987 fw_data = (const __be32 *)rdev->ce_fw->data;
3988 WREG32(CP_CE_UCODE_ADDR, 0);
3989 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3990 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3991 WREG32(CP_CE_UCODE_ADDR, 0);
3994 fw_data = (const __be32 *)rdev->me_fw->data;
3995 WREG32(CP_ME_RAM_WADDR, 0);
3996 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3997 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3998 WREG32(CP_ME_RAM_WADDR, 0);
4000 WREG32(CP_PFP_UCODE_ADDR, 0);
4001 WREG32(CP_CE_UCODE_ADDR, 0);
4002 WREG32(CP_ME_RAM_WADDR, 0);
4003 WREG32(CP_ME_RAM_RADDR, 0);
4008 * cik_cp_gfx_start - start the gfx ring
4010 * @rdev: radeon_device pointer
4012 * Enables the ring and loads the clear state context and other
4013 * packets required to init the ring.
4014 * Returns 0 for success, error for failure.
4016 static int cik_cp_gfx_start(struct radeon_device *rdev)
4018 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4022 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4023 WREG32(CP_ENDIAN_SWAP, 0);
4024 WREG32(CP_DEVICE_ID, 1);
4026 cik_cp_gfx_enable(rdev, true);
4028 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4030 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4034 /* init the CE partitions. CE only used for gfx on CIK */
4035 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4036 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4037 radeon_ring_write(ring, 0xc000);
4038 radeon_ring_write(ring, 0xc000);
4040 /* setup clear context state */
4041 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4042 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4044 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4045 radeon_ring_write(ring, 0x80000000);
4046 radeon_ring_write(ring, 0x80000000);
4048 for (i = 0; i < cik_default_size; i++)
4049 radeon_ring_write(ring, cik_default_state[i]);
4051 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4052 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4054 /* set clear context state */
4055 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4056 radeon_ring_write(ring, 0);
4058 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4059 radeon_ring_write(ring, 0x00000316);
4060 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4061 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4063 radeon_ring_unlock_commit(rdev, ring);
4069 * cik_cp_gfx_fini - stop the gfx ring
4071 * @rdev: radeon_device pointer
4073 * Stop the gfx ring and tear down the driver ring
4076 static void cik_cp_gfx_fini(struct radeon_device *rdev)
4078 cik_cp_gfx_enable(rdev, false);
4079 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4083 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4085 * @rdev: radeon_device pointer
4087 * Program the location and size of the gfx ring buffer
4088 * and test it to make sure it's working.
4089 * Returns 0 for success, error for failure.
4091 static int cik_cp_gfx_resume(struct radeon_device *rdev)
4093 struct radeon_ring *ring;
4099 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4100 if (rdev->family != CHIP_HAWAII)
4101 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4103 /* Set the write pointer delay */
4104 WREG32(CP_RB_WPTR_DELAY, 0);
4106 /* set the RB to use vmid 0 */
4107 WREG32(CP_RB_VMID, 0);
4109 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4111 /* ring 0 - compute and gfx */
4112 /* Set ring buffer size */
4113 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4114 rb_bufsz = order_base_2(ring->ring_size / 8);
4115 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4117 tmp |= BUF_SWAP_32BIT;
4119 WREG32(CP_RB0_CNTL, tmp);
4121 /* Initialize the ring buffer's read and write pointers */
4122 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4124 WREG32(CP_RB0_WPTR, ring->wptr);
4126 /* set the wb address wether it's enabled or not */
4127 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4128 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4130 /* scratch register shadowing is no longer supported */
4131 WREG32(SCRATCH_UMSK, 0);
4133 if (!rdev->wb.enabled)
4134 tmp |= RB_NO_UPDATE;
4137 WREG32(CP_RB0_CNTL, tmp);
4139 rb_addr = ring->gpu_addr >> 8;
4140 WREG32(CP_RB0_BASE, rb_addr);
4141 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4143 /* start the ring */
4144 cik_cp_gfx_start(rdev);
4145 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4146 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4148 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4152 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4153 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4158 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4159 struct radeon_ring *ring)
4163 if (rdev->wb.enabled)
4164 rptr = rdev->wb.wb[ring->rptr_offs/4];
4166 rptr = RREG32(CP_RB0_RPTR);
4171 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4172 struct radeon_ring *ring)
4176 wptr = RREG32(CP_RB0_WPTR);
4181 void cik_gfx_set_wptr(struct radeon_device *rdev,
4182 struct radeon_ring *ring)
4184 WREG32(CP_RB0_WPTR, ring->wptr);
4185 (void)RREG32(CP_RB0_WPTR);
4188 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4189 struct radeon_ring *ring)
4193 if (rdev->wb.enabled) {
4194 rptr = rdev->wb.wb[ring->rptr_offs/4];
4196 mutex_lock(&rdev->srbm_mutex);
4197 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4198 rptr = RREG32(CP_HQD_PQ_RPTR);
4199 cik_srbm_select(rdev, 0, 0, 0, 0);
4200 mutex_unlock(&rdev->srbm_mutex);
4206 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4207 struct radeon_ring *ring)
4211 if (rdev->wb.enabled) {
4212 /* XXX check if swapping is necessary on BE */
4213 wptr = rdev->wb.wb[ring->wptr_offs/4];
4215 mutex_lock(&rdev->srbm_mutex);
4216 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4217 wptr = RREG32(CP_HQD_PQ_WPTR);
4218 cik_srbm_select(rdev, 0, 0, 0, 0);
4219 mutex_unlock(&rdev->srbm_mutex);
4225 void cik_compute_set_wptr(struct radeon_device *rdev,
4226 struct radeon_ring *ring)
4228 /* XXX check if swapping is necessary on BE */
4229 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4230 WDOORBELL32(ring->doorbell_index, ring->wptr);
4234 * cik_cp_compute_enable - enable/disable the compute CP MEs
4236 * @rdev: radeon_device pointer
4237 * @enable: enable or disable the MEs
4239 * Halts or unhalts the compute MEs.
4241 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4244 WREG32(CP_MEC_CNTL, 0);
4246 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4247 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4248 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4254 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4256 * @rdev: radeon_device pointer
4258 * Loads the compute MEC1&2 ucode.
4259 * Returns 0 for success, -EINVAL if the ucode is not available.
4261 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4263 const __be32 *fw_data;
4269 cik_cp_compute_enable(rdev, false);
4272 fw_data = (const __be32 *)rdev->mec_fw->data;
4273 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4274 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4275 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4276 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4278 if (rdev->family == CHIP_KAVERI) {
4280 fw_data = (const __be32 *)rdev->mec_fw->data;
4281 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4282 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4283 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4284 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4291 * cik_cp_compute_start - start the compute queues
4293 * @rdev: radeon_device pointer
4295 * Enable the compute queues.
4296 * Returns 0 for success, error for failure.
4298 static int cik_cp_compute_start(struct radeon_device *rdev)
4300 cik_cp_compute_enable(rdev, true);
4306 * cik_cp_compute_fini - stop the compute queues
4308 * @rdev: radeon_device pointer
4310 * Stop the compute queues and tear down the driver queue
4313 static void cik_cp_compute_fini(struct radeon_device *rdev)
4317 cik_cp_compute_enable(rdev, false);
4319 for (i = 0; i < 2; i++) {
4321 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4323 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4325 if (rdev->ring[idx].mqd_obj) {
4326 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4327 if (unlikely(r != 0))
4328 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4330 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4331 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4333 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4334 rdev->ring[idx].mqd_obj = NULL;
4339 static void cik_mec_fini(struct radeon_device *rdev)
4343 if (rdev->mec.hpd_eop_obj) {
4344 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4345 if (unlikely(r != 0))
4346 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4347 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4348 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4350 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4351 rdev->mec.hpd_eop_obj = NULL;
4355 #define MEC_HPD_SIZE 2048
4357 static int cik_mec_init(struct radeon_device *rdev)
4363 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4364 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4366 if (rdev->family == CHIP_KAVERI)
4367 rdev->mec.num_mec = 2;
4369 rdev->mec.num_mec = 1;
4370 rdev->mec.num_pipe = 4;
4371 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4373 if (rdev->mec.hpd_eop_obj == NULL) {
4374 r = radeon_bo_create(rdev,
4375 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4377 RADEON_GEM_DOMAIN_GTT, NULL,
4378 &rdev->mec.hpd_eop_obj);
4380 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4385 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4386 if (unlikely(r != 0)) {
4390 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4391 &rdev->mec.hpd_eop_gpu_addr);
4393 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4397 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4399 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4404 /* clear memory. Not sure if this is required or not */
4405 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4407 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4408 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4413 struct hqd_registers
4415 u32 cp_mqd_base_addr;
4416 u32 cp_mqd_base_addr_hi;
4419 u32 cp_hqd_persistent_state;
4420 u32 cp_hqd_pipe_priority;
4421 u32 cp_hqd_queue_priority;
4424 u32 cp_hqd_pq_base_hi;
4426 u32 cp_hqd_pq_rptr_report_addr;
4427 u32 cp_hqd_pq_rptr_report_addr_hi;
4428 u32 cp_hqd_pq_wptr_poll_addr;
4429 u32 cp_hqd_pq_wptr_poll_addr_hi;
4430 u32 cp_hqd_pq_doorbell_control;
4432 u32 cp_hqd_pq_control;
4433 u32 cp_hqd_ib_base_addr;
4434 u32 cp_hqd_ib_base_addr_hi;
4436 u32 cp_hqd_ib_control;
4437 u32 cp_hqd_iq_timer;
4439 u32 cp_hqd_dequeue_request;
4440 u32 cp_hqd_dma_offload;
4441 u32 cp_hqd_sema_cmd;
4442 u32 cp_hqd_msg_type;
4443 u32 cp_hqd_atomic0_preop_lo;
4444 u32 cp_hqd_atomic0_preop_hi;
4445 u32 cp_hqd_atomic1_preop_lo;
4446 u32 cp_hqd_atomic1_preop_hi;
4447 u32 cp_hqd_hq_scheduler0;
4448 u32 cp_hqd_hq_scheduler1;
4455 u32 dispatch_initiator;
4459 u32 pipeline_stat_enable;
4460 u32 perf_counter_enable;
4466 u32 resource_limits;
4467 u32 static_thread_mgmt01[2];
4469 u32 static_thread_mgmt23[2];
4471 u32 thread_trace_enable;
4474 u32 vgtcs_invoke_count[2];
4475 struct hqd_registers queue_state;
4477 u32 interrupt_queue[64];
4481 * cik_cp_compute_resume - setup the compute queue registers
4483 * @rdev: radeon_device pointer
4485 * Program the compute queues and test them to make sure they
4487 * Returns 0 for success, error for failure.
4489 static int cik_cp_compute_resume(struct radeon_device *rdev)
4493 bool use_doorbell = true;
4499 struct bonaire_mqd *mqd;
4501 r = cik_cp_compute_start(rdev);
4505 /* fix up chicken bits */
4506 tmp = RREG32(CP_CPF_DEBUG);
4508 WREG32(CP_CPF_DEBUG, tmp);
4510 /* init the pipes */
4511 mutex_lock(&rdev->srbm_mutex);
4512 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4513 int me = (i < 4) ? 1 : 2;
4514 int pipe = (i < 4) ? i : (i - 4);
4516 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4518 cik_srbm_select(rdev, me, pipe, 0, 0);
4520 /* write the EOP addr */
4521 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4522 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4524 /* set the VMID assigned */
4525 WREG32(CP_HPD_EOP_VMID, 0);
4527 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4528 tmp = RREG32(CP_HPD_EOP_CONTROL);
4529 tmp &= ~EOP_SIZE_MASK;
4530 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4531 WREG32(CP_HPD_EOP_CONTROL, tmp);
4533 cik_srbm_select(rdev, 0, 0, 0, 0);
4534 mutex_unlock(&rdev->srbm_mutex);
4536 /* init the queues. Just two for now. */
4537 for (i = 0; i < 2; i++) {
4539 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4541 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4543 if (rdev->ring[idx].mqd_obj == NULL) {
4544 r = radeon_bo_create(rdev,
4545 sizeof(struct bonaire_mqd),
4547 RADEON_GEM_DOMAIN_GTT, NULL,
4548 &rdev->ring[idx].mqd_obj);
4550 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4555 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4556 if (unlikely(r != 0)) {
4557 cik_cp_compute_fini(rdev);
4560 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4563 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4564 cik_cp_compute_fini(rdev);
4567 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4569 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4570 cik_cp_compute_fini(rdev);
4574 /* init the mqd struct */
4575 memset(buf, 0, sizeof(struct bonaire_mqd));
4577 mqd = (struct bonaire_mqd *)buf;
4578 mqd->header = 0xC0310800;
4579 mqd->static_thread_mgmt01[0] = 0xffffffff;
4580 mqd->static_thread_mgmt01[1] = 0xffffffff;
4581 mqd->static_thread_mgmt23[0] = 0xffffffff;
4582 mqd->static_thread_mgmt23[1] = 0xffffffff;
4584 mutex_lock(&rdev->srbm_mutex);
4585 cik_srbm_select(rdev, rdev->ring[idx].me,
4586 rdev->ring[idx].pipe,
4587 rdev->ring[idx].queue, 0);
4589 /* disable wptr polling */
4590 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4591 tmp &= ~WPTR_POLL_EN;
4592 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4594 /* enable doorbell? */
4595 mqd->queue_state.cp_hqd_pq_doorbell_control =
4596 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4598 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4600 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4601 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4602 mqd->queue_state.cp_hqd_pq_doorbell_control);
4604 /* disable the queue if it's active */
4605 mqd->queue_state.cp_hqd_dequeue_request = 0;
4606 mqd->queue_state.cp_hqd_pq_rptr = 0;
4607 mqd->queue_state.cp_hqd_pq_wptr= 0;
4608 if (RREG32(CP_HQD_ACTIVE) & 1) {
4609 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4610 for (i = 0; i < rdev->usec_timeout; i++) {
4611 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4615 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4616 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4617 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4620 /* set the pointer to the MQD */
4621 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4622 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4623 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4624 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4625 /* set MQD vmid to 0 */
4626 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4627 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4628 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4630 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4631 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4632 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4633 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4634 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4635 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4637 /* set up the HQD, this is similar to CP_RB0_CNTL */
4638 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4639 mqd->queue_state.cp_hqd_pq_control &=
4640 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4642 mqd->queue_state.cp_hqd_pq_control |=
4643 order_base_2(rdev->ring[idx].ring_size / 8);
4644 mqd->queue_state.cp_hqd_pq_control |=
4645 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4647 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4649 mqd->queue_state.cp_hqd_pq_control &=
4650 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4651 mqd->queue_state.cp_hqd_pq_control |=
4652 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4653 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4655 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4657 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4659 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4660 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4661 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4662 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4663 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4664 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4666 /* set the wb address wether it's enabled or not */
4668 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4670 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4671 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4672 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4673 upper_32_bits(wb_gpu_addr) & 0xffff;
4674 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4675 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4676 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4677 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4679 /* enable the doorbell if requested */
4681 mqd->queue_state.cp_hqd_pq_doorbell_control =
4682 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4683 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4684 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4685 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4686 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4687 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4688 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4691 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4693 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4694 mqd->queue_state.cp_hqd_pq_doorbell_control);
4696 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4697 rdev->ring[idx].wptr = 0;
4698 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4699 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4700 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
4702 /* set the vmid for the queue */
4703 mqd->queue_state.cp_hqd_vmid = 0;
4704 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4706 /* activate the queue */
4707 mqd->queue_state.cp_hqd_active = 1;
4708 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4710 cik_srbm_select(rdev, 0, 0, 0, 0);
4711 mutex_unlock(&rdev->srbm_mutex);
4713 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4714 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4716 rdev->ring[idx].ready = true;
4717 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4719 rdev->ring[idx].ready = false;
4725 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4727 cik_cp_gfx_enable(rdev, enable);
4728 cik_cp_compute_enable(rdev, enable);
4731 static int cik_cp_load_microcode(struct radeon_device *rdev)
4735 r = cik_cp_gfx_load_microcode(rdev);
4738 r = cik_cp_compute_load_microcode(rdev);
4745 static void cik_cp_fini(struct radeon_device *rdev)
4747 cik_cp_gfx_fini(rdev);
4748 cik_cp_compute_fini(rdev);
4751 static int cik_cp_resume(struct radeon_device *rdev)
4755 cik_enable_gui_idle_interrupt(rdev, false);
4757 r = cik_cp_load_microcode(rdev);
4761 r = cik_cp_gfx_resume(rdev);
4764 r = cik_cp_compute_resume(rdev);
4768 cik_enable_gui_idle_interrupt(rdev, true);
4773 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4775 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4776 RREG32(GRBM_STATUS));
4777 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4778 RREG32(GRBM_STATUS2));
4779 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4780 RREG32(GRBM_STATUS_SE0));
4781 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4782 RREG32(GRBM_STATUS_SE1));
4783 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4784 RREG32(GRBM_STATUS_SE2));
4785 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4786 RREG32(GRBM_STATUS_SE3));
4787 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4788 RREG32(SRBM_STATUS));
4789 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4790 RREG32(SRBM_STATUS2));
4791 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4792 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4793 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4794 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4795 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4796 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4797 RREG32(CP_STALLED_STAT1));
4798 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4799 RREG32(CP_STALLED_STAT2));
4800 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4801 RREG32(CP_STALLED_STAT3));
4802 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4803 RREG32(CP_CPF_BUSY_STAT));
4804 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4805 RREG32(CP_CPF_STALLED_STAT1));
4806 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4807 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4808 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4809 RREG32(CP_CPC_STALLED_STAT1));
4810 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4814 * cik_gpu_check_soft_reset - check which blocks are busy
4816 * @rdev: radeon_device pointer
4818 * Check which blocks are busy and return the relevant reset
4819 * mask to be used by cik_gpu_soft_reset().
4820 * Returns a mask of the blocks to be reset.
4822 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4828 tmp = RREG32(GRBM_STATUS);
4829 if (tmp & (PA_BUSY | SC_BUSY |
4830 BCI_BUSY | SX_BUSY |
4831 TA_BUSY | VGT_BUSY |
4833 GDS_BUSY | SPI_BUSY |
4834 IA_BUSY | IA_BUSY_NO_DMA))
4835 reset_mask |= RADEON_RESET_GFX;
4837 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4838 reset_mask |= RADEON_RESET_CP;
4841 tmp = RREG32(GRBM_STATUS2);
4843 reset_mask |= RADEON_RESET_RLC;
4845 /* SDMA0_STATUS_REG */
4846 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4847 if (!(tmp & SDMA_IDLE))
4848 reset_mask |= RADEON_RESET_DMA;
4850 /* SDMA1_STATUS_REG */
4851 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4852 if (!(tmp & SDMA_IDLE))
4853 reset_mask |= RADEON_RESET_DMA1;
4856 tmp = RREG32(SRBM_STATUS2);
4857 if (tmp & SDMA_BUSY)
4858 reset_mask |= RADEON_RESET_DMA;
4860 if (tmp & SDMA1_BUSY)
4861 reset_mask |= RADEON_RESET_DMA1;
4864 tmp = RREG32(SRBM_STATUS);
4867 reset_mask |= RADEON_RESET_IH;
4870 reset_mask |= RADEON_RESET_SEM;
4872 if (tmp & GRBM_RQ_PENDING)
4873 reset_mask |= RADEON_RESET_GRBM;
4876 reset_mask |= RADEON_RESET_VMC;
4878 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4879 MCC_BUSY | MCD_BUSY))
4880 reset_mask |= RADEON_RESET_MC;
4882 if (evergreen_is_display_hung(rdev))
4883 reset_mask |= RADEON_RESET_DISPLAY;
4885 /* Skip MC reset as it's mostly likely not hung, just busy */
4886 if (reset_mask & RADEON_RESET_MC) {
4887 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4888 reset_mask &= ~RADEON_RESET_MC;
4895 * cik_gpu_soft_reset - soft reset GPU
4897 * @rdev: radeon_device pointer
4898 * @reset_mask: mask of which blocks to reset
4900 * Soft reset the blocks specified in @reset_mask.
4902 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4904 struct evergreen_mc_save save;
4905 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4908 if (reset_mask == 0)
4911 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4913 cik_print_gpu_status_regs(rdev);
4914 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4915 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4916 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4917 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4926 /* Disable GFX parsing/prefetching */
4927 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4929 /* Disable MEC parsing/prefetching */
4930 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4932 if (reset_mask & RADEON_RESET_DMA) {
4934 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4936 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4938 if (reset_mask & RADEON_RESET_DMA1) {
4940 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4942 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4945 evergreen_mc_stop(rdev, &save);
4946 if (evergreen_mc_wait_for_idle(rdev)) {
4947 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4950 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4951 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4953 if (reset_mask & RADEON_RESET_CP) {
4954 grbm_soft_reset |= SOFT_RESET_CP;
4956 srbm_soft_reset |= SOFT_RESET_GRBM;
4959 if (reset_mask & RADEON_RESET_DMA)
4960 srbm_soft_reset |= SOFT_RESET_SDMA;
4962 if (reset_mask & RADEON_RESET_DMA1)
4963 srbm_soft_reset |= SOFT_RESET_SDMA1;
4965 if (reset_mask & RADEON_RESET_DISPLAY)
4966 srbm_soft_reset |= SOFT_RESET_DC;
4968 if (reset_mask & RADEON_RESET_RLC)
4969 grbm_soft_reset |= SOFT_RESET_RLC;
4971 if (reset_mask & RADEON_RESET_SEM)
4972 srbm_soft_reset |= SOFT_RESET_SEM;
4974 if (reset_mask & RADEON_RESET_IH)
4975 srbm_soft_reset |= SOFT_RESET_IH;
4977 if (reset_mask & RADEON_RESET_GRBM)
4978 srbm_soft_reset |= SOFT_RESET_GRBM;
4980 if (reset_mask & RADEON_RESET_VMC)
4981 srbm_soft_reset |= SOFT_RESET_VMC;
4983 if (!(rdev->flags & RADEON_IS_IGP)) {
4984 if (reset_mask & RADEON_RESET_MC)
4985 srbm_soft_reset |= SOFT_RESET_MC;
4988 if (grbm_soft_reset) {
4989 tmp = RREG32(GRBM_SOFT_RESET);
4990 tmp |= grbm_soft_reset;
4991 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4992 WREG32(GRBM_SOFT_RESET, tmp);
4993 tmp = RREG32(GRBM_SOFT_RESET);
4997 tmp &= ~grbm_soft_reset;
4998 WREG32(GRBM_SOFT_RESET, tmp);
4999 tmp = RREG32(GRBM_SOFT_RESET);
5002 if (srbm_soft_reset) {
5003 tmp = RREG32(SRBM_SOFT_RESET);
5004 tmp |= srbm_soft_reset;
5005 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5006 WREG32(SRBM_SOFT_RESET, tmp);
5007 tmp = RREG32(SRBM_SOFT_RESET);
5011 tmp &= ~srbm_soft_reset;
5012 WREG32(SRBM_SOFT_RESET, tmp);
5013 tmp = RREG32(SRBM_SOFT_RESET);
5016 /* Wait a little for things to settle down */
5019 evergreen_mc_resume(rdev, &save);
5022 cik_print_gpu_status_regs(rdev);
5025 struct kv_reset_save_regs {
5026 u32 gmcon_reng_execute;
5031 static void kv_save_regs_for_reset(struct radeon_device *rdev,
5032 struct kv_reset_save_regs *save)
5034 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5035 save->gmcon_misc = RREG32(GMCON_MISC);
5036 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5038 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5039 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5040 STCTRL_STUTTER_EN));
5043 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5044 struct kv_reset_save_regs *save)
5048 WREG32(GMCON_PGFSM_WRITE, 0);
5049 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5051 for (i = 0; i < 5; i++)
5052 WREG32(GMCON_PGFSM_WRITE, 0);
5054 WREG32(GMCON_PGFSM_WRITE, 0);
5055 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5057 for (i = 0; i < 5; i++)
5058 WREG32(GMCON_PGFSM_WRITE, 0);
5060 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5061 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5063 for (i = 0; i < 5; i++)
5064 WREG32(GMCON_PGFSM_WRITE, 0);
5066 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5067 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5069 for (i = 0; i < 5; i++)
5070 WREG32(GMCON_PGFSM_WRITE, 0);
5072 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5073 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5075 for (i = 0; i < 5; i++)
5076 WREG32(GMCON_PGFSM_WRITE, 0);
5078 WREG32(GMCON_PGFSM_WRITE, 0);
5079 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5081 for (i = 0; i < 5; i++)
5082 WREG32(GMCON_PGFSM_WRITE, 0);
5084 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5085 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5087 for (i = 0; i < 5; i++)
5088 WREG32(GMCON_PGFSM_WRITE, 0);
5090 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5091 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5093 for (i = 0; i < 5; i++)
5094 WREG32(GMCON_PGFSM_WRITE, 0);
5096 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5097 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5099 for (i = 0; i < 5; i++)
5100 WREG32(GMCON_PGFSM_WRITE, 0);
5102 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5103 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5105 for (i = 0; i < 5; i++)
5106 WREG32(GMCON_PGFSM_WRITE, 0);
5108 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5109 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5111 WREG32(GMCON_MISC3, save->gmcon_misc3);
5112 WREG32(GMCON_MISC, save->gmcon_misc);
5113 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5116 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5118 struct evergreen_mc_save save;
5119 struct kv_reset_save_regs kv_save = { 0 };
5122 dev_info(rdev->dev, "GPU pci config reset\n");
5130 /* Disable GFX parsing/prefetching */
5131 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5133 /* Disable MEC parsing/prefetching */
5134 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5137 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5139 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5141 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5143 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5144 /* XXX other engines? */
5146 /* halt the rlc, disable cp internal ints */
5151 /* disable mem access */
5152 evergreen_mc_stop(rdev, &save);
5153 if (evergreen_mc_wait_for_idle(rdev)) {
5154 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5157 if (rdev->flags & RADEON_IS_IGP)
5158 kv_save_regs_for_reset(rdev, &kv_save);
5161 pci_clear_master(rdev->pdev);
5163 radeon_pci_config_reset(rdev);
5167 /* wait for asic to come out of reset */
5168 for (i = 0; i < rdev->usec_timeout; i++) {
5169 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5174 /* does asic init need to be run first??? */
5175 if (rdev->flags & RADEON_IS_IGP)
5176 kv_restore_regs_for_reset(rdev, &kv_save);
5180 * cik_asic_reset - soft reset GPU
5182 * @rdev: radeon_device pointer
5184 * Look up which blocks are hung and attempt
5186 * Returns 0 for success.
5188 int cik_asic_reset(struct radeon_device *rdev)
5192 reset_mask = cik_gpu_check_soft_reset(rdev);
5195 r600_set_bios_scratch_engine_hung(rdev, true);
5197 /* try soft reset */
5198 cik_gpu_soft_reset(rdev, reset_mask);
5200 reset_mask = cik_gpu_check_soft_reset(rdev);
5202 /* try pci config reset */
5203 if (reset_mask && radeon_hard_reset)
5204 cik_gpu_pci_config_reset(rdev);
5206 reset_mask = cik_gpu_check_soft_reset(rdev);
5209 r600_set_bios_scratch_engine_hung(rdev, false);
5215 * cik_gfx_is_lockup - check if the 3D engine is locked up
5217 * @rdev: radeon_device pointer
5218 * @ring: radeon_ring structure holding ring information
5220 * Check if the 3D engine is locked up (CIK).
5221 * Returns true if the engine is locked, false if not.
5223 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5225 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5227 if (!(reset_mask & (RADEON_RESET_GFX |
5228 RADEON_RESET_COMPUTE |
5229 RADEON_RESET_CP))) {
5230 radeon_ring_lockup_update(rdev, ring);
5233 return radeon_ring_test_lockup(rdev, ring);
5238 * cik_mc_program - program the GPU memory controller
5240 * @rdev: radeon_device pointer
5242 * Set the location of vram, gart, and AGP in the GPU's
5243 * physical address space (CIK).
5245 static void cik_mc_program(struct radeon_device *rdev)
5247 struct evergreen_mc_save save;
5251 /* Initialize HDP */
5252 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5253 WREG32((0x2c14 + j), 0x00000000);
5254 WREG32((0x2c18 + j), 0x00000000);
5255 WREG32((0x2c1c + j), 0x00000000);
5256 WREG32((0x2c20 + j), 0x00000000);
5257 WREG32((0x2c24 + j), 0x00000000);
5259 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5261 evergreen_mc_stop(rdev, &save);
5262 if (radeon_mc_wait_for_idle(rdev)) {
5263 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5265 /* Lockout access through VGA aperture*/
5266 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5267 /* Update configuration */
5268 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5269 rdev->mc.vram_start >> 12);
5270 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5271 rdev->mc.vram_end >> 12);
5272 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5273 rdev->vram_scratch.gpu_addr >> 12);
5274 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5275 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5276 WREG32(MC_VM_FB_LOCATION, tmp);
5277 /* XXX double check these! */
5278 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5279 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5280 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5281 WREG32(MC_VM_AGP_BASE, 0);
5282 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5283 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5284 if (radeon_mc_wait_for_idle(rdev)) {
5285 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5287 evergreen_mc_resume(rdev, &save);
5288 /* we need to own VRAM, so turn off the VGA renderer here
5289 * to stop it overwriting our objects */
5290 rv515_vga_render_disable(rdev);
5294 * cik_mc_init - initialize the memory controller driver params
5296 * @rdev: radeon_device pointer
5298 * Look up the amount of vram, vram width, and decide how to place
5299 * vram and gart within the GPU's physical address space (CIK).
5300 * Returns 0 for success.
5302 static int cik_mc_init(struct radeon_device *rdev)
5305 int chansize, numchan;
5307 /* Get VRAM informations */
5308 rdev->mc.vram_is_ddr = true;
5309 tmp = RREG32(MC_ARB_RAMCFG);
5310 if (tmp & CHANSIZE_MASK) {
5315 tmp = RREG32(MC_SHARED_CHMAP);
5316 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5346 rdev->mc.vram_width = numchan * chansize;
5347 /* Could aper size report 0 ? */
5348 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5349 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5350 /* size in MB on si */
5351 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5352 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5353 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5354 si_vram_gtt_location(rdev, &rdev->mc);
5355 radeon_update_bandwidth_info(rdev);
5362 * VMID 0 is the physical GPU addresses as used by the kernel.
5363 * VMIDs 1-15 are used for userspace clients and are handled
5364 * by the radeon vm/hsa code.
5367 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5369 * @rdev: radeon_device pointer
5371 * Flush the TLB for the VMID 0 page table (CIK).
5373 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5375 /* flush hdp cache */
5376 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5378 /* bits 0-15 are the VM contexts0-15 */
5379 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5383 * cik_pcie_gart_enable - gart enable
5385 * @rdev: radeon_device pointer
5387 * This sets up the TLBs, programs the page tables for VMID0,
5388 * sets up the hw for VMIDs 1-15 which are allocated on
5389 * demand, and sets up the global locations for the LDS, GDS,
5390 * and GPUVM for FSA64 clients (CIK).
5391 * Returns 0 for success, errors for failure.
5393 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5397 if (rdev->gart.robj == NULL) {
5398 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5401 r = radeon_gart_table_vram_pin(rdev);
5404 radeon_gart_restore(rdev);
5405 /* Setup TLB control */
5406 WREG32(MC_VM_MX_L1_TLB_CNTL,
5409 ENABLE_L1_FRAGMENT_PROCESSING |
5410 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5411 ENABLE_ADVANCED_DRIVER_MODEL |
5412 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5413 /* Setup L2 cache */
5414 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5415 ENABLE_L2_FRAGMENT_PROCESSING |
5416 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5417 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5418 EFFECTIVE_L2_QUEUE_SIZE(7) |
5419 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5420 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5421 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5423 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
5424 /* setup context0 */
5425 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5426 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5427 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5428 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5429 (u32)(rdev->dummy_page.addr >> 12));
5430 WREG32(VM_CONTEXT0_CNTL2, 0);
5431 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5432 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5438 /* empty context1-15 */
5439 /* FIXME start with 4G, once using 2 level pt switch to full
5442 /* set vm size, must be a multiple of 4 */
5443 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5444 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5445 for (i = 1; i < 16; i++) {
5447 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5448 rdev->gart.table_addr >> 12);
5450 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5451 rdev->gart.table_addr >> 12);
5454 /* enable context1-15 */
5455 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5456 (u32)(rdev->dummy_page.addr >> 12));
5457 WREG32(VM_CONTEXT1_CNTL2, 4);
5458 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5459 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
5460 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5461 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5462 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5463 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5464 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5465 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5466 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5467 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5468 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5469 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5470 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5471 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5473 if (rdev->family == CHIP_KAVERI) {
5474 u32 tmp = RREG32(CHUB_CONTROL);
5476 WREG32(CHUB_CONTROL, tmp);
5479 /* XXX SH_MEM regs */
5480 /* where to put LDS, scratch, GPUVM in FSA64 space */
5481 mutex_lock(&rdev->srbm_mutex);
5482 for (i = 0; i < 16; i++) {
5483 cik_srbm_select(rdev, 0, 0, 0, i);
5484 /* CP and shaders */
5485 WREG32(SH_MEM_CONFIG, 0);
5486 WREG32(SH_MEM_APE1_BASE, 1);
5487 WREG32(SH_MEM_APE1_LIMIT, 0);
5488 WREG32(SH_MEM_BASES, 0);
5490 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5491 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5492 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5493 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5494 /* XXX SDMA RLC - todo */
5496 cik_srbm_select(rdev, 0, 0, 0, 0);
5497 mutex_unlock(&rdev->srbm_mutex);
5499 cik_pcie_gart_tlb_flush(rdev);
5500 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5501 (unsigned)(rdev->mc.gtt_size >> 20),
5502 (unsigned long long)rdev->gart.table_addr);
5503 rdev->gart.ready = true;
5508 * cik_pcie_gart_disable - gart disable
5510 * @rdev: radeon_device pointer
5512 * This disables all VM page table (CIK).
5514 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5516 /* Disable all tables */
5517 WREG32(VM_CONTEXT0_CNTL, 0);
5518 WREG32(VM_CONTEXT1_CNTL, 0);
5519 /* Setup TLB control */
5520 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5521 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5522 /* Setup L2 cache */
5524 ENABLE_L2_FRAGMENT_PROCESSING |
5525 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5526 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5527 EFFECTIVE_L2_QUEUE_SIZE(7) |
5528 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5529 WREG32(VM_L2_CNTL2, 0);
5530 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5531 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5532 radeon_gart_table_vram_unpin(rdev);
5536 * cik_pcie_gart_fini - vm fini callback
5538 * @rdev: radeon_device pointer
5540 * Tears down the driver GART/VM setup (CIK).
5542 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5544 cik_pcie_gart_disable(rdev);
5545 radeon_gart_table_vram_free(rdev);
5546 radeon_gart_fini(rdev);
5551 * cik_ib_parse - vm ib_parse callback
5553 * @rdev: radeon_device pointer
5554 * @ib: indirect buffer pointer
5556 * CIK uses hw IB checking so this is a nop (CIK).
5558 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5565 * VMID 0 is the physical GPU addresses as used by the kernel.
5566 * VMIDs 1-15 are used for userspace clients and are handled
5567 * by the radeon vm/hsa code.
5570 * cik_vm_init - cik vm init callback
5572 * @rdev: radeon_device pointer
5574 * Inits cik specific vm parameters (number of VMs, base of vram for
5575 * VMIDs 1-15) (CIK).
5576 * Returns 0 for success.
5578 int cik_vm_init(struct radeon_device *rdev)
5581 rdev->vm_manager.nvm = 16;
5582 /* base offset of vram pages */
5583 if (rdev->flags & RADEON_IS_IGP) {
5584 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5586 rdev->vm_manager.vram_base_offset = tmp;
5588 rdev->vm_manager.vram_base_offset = 0;
5594 * cik_vm_fini - cik vm fini callback
5596 * @rdev: radeon_device pointer
5598 * Tear down any asic specific VM setup (CIK).
5600 void cik_vm_fini(struct radeon_device *rdev)
5605 * cik_vm_decode_fault - print human readable fault info
5607 * @rdev: radeon_device pointer
5608 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5609 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5611 * Print human readable fault information (CIK).
5613 static void cik_vm_decode_fault(struct radeon_device *rdev,
5614 u32 status, u32 addr, u32 mc_client)
5617 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5618 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5619 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5620 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5622 if (rdev->family == CHIP_HAWAII)
5623 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5625 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5627 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5628 protections, vmid, addr,
5629 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5630 block, mc_client, mc_id);
5634 * cik_vm_flush - cik vm flush using the CP
5636 * @rdev: radeon_device pointer
5638 * Update the page table base and flush the VM TLB
5639 * using the CP (CIK).
5641 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5643 struct radeon_ring *ring = &rdev->ring[ridx];
5648 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5649 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5650 WRITE_DATA_DST_SEL(0)));
5652 radeon_ring_write(ring,
5653 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5655 radeon_ring_write(ring,
5656 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5658 radeon_ring_write(ring, 0);
5659 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5661 /* update SH_MEM_* regs */
5662 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5663 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5664 WRITE_DATA_DST_SEL(0)));
5665 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5666 radeon_ring_write(ring, 0);
5667 radeon_ring_write(ring, VMID(vm->id));
5669 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5670 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5671 WRITE_DATA_DST_SEL(0)));
5672 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5673 radeon_ring_write(ring, 0);
5675 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5676 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5677 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5678 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5680 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5681 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5682 WRITE_DATA_DST_SEL(0)));
5683 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5684 radeon_ring_write(ring, 0);
5685 radeon_ring_write(ring, VMID(0));
5688 cik_hdp_flush_cp_ring_emit(rdev, ridx);
5690 /* bits 0-15 are the VM contexts0-15 */
5691 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5692 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5693 WRITE_DATA_DST_SEL(0)));
5694 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5695 radeon_ring_write(ring, 0);
5696 radeon_ring_write(ring, 1 << vm->id);
5698 /* compute doesn't have PFP */
5699 if (ridx == RADEON_RING_TYPE_GFX_INDEX) {
5700 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5701 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5702 radeon_ring_write(ring, 0x0);
5708 * The RLC is a multi-purpose microengine that handles a
5709 * variety of functions, the most important of which is
5710 * the interrupt controller.
5712 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5715 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5718 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5720 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5721 WREG32(CP_INT_CNTL_RING0, tmp);
5724 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5728 tmp = RREG32(RLC_LB_CNTL);
5730 tmp |= LOAD_BALANCE_ENABLE;
5732 tmp &= ~LOAD_BALANCE_ENABLE;
5733 WREG32(RLC_LB_CNTL, tmp);
5736 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5741 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5742 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5743 cik_select_se_sh(rdev, i, j);
5744 for (k = 0; k < rdev->usec_timeout; k++) {
5745 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5751 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5753 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5754 for (k = 0; k < rdev->usec_timeout; k++) {
5755 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5761 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5765 tmp = RREG32(RLC_CNTL);
5767 WREG32(RLC_CNTL, rlc);
5770 static u32 cik_halt_rlc(struct radeon_device *rdev)
5774 orig = data = RREG32(RLC_CNTL);
5776 if (data & RLC_ENABLE) {
5779 data &= ~RLC_ENABLE;
5780 WREG32(RLC_CNTL, data);
5782 for (i = 0; i < rdev->usec_timeout; i++) {
5783 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5788 cik_wait_for_rlc_serdes(rdev);
5794 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5798 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5799 WREG32(RLC_GPR_REG2, tmp);
5801 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5802 for (i = 0; i < rdev->usec_timeout; i++) {
5803 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5808 for (i = 0; i < rdev->usec_timeout; i++) {
5809 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5815 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5819 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5820 WREG32(RLC_GPR_REG2, tmp);
5824 * cik_rlc_stop - stop the RLC ME
5826 * @rdev: radeon_device pointer
5828 * Halt the RLC ME (MicroEngine) (CIK).
5830 static void cik_rlc_stop(struct radeon_device *rdev)
5832 WREG32(RLC_CNTL, 0);
5834 cik_enable_gui_idle_interrupt(rdev, false);
5836 cik_wait_for_rlc_serdes(rdev);
5840 * cik_rlc_start - start the RLC ME
5842 * @rdev: radeon_device pointer
5844 * Unhalt the RLC ME (MicroEngine) (CIK).
5846 static void cik_rlc_start(struct radeon_device *rdev)
5848 WREG32(RLC_CNTL, RLC_ENABLE);
5850 cik_enable_gui_idle_interrupt(rdev, true);
5856 * cik_rlc_resume - setup the RLC hw
5858 * @rdev: radeon_device pointer
5860 * Initialize the RLC registers, load the ucode,
5861 * and start the RLC (CIK).
5862 * Returns 0 for success, -EINVAL if the ucode is not available.
5864 static int cik_rlc_resume(struct radeon_device *rdev)
5867 const __be32 *fw_data;
5872 switch (rdev->family) {
5876 size = BONAIRE_RLC_UCODE_SIZE;
5879 size = KV_RLC_UCODE_SIZE;
5882 size = KB_RLC_UCODE_SIZE;
5885 size = ML_RLC_UCODE_SIZE;
5892 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5893 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5901 WREG32(RLC_LB_CNTR_INIT, 0);
5902 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5904 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5905 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5906 WREG32(RLC_LB_PARAMS, 0x00600408);
5907 WREG32(RLC_LB_CNTL, 0x80000004);
5909 WREG32(RLC_MC_CNTL, 0);
5910 WREG32(RLC_UCODE_CNTL, 0);
5912 fw_data = (const __be32 *)rdev->rlc_fw->data;
5913 WREG32(RLC_GPM_UCODE_ADDR, 0);
5914 for (i = 0; i < size; i++)
5915 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5916 WREG32(RLC_GPM_UCODE_ADDR, 0);
5918 /* XXX - find out what chips support lbpw */
5919 cik_enable_lbpw(rdev, false);
5921 if (rdev->family == CHIP_BONAIRE)
5922 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5924 cik_rlc_start(rdev);
5929 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5931 u32 data, orig, tmp, tmp2;
5933 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5935 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5936 cik_enable_gui_idle_interrupt(rdev, true);
5938 tmp = cik_halt_rlc(rdev);
5940 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5941 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5942 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5943 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5944 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5946 cik_update_rlc(rdev, tmp);
5948 data |= CGCG_EN | CGLS_EN;
5950 cik_enable_gui_idle_interrupt(rdev, false);
5952 RREG32(CB_CGTT_SCLK_CTRL);
5953 RREG32(CB_CGTT_SCLK_CTRL);
5954 RREG32(CB_CGTT_SCLK_CTRL);
5955 RREG32(CB_CGTT_SCLK_CTRL);
5957 data &= ~(CGCG_EN | CGLS_EN);
5961 WREG32(RLC_CGCG_CGLS_CTRL, data);
5965 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5967 u32 data, orig, tmp = 0;
5969 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5970 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5971 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5972 orig = data = RREG32(CP_MEM_SLP_CNTL);
5973 data |= CP_MEM_LS_EN;
5975 WREG32(CP_MEM_SLP_CNTL, data);
5979 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5982 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5984 tmp = cik_halt_rlc(rdev);
5986 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5987 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5988 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5989 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5990 WREG32(RLC_SERDES_WR_CTRL, data);
5992 cik_update_rlc(rdev, tmp);
5994 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5995 orig = data = RREG32(CGTS_SM_CTRL_REG);
5996 data &= ~SM_MODE_MASK;
5997 data |= SM_MODE(0x2);
5998 data |= SM_MODE_ENABLE;
5999 data &= ~CGTS_OVERRIDE;
6000 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6001 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6002 data &= ~CGTS_LS_OVERRIDE;
6003 data &= ~ON_MONITOR_ADD_MASK;
6004 data |= ON_MONITOR_ADD_EN;
6005 data |= ON_MONITOR_ADD(0x96);
6007 WREG32(CGTS_SM_CTRL_REG, data);
6010 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6013 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6015 data = RREG32(RLC_MEM_SLP_CNTL);
6016 if (data & RLC_MEM_LS_EN) {
6017 data &= ~RLC_MEM_LS_EN;
6018 WREG32(RLC_MEM_SLP_CNTL, data);
6021 data = RREG32(CP_MEM_SLP_CNTL);
6022 if (data & CP_MEM_LS_EN) {
6023 data &= ~CP_MEM_LS_EN;
6024 WREG32(CP_MEM_SLP_CNTL, data);
6027 orig = data = RREG32(CGTS_SM_CTRL_REG);
6028 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6030 WREG32(CGTS_SM_CTRL_REG, data);
6032 tmp = cik_halt_rlc(rdev);
6034 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6035 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6036 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6037 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6038 WREG32(RLC_SERDES_WR_CTRL, data);
6040 cik_update_rlc(rdev, tmp);
6044 static const u32 mc_cg_registers[] =
6057 static void cik_enable_mc_ls(struct radeon_device *rdev,
6063 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6064 orig = data = RREG32(mc_cg_registers[i]);
6065 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
6066 data |= MC_LS_ENABLE;
6068 data &= ~MC_LS_ENABLE;
6070 WREG32(mc_cg_registers[i], data);
6074 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6080 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6081 orig = data = RREG32(mc_cg_registers[i]);
6082 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
6083 data |= MC_CG_ENABLE;
6085 data &= ~MC_CG_ENABLE;
6087 WREG32(mc_cg_registers[i], data);
6091 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6096 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
6097 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6098 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6100 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6103 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6105 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6108 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6112 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6117 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6118 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6121 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6123 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6126 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6128 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6131 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6133 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6136 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6140 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6145 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6146 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6148 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6150 orig = data = RREG32(UVD_CGC_CTRL);
6153 WREG32(UVD_CGC_CTRL, data);
6155 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6157 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6159 orig = data = RREG32(UVD_CGC_CTRL);
6162 WREG32(UVD_CGC_CTRL, data);
6166 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6171 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6173 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6174 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6175 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6177 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6178 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6181 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6184 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6189 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6191 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6192 data &= ~CLOCK_GATING_DIS;
6194 data |= CLOCK_GATING_DIS;
6197 WREG32(HDP_HOST_PATH_CNTL, data);
6200 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6205 orig = data = RREG32(HDP_MEM_POWER_LS);
6207 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6208 data |= HDP_LS_ENABLE;
6210 data &= ~HDP_LS_ENABLE;
6213 WREG32(HDP_MEM_POWER_LS, data);
6216 void cik_update_cg(struct radeon_device *rdev,
6217 u32 block, bool enable)
6220 if (block & RADEON_CG_BLOCK_GFX) {
6221 cik_enable_gui_idle_interrupt(rdev, false);
6222 /* order matters! */
6224 cik_enable_mgcg(rdev, true);
6225 cik_enable_cgcg(rdev, true);
6227 cik_enable_cgcg(rdev, false);
6228 cik_enable_mgcg(rdev, false);
6230 cik_enable_gui_idle_interrupt(rdev, true);
6233 if (block & RADEON_CG_BLOCK_MC) {
6234 if (!(rdev->flags & RADEON_IS_IGP)) {
6235 cik_enable_mc_mgcg(rdev, enable);
6236 cik_enable_mc_ls(rdev, enable);
6240 if (block & RADEON_CG_BLOCK_SDMA) {
6241 cik_enable_sdma_mgcg(rdev, enable);
6242 cik_enable_sdma_mgls(rdev, enable);
6245 if (block & RADEON_CG_BLOCK_BIF) {
6246 cik_enable_bif_mgls(rdev, enable);
6249 if (block & RADEON_CG_BLOCK_UVD) {
6251 cik_enable_uvd_mgcg(rdev, enable);
6254 if (block & RADEON_CG_BLOCK_HDP) {
6255 cik_enable_hdp_mgcg(rdev, enable);
6256 cik_enable_hdp_ls(rdev, enable);
6259 if (block & RADEON_CG_BLOCK_VCE) {
6260 vce_v2_0_enable_mgcg(rdev, enable);
6264 static void cik_init_cg(struct radeon_device *rdev)
6267 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6270 si_init_uvd_internal_cg(rdev);
6272 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6273 RADEON_CG_BLOCK_SDMA |
6274 RADEON_CG_BLOCK_BIF |
6275 RADEON_CG_BLOCK_UVD |
6276 RADEON_CG_BLOCK_HDP), true);
6279 static void cik_fini_cg(struct radeon_device *rdev)
6281 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6282 RADEON_CG_BLOCK_SDMA |
6283 RADEON_CG_BLOCK_BIF |
6284 RADEON_CG_BLOCK_UVD |
6285 RADEON_CG_BLOCK_HDP), false);
6287 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6290 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6295 orig = data = RREG32(RLC_PG_CNTL);
6296 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6297 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6299 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6301 WREG32(RLC_PG_CNTL, data);
6304 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6309 orig = data = RREG32(RLC_PG_CNTL);
6310 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6311 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6313 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6315 WREG32(RLC_PG_CNTL, data);
6318 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6322 orig = data = RREG32(RLC_PG_CNTL);
6323 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6324 data &= ~DISABLE_CP_PG;
6326 data |= DISABLE_CP_PG;
6328 WREG32(RLC_PG_CNTL, data);
6331 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6335 orig = data = RREG32(RLC_PG_CNTL);
6336 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6337 data &= ~DISABLE_GDS_PG;
6339 data |= DISABLE_GDS_PG;
6341 WREG32(RLC_PG_CNTL, data);
6344 #define CP_ME_TABLE_SIZE 96
6345 #define CP_ME_TABLE_OFFSET 2048
6346 #define CP_MEC_TABLE_OFFSET 4096
6348 void cik_init_cp_pg_table(struct radeon_device *rdev)
6350 const __be32 *fw_data;
6351 volatile u32 *dst_ptr;
6352 int me, i, max_me = 4;
6356 if (rdev->family == CHIP_KAVERI)
6359 if (rdev->rlc.cp_table_ptr == NULL)
6362 /* write the cp table buffer */
6363 dst_ptr = rdev->rlc.cp_table_ptr;
6364 for (me = 0; me < max_me; me++) {
6366 fw_data = (const __be32 *)rdev->ce_fw->data;
6367 table_offset = CP_ME_TABLE_OFFSET;
6368 } else if (me == 1) {
6369 fw_data = (const __be32 *)rdev->pfp_fw->data;
6370 table_offset = CP_ME_TABLE_OFFSET;
6371 } else if (me == 2) {
6372 fw_data = (const __be32 *)rdev->me_fw->data;
6373 table_offset = CP_ME_TABLE_OFFSET;
6375 fw_data = (const __be32 *)rdev->mec_fw->data;
6376 table_offset = CP_MEC_TABLE_OFFSET;
6379 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6380 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6382 bo_offset += CP_ME_TABLE_SIZE;
6386 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6391 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6392 orig = data = RREG32(RLC_PG_CNTL);
6393 data |= GFX_PG_ENABLE;
6395 WREG32(RLC_PG_CNTL, data);
6397 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6400 WREG32(RLC_AUTO_PG_CTRL, data);
6402 orig = data = RREG32(RLC_PG_CNTL);
6403 data &= ~GFX_PG_ENABLE;
6405 WREG32(RLC_PG_CNTL, data);
6407 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6408 data &= ~AUTO_PG_EN;
6410 WREG32(RLC_AUTO_PG_CTRL, data);
6412 data = RREG32(DB_RENDER_CONTROL);
6416 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6418 u32 mask = 0, tmp, tmp1;
6421 cik_select_se_sh(rdev, se, sh);
6422 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6423 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6424 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6431 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6436 return (~tmp) & mask;
6439 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6441 u32 i, j, k, active_cu_number = 0;
6442 u32 mask, counter, cu_bitmap;
6445 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6446 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6450 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6451 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6459 active_cu_number += counter;
6460 tmp |= (cu_bitmap << (i * 16 + j * 8));
6464 WREG32(RLC_PG_AO_CU_MASK, tmp);
6466 tmp = RREG32(RLC_MAX_PG_CU);
6467 tmp &= ~MAX_PU_CU_MASK;
6468 tmp |= MAX_PU_CU(active_cu_number);
6469 WREG32(RLC_MAX_PG_CU, tmp);
6472 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6477 orig = data = RREG32(RLC_PG_CNTL);
6478 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6479 data |= STATIC_PER_CU_PG_ENABLE;
6481 data &= ~STATIC_PER_CU_PG_ENABLE;
6483 WREG32(RLC_PG_CNTL, data);
6486 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6491 orig = data = RREG32(RLC_PG_CNTL);
6492 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6493 data |= DYN_PER_CU_PG_ENABLE;
6495 data &= ~DYN_PER_CU_PG_ENABLE;
6497 WREG32(RLC_PG_CNTL, data);
6500 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6501 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6503 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6508 if (rdev->rlc.cs_data) {
6509 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6510 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6511 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6512 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6514 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6515 for (i = 0; i < 3; i++)
6516 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6518 if (rdev->rlc.reg_list) {
6519 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6520 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6521 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6524 orig = data = RREG32(RLC_PG_CNTL);
6527 WREG32(RLC_PG_CNTL, data);
6529 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6530 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6532 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6533 data &= ~IDLE_POLL_COUNT_MASK;
6534 data |= IDLE_POLL_COUNT(0x60);
6535 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6538 WREG32(RLC_PG_DELAY, data);
6540 data = RREG32(RLC_PG_DELAY_2);
6543 WREG32(RLC_PG_DELAY_2, data);
6545 data = RREG32(RLC_AUTO_PG_CTRL);
6546 data &= ~GRBM_REG_SGIT_MASK;
6547 data |= GRBM_REG_SGIT(0x700);
6548 WREG32(RLC_AUTO_PG_CTRL, data);
6552 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6554 cik_enable_gfx_cgpg(rdev, enable);
6555 cik_enable_gfx_static_mgpg(rdev, enable);
6556 cik_enable_gfx_dynamic_mgpg(rdev, enable);
6559 u32 cik_get_csb_size(struct radeon_device *rdev)
6562 const struct cs_section_def *sect = NULL;
6563 const struct cs_extent_def *ext = NULL;
6565 if (rdev->rlc.cs_data == NULL)
6568 /* begin clear state */
6570 /* context control state */
6573 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6574 for (ext = sect->section; ext->extent != NULL; ++ext) {
6575 if (sect->id == SECT_CONTEXT)
6576 count += 2 + ext->reg_count;
6581 /* pa_sc_raster_config/pa_sc_raster_config1 */
6583 /* end clear state */
6591 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6594 const struct cs_section_def *sect = NULL;
6595 const struct cs_extent_def *ext = NULL;
6597 if (rdev->rlc.cs_data == NULL)
6602 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6603 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6605 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6606 buffer[count++] = cpu_to_le32(0x80000000);
6607 buffer[count++] = cpu_to_le32(0x80000000);
6609 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6610 for (ext = sect->section; ext->extent != NULL; ++ext) {
6611 if (sect->id == SECT_CONTEXT) {
6613 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6614 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6615 for (i = 0; i < ext->reg_count; i++)
6616 buffer[count++] = cpu_to_le32(ext->extent[i]);
6623 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6624 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6625 switch (rdev->family) {
6627 buffer[count++] = cpu_to_le32(0x16000012);
6628 buffer[count++] = cpu_to_le32(0x00000000);
6631 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6632 buffer[count++] = cpu_to_le32(0x00000000);
6636 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6637 buffer[count++] = cpu_to_le32(0x00000000);
6640 buffer[count++] = cpu_to_le32(0x3a00161a);
6641 buffer[count++] = cpu_to_le32(0x0000002e);
6644 buffer[count++] = cpu_to_le32(0x00000000);
6645 buffer[count++] = cpu_to_le32(0x00000000);
6649 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6650 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6652 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6653 buffer[count++] = cpu_to_le32(0);
6656 static void cik_init_pg(struct radeon_device *rdev)
6658 if (rdev->pg_flags) {
6659 cik_enable_sck_slowdown_on_pu(rdev, true);
6660 cik_enable_sck_slowdown_on_pd(rdev, true);
6661 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6662 cik_init_gfx_cgpg(rdev);
6663 cik_enable_cp_pg(rdev, true);
6664 cik_enable_gds_pg(rdev, true);
6666 cik_init_ao_cu_mask(rdev);
6667 cik_update_gfx_pg(rdev, true);
6671 static void cik_fini_pg(struct radeon_device *rdev)
6673 if (rdev->pg_flags) {
6674 cik_update_gfx_pg(rdev, false);
6675 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6676 cik_enable_cp_pg(rdev, false);
6677 cik_enable_gds_pg(rdev, false);
6684 * Starting with r6xx, interrupts are handled via a ring buffer.
6685 * Ring buffers are areas of GPU accessible memory that the GPU
6686 * writes interrupt vectors into and the host reads vectors out of.
6687 * There is a rptr (read pointer) that determines where the
6688 * host is currently reading, and a wptr (write pointer)
6689 * which determines where the GPU has written. When the
6690 * pointers are equal, the ring is idle. When the GPU
6691 * writes vectors to the ring buffer, it increments the
6692 * wptr. When there is an interrupt, the host then starts
6693 * fetching commands and processing them until the pointers are
6694 * equal again at which point it updates the rptr.
6698 * cik_enable_interrupts - Enable the interrupt ring buffer
6700 * @rdev: radeon_device pointer
6702 * Enable the interrupt ring buffer (CIK).
6704 static void cik_enable_interrupts(struct radeon_device *rdev)
6706 u32 ih_cntl = RREG32(IH_CNTL);
6707 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6709 ih_cntl |= ENABLE_INTR;
6710 ih_rb_cntl |= IH_RB_ENABLE;
6711 WREG32(IH_CNTL, ih_cntl);
6712 WREG32(IH_RB_CNTL, ih_rb_cntl);
6713 rdev->ih.enabled = true;
6717 * cik_disable_interrupts - Disable the interrupt ring buffer
6719 * @rdev: radeon_device pointer
6721 * Disable the interrupt ring buffer (CIK).
6723 static void cik_disable_interrupts(struct radeon_device *rdev)
6725 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6726 u32 ih_cntl = RREG32(IH_CNTL);
6728 ih_rb_cntl &= ~IH_RB_ENABLE;
6729 ih_cntl &= ~ENABLE_INTR;
6730 WREG32(IH_RB_CNTL, ih_rb_cntl);
6731 WREG32(IH_CNTL, ih_cntl);
6732 /* set rptr, wptr to 0 */
6733 WREG32(IH_RB_RPTR, 0);
6734 WREG32(IH_RB_WPTR, 0);
6735 rdev->ih.enabled = false;
6740 * cik_disable_interrupt_state - Disable all interrupt sources
6742 * @rdev: radeon_device pointer
6744 * Clear all interrupt enable bits used by the driver (CIK).
6746 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6751 tmp = RREG32(CP_INT_CNTL_RING0) &
6752 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6753 WREG32(CP_INT_CNTL_RING0, tmp);
6755 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6756 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6757 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6758 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6759 /* compute queues */
6760 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6761 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6762 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6763 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6764 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6765 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6766 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6767 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6769 WREG32(GRBM_INT_CNTL, 0);
6770 /* vline/vblank, etc. */
6771 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6772 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6773 if (rdev->num_crtc >= 4) {
6774 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6775 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6777 if (rdev->num_crtc >= 6) {
6778 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6779 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6782 if (rdev->num_crtc >= 2) {
6783 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6784 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6786 if (rdev->num_crtc >= 4) {
6787 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6788 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6790 if (rdev->num_crtc >= 6) {
6791 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6792 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6796 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6798 /* digital hotplug */
6799 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6800 WREG32(DC_HPD1_INT_CONTROL, tmp);
6801 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6802 WREG32(DC_HPD2_INT_CONTROL, tmp);
6803 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6804 WREG32(DC_HPD3_INT_CONTROL, tmp);
6805 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6806 WREG32(DC_HPD4_INT_CONTROL, tmp);
6807 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6808 WREG32(DC_HPD5_INT_CONTROL, tmp);
6809 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6810 WREG32(DC_HPD6_INT_CONTROL, tmp);
6815 * cik_irq_init - init and enable the interrupt ring
6817 * @rdev: radeon_device pointer
6819 * Allocate a ring buffer for the interrupt controller,
6820 * enable the RLC, disable interrupts, enable the IH
6821 * ring buffer and enable it (CIK).
6822 * Called at device load and reume.
6823 * Returns 0 for success, errors for failure.
6825 static int cik_irq_init(struct radeon_device *rdev)
6829 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6832 ret = r600_ih_ring_alloc(rdev);
6837 cik_disable_interrupts(rdev);
6840 ret = cik_rlc_resume(rdev);
6842 r600_ih_ring_fini(rdev);
6846 /* setup interrupt control */
6847 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6848 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6849 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6850 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6851 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6853 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6854 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6855 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6856 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6858 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6859 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6861 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6862 IH_WPTR_OVERFLOW_CLEAR |
6865 if (rdev->wb.enabled)
6866 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6868 /* set the writeback address whether it's enabled or not */
6869 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6870 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6872 WREG32(IH_RB_CNTL, ih_rb_cntl);
6874 /* set rptr, wptr to 0 */
6875 WREG32(IH_RB_RPTR, 0);
6876 WREG32(IH_RB_WPTR, 0);
6878 /* Default settings for IH_CNTL (disabled at first) */
6879 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6880 /* RPTR_REARM only works if msi's are enabled */
6881 if (rdev->msi_enabled)
6882 ih_cntl |= RPTR_REARM;
6883 WREG32(IH_CNTL, ih_cntl);
6885 /* force the active interrupt state to all disabled */
6886 cik_disable_interrupt_state(rdev);
6888 pci_set_master(rdev->pdev);
6891 cik_enable_interrupts(rdev);
6897 * cik_irq_set - enable/disable interrupt sources
6899 * @rdev: radeon_device pointer
6901 * Enable interrupt sources on the GPU (vblanks, hpd,
6903 * Returns 0 for success, errors for failure.
6905 int cik_irq_set(struct radeon_device *rdev)
6908 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6909 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6910 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6911 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6912 u32 grbm_int_cntl = 0;
6913 u32 dma_cntl, dma_cntl1;
6916 if (!rdev->irq.installed) {
6917 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6920 /* don't enable anything if the ih is disabled */
6921 if (!rdev->ih.enabled) {
6922 cik_disable_interrupts(rdev);
6923 /* force the active interrupt state to all disabled */
6924 cik_disable_interrupt_state(rdev);
6928 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6929 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6930 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6932 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6933 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6934 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6935 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6936 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6937 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6939 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6940 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6942 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6943 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6944 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6945 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6946 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6947 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6948 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6949 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6951 if (rdev->flags & RADEON_IS_IGP)
6952 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6953 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6955 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6956 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6958 /* enable CP interrupts on all rings */
6959 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6960 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6961 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6963 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6964 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6965 DRM_DEBUG("si_irq_set: sw int cp1\n");
6966 if (ring->me == 1) {
6967 switch (ring->pipe) {
6969 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6972 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6975 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6978 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6981 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6984 } else if (ring->me == 2) {
6985 switch (ring->pipe) {
6987 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6990 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6993 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6996 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6999 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7003 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7006 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7007 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7008 DRM_DEBUG("si_irq_set: sw int cp2\n");
7009 if (ring->me == 1) {
7010 switch (ring->pipe) {
7012 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7015 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7018 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7021 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7024 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7027 } else if (ring->me == 2) {
7028 switch (ring->pipe) {
7030 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7033 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7036 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7039 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7042 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7046 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7050 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7051 DRM_DEBUG("cik_irq_set: sw int dma\n");
7052 dma_cntl |= TRAP_ENABLE;
7055 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7056 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7057 dma_cntl1 |= TRAP_ENABLE;
7060 if (rdev->irq.crtc_vblank_int[0] ||
7061 atomic_read(&rdev->irq.pflip[0])) {
7062 DRM_DEBUG("cik_irq_set: vblank 0\n");
7063 crtc1 |= VBLANK_INTERRUPT_MASK;
7065 if (rdev->irq.crtc_vblank_int[1] ||
7066 atomic_read(&rdev->irq.pflip[1])) {
7067 DRM_DEBUG("cik_irq_set: vblank 1\n");
7068 crtc2 |= VBLANK_INTERRUPT_MASK;
7070 if (rdev->irq.crtc_vblank_int[2] ||
7071 atomic_read(&rdev->irq.pflip[2])) {
7072 DRM_DEBUG("cik_irq_set: vblank 2\n");
7073 crtc3 |= VBLANK_INTERRUPT_MASK;
7075 if (rdev->irq.crtc_vblank_int[3] ||
7076 atomic_read(&rdev->irq.pflip[3])) {
7077 DRM_DEBUG("cik_irq_set: vblank 3\n");
7078 crtc4 |= VBLANK_INTERRUPT_MASK;
7080 if (rdev->irq.crtc_vblank_int[4] ||
7081 atomic_read(&rdev->irq.pflip[4])) {
7082 DRM_DEBUG("cik_irq_set: vblank 4\n");
7083 crtc5 |= VBLANK_INTERRUPT_MASK;
7085 if (rdev->irq.crtc_vblank_int[5] ||
7086 atomic_read(&rdev->irq.pflip[5])) {
7087 DRM_DEBUG("cik_irq_set: vblank 5\n");
7088 crtc6 |= VBLANK_INTERRUPT_MASK;
7090 if (rdev->irq.hpd[0]) {
7091 DRM_DEBUG("cik_irq_set: hpd 1\n");
7092 hpd1 |= DC_HPDx_INT_EN;
7094 if (rdev->irq.hpd[1]) {
7095 DRM_DEBUG("cik_irq_set: hpd 2\n");
7096 hpd2 |= DC_HPDx_INT_EN;
7098 if (rdev->irq.hpd[2]) {
7099 DRM_DEBUG("cik_irq_set: hpd 3\n");
7100 hpd3 |= DC_HPDx_INT_EN;
7102 if (rdev->irq.hpd[3]) {
7103 DRM_DEBUG("cik_irq_set: hpd 4\n");
7104 hpd4 |= DC_HPDx_INT_EN;
7106 if (rdev->irq.hpd[4]) {
7107 DRM_DEBUG("cik_irq_set: hpd 5\n");
7108 hpd5 |= DC_HPDx_INT_EN;
7110 if (rdev->irq.hpd[5]) {
7111 DRM_DEBUG("cik_irq_set: hpd 6\n");
7112 hpd6 |= DC_HPDx_INT_EN;
7115 if (rdev->irq.dpm_thermal) {
7116 DRM_DEBUG("dpm thermal\n");
7117 if (rdev->flags & RADEON_IS_IGP)
7118 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7120 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7123 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7125 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7126 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7128 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7129 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7130 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7131 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7132 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7133 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7134 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7135 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7137 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7139 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7140 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7141 if (rdev->num_crtc >= 4) {
7142 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7143 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7145 if (rdev->num_crtc >= 6) {
7146 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7147 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7150 if (rdev->num_crtc >= 2) {
7151 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7152 GRPH_PFLIP_INT_MASK);
7153 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7154 GRPH_PFLIP_INT_MASK);
7156 if (rdev->num_crtc >= 4) {
7157 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7158 GRPH_PFLIP_INT_MASK);
7159 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7160 GRPH_PFLIP_INT_MASK);
7162 if (rdev->num_crtc >= 6) {
7163 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7164 GRPH_PFLIP_INT_MASK);
7165 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7166 GRPH_PFLIP_INT_MASK);
7169 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7170 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7171 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7172 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7173 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7174 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7176 if (rdev->flags & RADEON_IS_IGP)
7177 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7179 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7185 * cik_irq_ack - ack interrupt sources
7187 * @rdev: radeon_device pointer
7189 * Ack interrupt sources on the GPU (vblanks, hpd,
7190 * etc.) (CIK). Certain interrupts sources are sw
7191 * generated and do not require an explicit ack.
7193 static inline void cik_irq_ack(struct radeon_device *rdev)
7197 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7198 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7199 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7200 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7201 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7202 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7203 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7205 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7206 EVERGREEN_CRTC0_REGISTER_OFFSET);
7207 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7208 EVERGREEN_CRTC1_REGISTER_OFFSET);
7209 if (rdev->num_crtc >= 4) {
7210 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7211 EVERGREEN_CRTC2_REGISTER_OFFSET);
7212 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7213 EVERGREEN_CRTC3_REGISTER_OFFSET);
7215 if (rdev->num_crtc >= 6) {
7216 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7217 EVERGREEN_CRTC4_REGISTER_OFFSET);
7218 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7219 EVERGREEN_CRTC5_REGISTER_OFFSET);
7222 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7223 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7224 GRPH_PFLIP_INT_CLEAR);
7225 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7226 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7227 GRPH_PFLIP_INT_CLEAR);
7228 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7229 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7230 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7231 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7232 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7233 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7234 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7235 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7237 if (rdev->num_crtc >= 4) {
7238 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7239 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7240 GRPH_PFLIP_INT_CLEAR);
7241 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7242 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7243 GRPH_PFLIP_INT_CLEAR);
7244 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7245 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7246 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7247 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7248 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7249 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7250 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7251 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7254 if (rdev->num_crtc >= 6) {
7255 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7256 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7257 GRPH_PFLIP_INT_CLEAR);
7258 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7259 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7260 GRPH_PFLIP_INT_CLEAR);
7261 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7262 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7263 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7264 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7265 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7266 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7267 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7268 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7271 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7272 tmp = RREG32(DC_HPD1_INT_CONTROL);
7273 tmp |= DC_HPDx_INT_ACK;
7274 WREG32(DC_HPD1_INT_CONTROL, tmp);
7276 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7277 tmp = RREG32(DC_HPD2_INT_CONTROL);
7278 tmp |= DC_HPDx_INT_ACK;
7279 WREG32(DC_HPD2_INT_CONTROL, tmp);
7281 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7282 tmp = RREG32(DC_HPD3_INT_CONTROL);
7283 tmp |= DC_HPDx_INT_ACK;
7284 WREG32(DC_HPD3_INT_CONTROL, tmp);
7286 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7287 tmp = RREG32(DC_HPD4_INT_CONTROL);
7288 tmp |= DC_HPDx_INT_ACK;
7289 WREG32(DC_HPD4_INT_CONTROL, tmp);
7291 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7292 tmp = RREG32(DC_HPD5_INT_CONTROL);
7293 tmp |= DC_HPDx_INT_ACK;
7294 WREG32(DC_HPD5_INT_CONTROL, tmp);
7296 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7297 tmp = RREG32(DC_HPD5_INT_CONTROL);
7298 tmp |= DC_HPDx_INT_ACK;
7299 WREG32(DC_HPD6_INT_CONTROL, tmp);
7304 * cik_irq_disable - disable interrupts
7306 * @rdev: radeon_device pointer
7308 * Disable interrupts on the hw (CIK).
7310 static void cik_irq_disable(struct radeon_device *rdev)
7312 cik_disable_interrupts(rdev);
7313 /* Wait and acknowledge irq */
7316 cik_disable_interrupt_state(rdev);
7320 * cik_irq_disable - disable interrupts for suspend
7322 * @rdev: radeon_device pointer
7324 * Disable interrupts and stop the RLC (CIK).
7327 static void cik_irq_suspend(struct radeon_device *rdev)
7329 cik_irq_disable(rdev);
7334 * cik_irq_fini - tear down interrupt support
7336 * @rdev: radeon_device pointer
7338 * Disable interrupts on the hw and free the IH ring
7340 * Used for driver unload.
7342 static void cik_irq_fini(struct radeon_device *rdev)
7344 cik_irq_suspend(rdev);
7345 r600_ih_ring_fini(rdev);
7349 * cik_get_ih_wptr - get the IH ring buffer wptr
7351 * @rdev: radeon_device pointer
7353 * Get the IH ring buffer wptr from either the register
7354 * or the writeback memory buffer (CIK). Also check for
7355 * ring buffer overflow and deal with it.
7356 * Used by cik_irq_process().
7357 * Returns the value of the wptr.
7359 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7363 if (rdev->wb.enabled)
7364 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7366 wptr = RREG32(IH_RB_WPTR);
7368 if (wptr & RB_OVERFLOW) {
7369 /* When a ring buffer overflow happen start parsing interrupt
7370 * from the last not overwritten vector (wptr + 16). Hopefully
7371 * this should allow us to catchup.
7373 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7374 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7375 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7376 tmp = RREG32(IH_RB_CNTL);
7377 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7378 WREG32(IH_RB_CNTL, tmp);
7380 return (wptr & rdev->ih.ptr_mask);
7384 * Each IV ring entry is 128 bits:
7385 * [7:0] - interrupt source id
7387 * [59:32] - interrupt source data
7388 * [63:60] - reserved
7391 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7392 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7393 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7394 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7395 * PIPE_ID - ME0 0=3D
7396 * - ME1&2 compute dispatcher (4 pipes each)
7398 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7399 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7400 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7403 * [127:96] - reserved
7406 * cik_irq_process - interrupt handler
7408 * @rdev: radeon_device pointer
7410 * Interrupt hander (CIK). Walk the IH ring,
7411 * ack interrupts and schedule work to handle
7413 * Returns irq process return code.
7415 int cik_irq_process(struct radeon_device *rdev)
7417 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7418 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7421 u32 src_id, src_data, ring_id;
7422 u8 me_id, pipe_id, queue_id;
7424 bool queue_hotplug = false;
7425 bool queue_reset = false;
7426 u32 addr, status, mc_client;
7427 bool queue_thermal = false;
7429 if (!rdev->ih.enabled || rdev->shutdown)
7432 wptr = cik_get_ih_wptr(rdev);
7435 /* is somebody else already processing irqs? */
7436 if (atomic_xchg(&rdev->ih.lock, 1))
7439 rptr = rdev->ih.rptr;
7440 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7442 /* Order reading of wptr vs. reading of IH ring data */
7445 /* display interrupts */
7448 while (rptr != wptr) {
7449 /* wptr/rptr are in bytes! */
7450 ring_index = rptr / 4;
7451 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7452 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7453 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7456 case 1: /* D1 vblank/vline */
7458 case 0: /* D1 vblank */
7459 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7460 if (rdev->irq.crtc_vblank_int[0]) {
7461 drm_handle_vblank(rdev->ddev, 0);
7462 rdev->pm.vblank_sync = true;
7463 wake_up(&rdev->irq.vblank_queue);
7465 if (atomic_read(&rdev->irq.pflip[0]))
7466 radeon_crtc_handle_vblank(rdev, 0);
7467 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7468 DRM_DEBUG("IH: D1 vblank\n");
7471 case 1: /* D1 vline */
7472 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7473 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7474 DRM_DEBUG("IH: D1 vline\n");
7478 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7482 case 2: /* D2 vblank/vline */
7484 case 0: /* D2 vblank */
7485 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7486 if (rdev->irq.crtc_vblank_int[1]) {
7487 drm_handle_vblank(rdev->ddev, 1);
7488 rdev->pm.vblank_sync = true;
7489 wake_up(&rdev->irq.vblank_queue);
7491 if (atomic_read(&rdev->irq.pflip[1]))
7492 radeon_crtc_handle_vblank(rdev, 1);
7493 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7494 DRM_DEBUG("IH: D2 vblank\n");
7497 case 1: /* D2 vline */
7498 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7499 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7500 DRM_DEBUG("IH: D2 vline\n");
7504 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7508 case 3: /* D3 vblank/vline */
7510 case 0: /* D3 vblank */
7511 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7512 if (rdev->irq.crtc_vblank_int[2]) {
7513 drm_handle_vblank(rdev->ddev, 2);
7514 rdev->pm.vblank_sync = true;
7515 wake_up(&rdev->irq.vblank_queue);
7517 if (atomic_read(&rdev->irq.pflip[2]))
7518 radeon_crtc_handle_vblank(rdev, 2);
7519 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7520 DRM_DEBUG("IH: D3 vblank\n");
7523 case 1: /* D3 vline */
7524 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7525 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7526 DRM_DEBUG("IH: D3 vline\n");
7530 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7534 case 4: /* D4 vblank/vline */
7536 case 0: /* D4 vblank */
7537 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7538 if (rdev->irq.crtc_vblank_int[3]) {
7539 drm_handle_vblank(rdev->ddev, 3);
7540 rdev->pm.vblank_sync = true;
7541 wake_up(&rdev->irq.vblank_queue);
7543 if (atomic_read(&rdev->irq.pflip[3]))
7544 radeon_crtc_handle_vblank(rdev, 3);
7545 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7546 DRM_DEBUG("IH: D4 vblank\n");
7549 case 1: /* D4 vline */
7550 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7551 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7552 DRM_DEBUG("IH: D4 vline\n");
7556 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7560 case 5: /* D5 vblank/vline */
7562 case 0: /* D5 vblank */
7563 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7564 if (rdev->irq.crtc_vblank_int[4]) {
7565 drm_handle_vblank(rdev->ddev, 4);
7566 rdev->pm.vblank_sync = true;
7567 wake_up(&rdev->irq.vblank_queue);
7569 if (atomic_read(&rdev->irq.pflip[4]))
7570 radeon_crtc_handle_vblank(rdev, 4);
7571 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7572 DRM_DEBUG("IH: D5 vblank\n");
7575 case 1: /* D5 vline */
7576 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7577 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7578 DRM_DEBUG("IH: D5 vline\n");
7582 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7586 case 6: /* D6 vblank/vline */
7588 case 0: /* D6 vblank */
7589 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7590 if (rdev->irq.crtc_vblank_int[5]) {
7591 drm_handle_vblank(rdev->ddev, 5);
7592 rdev->pm.vblank_sync = true;
7593 wake_up(&rdev->irq.vblank_queue);
7595 if (atomic_read(&rdev->irq.pflip[5]))
7596 radeon_crtc_handle_vblank(rdev, 5);
7597 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7598 DRM_DEBUG("IH: D6 vblank\n");
7601 case 1: /* D6 vline */
7602 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7603 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7604 DRM_DEBUG("IH: D6 vline\n");
7608 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7612 case 8: /* D1 page flip */
7613 case 10: /* D2 page flip */
7614 case 12: /* D3 page flip */
7615 case 14: /* D4 page flip */
7616 case 16: /* D5 page flip */
7617 case 18: /* D6 page flip */
7618 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7619 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7621 case 42: /* HPD hotplug */
7624 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7625 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7626 queue_hotplug = true;
7627 DRM_DEBUG("IH: HPD1\n");
7631 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7632 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7633 queue_hotplug = true;
7634 DRM_DEBUG("IH: HPD2\n");
7638 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7639 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7640 queue_hotplug = true;
7641 DRM_DEBUG("IH: HPD3\n");
7645 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7646 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7647 queue_hotplug = true;
7648 DRM_DEBUG("IH: HPD4\n");
7652 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7653 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7654 queue_hotplug = true;
7655 DRM_DEBUG("IH: HPD5\n");
7659 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7660 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7661 queue_hotplug = true;
7662 DRM_DEBUG("IH: HPD6\n");
7666 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7671 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7672 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7676 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7677 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7678 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7679 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7680 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
7682 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7684 cik_vm_decode_fault(rdev, status, addr, mc_client);
7685 /* reset addr and status */
7686 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7689 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7692 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7695 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7698 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7702 case 176: /* GFX RB CP_INT */
7703 case 177: /* GFX IB CP_INT */
7704 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7706 case 181: /* CP EOP event */
7707 DRM_DEBUG("IH: CP EOP\n");
7708 /* XXX check the bitfield order! */
7709 me_id = (ring_id & 0x60) >> 5;
7710 pipe_id = (ring_id & 0x18) >> 3;
7711 queue_id = (ring_id & 0x7) >> 0;
7714 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7718 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7719 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7720 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7721 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7725 case 184: /* CP Privileged reg access */
7726 DRM_ERROR("Illegal register access in command stream\n");
7727 /* XXX check the bitfield order! */
7728 me_id = (ring_id & 0x60) >> 5;
7729 pipe_id = (ring_id & 0x18) >> 3;
7730 queue_id = (ring_id & 0x7) >> 0;
7733 /* This results in a full GPU reset, but all we need to do is soft
7734 * reset the CP for gfx
7748 case 185: /* CP Privileged inst */
7749 DRM_ERROR("Illegal instruction in command stream\n");
7750 /* XXX check the bitfield order! */
7751 me_id = (ring_id & 0x60) >> 5;
7752 pipe_id = (ring_id & 0x18) >> 3;
7753 queue_id = (ring_id & 0x7) >> 0;
7756 /* This results in a full GPU reset, but all we need to do is soft
7757 * reset the CP for gfx
7771 case 224: /* SDMA trap event */
7772 /* XXX check the bitfield order! */
7773 me_id = (ring_id & 0x3) >> 0;
7774 queue_id = (ring_id & 0xc) >> 2;
7775 DRM_DEBUG("IH: SDMA trap\n");
7780 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7793 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7805 case 230: /* thermal low to high */
7806 DRM_DEBUG("IH: thermal low to high\n");
7807 rdev->pm.dpm.thermal.high_to_low = false;
7808 queue_thermal = true;
7810 case 231: /* thermal high to low */
7811 DRM_DEBUG("IH: thermal high to low\n");
7812 rdev->pm.dpm.thermal.high_to_low = true;
7813 queue_thermal = true;
7815 case 233: /* GUI IDLE */
7816 DRM_DEBUG("IH: GUI idle\n");
7818 case 241: /* SDMA Privileged inst */
7819 case 247: /* SDMA Privileged inst */
7820 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7821 /* XXX check the bitfield order! */
7822 me_id = (ring_id & 0x3) >> 0;
7823 queue_id = (ring_id & 0xc) >> 2;
7858 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7862 /* wptr/rptr are in bytes! */
7864 rptr &= rdev->ih.ptr_mask;
7867 schedule_work(&rdev->hotplug_work);
7869 schedule_work(&rdev->reset_work);
7871 schedule_work(&rdev->pm.dpm.thermal.work);
7872 rdev->ih.rptr = rptr;
7873 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7874 atomic_set(&rdev->ih.lock, 0);
7876 /* make sure wptr hasn't changed while processing */
7877 wptr = cik_get_ih_wptr(rdev);
7885 * startup/shutdown callbacks
7888 * cik_startup - program the asic to a functional state
7890 * @rdev: radeon_device pointer
7892 * Programs the asic to a functional state (CIK).
7893 * Called by cik_init() and cik_resume().
7894 * Returns 0 for success, error for failure.
7896 static int cik_startup(struct radeon_device *rdev)
7898 struct radeon_ring *ring;
7901 /* enable pcie gen2/3 link */
7902 cik_pcie_gen3_enable(rdev);
7904 cik_program_aspm(rdev);
7906 /* scratch needs to be initialized before MC */
7907 r = r600_vram_scratch_init(rdev);
7911 cik_mc_program(rdev);
7913 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7914 r = ci_mc_load_microcode(rdev);
7916 DRM_ERROR("Failed to load MC firmware!\n");
7921 r = cik_pcie_gart_enable(rdev);
7926 /* allocate rlc buffers */
7927 if (rdev->flags & RADEON_IS_IGP) {
7928 if (rdev->family == CHIP_KAVERI) {
7929 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7930 rdev->rlc.reg_list_size =
7931 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7933 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7934 rdev->rlc.reg_list_size =
7935 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7938 rdev->rlc.cs_data = ci_cs_data;
7939 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7940 r = sumo_rlc_init(rdev);
7942 DRM_ERROR("Failed to init rlc BOs!\n");
7946 /* allocate wb buffer */
7947 r = radeon_wb_init(rdev);
7951 /* allocate mec buffers */
7952 r = cik_mec_init(rdev);
7954 DRM_ERROR("Failed to init MEC BOs!\n");
7958 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7960 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7964 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7966 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7970 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7972 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7976 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7978 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7982 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7984 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7988 r = radeon_uvd_resume(rdev);
7990 r = uvd_v4_2_resume(rdev);
7992 r = radeon_fence_driver_start_ring(rdev,
7993 R600_RING_TYPE_UVD_INDEX);
7995 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7999 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8001 r = radeon_vce_resume(rdev);
8003 r = vce_v2_0_resume(rdev);
8005 r = radeon_fence_driver_start_ring(rdev,
8006 TN_RING_TYPE_VCE1_INDEX);
8008 r = radeon_fence_driver_start_ring(rdev,
8009 TN_RING_TYPE_VCE2_INDEX);
8012 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8013 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8014 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8018 if (!rdev->irq.installed) {
8019 r = radeon_irq_kms_init(rdev);
8024 r = cik_irq_init(rdev);
8026 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8027 radeon_irq_kms_fini(rdev);
8032 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8033 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
8034 PACKET3(PACKET3_NOP, 0x3FFF));
8038 /* set up the compute queues */
8039 /* type-2 packets are deprecated on MEC, use type-3 instead */
8040 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8041 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
8042 PACKET3(PACKET3_NOP, 0x3FFF));
8045 ring->me = 1; /* first MEC */
8046 ring->pipe = 0; /* first pipe */
8047 ring->queue = 0; /* first queue */
8048 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8050 /* type-2 packets are deprecated on MEC, use type-3 instead */
8051 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8052 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
8053 PACKET3(PACKET3_NOP, 0x3FFF));
8056 /* dGPU only have 1 MEC */
8057 ring->me = 1; /* first MEC */
8058 ring->pipe = 0; /* first pipe */
8059 ring->queue = 1; /* second queue */
8060 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8062 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8063 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
8064 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8068 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8069 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
8070 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
8074 r = cik_cp_resume(rdev);
8078 r = cik_sdma_resume(rdev);
8082 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8083 if (ring->ring_size) {
8084 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8087 r = uvd_v1_0_init(rdev);
8089 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8094 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8095 if (ring->ring_size)
8096 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8099 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8100 if (ring->ring_size)
8101 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8105 r = vce_v1_0_init(rdev);
8106 else if (r != -ENOENT)
8107 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8109 r = radeon_ib_pool_init(rdev);
8111 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8115 r = radeon_vm_manager_init(rdev);
8117 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8121 r = dce6_audio_init(rdev);
8129 * cik_resume - resume the asic to a functional state
8131 * @rdev: radeon_device pointer
8133 * Programs the asic to a functional state (CIK).
8135 * Returns 0 for success, error for failure.
8137 int cik_resume(struct radeon_device *rdev)
8142 atom_asic_init(rdev->mode_info.atom_context);
8144 /* init golden registers */
8145 cik_init_golden_registers(rdev);
8147 if (rdev->pm.pm_method == PM_METHOD_DPM)
8148 radeon_pm_resume(rdev);
8150 rdev->accel_working = true;
8151 r = cik_startup(rdev);
8153 DRM_ERROR("cik startup failed on resume\n");
8154 rdev->accel_working = false;
8163 * cik_suspend - suspend the asic
8165 * @rdev: radeon_device pointer
8167 * Bring the chip into a state suitable for suspend (CIK).
8168 * Called at suspend.
8169 * Returns 0 for success.
8171 int cik_suspend(struct radeon_device *rdev)
8173 radeon_pm_suspend(rdev);
8174 dce6_audio_fini(rdev);
8175 radeon_vm_manager_fini(rdev);
8176 cik_cp_enable(rdev, false);
8177 cik_sdma_enable(rdev, false);
8178 uvd_v1_0_fini(rdev);
8179 radeon_uvd_suspend(rdev);
8180 radeon_vce_suspend(rdev);
8183 cik_irq_suspend(rdev);
8184 radeon_wb_disable(rdev);
8185 cik_pcie_gart_disable(rdev);
8189 /* Plan is to move initialization in that function and use
8190 * helper function so that radeon_device_init pretty much
8191 * do nothing more than calling asic specific function. This
8192 * should also allow to remove a bunch of callback function
8196 * cik_init - asic specific driver and hw init
8198 * @rdev: radeon_device pointer
8200 * Setup asic specific driver variables and program the hw
8201 * to a functional state (CIK).
8202 * Called at driver startup.
8203 * Returns 0 for success, errors for failure.
8205 int cik_init(struct radeon_device *rdev)
8207 struct radeon_ring *ring;
8211 if (!radeon_get_bios(rdev)) {
8212 if (ASIC_IS_AVIVO(rdev))
8215 /* Must be an ATOMBIOS */
8216 if (!rdev->is_atom_bios) {
8217 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8220 r = radeon_atombios_init(rdev);
8224 /* Post card if necessary */
8225 if (!radeon_card_posted(rdev)) {
8227 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8230 DRM_INFO("GPU not posted. posting now...\n");
8231 atom_asic_init(rdev->mode_info.atom_context);
8233 /* init golden registers */
8234 cik_init_golden_registers(rdev);
8235 /* Initialize scratch registers */
8236 cik_scratch_init(rdev);
8237 /* Initialize surface registers */
8238 radeon_surface_init(rdev);
8239 /* Initialize clocks */
8240 radeon_get_clock_info(rdev->ddev);
8243 r = radeon_fence_driver_init(rdev);
8247 /* initialize memory controller */
8248 r = cik_mc_init(rdev);
8251 /* Memory manager */
8252 r = radeon_bo_init(rdev);
8256 if (rdev->flags & RADEON_IS_IGP) {
8257 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8258 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8259 r = cik_init_microcode(rdev);
8261 DRM_ERROR("Failed to load firmware!\n");
8266 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8267 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8269 r = cik_init_microcode(rdev);
8271 DRM_ERROR("Failed to load firmware!\n");
8277 /* Initialize power management */
8278 radeon_pm_init(rdev);
8280 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8281 ring->ring_obj = NULL;
8282 r600_ring_init(rdev, ring, 1024 * 1024);
8284 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8285 ring->ring_obj = NULL;
8286 r600_ring_init(rdev, ring, 1024 * 1024);
8287 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8291 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8292 ring->ring_obj = NULL;
8293 r600_ring_init(rdev, ring, 1024 * 1024);
8294 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8298 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8299 ring->ring_obj = NULL;
8300 r600_ring_init(rdev, ring, 256 * 1024);
8302 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8303 ring->ring_obj = NULL;
8304 r600_ring_init(rdev, ring, 256 * 1024);
8306 r = radeon_uvd_init(rdev);
8308 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8309 ring->ring_obj = NULL;
8310 r600_ring_init(rdev, ring, 4096);
8313 r = radeon_vce_init(rdev);
8315 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8316 ring->ring_obj = NULL;
8317 r600_ring_init(rdev, ring, 4096);
8319 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8320 ring->ring_obj = NULL;
8321 r600_ring_init(rdev, ring, 4096);
8324 rdev->ih.ring_obj = NULL;
8325 r600_ih_ring_init(rdev, 64 * 1024);
8327 r = r600_pcie_gart_init(rdev);
8331 rdev->accel_working = true;
8332 r = cik_startup(rdev);
8334 dev_err(rdev->dev, "disabling GPU acceleration\n");
8336 cik_sdma_fini(rdev);
8338 sumo_rlc_fini(rdev);
8340 radeon_wb_fini(rdev);
8341 radeon_ib_pool_fini(rdev);
8342 radeon_vm_manager_fini(rdev);
8343 radeon_irq_kms_fini(rdev);
8344 cik_pcie_gart_fini(rdev);
8345 rdev->accel_working = false;
8348 /* Don't start up if the MC ucode is missing.
8349 * The default clocks and voltages before the MC ucode
8350 * is loaded are not suffient for advanced operations.
8352 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8353 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8361 * cik_fini - asic specific driver and hw fini
8363 * @rdev: radeon_device pointer
8365 * Tear down the asic specific driver variables and program the hw
8366 * to an idle state (CIK).
8367 * Called at driver unload.
8369 void cik_fini(struct radeon_device *rdev)
8371 radeon_pm_fini(rdev);
8373 cik_sdma_fini(rdev);
8377 sumo_rlc_fini(rdev);
8379 radeon_wb_fini(rdev);
8380 radeon_vm_manager_fini(rdev);
8381 radeon_ib_pool_fini(rdev);
8382 radeon_irq_kms_fini(rdev);
8383 uvd_v1_0_fini(rdev);
8384 radeon_uvd_fini(rdev);
8385 radeon_vce_fini(rdev);
8386 cik_pcie_gart_fini(rdev);
8387 r600_vram_scratch_fini(rdev);
8388 radeon_gem_fini(rdev);
8389 radeon_fence_driver_fini(rdev);
8390 radeon_bo_fini(rdev);
8391 radeon_atombios_fini(rdev);
8396 void dce8_program_fmt(struct drm_encoder *encoder)
8398 struct drm_device *dev = encoder->dev;
8399 struct radeon_device *rdev = dev->dev_private;
8400 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8401 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8402 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8405 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8408 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8409 bpc = radeon_get_monitor_bpc(connector);
8410 dither = radeon_connector->dither;
8413 /* LVDS/eDP FMT is set up by atom */
8414 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8417 /* not needed for analog */
8418 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8419 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8427 if (dither == RADEON_FMT_DITHER_ENABLE)
8428 /* XXX sort out optimal dither settings */
8429 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8430 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8432 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8435 if (dither == RADEON_FMT_DITHER_ENABLE)
8436 /* XXX sort out optimal dither settings */
8437 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8438 FMT_RGB_RANDOM_ENABLE |
8439 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8441 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8444 if (dither == RADEON_FMT_DITHER_ENABLE)
8445 /* XXX sort out optimal dither settings */
8446 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8447 FMT_RGB_RANDOM_ENABLE |
8448 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8450 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8457 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8460 /* display watermark setup */
8462 * dce8_line_buffer_adjust - Set up the line buffer
8464 * @rdev: radeon_device pointer
8465 * @radeon_crtc: the selected display controller
8466 * @mode: the current display mode on the selected display
8469 * Setup up the line buffer allocation for
8470 * the selected display controller (CIK).
8471 * Returns the line buffer size in pixels.
8473 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8474 struct radeon_crtc *radeon_crtc,
8475 struct drm_display_mode *mode)
8477 u32 tmp, buffer_alloc, i;
8478 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8481 * There are 6 line buffers, one for each display controllers.
8482 * There are 3 partitions per LB. Select the number of partitions
8483 * to enable based on the display width. For display widths larger
8484 * than 4096, you need use to use 2 display controllers and combine
8485 * them using the stereo blender.
8487 if (radeon_crtc->base.enabled && mode) {
8488 if (mode->crtc_hdisplay < 1920) {
8491 } else if (mode->crtc_hdisplay < 2560) {
8494 } else if (mode->crtc_hdisplay < 4096) {
8496 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8498 DRM_DEBUG_KMS("Mode too big for LB!\n");
8500 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8507 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8508 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8510 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8511 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8512 for (i = 0; i < rdev->usec_timeout; i++) {
8513 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8514 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8519 if (radeon_crtc->base.enabled && mode) {
8531 /* controller not enabled, so no lb used */
8536 * cik_get_number_of_dram_channels - get the number of dram channels
8538 * @rdev: radeon_device pointer
8540 * Look up the number of video ram channels (CIK).
8541 * Used for display watermark bandwidth calculations
8542 * Returns the number of dram channels
8544 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8546 u32 tmp = RREG32(MC_SHARED_CHMAP);
8548 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8571 struct dce8_wm_params {
8572 u32 dram_channels; /* number of dram channels */
8573 u32 yclk; /* bandwidth per dram data pin in kHz */
8574 u32 sclk; /* engine clock in kHz */
8575 u32 disp_clk; /* display clock in kHz */
8576 u32 src_width; /* viewport width */
8577 u32 active_time; /* active display time in ns */
8578 u32 blank_time; /* blank time in ns */
8579 bool interlaced; /* mode is interlaced */
8580 fixed20_12 vsc; /* vertical scale ratio */
8581 u32 num_heads; /* number of active crtcs */
8582 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8583 u32 lb_size; /* line buffer allocated to pipe */
8584 u32 vtaps; /* vertical scaler taps */
8588 * dce8_dram_bandwidth - get the dram bandwidth
8590 * @wm: watermark calculation data
8592 * Calculate the raw dram bandwidth (CIK).
8593 * Used for display watermark bandwidth calculations
8594 * Returns the dram bandwidth in MBytes/s
8596 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8598 /* Calculate raw DRAM Bandwidth */
8599 fixed20_12 dram_efficiency; /* 0.7 */
8600 fixed20_12 yclk, dram_channels, bandwidth;
8603 a.full = dfixed_const(1000);
8604 yclk.full = dfixed_const(wm->yclk);
8605 yclk.full = dfixed_div(yclk, a);
8606 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8607 a.full = dfixed_const(10);
8608 dram_efficiency.full = dfixed_const(7);
8609 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8610 bandwidth.full = dfixed_mul(dram_channels, yclk);
8611 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8613 return dfixed_trunc(bandwidth);
8617 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8619 * @wm: watermark calculation data
8621 * Calculate the dram bandwidth used for display (CIK).
8622 * Used for display watermark bandwidth calculations
8623 * Returns the dram bandwidth for display in MBytes/s
8625 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8627 /* Calculate DRAM Bandwidth and the part allocated to display. */
8628 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8629 fixed20_12 yclk, dram_channels, bandwidth;
8632 a.full = dfixed_const(1000);
8633 yclk.full = dfixed_const(wm->yclk);
8634 yclk.full = dfixed_div(yclk, a);
8635 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8636 a.full = dfixed_const(10);
8637 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8638 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8639 bandwidth.full = dfixed_mul(dram_channels, yclk);
8640 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8642 return dfixed_trunc(bandwidth);
8646 * dce8_data_return_bandwidth - get the data return bandwidth
8648 * @wm: watermark calculation data
8650 * Calculate the data return bandwidth used for display (CIK).
8651 * Used for display watermark bandwidth calculations
8652 * Returns the data return bandwidth in MBytes/s
8654 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8656 /* Calculate the display Data return Bandwidth */
8657 fixed20_12 return_efficiency; /* 0.8 */
8658 fixed20_12 sclk, bandwidth;
8661 a.full = dfixed_const(1000);
8662 sclk.full = dfixed_const(wm->sclk);
8663 sclk.full = dfixed_div(sclk, a);
8664 a.full = dfixed_const(10);
8665 return_efficiency.full = dfixed_const(8);
8666 return_efficiency.full = dfixed_div(return_efficiency, a);
8667 a.full = dfixed_const(32);
8668 bandwidth.full = dfixed_mul(a, sclk);
8669 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8671 return dfixed_trunc(bandwidth);
8675 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8677 * @wm: watermark calculation data
8679 * Calculate the dmif bandwidth used for display (CIK).
8680 * Used for display watermark bandwidth calculations
8681 * Returns the dmif bandwidth in MBytes/s
8683 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8685 /* Calculate the DMIF Request Bandwidth */
8686 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8687 fixed20_12 disp_clk, bandwidth;
8690 a.full = dfixed_const(1000);
8691 disp_clk.full = dfixed_const(wm->disp_clk);
8692 disp_clk.full = dfixed_div(disp_clk, a);
8693 a.full = dfixed_const(32);
8694 b.full = dfixed_mul(a, disp_clk);
8696 a.full = dfixed_const(10);
8697 disp_clk_request_efficiency.full = dfixed_const(8);
8698 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8700 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8702 return dfixed_trunc(bandwidth);
8706 * dce8_available_bandwidth - get the min available bandwidth
8708 * @wm: watermark calculation data
8710 * Calculate the min available bandwidth used for display (CIK).
8711 * Used for display watermark bandwidth calculations
8712 * Returns the min available bandwidth in MBytes/s
8714 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8716 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8717 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8718 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8719 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8721 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8725 * dce8_average_bandwidth - get the average available bandwidth
8727 * @wm: watermark calculation data
8729 * Calculate the average available bandwidth used for display (CIK).
8730 * Used for display watermark bandwidth calculations
8731 * Returns the average available bandwidth in MBytes/s
8733 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8735 /* Calculate the display mode Average Bandwidth
8736 * DisplayMode should contain the source and destination dimensions,
8740 fixed20_12 line_time;
8741 fixed20_12 src_width;
8742 fixed20_12 bandwidth;
8745 a.full = dfixed_const(1000);
8746 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8747 line_time.full = dfixed_div(line_time, a);
8748 bpp.full = dfixed_const(wm->bytes_per_pixel);
8749 src_width.full = dfixed_const(wm->src_width);
8750 bandwidth.full = dfixed_mul(src_width, bpp);
8751 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8752 bandwidth.full = dfixed_div(bandwidth, line_time);
8754 return dfixed_trunc(bandwidth);
8758 * dce8_latency_watermark - get the latency watermark
8760 * @wm: watermark calculation data
8762 * Calculate the latency watermark (CIK).
8763 * Used for display watermark bandwidth calculations
8764 * Returns the latency watermark in ns
8766 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8768 /* First calculate the latency in ns */
8769 u32 mc_latency = 2000; /* 2000 ns. */
8770 u32 available_bandwidth = dce8_available_bandwidth(wm);
8771 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8772 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8773 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8774 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8775 (wm->num_heads * cursor_line_pair_return_time);
8776 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8777 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8778 u32 tmp, dmif_size = 12288;
8781 if (wm->num_heads == 0)
8784 a.full = dfixed_const(2);
8785 b.full = dfixed_const(1);
8786 if ((wm->vsc.full > a.full) ||
8787 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8789 ((wm->vsc.full >= a.full) && wm->interlaced))
8790 max_src_lines_per_dst_line = 4;
8792 max_src_lines_per_dst_line = 2;
8794 a.full = dfixed_const(available_bandwidth);
8795 b.full = dfixed_const(wm->num_heads);
8796 a.full = dfixed_div(a, b);
8798 b.full = dfixed_const(mc_latency + 512);
8799 c.full = dfixed_const(wm->disp_clk);
8800 b.full = dfixed_div(b, c);
8802 c.full = dfixed_const(dmif_size);
8803 b.full = dfixed_div(c, b);
8805 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8807 b.full = dfixed_const(1000);
8808 c.full = dfixed_const(wm->disp_clk);
8809 b.full = dfixed_div(c, b);
8810 c.full = dfixed_const(wm->bytes_per_pixel);
8811 b.full = dfixed_mul(b, c);
8813 lb_fill_bw = min(tmp, dfixed_trunc(b));
8815 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8816 b.full = dfixed_const(1000);
8817 c.full = dfixed_const(lb_fill_bw);
8818 b.full = dfixed_div(c, b);
8819 a.full = dfixed_div(a, b);
8820 line_fill_time = dfixed_trunc(a);
8822 if (line_fill_time < wm->active_time)
8825 return latency + (line_fill_time - wm->active_time);
8830 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8831 * average and available dram bandwidth
8833 * @wm: watermark calculation data
8835 * Check if the display average bandwidth fits in the display
8836 * dram bandwidth (CIK).
8837 * Used for display watermark bandwidth calculations
8838 * Returns true if the display fits, false if not.
8840 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8842 if (dce8_average_bandwidth(wm) <=
8843 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8850 * dce8_average_bandwidth_vs_available_bandwidth - check
8851 * average and available bandwidth
8853 * @wm: watermark calculation data
8855 * Check if the display average bandwidth fits in the display
8856 * available bandwidth (CIK).
8857 * Used for display watermark bandwidth calculations
8858 * Returns true if the display fits, false if not.
8860 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8862 if (dce8_average_bandwidth(wm) <=
8863 (dce8_available_bandwidth(wm) / wm->num_heads))
8870 * dce8_check_latency_hiding - check latency hiding
8872 * @wm: watermark calculation data
8874 * Check latency hiding (CIK).
8875 * Used for display watermark bandwidth calculations
8876 * Returns true if the display fits, false if not.
8878 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8880 u32 lb_partitions = wm->lb_size / wm->src_width;
8881 u32 line_time = wm->active_time + wm->blank_time;
8882 u32 latency_tolerant_lines;
8886 a.full = dfixed_const(1);
8887 if (wm->vsc.full > a.full)
8888 latency_tolerant_lines = 1;
8890 if (lb_partitions <= (wm->vtaps + 1))
8891 latency_tolerant_lines = 1;
8893 latency_tolerant_lines = 2;
8896 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8898 if (dce8_latency_watermark(wm) <= latency_hiding)
8905 * dce8_program_watermarks - program display watermarks
8907 * @rdev: radeon_device pointer
8908 * @radeon_crtc: the selected display controller
8909 * @lb_size: line buffer size
8910 * @num_heads: number of display controllers in use
8912 * Calculate and program the display watermarks for the
8913 * selected display controller (CIK).
8915 static void dce8_program_watermarks(struct radeon_device *rdev,
8916 struct radeon_crtc *radeon_crtc,
8917 u32 lb_size, u32 num_heads)
8919 struct drm_display_mode *mode = &radeon_crtc->base.mode;
8920 struct dce8_wm_params wm_low, wm_high;
8923 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8926 if (radeon_crtc->base.enabled && num_heads && mode) {
8927 pixel_period = 1000000 / (u32)mode->clock;
8928 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8930 /* watermark for high clocks */
8931 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8932 rdev->pm.dpm_enabled) {
8934 radeon_dpm_get_mclk(rdev, false) * 10;
8936 radeon_dpm_get_sclk(rdev, false) * 10;
8938 wm_high.yclk = rdev->pm.current_mclk * 10;
8939 wm_high.sclk = rdev->pm.current_sclk * 10;
8942 wm_high.disp_clk = mode->clock;
8943 wm_high.src_width = mode->crtc_hdisplay;
8944 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8945 wm_high.blank_time = line_time - wm_high.active_time;
8946 wm_high.interlaced = false;
8947 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8948 wm_high.interlaced = true;
8949 wm_high.vsc = radeon_crtc->vsc;
8951 if (radeon_crtc->rmx_type != RMX_OFF)
8953 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8954 wm_high.lb_size = lb_size;
8955 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8956 wm_high.num_heads = num_heads;
8958 /* set for high clocks */
8959 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8961 /* possibly force display priority to high */
8962 /* should really do this at mode validation time... */
8963 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8964 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8965 !dce8_check_latency_hiding(&wm_high) ||
8966 (rdev->disp_priority == 2)) {
8967 DRM_DEBUG_KMS("force priority to high\n");
8970 /* watermark for low clocks */
8971 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8972 rdev->pm.dpm_enabled) {
8974 radeon_dpm_get_mclk(rdev, true) * 10;
8976 radeon_dpm_get_sclk(rdev, true) * 10;
8978 wm_low.yclk = rdev->pm.current_mclk * 10;
8979 wm_low.sclk = rdev->pm.current_sclk * 10;
8982 wm_low.disp_clk = mode->clock;
8983 wm_low.src_width = mode->crtc_hdisplay;
8984 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8985 wm_low.blank_time = line_time - wm_low.active_time;
8986 wm_low.interlaced = false;
8987 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8988 wm_low.interlaced = true;
8989 wm_low.vsc = radeon_crtc->vsc;
8991 if (radeon_crtc->rmx_type != RMX_OFF)
8993 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8994 wm_low.lb_size = lb_size;
8995 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8996 wm_low.num_heads = num_heads;
8998 /* set for low clocks */
8999 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
9001 /* possibly force display priority to high */
9002 /* should really do this at mode validation time... */
9003 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9004 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9005 !dce8_check_latency_hiding(&wm_low) ||
9006 (rdev->disp_priority == 2)) {
9007 DRM_DEBUG_KMS("force priority to high\n");
9012 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9014 tmp &= ~LATENCY_WATERMARK_MASK(3);
9015 tmp |= LATENCY_WATERMARK_MASK(1);
9016 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9017 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9018 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9019 LATENCY_HIGH_WATERMARK(line_time)));
9021 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9022 tmp &= ~LATENCY_WATERMARK_MASK(3);
9023 tmp |= LATENCY_WATERMARK_MASK(2);
9024 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9025 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9026 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9027 LATENCY_HIGH_WATERMARK(line_time)));
9028 /* restore original selection */
9029 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
9031 /* save values for DPM */
9032 radeon_crtc->line_time = line_time;
9033 radeon_crtc->wm_high = latency_watermark_a;
9034 radeon_crtc->wm_low = latency_watermark_b;
9038 * dce8_bandwidth_update - program display watermarks
9040 * @rdev: radeon_device pointer
9042 * Calculate and program the display watermarks and line
9043 * buffer allocation (CIK).
9045 void dce8_bandwidth_update(struct radeon_device *rdev)
9047 struct drm_display_mode *mode = NULL;
9048 u32 num_heads = 0, lb_size;
9051 radeon_update_display_priority(rdev);
9053 for (i = 0; i < rdev->num_crtc; i++) {
9054 if (rdev->mode_info.crtcs[i]->base.enabled)
9057 for (i = 0; i < rdev->num_crtc; i++) {
9058 mode = &rdev->mode_info.crtcs[i]->base.mode;
9059 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9060 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9065 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9067 * @rdev: radeon_device pointer
9069 * Fetches a GPU clock counter snapshot (SI).
9070 * Returns the 64 bit clock counter snapshot.
9072 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9076 mutex_lock(&rdev->gpu_clock_mutex);
9077 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9078 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9079 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9080 mutex_unlock(&rdev->gpu_clock_mutex);
9084 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9085 u32 cntl_reg, u32 status_reg)
9088 struct atom_clock_dividers dividers;
9091 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9092 clock, false, ÷rs);
9096 tmp = RREG32_SMC(cntl_reg);
9097 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9098 tmp |= dividers.post_divider;
9099 WREG32_SMC(cntl_reg, tmp);
9101 for (i = 0; i < 100; i++) {
9102 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9112 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9116 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9120 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9124 int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9127 struct atom_clock_dividers dividers;
9130 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9131 ecclk, false, ÷rs);
9135 for (i = 0; i < 100; i++) {
9136 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9143 tmp = RREG32_SMC(CG_ECLK_CNTL);
9144 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9145 tmp |= dividers.post_divider;
9146 WREG32_SMC(CG_ECLK_CNTL, tmp);
9148 for (i = 0; i < 100; i++) {
9149 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9159 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
9161 struct pci_dev *root = rdev->pdev->bus->self;
9162 int bridge_pos, gpu_pos;
9163 u32 speed_cntl, mask, current_data_rate;
9167 if (radeon_pcie_gen2 == 0)
9170 if (rdev->flags & RADEON_IS_IGP)
9173 if (!(rdev->flags & RADEON_IS_PCIE))
9176 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9180 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9183 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9184 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9185 LC_CURRENT_DATA_RATE_SHIFT;
9186 if (mask & DRM_PCIE_SPEED_80) {
9187 if (current_data_rate == 2) {
9188 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9191 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9192 } else if (mask & DRM_PCIE_SPEED_50) {
9193 if (current_data_rate == 1) {
9194 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9197 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9200 bridge_pos = pci_pcie_cap(root);
9204 gpu_pos = pci_pcie_cap(rdev->pdev);
9208 if (mask & DRM_PCIE_SPEED_80) {
9209 /* re-try equalization if gen3 is not already enabled */
9210 if (current_data_rate != 2) {
9211 u16 bridge_cfg, gpu_cfg;
9212 u16 bridge_cfg2, gpu_cfg2;
9213 u32 max_lw, current_lw, tmp;
9215 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9216 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9218 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9219 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9221 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9222 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9224 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9225 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9226 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9228 if (current_lw < max_lw) {
9229 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9230 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9231 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9232 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9233 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9234 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9238 for (i = 0; i < 10; i++) {
9240 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9241 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9244 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9245 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9247 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9248 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9250 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9251 tmp |= LC_SET_QUIESCE;
9252 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9254 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9256 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9261 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9262 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9263 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9264 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9266 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9267 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9268 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9269 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9272 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9273 tmp16 &= ~((1 << 4) | (7 << 9));
9274 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9275 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9277 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9278 tmp16 &= ~((1 << 4) | (7 << 9));
9279 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9280 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9282 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9283 tmp &= ~LC_SET_QUIESCE;
9284 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9289 /* set the link speed */
9290 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9291 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9292 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9294 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9296 if (mask & DRM_PCIE_SPEED_80)
9297 tmp16 |= 3; /* gen3 */
9298 else if (mask & DRM_PCIE_SPEED_50)
9299 tmp16 |= 2; /* gen2 */
9301 tmp16 |= 1; /* gen1 */
9302 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9304 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9305 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9306 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9308 for (i = 0; i < rdev->usec_timeout; i++) {
9309 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9310 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9316 static void cik_program_aspm(struct radeon_device *rdev)
9319 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9320 bool disable_clkreq = false;
9322 if (radeon_aspm == 0)
9325 /* XXX double check IGPs */
9326 if (rdev->flags & RADEON_IS_IGP)
9329 if (!(rdev->flags & RADEON_IS_PCIE))
9332 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9333 data &= ~LC_XMIT_N_FTS_MASK;
9334 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9336 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9338 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9339 data |= LC_GO_TO_RECOVERY;
9341 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9343 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9344 data |= P_IGNORE_EDB_ERR;
9346 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9348 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9349 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9350 data |= LC_PMI_TO_L1_DIS;
9352 data |= LC_L0S_INACTIVITY(7);
9355 data |= LC_L1_INACTIVITY(7);
9356 data &= ~LC_PMI_TO_L1_DIS;
9358 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9360 if (!disable_plloff_in_l1) {
9361 bool clk_req_support;
9363 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9364 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9365 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9367 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9369 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9370 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9371 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9373 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9375 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9376 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9377 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9379 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9381 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9382 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9383 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9385 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9387 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9388 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9389 data |= LC_DYN_LANES_PWR_STATE(3);
9391 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9393 if (!disable_clkreq) {
9394 struct pci_dev *root = rdev->pdev->bus->self;
9397 clk_req_support = false;
9398 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9399 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9400 clk_req_support = true;
9402 clk_req_support = false;
9405 if (clk_req_support) {
9406 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9407 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9409 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9411 orig = data = RREG32_SMC(THM_CLK_CNTL);
9412 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9413 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9415 WREG32_SMC(THM_CLK_CNTL, data);
9417 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9418 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9419 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9421 WREG32_SMC(MISC_CLK_CTRL, data);
9423 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9424 data &= ~BCLK_AS_XCLK;
9426 WREG32_SMC(CG_CLKPIN_CNTL, data);
9428 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9429 data &= ~FORCE_BIF_REFCLK_EN;
9431 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9433 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9434 data &= ~MPLL_CLKOUT_SEL_MASK;
9435 data |= MPLL_CLKOUT_SEL(4);
9437 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9442 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9445 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9446 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9448 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9451 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9452 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9453 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9454 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9455 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9456 data &= ~LC_L0S_INACTIVITY_MASK;
9458 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);