projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'driver-core-4.9-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-2.6-block.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
bfc1f97d 30#include "radeon_audio.h"
8cc1a532
AD		 31#include "cikd.h"
32#include "atom.h"
841cf442 33#include "cik_blit_shaders.h"
8c68e393 34#include "radeon_ucode.h"
22c775ce 35#include "clearstate_ci.h"
e28740ec 36#include "radeon_kfd.h"
02c81327
AD		 37
38MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
41MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
42MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
277babc3 43MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
02c81327 44MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 45MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 46MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
f2c6b0f4
AD		 47
48MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
49MODULE_FIRMWARE("radeon/bonaire_me.bin");
50MODULE_FIRMWARE("radeon/bonaire_ce.bin");
51MODULE_FIRMWARE("radeon/bonaire_mec.bin");
52MODULE_FIRMWARE("radeon/bonaire_mc.bin");
53MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
54MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
55MODULE_FIRMWARE("radeon/bonaire_smc.bin");
b2ea0dcd 56MODULE_FIRMWARE("radeon/bonaire_k_smc.bin");
f2c6b0f4 57
d4775655
AD		 58MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
59MODULE_FIRMWARE("radeon/HAWAII_me.bin");
60MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
61MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
62MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
277babc3 63MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
d4775655
AD		 64MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
65MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
66MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
f2c6b0f4
AD		 67
68MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
69MODULE_FIRMWARE("radeon/hawaii_me.bin");
70MODULE_FIRMWARE("radeon/hawaii_ce.bin");
71MODULE_FIRMWARE("radeon/hawaii_mec.bin");
72MODULE_FIRMWARE("radeon/hawaii_mc.bin");
73MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
74MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
75MODULE_FIRMWARE("radeon/hawaii_smc.bin");
b2ea0dcd 76MODULE_FIRMWARE("radeon/hawaii_k_smc.bin");
f2c6b0f4 77
02c81327
AD		 78MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
79MODULE_FIRMWARE("radeon/KAVERI_me.bin");
80MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
81MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
82MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 83MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
f2c6b0f4
AD		 84
85MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
86MODULE_FIRMWARE("radeon/kaveri_me.bin");
87MODULE_FIRMWARE("radeon/kaveri_ce.bin");
88MODULE_FIRMWARE("radeon/kaveri_mec.bin");
89MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
90MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
91MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
92
02c81327
AD		 93MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
94MODULE_FIRMWARE("radeon/KABINI_me.bin");
95MODULE_FIRMWARE("radeon/KABINI_ce.bin");
96MODULE_FIRMWARE("radeon/KABINI_mec.bin");
97MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 98MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
f2c6b0f4
AD		 99
100MODULE_FIRMWARE("radeon/kabini_pfp.bin");
101MODULE_FIRMWARE("radeon/kabini_me.bin");
102MODULE_FIRMWARE("radeon/kabini_ce.bin");
103MODULE_FIRMWARE("radeon/kabini_mec.bin");
104MODULE_FIRMWARE("radeon/kabini_rlc.bin");
105MODULE_FIRMWARE("radeon/kabini_sdma.bin");
106
f73a9e83
SL		 107MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
108MODULE_FIRMWARE("radeon/MULLINS_me.bin");
109MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
110MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
111MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
112MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
02c81327 113
f2c6b0f4
AD		 114MODULE_FIRMWARE("radeon/mullins_pfp.bin");
115MODULE_FIRMWARE("radeon/mullins_me.bin");
116MODULE_FIRMWARE("radeon/mullins_ce.bin");
117MODULE_FIRMWARE("radeon/mullins_mec.bin");
118MODULE_FIRMWARE("radeon/mullins_rlc.bin");
119MODULE_FIRMWARE("radeon/mullins_sdma.bin");
120
a59781bb
AD		 121extern int r600_ih_ring_alloc(struct radeon_device *rdev);
122extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 123extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
124extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 125extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 126extern void sumo_rlc_fini(struct radeon_device *rdev);
127extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 128extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 129extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 130extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
65fcf668 131static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
2483b4ea
CK		 132extern int cik_sdma_resume(struct radeon_device *rdev);
133extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
134extern void cik_sdma_fini(struct radeon_device *rdev);
a1d6f97c 135extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
cc066715 136static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 137static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 138static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 139static void cik_init_pg(struct radeon_device *rdev);
140static void cik_init_cg(struct radeon_device *rdev);
fb2c7f4d
AD		 141static void cik_fini_pg(struct radeon_device *rdev);
142static void cik_fini_cg(struct radeon_device *rdev);
4214faf6
AD		 143static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
144 bool enable);
6f2043ce 145
353eec2a
AD		 146/**
147 * cik_get_allowed_info_register - fetch the register for the info ioctl
148 *
149 * @rdev: radeon_device pointer
150 * @reg: register offset in bytes
151 * @val: register value
152 *
153 * Returns 0 for success or -EINVAL for an invalid register
154 *
155 */
156int cik_get_allowed_info_register(struct radeon_device *rdev,
157 u32 reg, u32 *val)
158{
159 switch (reg) {
160 case GRBM_STATUS:
161 case GRBM_STATUS2:
162 case GRBM_STATUS_SE0:
163 case GRBM_STATUS_SE1:
164 case GRBM_STATUS_SE2:
165 case GRBM_STATUS_SE3:
166 case SRBM_STATUS:
167 case SRBM_STATUS2:
168 case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET):
169 case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET):
170 case UVD_STATUS:
171 /* TODO VCE */
172 *val = RREG32(reg);
173 return 0;
174 default:
175 return -EINVAL;
176 }
177}
178
9e5acbc2
DV		 179/*
180 * Indirect registers accessor
181 */
182u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)
183{
184 unsigned long flags;
185 u32 r;
186
187 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
188 WREG32(CIK_DIDT_IND_INDEX, (reg));
189 r = RREG32(CIK_DIDT_IND_DATA);
190 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
191 return r;
192}
193
194void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)
195{
196 unsigned long flags;
197
198 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
199 WREG32(CIK_DIDT_IND_INDEX, (reg));
200 WREG32(CIK_DIDT_IND_DATA, (v));
201 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
202}
203
286d9cc6
AD		 204/* get temperature in millidegrees */
205int ci_get_temp(struct radeon_device *rdev)
206{
207 u32 temp;
208 int actual_temp = 0;
209
210 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
211 CTF_TEMP_SHIFT;
212
213 if (temp & 0x200)
214 actual_temp = 255;
215 else
216 actual_temp = temp & 0x1ff;
217
218 actual_temp = actual_temp * 1000;
219
220 return actual_temp;
221}
222
223/* get temperature in millidegrees */
224int kv_get_temp(struct radeon_device *rdev)
225{
226 u32 temp;
227 int actual_temp = 0;
228
229 temp = RREG32_SMC(0xC0300E0C);
230
231 if (temp)
232 actual_temp = (temp / 8) - 49;
233 else
234 actual_temp = 0;
235
236 actual_temp = actual_temp * 1000;
237
238 return actual_temp;
239}
6f2043ce 240
6e2c3c0a
AD		 241/*
242 * Indirect registers accessor
243 */
244u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
245{
0a5b7b0b 246 unsigned long flags;
6e2c3c0a
AD		 247 u32 r;
248
0a5b7b0b 249 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 250 WREG32(PCIE_INDEX, reg);
251 (void)RREG32(PCIE_INDEX);
252 r = RREG32(PCIE_DATA);
0a5b7b0b 253 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 254 return r;
255}
256
257void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
258{
0a5b7b0b
AD		 259 unsigned long flags;
260
261 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 262 WREG32(PCIE_INDEX, reg);
263 (void)RREG32(PCIE_INDEX);
264 WREG32(PCIE_DATA, v);
265 (void)RREG32(PCIE_DATA);
0a5b7b0b 266 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 267}
268
22c775ce
AD		 269static const u32 spectre_rlc_save_restore_register_list[] =
270{
271 (0x0e00 << 16) | (0xc12c >> 2),
272 0x00000000,
273 (0x0e00 << 16) | (0xc140 >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0xc150 >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0xc15c >> 2),
278 0x00000000,
279 (0x0e00 << 16) | (0xc168 >> 2),
280 0x00000000,
281 (0x0e00 << 16) | (0xc170 >> 2),
282 0x00000000,
283 (0x0e00 << 16) | (0xc178 >> 2),
284 0x00000000,
285 (0x0e00 << 16) | (0xc204 >> 2),
286 0x00000000,
287 (0x0e00 << 16) | (0xc2b4 >> 2),
288 0x00000000,
289 (0x0e00 << 16) | (0xc2b8 >> 2),
290 0x00000000,
291 (0x0e00 << 16) | (0xc2bc >> 2),
292 0x00000000,
293 (0x0e00 << 16) | (0xc2c0 >> 2),
294 0x00000000,
295 (0x0e00 << 16) | (0x8228 >> 2),
296 0x00000000,
297 (0x0e00 << 16) | (0x829c >> 2),
298 0x00000000,
299 (0x0e00 << 16) | (0x869c >> 2),
300 0x00000000,
301 (0x0600 << 16) | (0x98f4 >> 2),
302 0x00000000,
303 (0x0e00 << 16) | (0x98f8 >> 2),
304 0x00000000,
305 (0x0e00 << 16) | (0x9900 >> 2),
306 0x00000000,
307 (0x0e00 << 16) | (0xc260 >> 2),
308 0x00000000,
309 (0x0e00 << 16) | (0x90e8 >> 2),
310 0x00000000,
311 (0x0e00 << 16) | (0x3c000 >> 2),
312 0x00000000,
313 (0x0e00 << 16) | (0x3c00c >> 2),
314 0x00000000,
315 (0x0e00 << 16) | (0x8c1c >> 2),
316 0x00000000,
317 (0x0e00 << 16) | (0x9700 >> 2),
318 0x00000000,
319 (0x0e00 << 16) | (0xcd20 >> 2),
320 0x00000000,
321 (0x4e00 << 16) | (0xcd20 >> 2),
322 0x00000000,
323 (0x5e00 << 16) | (0xcd20 >> 2),
324 0x00000000,
325 (0x6e00 << 16) | (0xcd20 >> 2),
326 0x00000000,
327 (0x7e00 << 16) | (0xcd20 >> 2),
328 0x00000000,
329 (0x8e00 << 16) | (0xcd20 >> 2),
330 0x00000000,
331 (0x9e00 << 16) | (0xcd20 >> 2),
332 0x00000000,
333 (0xae00 << 16) | (0xcd20 >> 2),
334 0x00000000,
335 (0xbe00 << 16) | (0xcd20 >> 2),
336 0x00000000,
337 (0x0e00 << 16) | (0x89bc >> 2),
338 0x00000000,
339 (0x0e00 << 16) | (0x8900 >> 2),
340 0x00000000,
341 0x3,
342 (0x0e00 << 16) | (0xc130 >> 2),
343 0x00000000,
344 (0x0e00 << 16) | (0xc134 >> 2),
345 0x00000000,
346 (0x0e00 << 16) | (0xc1fc >> 2),
347 0x00000000,
348 (0x0e00 << 16) | (0xc208 >> 2),
349 0x00000000,
350 (0x0e00 << 16) | (0xc264 >> 2),
351 0x00000000,
352 (0x0e00 << 16) | (0xc268 >> 2),
353 0x00000000,
354 (0x0e00 << 16) | (0xc26c >> 2),
355 0x00000000,
356 (0x0e00 << 16) | (0xc270 >> 2),
357 0x00000000,
358 (0x0e00 << 16) | (0xc274 >> 2),
359 0x00000000,
360 (0x0e00 << 16) | (0xc278 >> 2),
361 0x00000000,
362 (0x0e00 << 16) | (0xc27c >> 2),
363 0x00000000,
364 (0x0e00 << 16) | (0xc280 >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc284 >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0xc288 >> 2),
369 0x00000000,
370 (0x0e00 << 16) | (0xc28c >> 2),
371 0x00000000,
372 (0x0e00 << 16) | (0xc290 >> 2),
373 0x00000000,
374 (0x0e00 << 16) | (0xc294 >> 2),
375 0x00000000,
376 (0x0e00 << 16) | (0xc298 >> 2),
377 0x00000000,
378 (0x0e00 << 16) | (0xc29c >> 2),
379 0x00000000,
380 (0x0e00 << 16) | (0xc2a0 >> 2),
381 0x00000000,
382 (0x0e00 << 16) | (0xc2a4 >> 2),
383 0x00000000,
384 (0x0e00 << 16) | (0xc2a8 >> 2),
385 0x00000000,
386 (0x0e00 << 16) | (0xc2ac >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0xc2b0 >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x301d0 >> 2),
391 0x00000000,
392 (0x0e00 << 16) | (0x30238 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x30250 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x30254 >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x30258 >> 2),
399 0x00000000,
400 (0x0e00 << 16) | (0x3025c >> 2),
401 0x00000000,
402 (0x4e00 << 16) | (0xc900 >> 2),
403 0x00000000,
404 (0x5e00 << 16) | (0xc900 >> 2),
405 0x00000000,
406 (0x6e00 << 16) | (0xc900 >> 2),
407 0x00000000,
408 (0x7e00 << 16) | (0xc900 >> 2),
409 0x00000000,
410 (0x8e00 << 16) | (0xc900 >> 2),
411 0x00000000,
412 (0x9e00 << 16) | (0xc900 >> 2),
413 0x00000000,
414 (0xae00 << 16) | (0xc900 >> 2),
415 0x00000000,
416 (0xbe00 << 16) | (0xc900 >> 2),
417 0x00000000,
418 (0x4e00 << 16) | (0xc904 >> 2),
419 0x00000000,
420 (0x5e00 << 16) | (0xc904 >> 2),
421 0x00000000,
422 (0x6e00 << 16) | (0xc904 >> 2),
423 0x00000000,
424 (0x7e00 << 16) | (0xc904 >> 2),
425 0x00000000,
426 (0x8e00 << 16) | (0xc904 >> 2),
427 0x00000000,
428 (0x9e00 << 16) | (0xc904 >> 2),
429 0x00000000,
430 (0xae00 << 16) | (0xc904 >> 2),
431 0x00000000,
432 (0xbe00 << 16) | (0xc904 >> 2),
433 0x00000000,
434 (0x4e00 << 16) | (0xc908 >> 2),
435 0x00000000,
436 (0x5e00 << 16) | (0xc908 >> 2),
437 0x00000000,
438 (0x6e00 << 16) | (0xc908 >> 2),
439 0x00000000,
440 (0x7e00 << 16) | (0xc908 >> 2),
441 0x00000000,
442 (0x8e00 << 16) | (0xc908 >> 2),
443 0x00000000,
444 (0x9e00 << 16) | (0xc908 >> 2),
445 0x00000000,
446 (0xae00 << 16) | (0xc908 >> 2),
447 0x00000000,
448 (0xbe00 << 16) | (0xc908 >> 2),
449 0x00000000,
450 (0x4e00 << 16) | (0xc90c >> 2),
451 0x00000000,
452 (0x5e00 << 16) | (0xc90c >> 2),
453 0x00000000,
454 (0x6e00 << 16) | (0xc90c >> 2),
455 0x00000000,
456 (0x7e00 << 16) | (0xc90c >> 2),
457 0x00000000,
458 (0x8e00 << 16) | (0xc90c >> 2),
459 0x00000000,
460 (0x9e00 << 16) | (0xc90c >> 2),
461 0x00000000,
462 (0xae00 << 16) | (0xc90c >> 2),
463 0x00000000,
464 (0xbe00 << 16) | (0xc90c >> 2),
465 0x00000000,
466 (0x4e00 << 16) | (0xc910 >> 2),
467 0x00000000,
468 (0x5e00 << 16) | (0xc910 >> 2),
469 0x00000000,
470 (0x6e00 << 16) | (0xc910 >> 2),
471 0x00000000,
472 (0x7e00 << 16) | (0xc910 >> 2),
473 0x00000000,
474 (0x8e00 << 16) | (0xc910 >> 2),
475 0x00000000,
476 (0x9e00 << 16) | (0xc910 >> 2),
477 0x00000000,
478 (0xae00 << 16) | (0xc910 >> 2),
479 0x00000000,
480 (0xbe00 << 16) | (0xc910 >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0xc99c >> 2),
483 0x00000000,
484 (0x0e00 << 16) | (0x9834 >> 2),
485 0x00000000,
486 (0x0000 << 16) | (0x30f00 >> 2),
487 0x00000000,
488 (0x0001 << 16) | (0x30f00 >> 2),
489 0x00000000,
490 (0x0000 << 16) | (0x30f04 >> 2),
491 0x00000000,
492 (0x0001 << 16) | (0x30f04 >> 2),
493 0x00000000,
494 (0x0000 << 16) | (0x30f08 >> 2),
495 0x00000000,
496 (0x0001 << 16) | (0x30f08 >> 2),
497 0x00000000,
498 (0x0000 << 16) | (0x30f0c >> 2),
499 0x00000000,
500 (0x0001 << 16) | (0x30f0c >> 2),
501 0x00000000,
502 (0x0600 << 16) | (0x9b7c >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0x8a14 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0x8a18 >> 2),
507 0x00000000,
508 (0x0600 << 16) | (0x30a00 >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x8bf0 >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x8bcc >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x8b24 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x30a04 >> 2),
517 0x00000000,
518 (0x0600 << 16) | (0x30a10 >> 2),
519 0x00000000,
520 (0x0600 << 16) | (0x30a14 >> 2),
521 0x00000000,
522 (0x0600 << 16) | (0x30a18 >> 2),
523 0x00000000,
524 (0x0600 << 16) | (0x30a2c >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0xc700 >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0xc704 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0xc708 >> 2),
531 0x00000000,
532 (0x0e00 << 16) | (0xc768 >> 2),
533 0x00000000,
534 (0x0400 << 16) | (0xc770 >> 2),
535 0x00000000,
536 (0x0400 << 16) | (0xc774 >> 2),
537 0x00000000,
538 (0x0400 << 16) | (0xc778 >> 2),
539 0x00000000,
540 (0x0400 << 16) | (0xc77c >> 2),
541 0x00000000,
542 (0x0400 << 16) | (0xc780 >> 2),
543 0x00000000,
544 (0x0400 << 16) | (0xc784 >> 2),
545 0x00000000,
546 (0x0400 << 16) | (0xc788 >> 2),
547 0x00000000,
548 (0x0400 << 16) | (0xc78c >> 2),
549 0x00000000,
550 (0x0400 << 16) | (0xc798 >> 2),
551 0x00000000,
552 (0x0400 << 16) | (0xc79c >> 2),
553 0x00000000,
554 (0x0400 << 16) | (0xc7a0 >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0xc7a4 >> 2),
557 0x00000000,
558 (0x0400 << 16) | (0xc7a8 >> 2),
559 0x00000000,
560 (0x0400 << 16) | (0xc7ac >> 2),
561 0x00000000,
562 (0x0400 << 16) | (0xc7b0 >> 2),
563 0x00000000,
564 (0x0400 << 16) | (0xc7b4 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x9100 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x3c010 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x92a8 >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x92ac >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x92b4 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x92b8 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x92bc >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x92c0 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x92c4 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x92c8 >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x92cc >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x92d0 >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x8c00 >> 2),
591 0x00000000,
592 (0x0e00 << 16) | (0x8c04 >> 2),
593 0x00000000,
594 (0x0e00 << 16) | (0x8c20 >> 2),
595 0x00000000,
596 (0x0e00 << 16) | (0x8c38 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0x8c3c >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0xae00 >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0x9604 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xac08 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xac0c >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xac10 >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xac14 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xac58 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xac68 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xac6c >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xac70 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xac74 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xac78 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0xac7c >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0xac80 >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0xac84 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0xac88 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0xac8c >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x970c >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x9714 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x9718 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x971c >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x31068 >> 2),
643 0x00000000,
644 (0x4e00 << 16) | (0x31068 >> 2),
645 0x00000000,
646 (0x5e00 << 16) | (0x31068 >> 2),
647 0x00000000,
648 (0x6e00 << 16) | (0x31068 >> 2),
649 0x00000000,
650 (0x7e00 << 16) | (0x31068 >> 2),
651 0x00000000,
652 (0x8e00 << 16) | (0x31068 >> 2),
653 0x00000000,
654 (0x9e00 << 16) | (0x31068 >> 2),
655 0x00000000,
656 (0xae00 << 16) | (0x31068 >> 2),
657 0x00000000,
658 (0xbe00 << 16) | (0x31068 >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0xcd10 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0xcd14 >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0x88b0 >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0x88b4 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0x88b8 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0x88bc >> 2),
671 0x00000000,
672 (0x0400 << 16) | (0x89c0 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0x88c4 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0x88c8 >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0x88d0 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0x88d4 >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0x88d8 >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0x8980 >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0x30938 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0x3093c >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0x30940 >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0x89a0 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x30900 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x30904 >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x89b4 >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x3c210 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0x3c214 >> 2),
703 0x00000000,
704 (0x0e00 << 16) | (0x3c218 >> 2),
705 0x00000000,
706 (0x0e00 << 16) | (0x8904 >> 2),
707 0x00000000,
708 0x5,
709 (0x0e00 << 16) | (0x8c28 >> 2),
710 (0x0e00 << 16) | (0x8c2c >> 2),
711 (0x0e00 << 16) | (0x8c30 >> 2),
712 (0x0e00 << 16) | (0x8c34 >> 2),
713 (0x0e00 << 16) | (0x9600 >> 2),
714};
715
716static const u32 kalindi_rlc_save_restore_register_list[] =
717{
718 (0x0e00 << 16) | (0xc12c >> 2),
719 0x00000000,
720 (0x0e00 << 16) | (0xc140 >> 2),
721 0x00000000,
722 (0x0e00 << 16) | (0xc150 >> 2),
723 0x00000000,
724 (0x0e00 << 16) | (0xc15c >> 2),
725 0x00000000,
726 (0x0e00 << 16) | (0xc168 >> 2),
727 0x00000000,
728 (0x0e00 << 16) | (0xc170 >> 2),
729 0x00000000,
730 (0x0e00 << 16) | (0xc204 >> 2),
731 0x00000000,
732 (0x0e00 << 16) | (0xc2b4 >> 2),
733 0x00000000,
734 (0x0e00 << 16) | (0xc2b8 >> 2),
735 0x00000000,
736 (0x0e00 << 16) | (0xc2bc >> 2),
737 0x00000000,
738 (0x0e00 << 16) | (0xc2c0 >> 2),
739 0x00000000,
740 (0x0e00 << 16) | (0x8228 >> 2),
741 0x00000000,
742 (0x0e00 << 16) | (0x829c >> 2),
743 0x00000000,
744 (0x0e00 << 16) | (0x869c >> 2),
745 0x00000000,
746 (0x0600 << 16) | (0x98f4 >> 2),
747 0x00000000,
748 (0x0e00 << 16) | (0x98f8 >> 2),
749 0x00000000,
750 (0x0e00 << 16) | (0x9900 >> 2),
751 0x00000000,
752 (0x0e00 << 16) | (0xc260 >> 2),
753 0x00000000,
754 (0x0e00 << 16) | (0x90e8 >> 2),
755 0x00000000,
756 (0x0e00 << 16) | (0x3c000 >> 2),
757 0x00000000,
758 (0x0e00 << 16) | (0x3c00c >> 2),
759 0x00000000,
760 (0x0e00 << 16) | (0x8c1c >> 2),
761 0x00000000,
762 (0x0e00 << 16) | (0x9700 >> 2),
763 0x00000000,
764 (0x0e00 << 16) | (0xcd20 >> 2),
765 0x00000000,
766 (0x4e00 << 16) | (0xcd20 >> 2),
767 0x00000000,
768 (0x5e00 << 16) | (0xcd20 >> 2),
769 0x00000000,
770 (0x6e00 << 16) | (0xcd20 >> 2),
771 0x00000000,
772 (0x7e00 << 16) | (0xcd20 >> 2),
773 0x00000000,
774 (0x0e00 << 16) | (0x89bc >> 2),
775 0x00000000,
776 (0x0e00 << 16) | (0x8900 >> 2),
777 0x00000000,
778 0x3,
779 (0x0e00 << 16) | (0xc130 >> 2),
780 0x00000000,
781 (0x0e00 << 16) | (0xc134 >> 2),
782 0x00000000,
783 (0x0e00 << 16) | (0xc1fc >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc208 >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc264 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc268 >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc26c >> 2),
792 0x00000000,
793 (0x0e00 << 16) | (0xc270 >> 2),
794 0x00000000,
795 (0x0e00 << 16) | (0xc274 >> 2),
796 0x00000000,
797 (0x0e00 << 16) | (0xc28c >> 2),
798 0x00000000,
799 (0x0e00 << 16) | (0xc290 >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0xc294 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0xc298 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0xc2a0 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0xc2a4 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0xc2a8 >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0xc2ac >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0x301d0 >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0x30238 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x30250 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0x30254 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0x30258 >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0x3025c >> 2),
824 0x00000000,
825 (0x4e00 << 16) | (0xc900 >> 2),
826 0x00000000,
827 (0x5e00 << 16) | (0xc900 >> 2),
828 0x00000000,
829 (0x6e00 << 16) | (0xc900 >> 2),
830 0x00000000,
831 (0x7e00 << 16) | (0xc900 >> 2),
832 0x00000000,
833 (0x4e00 << 16) | (0xc904 >> 2),
834 0x00000000,
835 (0x5e00 << 16) | (0xc904 >> 2),
836 0x00000000,
837 (0x6e00 << 16) | (0xc904 >> 2),
838 0x00000000,
839 (0x7e00 << 16) | (0xc904 >> 2),
840 0x00000000,
841 (0x4e00 << 16) | (0xc908 >> 2),
842 0x00000000,
843 (0x5e00 << 16) | (0xc908 >> 2),
844 0x00000000,
845 (0x6e00 << 16) | (0xc908 >> 2),
846 0x00000000,
847 (0x7e00 << 16) | (0xc908 >> 2),
848 0x00000000,
849 (0x4e00 << 16) | (0xc90c >> 2),
850 0x00000000,
851 (0x5e00 << 16) | (0xc90c >> 2),
852 0x00000000,
853 (0x6e00 << 16) | (0xc90c >> 2),
854 0x00000000,
855 (0x7e00 << 16) | (0xc90c >> 2),
856 0x00000000,
857 (0x4e00 << 16) | (0xc910 >> 2),
858 0x00000000,
859 (0x5e00 << 16) | (0xc910 >> 2),
860 0x00000000,
861 (0x6e00 << 16) | (0xc910 >> 2),
862 0x00000000,
863 (0x7e00 << 16) | (0xc910 >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0xc99c >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0x9834 >> 2),
868 0x00000000,
869 (0x0000 << 16) | (0x30f00 >> 2),
870 0x00000000,
871 (0x0000 << 16) | (0x30f04 >> 2),
872 0x00000000,
873 (0x0000 << 16) | (0x30f08 >> 2),
874 0x00000000,
875 (0x0000 << 16) | (0x30f0c >> 2),
876 0x00000000,
877 (0x0600 << 16) | (0x9b7c >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0x8a14 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0x8a18 >> 2),
882 0x00000000,
883 (0x0600 << 16) | (0x30a00 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x8bf0 >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x8bcc >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x8b24 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0x30a04 >> 2),
892 0x00000000,
893 (0x0600 << 16) | (0x30a10 >> 2),
894 0x00000000,
895 (0x0600 << 16) | (0x30a14 >> 2),
896 0x00000000,
897 (0x0600 << 16) | (0x30a18 >> 2),
898 0x00000000,
899 (0x0600 << 16) | (0x30a2c >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0xc700 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0xc704 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0xc708 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0xc768 >> 2),
908 0x00000000,
909 (0x0400 << 16) | (0xc770 >> 2),
910 0x00000000,
911 (0x0400 << 16) | (0xc774 >> 2),
912 0x00000000,
913 (0x0400 << 16) | (0xc798 >> 2),
914 0x00000000,
915 (0x0400 << 16) | (0xc79c >> 2),
916 0x00000000,
917 (0x0e00 << 16) | (0x9100 >> 2),
918 0x00000000,
919 (0x0e00 << 16) | (0x3c010 >> 2),
920 0x00000000,
921 (0x0e00 << 16) | (0x8c00 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0x8c04 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0x8c20 >> 2),
926 0x00000000,
927 (0x0e00 << 16) | (0x8c38 >> 2),
928 0x00000000,
929 (0x0e00 << 16) | (0x8c3c >> 2),
930 0x00000000,
931 (0x0e00 << 16) | (0xae00 >> 2),
932 0x00000000,
933 (0x0e00 << 16) | (0x9604 >> 2),
934 0x00000000,
935 (0x0e00 << 16) | (0xac08 >> 2),
936 0x00000000,
937 (0x0e00 << 16) | (0xac0c >> 2),
938 0x00000000,
939 (0x0e00 << 16) | (0xac10 >> 2),
940 0x00000000,
941 (0x0e00 << 16) | (0xac14 >> 2),
942 0x00000000,
943 (0x0e00 << 16) | (0xac58 >> 2),
944 0x00000000,
945 (0x0e00 << 16) | (0xac68 >> 2),
946 0x00000000,
947 (0x0e00 << 16) | (0xac6c >> 2),
948 0x00000000,
949 (0x0e00 << 16) | (0xac70 >> 2),
950 0x00000000,
951 (0x0e00 << 16) | (0xac74 >> 2),
952 0x00000000,
953 (0x0e00 << 16) | (0xac78 >> 2),
954 0x00000000,
955 (0x0e00 << 16) | (0xac7c >> 2),
956 0x00000000,
957 (0x0e00 << 16) | (0xac80 >> 2),
958 0x00000000,
959 (0x0e00 << 16) | (0xac84 >> 2),
960 0x00000000,
961 (0x0e00 << 16) | (0xac88 >> 2),
962 0x00000000,
963 (0x0e00 << 16) | (0xac8c >> 2),
964 0x00000000,
965 (0x0e00 << 16) | (0x970c >> 2),
966 0x00000000,
967 (0x0e00 << 16) | (0x9714 >> 2),
968 0x00000000,
969 (0x0e00 << 16) | (0x9718 >> 2),
970 0x00000000,
971 (0x0e00 << 16) | (0x971c >> 2),
972 0x00000000,
973 (0x0e00 << 16) | (0x31068 >> 2),
974 0x00000000,
975 (0x4e00 << 16) | (0x31068 >> 2),
976 0x00000000,
977 (0x5e00 << 16) | (0x31068 >> 2),
978 0x00000000,
979 (0x6e00 << 16) | (0x31068 >> 2),
980 0x00000000,
981 (0x7e00 << 16) | (0x31068 >> 2),
982 0x00000000,
983 (0x0e00 << 16) | (0xcd10 >> 2),
984 0x00000000,
985 (0x0e00 << 16) | (0xcd14 >> 2),
986 0x00000000,
987 (0x0e00 << 16) | (0x88b0 >> 2),
988 0x00000000,
989 (0x0e00 << 16) | (0x88b4 >> 2),
990 0x00000000,
991 (0x0e00 << 16) | (0x88b8 >> 2),
992 0x00000000,
993 (0x0e00 << 16) | (0x88bc >> 2),
994 0x00000000,
995 (0x0400 << 16) | (0x89c0 >> 2),
996 0x00000000,
997 (0x0e00 << 16) | (0x88c4 >> 2),
998 0x00000000,
999 (0x0e00 << 16) | (0x88c8 >> 2),
1000 0x00000000,
1001 (0x0e00 << 16) | (0x88d0 >> 2),
1002 0x00000000,
1003 (0x0e00 << 16) | (0x88d4 >> 2),
1004 0x00000000,
1005 (0x0e00 << 16) | (0x88d8 >> 2),
1006 0x00000000,
1007 (0x0e00 << 16) | (0x8980 >> 2),
1008 0x00000000,
1009 (0x0e00 << 16) | (0x30938 >> 2),
1010 0x00000000,
1011 (0x0e00 << 16) | (0x3093c >> 2),
1012 0x00000000,
1013 (0x0e00 << 16) | (0x30940 >> 2),
1014 0x00000000,
1015 (0x0e00 << 16) | (0x89a0 >> 2),
1016 0x00000000,
1017 (0x0e00 << 16) | (0x30900 >> 2),
1018 0x00000000,
1019 (0x0e00 << 16) | (0x30904 >> 2),
1020 0x00000000,
1021 (0x0e00 << 16) | (0x89b4 >> 2),
1022 0x00000000,
1023 (0x0e00 << 16) | (0x3e1fc >> 2),
1024 0x00000000,
1025 (0x0e00 << 16) | (0x3c210 >> 2),
1026 0x00000000,
1027 (0x0e00 << 16) | (0x3c214 >> 2),
1028 0x00000000,
1029 (0x0e00 << 16) | (0x3c218 >> 2),
1030 0x00000000,
1031 (0x0e00 << 16) | (0x8904 >> 2),
1032 0x00000000,
1033 0x5,
1034 (0x0e00 << 16) | (0x8c28 >> 2),
1035 (0x0e00 << 16) | (0x8c2c >> 2),
1036 (0x0e00 << 16) | (0x8c30 >> 2),
1037 (0x0e00 << 16) | (0x8c34 >> 2),
1038 (0x0e00 << 16) | (0x9600 >> 2),
1039};
1040
0aafd313
AD		 1041static const u32 bonaire_golden_spm_registers[] =
1042{
1043 0x30800, 0xe0ffffff, 0xe0000000
1044};
1045
1046static const u32 bonaire_golden_common_registers[] =
1047{
1048 0xc770, 0xffffffff, 0x00000800,
1049 0xc774, 0xffffffff, 0x00000800,
1050 0xc798, 0xffffffff, 0x00007fbf,
1051 0xc79c, 0xffffffff, 0x00007faf
1052};
1053
1054static const u32 bonaire_golden_registers[] =
1055{
1056 0x3354, 0x00000333, 0x00000333,
1057 0x3350, 0x000c0fc0, 0x00040200,
1058 0x9a10, 0x00010000, 0x00058208,
1059 0x3c000, 0xffff1fff, 0x00140000,
1060 0x3c200, 0xfdfc0fff, 0x00000100,
1061 0x3c234, 0x40000000, 0x40000200,
1062 0x9830, 0xffffffff, 0x00000000,
1063 0x9834, 0xf00fffff, 0x00000400,
1064 0x9838, 0x0002021c, 0x00020200,
1065 0xc78, 0x00000080, 0x00000000,
1066 0x5bb0, 0x000000f0, 0x00000070,
1067 0x5bc0, 0xf0311fff, 0x80300000,
1068 0x98f8, 0x73773777, 0x12010001,
1069 0x350c, 0x00810000, 0x408af000,
1070 0x7030, 0x31000111, 0x00000011,
1071 0x2f48, 0x73773777, 0x12010001,
1072 0x220c, 0x00007fb6, 0x0021a1b1,
1073 0x2210, 0x00007fb6, 0x002021b1,
1074 0x2180, 0x00007fb6, 0x00002191,
1075 0x2218, 0x00007fb6, 0x002121b1,
1076 0x221c, 0x00007fb6, 0x002021b1,
1077 0x21dc, 0x00007fb6, 0x00002191,
1078 0x21e0, 0x00007fb6, 0x00002191,
1079 0x3628, 0x0000003f, 0x0000000a,
1080 0x362c, 0x0000003f, 0x0000000a,
1081 0x2ae4, 0x00073ffe, 0x000022a2,
1082 0x240c, 0x000007ff, 0x00000000,
1083 0x8a14, 0xf000003f, 0x00000007,
1084 0x8bf0, 0x00002001, 0x00000001,
1085 0x8b24, 0xffffffff, 0x00ffffff,
1086 0x30a04, 0x0000ff0f, 0x00000000,
1087 0x28a4c, 0x07ffffff, 0x06000000,
1088 0x4d8, 0x00000fff, 0x00000100,
1089 0x3e78, 0x00000001, 0x00000002,
1090 0x9100, 0x03000000, 0x0362c688,
1091 0x8c00, 0x000000ff, 0x00000001,
1092 0xe40, 0x00001fff, 0x00001fff,
1093 0x9060, 0x0000007f, 0x00000020,
1094 0x9508, 0x00010000, 0x00010000,
1095 0xac14, 0x000003ff, 0x000000f3,
1096 0xac0c, 0xffffffff, 0x00001032
1097};
1098
1099static const u32 bonaire_mgcg_cgcg_init[] =
1100{
1101 0xc420, 0xffffffff, 0xfffffffc,
1102 0x30800, 0xffffffff, 0xe0000000,
1103 0x3c2a0, 0xffffffff, 0x00000100,
1104 0x3c208, 0xffffffff, 0x00000100,
1105 0x3c2c0, 0xffffffff, 0xc0000100,
1106 0x3c2c8, 0xffffffff, 0xc0000100,
1107 0x3c2c4, 0xffffffff, 0xc0000100,
1108 0x55e4, 0xffffffff, 0x00600100,
1109 0x3c280, 0xffffffff, 0x00000100,
1110 0x3c214, 0xffffffff, 0x06000100,
1111 0x3c220, 0xffffffff, 0x00000100,
1112 0x3c218, 0xffffffff, 0x06000100,
1113 0x3c204, 0xffffffff, 0x00000100,
1114 0x3c2e0, 0xffffffff, 0x00000100,
1115 0x3c224, 0xffffffff, 0x00000100,
1116 0x3c200, 0xffffffff, 0x00000100,
1117 0x3c230, 0xffffffff, 0x00000100,
1118 0x3c234, 0xffffffff, 0x00000100,
1119 0x3c250, 0xffffffff, 0x00000100,
1120 0x3c254, 0xffffffff, 0x00000100,
1121 0x3c258, 0xffffffff, 0x00000100,
1122 0x3c25c, 0xffffffff, 0x00000100,
1123 0x3c260, 0xffffffff, 0x00000100,
1124 0x3c27c, 0xffffffff, 0x00000100,
1125 0x3c278, 0xffffffff, 0x00000100,
1126 0x3c210, 0xffffffff, 0x06000100,
1127 0x3c290, 0xffffffff, 0x00000100,
1128 0x3c274, 0xffffffff, 0x00000100,
1129 0x3c2b4, 0xffffffff, 0x00000100,
1130 0x3c2b0, 0xffffffff, 0x00000100,
1131 0x3c270, 0xffffffff, 0x00000100,
1132 0x30800, 0xffffffff, 0xe0000000,
1133 0x3c020, 0xffffffff, 0x00010000,
1134 0x3c024, 0xffffffff, 0x00030002,
1135 0x3c028, 0xffffffff, 0x00040007,
1136 0x3c02c, 0xffffffff, 0x00060005,
1137 0x3c030, 0xffffffff, 0x00090008,
1138 0x3c034, 0xffffffff, 0x00010000,
1139 0x3c038, 0xffffffff, 0x00030002,
1140 0x3c03c, 0xffffffff, 0x00040007,
1141 0x3c040, 0xffffffff, 0x00060005,
1142 0x3c044, 0xffffffff, 0x00090008,
1143 0x3c048, 0xffffffff, 0x00010000,
1144 0x3c04c, 0xffffffff, 0x00030002,
1145 0x3c050, 0xffffffff, 0x00040007,
1146 0x3c054, 0xffffffff, 0x00060005,
1147 0x3c058, 0xffffffff, 0x00090008,
1148 0x3c05c, 0xffffffff, 0x00010000,
1149 0x3c060, 0xffffffff, 0x00030002,
1150 0x3c064, 0xffffffff, 0x00040007,
1151 0x3c068, 0xffffffff, 0x00060005,
1152 0x3c06c, 0xffffffff, 0x00090008,
1153 0x3c070, 0xffffffff, 0x00010000,
1154 0x3c074, 0xffffffff, 0x00030002,
1155 0x3c078, 0xffffffff, 0x00040007,
1156 0x3c07c, 0xffffffff, 0x00060005,
1157 0x3c080, 0xffffffff, 0x00090008,
1158 0x3c084, 0xffffffff, 0x00010000,
1159 0x3c088, 0xffffffff, 0x00030002,
1160 0x3c08c, 0xffffffff, 0x00040007,
1161 0x3c090, 0xffffffff, 0x00060005,
1162 0x3c094, 0xffffffff, 0x00090008,
1163 0x3c098, 0xffffffff, 0x00010000,
1164 0x3c09c, 0xffffffff, 0x00030002,
1165 0x3c0a0, 0xffffffff, 0x00040007,
1166 0x3c0a4, 0xffffffff, 0x00060005,
1167 0x3c0a8, 0xffffffff, 0x00090008,
1168 0x3c000, 0xffffffff, 0x96e00200,
1169 0x8708, 0xffffffff, 0x00900100,
1170 0xc424, 0xffffffff, 0x0020003f,
1171 0x38, 0xffffffff, 0x0140001c,
1172 0x3c, 0x000f0000, 0x000f0000,
1173 0x220, 0xffffffff, 0xC060000C,
1174 0x224, 0xc0000fff, 0x00000100,
1175 0xf90, 0xffffffff, 0x00000100,
1176 0xf98, 0x00000101, 0x00000000,
1177 0x20a8, 0xffffffff, 0x00000104,
1178 0x55e4, 0xff000fff, 0x00000100,
1179 0x30cc, 0xc0000fff, 0x00000104,
1180 0xc1e4, 0x00000001, 0x00000001,
1181 0xd00c, 0xff000ff0, 0x00000100,
1182 0xd80c, 0xff000ff0, 0x00000100
1183};
1184
1185static const u32 spectre_golden_spm_registers[] =
1186{
1187 0x30800, 0xe0ffffff, 0xe0000000
1188};
1189
1190static const u32 spectre_golden_common_registers[] =
1191{
1192 0xc770, 0xffffffff, 0x00000800,
1193 0xc774, 0xffffffff, 0x00000800,
1194 0xc798, 0xffffffff, 0x00007fbf,
1195 0xc79c, 0xffffffff, 0x00007faf
1196};
1197
1198static const u32 spectre_golden_registers[] =
1199{
1200 0x3c000, 0xffff1fff, 0x96940200,
1201 0x3c00c, 0xffff0001, 0xff000000,
1202 0x3c200, 0xfffc0fff, 0x00000100,
1203 0x6ed8, 0x00010101, 0x00010000,
1204 0x9834, 0xf00fffff, 0x00000400,
1205 0x9838, 0xfffffffc, 0x00020200,
1206 0x5bb0, 0x000000f0, 0x00000070,
1207 0x5bc0, 0xf0311fff, 0x80300000,
1208 0x98f8, 0x73773777, 0x12010001,
1209 0x9b7c, 0x00ff0000, 0x00fc0000,
1210 0x2f48, 0x73773777, 0x12010001,
1211 0x8a14, 0xf000003f, 0x00000007,
1212 0x8b24, 0xffffffff, 0x00ffffff,
1213 0x28350, 0x3f3f3fff, 0x00000082,
f1553174 1214 0x28354, 0x0000003f, 0x00000000,
0aafd313
AD		 1215 0x3e78, 0x00000001, 0x00000002,
1216 0x913c, 0xffff03df, 0x00000004,
1217 0xc768, 0x00000008, 0x00000008,
1218 0x8c00, 0x000008ff, 0x00000800,
1219 0x9508, 0x00010000, 0x00010000,
1220 0xac0c, 0xffffffff, 0x54763210,
1221 0x214f8, 0x01ff01ff, 0x00000002,
1222 0x21498, 0x007ff800, 0x00200000,
1223 0x2015c, 0xffffffff, 0x00000f40,
1224 0x30934, 0xffffffff, 0x00000001
1225};
1226
1227static const u32 spectre_mgcg_cgcg_init[] =
1228{
1229 0xc420, 0xffffffff, 0xfffffffc,
1230 0x30800, 0xffffffff, 0xe0000000,
1231 0x3c2a0, 0xffffffff, 0x00000100,
1232 0x3c208, 0xffffffff, 0x00000100,
1233 0x3c2c0, 0xffffffff, 0x00000100,
1234 0x3c2c8, 0xffffffff, 0x00000100,
1235 0x3c2c4, 0xffffffff, 0x00000100,
1236 0x55e4, 0xffffffff, 0x00600100,
1237 0x3c280, 0xffffffff, 0x00000100,
1238 0x3c214, 0xffffffff, 0x06000100,
1239 0x3c220, 0xffffffff, 0x00000100,
1240 0x3c218, 0xffffffff, 0x06000100,
1241 0x3c204, 0xffffffff, 0x00000100,
1242 0x3c2e0, 0xffffffff, 0x00000100,
1243 0x3c224, 0xffffffff, 0x00000100,
1244 0x3c200, 0xffffffff, 0x00000100,
1245 0x3c230, 0xffffffff, 0x00000100,
1246 0x3c234, 0xffffffff, 0x00000100,
1247 0x3c250, 0xffffffff, 0x00000100,
1248 0x3c254, 0xffffffff, 0x00000100,
1249 0x3c258, 0xffffffff, 0x00000100,
1250 0x3c25c, 0xffffffff, 0x00000100,
1251 0x3c260, 0xffffffff, 0x00000100,
1252 0x3c27c, 0xffffffff, 0x00000100,
1253 0x3c278, 0xffffffff, 0x00000100,
1254 0x3c210, 0xffffffff, 0x06000100,
1255 0x3c290, 0xffffffff, 0x00000100,
1256 0x3c274, 0xffffffff, 0x00000100,
1257 0x3c2b4, 0xffffffff, 0x00000100,
1258 0x3c2b0, 0xffffffff, 0x00000100,
1259 0x3c270, 0xffffffff, 0x00000100,
1260 0x30800, 0xffffffff, 0xe0000000,
1261 0x3c020, 0xffffffff, 0x00010000,
1262 0x3c024, 0xffffffff, 0x00030002,
1263 0x3c028, 0xffffffff, 0x00040007,
1264 0x3c02c, 0xffffffff, 0x00060005,
1265 0x3c030, 0xffffffff, 0x00090008,
1266 0x3c034, 0xffffffff, 0x00010000,
1267 0x3c038, 0xffffffff, 0x00030002,
1268 0x3c03c, 0xffffffff, 0x00040007,
1269 0x3c040, 0xffffffff, 0x00060005,
1270 0x3c044, 0xffffffff, 0x00090008,
1271 0x3c048, 0xffffffff, 0x00010000,
1272 0x3c04c, 0xffffffff, 0x00030002,
1273 0x3c050, 0xffffffff, 0x00040007,
1274 0x3c054, 0xffffffff, 0x00060005,
1275 0x3c058, 0xffffffff, 0x00090008,
1276 0x3c05c, 0xffffffff, 0x00010000,
1277 0x3c060, 0xffffffff, 0x00030002,
1278 0x3c064, 0xffffffff, 0x00040007,
1279 0x3c068, 0xffffffff, 0x00060005,
1280 0x3c06c, 0xffffffff, 0x00090008,
1281 0x3c070, 0xffffffff, 0x00010000,
1282 0x3c074, 0xffffffff, 0x00030002,
1283 0x3c078, 0xffffffff, 0x00040007,
1284 0x3c07c, 0xffffffff, 0x00060005,
1285 0x3c080, 0xffffffff, 0x00090008,
1286 0x3c084, 0xffffffff, 0x00010000,
1287 0x3c088, 0xffffffff, 0x00030002,
1288 0x3c08c, 0xffffffff, 0x00040007,
1289 0x3c090, 0xffffffff, 0x00060005,
1290 0x3c094, 0xffffffff, 0x00090008,
1291 0x3c098, 0xffffffff, 0x00010000,
1292 0x3c09c, 0xffffffff, 0x00030002,
1293 0x3c0a0, 0xffffffff, 0x00040007,
1294 0x3c0a4, 0xffffffff, 0x00060005,
1295 0x3c0a8, 0xffffffff, 0x00090008,
1296 0x3c0ac, 0xffffffff, 0x00010000,
1297 0x3c0b0, 0xffffffff, 0x00030002,
1298 0x3c0b4, 0xffffffff, 0x00040007,
1299 0x3c0b8, 0xffffffff, 0x00060005,
1300 0x3c0bc, 0xffffffff, 0x00090008,
1301 0x3c000, 0xffffffff, 0x96e00200,
1302 0x8708, 0xffffffff, 0x00900100,
1303 0xc424, 0xffffffff, 0x0020003f,
1304 0x38, 0xffffffff, 0x0140001c,
1305 0x3c, 0x000f0000, 0x000f0000,
1306 0x220, 0xffffffff, 0xC060000C,
1307 0x224, 0xc0000fff, 0x00000100,
1308 0xf90, 0xffffffff, 0x00000100,
1309 0xf98, 0x00000101, 0x00000000,
1310 0x20a8, 0xffffffff, 0x00000104,
1311 0x55e4, 0xff000fff, 0x00000100,
1312 0x30cc, 0xc0000fff, 0x00000104,
1313 0xc1e4, 0x00000001, 0x00000001,
1314 0xd00c, 0xff000ff0, 0x00000100,
1315 0xd80c, 0xff000ff0, 0x00000100
1316};
1317
1318static const u32 kalindi_golden_spm_registers[] =
1319{
1320 0x30800, 0xe0ffffff, 0xe0000000
1321};
1322
1323static const u32 kalindi_golden_common_registers[] =
1324{
1325 0xc770, 0xffffffff, 0x00000800,
1326 0xc774, 0xffffffff, 0x00000800,
1327 0xc798, 0xffffffff, 0x00007fbf,
1328 0xc79c, 0xffffffff, 0x00007faf
1329};
1330
1331static const u32 kalindi_golden_registers[] =
1332{
1333 0x3c000, 0xffffdfff, 0x6e944040,
1334 0x55e4, 0xff607fff, 0xfc000100,
1335 0x3c220, 0xff000fff, 0x00000100,
1336 0x3c224, 0xff000fff, 0x00000100,
1337 0x3c200, 0xfffc0fff, 0x00000100,
1338 0x6ed8, 0x00010101, 0x00010000,
1339 0x9830, 0xffffffff, 0x00000000,
1340 0x9834, 0xf00fffff, 0x00000400,
1341 0x5bb0, 0x000000f0, 0x00000070,
1342 0x5bc0, 0xf0311fff, 0x80300000,
1343 0x98f8, 0x73773777, 0x12010001,
1344 0x98fc, 0xffffffff, 0x00000010,
1345 0x9b7c, 0x00ff0000, 0x00fc0000,
1346 0x8030, 0x00001f0f, 0x0000100a,
1347 0x2f48, 0x73773777, 0x12010001,
1348 0x2408, 0x000fffff, 0x000c007f,
1349 0x8a14, 0xf000003f, 0x00000007,
1350 0x8b24, 0x3fff3fff, 0x00ffcfff,
1351 0x30a04, 0x0000ff0f, 0x00000000,
1352 0x28a4c, 0x07ffffff, 0x06000000,
1353 0x4d8, 0x00000fff, 0x00000100,
1354 0x3e78, 0x00000001, 0x00000002,
1355 0xc768, 0x00000008, 0x00000008,
1356 0x8c00, 0x000000ff, 0x00000003,
1357 0x214f8, 0x01ff01ff, 0x00000002,
1358 0x21498, 0x007ff800, 0x00200000,
1359 0x2015c, 0xffffffff, 0x00000f40,
1360 0x88c4, 0x001f3ae3, 0x00000082,
1361 0x88d4, 0x0000001f, 0x00000010,
1362 0x30934, 0xffffffff, 0x00000000
1363};
1364
1365static const u32 kalindi_mgcg_cgcg_init[] =
1366{
1367 0xc420, 0xffffffff, 0xfffffffc,
1368 0x30800, 0xffffffff, 0xe0000000,
1369 0x3c2a0, 0xffffffff, 0x00000100,
1370 0x3c208, 0xffffffff, 0x00000100,
1371 0x3c2c0, 0xffffffff, 0x00000100,
1372 0x3c2c8, 0xffffffff, 0x00000100,
1373 0x3c2c4, 0xffffffff, 0x00000100,
1374 0x55e4, 0xffffffff, 0x00600100,
1375 0x3c280, 0xffffffff, 0x00000100,
1376 0x3c214, 0xffffffff, 0x06000100,
1377 0x3c220, 0xffffffff, 0x00000100,
1378 0x3c218, 0xffffffff, 0x06000100,
1379 0x3c204, 0xffffffff, 0x00000100,
1380 0x3c2e0, 0xffffffff, 0x00000100,
1381 0x3c224, 0xffffffff, 0x00000100,
1382 0x3c200, 0xffffffff, 0x00000100,
1383 0x3c230, 0xffffffff, 0x00000100,
1384 0x3c234, 0xffffffff, 0x00000100,
1385 0x3c250, 0xffffffff, 0x00000100,
1386 0x3c254, 0xffffffff, 0x00000100,
1387 0x3c258, 0xffffffff, 0x00000100,
1388 0x3c25c, 0xffffffff, 0x00000100,
1389 0x3c260, 0xffffffff, 0x00000100,
1390 0x3c27c, 0xffffffff, 0x00000100,
1391 0x3c278, 0xffffffff, 0x00000100,
1392 0x3c210, 0xffffffff, 0x06000100,
1393 0x3c290, 0xffffffff, 0x00000100,
1394 0x3c274, 0xffffffff, 0x00000100,
1395 0x3c2b4, 0xffffffff, 0x00000100,
1396 0x3c2b0, 0xffffffff, 0x00000100,
1397 0x3c270, 0xffffffff, 0x00000100,
1398 0x30800, 0xffffffff, 0xe0000000,
1399 0x3c020, 0xffffffff, 0x00010000,
1400 0x3c024, 0xffffffff, 0x00030002,
1401 0x3c028, 0xffffffff, 0x00040007,
1402 0x3c02c, 0xffffffff, 0x00060005,
1403 0x3c030, 0xffffffff, 0x00090008,
1404 0x3c034, 0xffffffff, 0x00010000,
1405 0x3c038, 0xffffffff, 0x00030002,
1406 0x3c03c, 0xffffffff, 0x00040007,
1407 0x3c040, 0xffffffff, 0x00060005,
1408 0x3c044, 0xffffffff, 0x00090008,
1409 0x3c000, 0xffffffff, 0x96e00200,
1410 0x8708, 0xffffffff, 0x00900100,
1411 0xc424, 0xffffffff, 0x0020003f,
1412 0x38, 0xffffffff, 0x0140001c,
1413 0x3c, 0x000f0000, 0x000f0000,
1414 0x220, 0xffffffff, 0xC060000C,
1415 0x224, 0xc0000fff, 0x00000100,
1416 0x20a8, 0xffffffff, 0x00000104,
1417 0x55e4, 0xff000fff, 0x00000100,
1418 0x30cc, 0xc0000fff, 0x00000104,
1419 0xc1e4, 0x00000001, 0x00000001,
1420 0xd00c, 0xff000ff0, 0x00000100,
1421 0xd80c, 0xff000ff0, 0x00000100
1422};
1423
8efff337
AD		 1424static const u32 hawaii_golden_spm_registers[] =
1425{
1426 0x30800, 0xe0ffffff, 0xe0000000
1427};
1428
1429static const u32 hawaii_golden_common_registers[] =
1430{
1431 0x30800, 0xffffffff, 0xe0000000,
1432 0x28350, 0xffffffff, 0x3a00161a,
1433 0x28354, 0xffffffff, 0x0000002e,
1434 0x9a10, 0xffffffff, 0x00018208,
1435 0x98f8, 0xffffffff, 0x12011003
1436};
1437
1438static const u32 hawaii_golden_registers[] =
1439{
1440 0x3354, 0x00000333, 0x00000333,
1441 0x9a10, 0x00010000, 0x00058208,
1442 0x9830, 0xffffffff, 0x00000000,
1443 0x9834, 0xf00fffff, 0x00000400,
1444 0x9838, 0x0002021c, 0x00020200,
1445 0xc78, 0x00000080, 0x00000000,
1446 0x5bb0, 0x000000f0, 0x00000070,
1447 0x5bc0, 0xf0311fff, 0x80300000,
1448 0x350c, 0x00810000, 0x408af000,
1449 0x7030, 0x31000111, 0x00000011,
1450 0x2f48, 0x73773777, 0x12010001,
1451 0x2120, 0x0000007f, 0x0000001b,
1452 0x21dc, 0x00007fb6, 0x00002191,
1453 0x3628, 0x0000003f, 0x0000000a,
1454 0x362c, 0x0000003f, 0x0000000a,
1455 0x2ae4, 0x00073ffe, 0x000022a2,
1456 0x240c, 0x000007ff, 0x00000000,
1457 0x8bf0, 0x00002001, 0x00000001,
1458 0x8b24, 0xffffffff, 0x00ffffff,
1459 0x30a04, 0x0000ff0f, 0x00000000,
1460 0x28a4c, 0x07ffffff, 0x06000000,
1461 0x3e78, 0x00000001, 0x00000002,
1462 0xc768, 0x00000008, 0x00000008,
1463 0xc770, 0x00000f00, 0x00000800,
1464 0xc774, 0x00000f00, 0x00000800,
1465 0xc798, 0x00ffffff, 0x00ff7fbf,
1466 0xc79c, 0x00ffffff, 0x00ff7faf,
1467 0x8c00, 0x000000ff, 0x00000800,
1468 0xe40, 0x00001fff, 0x00001fff,
1469 0x9060, 0x0000007f, 0x00000020,
1470 0x9508, 0x00010000, 0x00010000,
1471 0xae00, 0x00100000, 0x000ff07c,
1472 0xac14, 0x000003ff, 0x0000000f,
1473 0xac10, 0xffffffff, 0x7564fdec,
1474 0xac0c, 0xffffffff, 0x3120b9a8,
1475 0xac08, 0x20000000, 0x0f9c0000
1476};
1477
1478static const u32 hawaii_mgcg_cgcg_init[] =
1479{
1480 0xc420, 0xffffffff, 0xfffffffd,
1481 0x30800, 0xffffffff, 0xe0000000,
1482 0x3c2a0, 0xffffffff, 0x00000100,
1483 0x3c208, 0xffffffff, 0x00000100,
1484 0x3c2c0, 0xffffffff, 0x00000100,
1485 0x3c2c8, 0xffffffff, 0x00000100,
1486 0x3c2c4, 0xffffffff, 0x00000100,
1487 0x55e4, 0xffffffff, 0x00200100,
1488 0x3c280, 0xffffffff, 0x00000100,
1489 0x3c214, 0xffffffff, 0x06000100,
1490 0x3c220, 0xffffffff, 0x00000100,
1491 0x3c218, 0xffffffff, 0x06000100,
1492 0x3c204, 0xffffffff, 0x00000100,
1493 0x3c2e0, 0xffffffff, 0x00000100,
1494 0x3c224, 0xffffffff, 0x00000100,
1495 0x3c200, 0xffffffff, 0x00000100,
1496 0x3c230, 0xffffffff, 0x00000100,
1497 0x3c234, 0xffffffff, 0x00000100,
1498 0x3c250, 0xffffffff, 0x00000100,
1499 0x3c254, 0xffffffff, 0x00000100,
1500 0x3c258, 0xffffffff, 0x00000100,
1501 0x3c25c, 0xffffffff, 0x00000100,
1502 0x3c260, 0xffffffff, 0x00000100,
1503 0x3c27c, 0xffffffff, 0x00000100,
1504 0x3c278, 0xffffffff, 0x00000100,
1505 0x3c210, 0xffffffff, 0x06000100,
1506 0x3c290, 0xffffffff, 0x00000100,
1507 0x3c274, 0xffffffff, 0x00000100,
1508 0x3c2b4, 0xffffffff, 0x00000100,
1509 0x3c2b0, 0xffffffff, 0x00000100,
1510 0x3c270, 0xffffffff, 0x00000100,
1511 0x30800, 0xffffffff, 0xe0000000,
1512 0x3c020, 0xffffffff, 0x00010000,
1513 0x3c024, 0xffffffff, 0x00030002,
1514 0x3c028, 0xffffffff, 0x00040007,
1515 0x3c02c, 0xffffffff, 0x00060005,
1516 0x3c030, 0xffffffff, 0x00090008,
1517 0x3c034, 0xffffffff, 0x00010000,
1518 0x3c038, 0xffffffff, 0x00030002,
1519 0x3c03c, 0xffffffff, 0x00040007,
1520 0x3c040, 0xffffffff, 0x00060005,
1521 0x3c044, 0xffffffff, 0x00090008,
1522 0x3c048, 0xffffffff, 0x00010000,
1523 0x3c04c, 0xffffffff, 0x00030002,
1524 0x3c050, 0xffffffff, 0x00040007,
1525 0x3c054, 0xffffffff, 0x00060005,
1526 0x3c058, 0xffffffff, 0x00090008,
1527 0x3c05c, 0xffffffff, 0x00010000,
1528 0x3c060, 0xffffffff, 0x00030002,
1529 0x3c064, 0xffffffff, 0x00040007,
1530 0x3c068, 0xffffffff, 0x00060005,
1531 0x3c06c, 0xffffffff, 0x00090008,
1532 0x3c070, 0xffffffff, 0x00010000,
1533 0x3c074, 0xffffffff, 0x00030002,
1534 0x3c078, 0xffffffff, 0x00040007,
1535 0x3c07c, 0xffffffff, 0x00060005,
1536 0x3c080, 0xffffffff, 0x00090008,
1537 0x3c084, 0xffffffff, 0x00010000,
1538 0x3c088, 0xffffffff, 0x00030002,
1539 0x3c08c, 0xffffffff, 0x00040007,
1540 0x3c090, 0xffffffff, 0x00060005,
1541 0x3c094, 0xffffffff, 0x00090008,
1542 0x3c098, 0xffffffff, 0x00010000,
1543 0x3c09c, 0xffffffff, 0x00030002,
1544 0x3c0a0, 0xffffffff, 0x00040007,
1545 0x3c0a4, 0xffffffff, 0x00060005,
1546 0x3c0a8, 0xffffffff, 0x00090008,
1547 0x3c0ac, 0xffffffff, 0x00010000,
1548 0x3c0b0, 0xffffffff, 0x00030002,
1549 0x3c0b4, 0xffffffff, 0x00040007,
1550 0x3c0b8, 0xffffffff, 0x00060005,
1551 0x3c0bc, 0xffffffff, 0x00090008,
1552 0x3c0c0, 0xffffffff, 0x00010000,
1553 0x3c0c4, 0xffffffff, 0x00030002,
1554 0x3c0c8, 0xffffffff, 0x00040007,
1555 0x3c0cc, 0xffffffff, 0x00060005,
1556 0x3c0d0, 0xffffffff, 0x00090008,
1557 0x3c0d4, 0xffffffff, 0x00010000,
1558 0x3c0d8, 0xffffffff, 0x00030002,
1559 0x3c0dc, 0xffffffff, 0x00040007,
1560 0x3c0e0, 0xffffffff, 0x00060005,
1561 0x3c0e4, 0xffffffff, 0x00090008,
1562 0x3c0e8, 0xffffffff, 0x00010000,
1563 0x3c0ec, 0xffffffff, 0x00030002,
1564 0x3c0f0, 0xffffffff, 0x00040007,
1565 0x3c0f4, 0xffffffff, 0x00060005,
1566 0x3c0f8, 0xffffffff, 0x00090008,
1567 0xc318, 0xffffffff, 0x00020200,
1568 0x3350, 0xffffffff, 0x00000200,
1569 0x15c0, 0xffffffff, 0x00000400,
1570 0x55e8, 0xffffffff, 0x00000000,
1571 0x2f50, 0xffffffff, 0x00000902,
1572 0x3c000, 0xffffffff, 0x96940200,
1573 0x8708, 0xffffffff, 0x00900100,
1574 0xc424, 0xffffffff, 0x0020003f,
1575 0x38, 0xffffffff, 0x0140001c,
1576 0x3c, 0x000f0000, 0x000f0000,
1577 0x220, 0xffffffff, 0xc060000c,
1578 0x224, 0xc0000fff, 0x00000100,
1579 0xf90, 0xffffffff, 0x00000100,
1580 0xf98, 0x00000101, 0x00000000,
1581 0x20a8, 0xffffffff, 0x00000104,
1582 0x55e4, 0xff000fff, 0x00000100,
1583 0x30cc, 0xc0000fff, 0x00000104,
1584 0xc1e4, 0x00000001, 0x00000001,
1585 0xd00c, 0xff000ff0, 0x00000100,
1586 0xd80c, 0xff000ff0, 0x00000100
1587};
1588
f73a9e83
SL		 1589static const u32 godavari_golden_registers[] =
1590{
1591 0x55e4, 0xff607fff, 0xfc000100,
1592 0x6ed8, 0x00010101, 0x00010000,
1593 0x9830, 0xffffffff, 0x00000000,
1594 0x98302, 0xf00fffff, 0x00000400,
1595 0x6130, 0xffffffff, 0x00010000,
1596 0x5bb0, 0x000000f0, 0x00000070,
1597 0x5bc0, 0xf0311fff, 0x80300000,
1598 0x98f8, 0x73773777, 0x12010001,
1599 0x98fc, 0xffffffff, 0x00000010,
1600 0x8030, 0x00001f0f, 0x0000100a,
1601 0x2f48, 0x73773777, 0x12010001,
1602 0x2408, 0x000fffff, 0x000c007f,
1603 0x8a14, 0xf000003f, 0x00000007,
1604 0x8b24, 0xffffffff, 0x00ff0fff,
1605 0x30a04, 0x0000ff0f, 0x00000000,
1606 0x28a4c, 0x07ffffff, 0x06000000,
1607 0x4d8, 0x00000fff, 0x00000100,
1608 0xd014, 0x00010000, 0x00810001,
1609 0xd814, 0x00010000, 0x00810001,
1610 0x3e78, 0x00000001, 0x00000002,
1611 0xc768, 0x00000008, 0x00000008,
1612 0xc770, 0x00000f00, 0x00000800,
1613 0xc774, 0x00000f00, 0x00000800,
1614 0xc798, 0x00ffffff, 0x00ff7fbf,
1615 0xc79c, 0x00ffffff, 0x00ff7faf,
1616 0x8c00, 0x000000ff, 0x00000001,
1617 0x214f8, 0x01ff01ff, 0x00000002,
1618 0x21498, 0x007ff800, 0x00200000,
1619 0x2015c, 0xffffffff, 0x00000f40,
1620 0x88c4, 0x001f3ae3, 0x00000082,
1621 0x88d4, 0x0000001f, 0x00000010,
1622 0x30934, 0xffffffff, 0x00000000
1623};
1624
1625
0aafd313
AD		 1626static void cik_init_golden_registers(struct radeon_device *rdev)
1627{
1c0a4625
OG		 1628 /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1629 mutex_lock(&rdev->grbm_idx_mutex);
0aafd313
AD		 1630 switch (rdev->family) {
1631 case CHIP_BONAIRE:
1632 radeon_program_register_sequence(rdev,
1633 bonaire_mgcg_cgcg_init,
1634 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1635 radeon_program_register_sequence(rdev,
1636 bonaire_golden_registers,
1637 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1638 radeon_program_register_sequence(rdev,
1639 bonaire_golden_common_registers,
1640 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1641 radeon_program_register_sequence(rdev,
1642 bonaire_golden_spm_registers,
1643 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1644 break;
1645 case CHIP_KABINI:
1646 radeon_program_register_sequence(rdev,
1647 kalindi_mgcg_cgcg_init,
1648 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1649 radeon_program_register_sequence(rdev,
1650 kalindi_golden_registers,
1651 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1652 radeon_program_register_sequence(rdev,
1653 kalindi_golden_common_registers,
1654 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1655 radeon_program_register_sequence(rdev,
1656 kalindi_golden_spm_registers,
1657 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1658 break;
f73a9e83
SL		 1659 case CHIP_MULLINS:
1660 radeon_program_register_sequence(rdev,
1661 kalindi_mgcg_cgcg_init,
1662 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1663 radeon_program_register_sequence(rdev,
1664 godavari_golden_registers,
1665 (const u32)ARRAY_SIZE(godavari_golden_registers));
1666 radeon_program_register_sequence(rdev,
1667 kalindi_golden_common_registers,
1668 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1669 radeon_program_register_sequence(rdev,
1670 kalindi_golden_spm_registers,
1671 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1672 break;
0aafd313
AD		 1673 case CHIP_KAVERI:
1674 radeon_program_register_sequence(rdev,
1675 spectre_mgcg_cgcg_init,
1676 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1677 radeon_program_register_sequence(rdev,
1678 spectre_golden_registers,
1679 (const u32)ARRAY_SIZE(spectre_golden_registers));
1680 radeon_program_register_sequence(rdev,
1681 spectre_golden_common_registers,
1682 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1683 radeon_program_register_sequence(rdev,
1684 spectre_golden_spm_registers,
1685 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1686 break;
8efff337
AD		 1687 case CHIP_HAWAII:
1688 radeon_program_register_sequence(rdev,
1689 hawaii_mgcg_cgcg_init,
1690 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1691 radeon_program_register_sequence(rdev,
1692 hawaii_golden_registers,
1693 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1694 radeon_program_register_sequence(rdev,
1695 hawaii_golden_common_registers,
1696 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1697 radeon_program_register_sequence(rdev,
1698 hawaii_golden_spm_registers,
1699 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1700 break;
0aafd313
AD		 1701 default:
1702 break;
1703 }
1c0a4625 1704 mutex_unlock(&rdev->grbm_idx_mutex);
0aafd313
AD		 1705}
1706
2c67912c
AD		 1707/**
1708 * cik_get_xclk - get the xclk
1709 *
1710 * @rdev: radeon_device pointer
1711 *
1712 * Returns the reference clock used by the gfx engine
1713 * (CIK).
1714 */
1715u32 cik_get_xclk(struct radeon_device *rdev)
1716{
3cf8bb1a 1717 u32 reference_clock = rdev->clock.spll.reference_freq;
2c67912c
AD		 1718
1719 if (rdev->flags & RADEON_IS_IGP) {
1720 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1721 return reference_clock / 2;
1722 } else {
1723 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1724 return reference_clock / 4;
1725 }
1726 return reference_clock;
1727}
1728
75efdee1
AD		 1729/**
1730 * cik_mm_rdoorbell - read a doorbell dword
1731 *
1732 * @rdev: radeon_device pointer
d5754ab8 1733 * @index: doorbell index
75efdee1
AD		 1734 *
1735 * Returns the value in the doorbell aperture at the
d5754ab8 1736 * requested doorbell index (CIK).
75efdee1 1737 */
d5754ab8 1738u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
75efdee1 1739{
d5754ab8
AL		 1740 if (index < rdev->doorbell.num_doorbells) {
1741 return readl(rdev->doorbell.ptr + index);
75efdee1 1742 } else {
d5754ab8 1743 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1744 return 0;
1745 }
1746}
1747
1748/**
1749 * cik_mm_wdoorbell - write a doorbell dword
1750 *
1751 * @rdev: radeon_device pointer
d5754ab8 1752 * @index: doorbell index
75efdee1
AD		 1753 * @v: value to write
1754 *
1755 * Writes @v to the doorbell aperture at the
d5754ab8 1756 * requested doorbell index (CIK).
75efdee1 1757 */
d5754ab8 1758void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
75efdee1 1759{
d5754ab8
AL		 1760 if (index < rdev->doorbell.num_doorbells) {
1761 writel(v, rdev->doorbell.ptr + index);
75efdee1 1762 } else {
d5754ab8 1763 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1764 }
1765}
1766
bc8273fe
AD		 1767#define BONAIRE_IO_MC_REGS_SIZE 36
1768
1769static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1770{
1771 {0x00000070, 0x04400000},
1772 {0x00000071, 0x80c01803},
1773 {0x00000072, 0x00004004},
1774 {0x00000073, 0x00000100},
1775 {0x00000074, 0x00ff0000},
1776 {0x00000075, 0x34000000},
1777 {0x00000076, 0x08000014},
1778 {0x00000077, 0x00cc08ec},
1779 {0x00000078, 0x00000400},
1780 {0x00000079, 0x00000000},
1781 {0x0000007a, 0x04090000},
1782 {0x0000007c, 0x00000000},
1783 {0x0000007e, 0x4408a8e8},
1784 {0x0000007f, 0x00000304},
1785 {0x00000080, 0x00000000},
1786 {0x00000082, 0x00000001},
1787 {0x00000083, 0x00000002},
1788 {0x00000084, 0xf3e4f400},
1789 {0x00000085, 0x052024e3},
1790 {0x00000087, 0x00000000},
1791 {0x00000088, 0x01000000},
1792 {0x0000008a, 0x1c0a0000},
1793 {0x0000008b, 0xff010000},
1794 {0x0000008d, 0xffffefff},
1795 {0x0000008e, 0xfff3efff},
1796 {0x0000008f, 0xfff3efbf},
1797 {0x00000092, 0xf7ffffff},
1798 {0x00000093, 0xffffff7f},
1799 {0x00000095, 0x00101101},
1800 {0x00000096, 0x00000fff},
1801 {0x00000097, 0x00116fff},
1802 {0x00000098, 0x60010000},
1803 {0x00000099, 0x10010000},
1804 {0x0000009a, 0x00006000},
1805 {0x0000009b, 0x00001000},
1806 {0x0000009f, 0x00b48000}
1807};
1808
d4775655
AD		 1809#define HAWAII_IO_MC_REGS_SIZE 22
1810
1811static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1812{
1813 {0x0000007d, 0x40000000},
1814 {0x0000007e, 0x40180304},
1815 {0x0000007f, 0x0000ff00},
1816 {0x00000081, 0x00000000},
1817 {0x00000083, 0x00000800},
1818 {0x00000086, 0x00000000},
1819 {0x00000087, 0x00000100},
1820 {0x00000088, 0x00020100},
1821 {0x00000089, 0x00000000},
1822 {0x0000008b, 0x00040000},
1823 {0x0000008c, 0x00000100},
1824 {0x0000008e, 0xff010000},
1825 {0x00000090, 0xffffefff},
1826 {0x00000091, 0xfff3efff},
1827 {0x00000092, 0xfff3efbf},
1828 {0x00000093, 0xf7ffffff},
1829 {0x00000094, 0xffffff7f},
1830 {0x00000095, 0x00000fff},
1831 {0x00000096, 0x00116fff},
1832 {0x00000097, 0x60010000},
1833 {0x00000098, 0x10010000},
1834 {0x0000009f, 0x00c79000}
1835};
1836
1837
b556b12e
AD		 1838/**
1839 * cik_srbm_select - select specific register instances
1840 *
1841 * @rdev: radeon_device pointer
1842 * @me: selected ME (micro engine)
1843 * @pipe: pipe
1844 * @queue: queue
1845 * @vmid: VMID
1846 *
1847 * Switches the currently active registers instances. Some
1848 * registers are instanced per VMID, others are instanced per
1849 * me/pipe/queue combination.
1850 */
1851static void cik_srbm_select(struct radeon_device *rdev,
1852 u32 me, u32 pipe, u32 queue, u32 vmid)
1853{
1854 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1855 MEID(me & 0x3) |
1856 VMID(vmid & 0xf) |
1857 QUEUEID(queue & 0x7));
1858 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1859}
1860
bc8273fe
AD		 1861/* ucode loading */
1862/**
1863 * ci_mc_load_microcode - load MC ucode into the hw
1864 *
1865 * @rdev: radeon_device pointer
1866 *
1867 * Load the GDDR MC ucode into the hw (CIK).
1868 * Returns 0 on success, error on failure.
1869 */
6c7bccea 1870int ci_mc_load_microcode(struct radeon_device *rdev)
bc8273fe 1871{
f2c6b0f4
AD		 1872 const __be32 *fw_data = NULL;
1873 const __le32 *new_fw_data = NULL;
9feb3dda 1874 u32 running, blackout = 0, tmp;
f2c6b0f4
AD		 1875 u32 *io_mc_regs = NULL;
1876 const __le32 *new_io_mc_regs = NULL;
bcddee29 1877 int i, regs_size, ucode_size;
bc8273fe
AD		 1878
1879 if (!rdev->mc_fw)
1880 return -EINVAL;
1881
f2c6b0f4
AD		 1882 if (rdev->new_fw) {
1883 const struct mc_firmware_header_v1_0 *hdr =
1884 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
bcddee29 1885
f2c6b0f4
AD		 1886 radeon_ucode_print_mc_hdr(&hdr->header);
1887
1888 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1889 new_io_mc_regs = (const __le32 *)
1890 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1891 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1892 new_fw_data = (const __le32 *)
1893 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1894 } else {
1895 ucode_size = rdev->mc_fw->size / 4;
1896
1897 switch (rdev->family) {
1898 case CHIP_BONAIRE:
1899 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1900 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1901 break;
1902 case CHIP_HAWAII:
1903 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1904 regs_size = HAWAII_IO_MC_REGS_SIZE;
1905 break;
1906 default:
1907 return -EINVAL;
1908 }
1909 fw_data = (const __be32 *)rdev->mc_fw->data;
bc8273fe
AD		 1910 }
1911
1912 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1913
1914 if (running == 0) {
1915 if (running) {
1916 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1917 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1918 }
1919
1920 /* reset the engine and set to writable */
1921 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1922 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1923
1924 /* load mc io regs */
1925 for (i = 0; i < regs_size; i++) {
f2c6b0f4
AD		 1926 if (rdev->new_fw) {
1927 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1928 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1929 } else {
1930 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1931 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1932 }
bc8273fe 1933 }
9feb3dda
AD		 1934
1935 tmp = RREG32(MC_SEQ_MISC0);
1936 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1937 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1938 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1939 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1940 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1941 }
1942
bc8273fe 1943 /* load the MC ucode */
f2c6b0f4
AD		 1944 for (i = 0; i < ucode_size; i++) {
1945 if (rdev->new_fw)
1946 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1947 else
1948 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1949 }
bc8273fe
AD		 1950
1951 /* put the engine back into the active state */
1952 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1953 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1954 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1955
1956 /* wait for training to complete */
1957 for (i = 0; i < rdev->usec_timeout; i++) {
1958 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1959 break;
1960 udelay(1);
1961 }
1962 for (i = 0; i < rdev->usec_timeout; i++) {
1963 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1964 break;
1965 udelay(1);
1966 }
1967
1968 if (running)
1969 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1970 }
1971
1972 return 0;
1973}
1974
02c81327
AD		 1975/**
1976 * cik_init_microcode - load ucode images from disk
1977 *
1978 * @rdev: radeon_device pointer
1979 *
1980 * Use the firmware interface to load the ucode images into
1981 * the driver (not loaded into hw).
1982 * Returns 0 on success, error on failure.
1983 */
1984static int cik_init_microcode(struct radeon_device *rdev)
1985{
02c81327 1986 const char *chip_name;
f2c6b0f4 1987 const char *new_chip_name;
02c81327 1988 size_t pfp_req_size, me_req_size, ce_req_size,
d4775655 1989 mec_req_size, rlc_req_size, mc_req_size = 0,
277babc3 1990 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
02c81327 1991 char fw_name[30];
f2c6b0f4 1992 int new_fw = 0;
02c81327 1993 int err;
f2c6b0f4 1994 int num_fw;
b2ea0dcd 1995 bool new_smc = false;
02c81327
AD		 1996
1997 DRM_DEBUG("\n");
1998
02c81327
AD		 1999 switch (rdev->family) {
2000 case CHIP_BONAIRE:
2001 chip_name = "BONAIRE";
b2ea0dcd
AD		 2002 if ((rdev->pdev->revision == 0x80) ||
2003 (rdev->pdev->revision == 0x81) ||
2004 (rdev->pdev->device == 0x665f))
2005 new_smc = true;
f2c6b0f4 2006 new_chip_name = "bonaire";
02c81327
AD		 2007 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2008 me_req_size = CIK_ME_UCODE_SIZE * 4;
2009 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2010 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2011 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
277babc3
AD		 2012 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
2013 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
21a93e13 2014 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 2015 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
f2c6b0f4 2016 num_fw = 8;
02c81327 2017 break;
d4775655
AD		 2018 case CHIP_HAWAII:
2019 chip_name = "HAWAII";
b2ea0dcd
AD		 2020 if (rdev->pdev->revision == 0x80)
2021 new_smc = true;
f2c6b0f4 2022 new_chip_name = "hawaii";
d4775655
AD		 2023 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2024 me_req_size = CIK_ME_UCODE_SIZE * 4;
2025 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2026 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2027 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2028 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
277babc3 2029 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
d4775655
AD		 2030 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2031 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
f2c6b0f4 2032 num_fw = 8;
d4775655 2033 break;
02c81327
AD		 2034 case CHIP_KAVERI:
2035 chip_name = "KAVERI";
f2c6b0f4 2036 new_chip_name = "kaveri";
02c81327
AD		 2037 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2038 me_req_size = CIK_ME_UCODE_SIZE * 4;
2039 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2040 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2041 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 2042 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2043 num_fw = 7;
02c81327
AD		 2044 break;
2045 case CHIP_KABINI:
2046 chip_name = "KABINI";
f2c6b0f4 2047 new_chip_name = "kabini";
02c81327
AD		 2048 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2049 me_req_size = CIK_ME_UCODE_SIZE * 4;
2050 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2051 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2052 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 2053 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2054 num_fw = 6;
02c81327 2055 break;
f73a9e83
SL		 2056 case CHIP_MULLINS:
2057 chip_name = "MULLINS";
f2c6b0f4 2058 new_chip_name = "mullins";
f73a9e83
SL		 2059 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2060 me_req_size = CIK_ME_UCODE_SIZE * 4;
2061 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2062 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2063 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
2064 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2065 num_fw = 6;
f73a9e83 2066 break;
02c81327
AD		 2067 default: BUG();
2068 }
2069
f2c6b0f4 2070 DRM_INFO("Loading %s Microcode\n", new_chip_name);
02c81327 2071
f2c6b0f4 2072 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
0a168933 2073 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2074 if (err) {
2075 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2076 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2077 if (err)
2078 goto out;
2079 if (rdev->pfp_fw->size != pfp_req_size) {
2080 printk(KERN_ERR
2081 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2082 rdev->pfp_fw->size, fw_name);
2083 err = -EINVAL;
2084 goto out;
2085 }
2086 } else {
2087 err = radeon_ucode_validate(rdev->pfp_fw);
2088 if (err) {
2089 printk(KERN_ERR
2090 "cik_fw: validation failed for firmware \"%s\"\n",
2091 fw_name);
2092 goto out;
2093 } else {
2094 new_fw++;
2095 }
02c81327
AD		 2096 }
2097
f2c6b0f4 2098 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
0a168933 2099 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2100 if (err) {
2101 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2102 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2103 if (err)
2104 goto out;
2105 if (rdev->me_fw->size != me_req_size) {
2106 printk(KERN_ERR
2107 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2108 rdev->me_fw->size, fw_name);
2109 err = -EINVAL;
2110 }
2111 } else {
2112 err = radeon_ucode_validate(rdev->me_fw);
2113 if (err) {
2114 printk(KERN_ERR
2115 "cik_fw: validation failed for firmware \"%s\"\n",
2116 fw_name);
2117 goto out;
2118 } else {
2119 new_fw++;
2120 }
02c81327
AD		 2121 }
2122
f2c6b0f4 2123 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
0a168933 2124 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2125 if (err) {
2126 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2127 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2128 if (err)
2129 goto out;
2130 if (rdev->ce_fw->size != ce_req_size) {
2131 printk(KERN_ERR
2132 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2133 rdev->ce_fw->size, fw_name);
2134 err = -EINVAL;
2135 }
2136 } else {
2137 err = radeon_ucode_validate(rdev->ce_fw);
2138 if (err) {
2139 printk(KERN_ERR
2140 "cik_fw: validation failed for firmware \"%s\"\n",
2141 fw_name);
2142 goto out;
2143 } else {
2144 new_fw++;
2145 }
02c81327
AD		 2146 }
2147
f2c6b0f4 2148 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
0a168933 2149 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2150 if (err) {
2151 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2152 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2153 if (err)
2154 goto out;
2155 if (rdev->mec_fw->size != mec_req_size) {
2156 printk(KERN_ERR
2157 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2158 rdev->mec_fw->size, fw_name);
2159 err = -EINVAL;
2160 }
2161 } else {
2162 err = radeon_ucode_validate(rdev->mec_fw);
2163 if (err) {
2164 printk(KERN_ERR
2165 "cik_fw: validation failed for firmware \"%s\"\n",
2166 fw_name);
2167 goto out;
2168 } else {
2169 new_fw++;
2170 }
2171 }
2172
2173 if (rdev->family == CHIP_KAVERI) {
2174 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2175 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2176 if (err) {
2177 goto out;
2178 } else {
2179 err = radeon_ucode_validate(rdev->mec2_fw);
2180 if (err) {
2181 goto out;
2182 } else {
2183 new_fw++;
2184 }
2185 }
02c81327
AD		 2186 }
2187
f2c6b0f4 2188 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
0a168933 2189 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2190 if (err) {
2191 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2192 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2193 if (err)
2194 goto out;
2195 if (rdev->rlc_fw->size != rlc_req_size) {
2196 printk(KERN_ERR
2197 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2198 rdev->rlc_fw->size, fw_name);
2199 err = -EINVAL;
2200 }
2201 } else {
2202 err = radeon_ucode_validate(rdev->rlc_fw);
2203 if (err) {
2204 printk(KERN_ERR
2205 "cik_fw: validation failed for firmware \"%s\"\n",
2206 fw_name);
2207 goto out;
2208 } else {
2209 new_fw++;
2210 }
02c81327
AD		 2211 }
2212
f2c6b0f4 2213 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
0a168933 2214 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2215 if (err) {
2216 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2217 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2218 if (err)
2219 goto out;
2220 if (rdev->sdma_fw->size != sdma_req_size) {
2221 printk(KERN_ERR
2222 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2223 rdev->sdma_fw->size, fw_name);
2224 err = -EINVAL;
2225 }
2226 } else {
2227 err = radeon_ucode_validate(rdev->sdma_fw);
2228 if (err) {
2229 printk(KERN_ERR
2230 "cik_fw: validation failed for firmware \"%s\"\n",
2231 fw_name);
2232 goto out;
2233 } else {
2234 new_fw++;
2235 }
21a93e13
AD		 2236 }
2237
cc8dbbb4 2238 /* No SMC, MC ucode on APUs */
02c81327 2239 if (!(rdev->flags & RADEON_IS_IGP)) {
f2c6b0f4 2240 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
0a168933 2241 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
277babc3 2242 if (err) {
f2c6b0f4 2243 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
277babc3 2244 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2245 if (err) {
2246 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2247 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2248 if (err)
2249 goto out;
2250 }
2251 if ((rdev->mc_fw->size != mc_req_size) &&
2252 (rdev->mc_fw->size != mc2_req_size)){
2253 printk(KERN_ERR
2254 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2255 rdev->mc_fw->size, fw_name);
2256 err = -EINVAL;
2257 }
2258 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2259 } else {
2260 err = radeon_ucode_validate(rdev->mc_fw);
2261 if (err) {
2262 printk(KERN_ERR
2263 "cik_fw: validation failed for firmware \"%s\"\n",
2264 fw_name);
277babc3 2265 goto out;
f2c6b0f4
AD		 2266 } else {
2267 new_fw++;
2268 }
277babc3 2269 }
cc8dbbb4 2270
b2ea0dcd
AD		 2271 if (new_smc)
2272 snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
2273 else
2274 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
cc8dbbb4
AD		 2275 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2276 if (err) {
f2c6b0f4
AD		 2277 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2278 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2279 if (err) {
2280 printk(KERN_ERR
2281 "smc: error loading firmware \"%s\"\n",
2282 fw_name);
2283 release_firmware(rdev->smc_fw);
2284 rdev->smc_fw = NULL;
2285 err = 0;
2286 } else if (rdev->smc_fw->size != smc_req_size) {
2287 printk(KERN_ERR
2288 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2289 rdev->smc_fw->size, fw_name);
2290 err = -EINVAL;
2291 }
2292 } else {
2293 err = radeon_ucode_validate(rdev->smc_fw);
2294 if (err) {
2295 printk(KERN_ERR
2296 "cik_fw: validation failed for firmware \"%s\"\n",
2297 fw_name);
2298 goto out;
2299 } else {
2300 new_fw++;
2301 }
cc8dbbb4 2302 }
02c81327
AD		 2303 }
2304
f2c6b0f4
AD		 2305 if (new_fw == 0) {
2306 rdev->new_fw = false;
2307 } else if (new_fw < num_fw) {
2308 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2309 err = -EINVAL;
2310 } else {
2311 rdev->new_fw = true;
2312 }
2313
02c81327 2314out:
02c81327
AD		 2315 if (err) {
2316 if (err != -EINVAL)
2317 printk(KERN_ERR
2318 "cik_cp: Failed to load firmware \"%s\"\n",
2319 fw_name);
2320 release_firmware(rdev->pfp_fw);
2321 rdev->pfp_fw = NULL;
2322 release_firmware(rdev->me_fw);
2323 rdev->me_fw = NULL;
2324 release_firmware(rdev->ce_fw);
2325 rdev->ce_fw = NULL;
f2c6b0f4
AD		 2326 release_firmware(rdev->mec_fw);
2327 rdev->mec_fw = NULL;
2328 release_firmware(rdev->mec2_fw);
2329 rdev->mec2_fw = NULL;
02c81327
AD		 2330 release_firmware(rdev->rlc_fw);
2331 rdev->rlc_fw = NULL;
f2c6b0f4
AD		 2332 release_firmware(rdev->sdma_fw);
2333 rdev->sdma_fw = NULL;
02c81327
AD		 2334 release_firmware(rdev->mc_fw);
2335 rdev->mc_fw = NULL;
cc8dbbb4
AD		 2336 release_firmware(rdev->smc_fw);
2337 rdev->smc_fw = NULL;
02c81327
AD		 2338 }
2339 return err;
2340}
2341
8cc1a532
AD		 2342/*
2343 * Core functions
2344 */
2345/**
2346 * cik_tiling_mode_table_init - init the hw tiling table
2347 *
2348 * @rdev: radeon_device pointer
2349 *
2350 * Starting with SI, the tiling setup is done globally in a
2351 * set of 32 tiling modes. Rather than selecting each set of
2352 * parameters per surface as on older asics, we just select
2353 * which index in the tiling table we want to use, and the
2354 * surface uses those parameters (CIK).
2355 */
2356static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2357{
f0e201f2
JP		 2358 u32 *tile = rdev->config.cik.tile_mode_array;
2359 u32 *macrotile = rdev->config.cik.macrotile_mode_array;
2360 const u32 num_tile_mode_states =
2361 ARRAY_SIZE(rdev->config.cik.tile_mode_array);
2362 const u32 num_secondary_tile_mode_states =
2363 ARRAY_SIZE(rdev->config.cik.macrotile_mode_array);
2364 u32 reg_offset, split_equal_to_row_size;
8cc1a532
AD		 2365 u32 num_pipe_configs;
2366 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2367 rdev->config.cik.max_shader_engines;
2368
2369 switch (rdev->config.cik.mem_row_size_in_kb) {
2370 case 1:
2371 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2372 break;
2373 case 2:
2374 default:
2375 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2376 break;
2377 case 4:
2378 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2379 break;
2380 }
2381
2382 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2383 if (num_pipe_configs > 8)
21e438af 2384 num_pipe_configs = 16;
8cc1a532 2385
f0e201f2
JP		 2386 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2387 tile[reg_offset] = 0;
2388 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2389 macrotile[reg_offset] = 0;
2390
2391 switch(num_pipe_configs) {
2392 case 16:
2393 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2394 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2395 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2397 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2398 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2399 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2400 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2401 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2402 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2403 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2404 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2405 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2406 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2407 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2408 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2409 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2410 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2411 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2412 TILE_SPLIT(split_equal_to_row_size));
2413 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2414 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2415 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2416 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2417 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2418 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2420 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2421 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2422 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2423 TILE_SPLIT(split_equal_to_row_size));
2424 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2425 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2426 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2427 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2428 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2429 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2430 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2431 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2432 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2433 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2434 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2435 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2436 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2437 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2438 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2439 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2440 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2441 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2442 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2443 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2444 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2445 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2446 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2447 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2448 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2449 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2450 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2451 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2452 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2453 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2454 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2455 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2456 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2457 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2458 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2459 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2460 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2461 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2462 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2463 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2464 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2465 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2466 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2467 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2468 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2469 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2470 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2471
2472 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2473 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2474 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2475 NUM_BANKS(ADDR_SURF_16_BANK));
2476 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2477 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2478 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2479 NUM_BANKS(ADDR_SURF_16_BANK));
2480 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2481 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2482 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2483 NUM_BANKS(ADDR_SURF_16_BANK));
2484 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2485 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2486 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2487 NUM_BANKS(ADDR_SURF_16_BANK));
2488 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2489 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2490 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2491 NUM_BANKS(ADDR_SURF_8_BANK));
2492 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2493 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2494 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2495 NUM_BANKS(ADDR_SURF_4_BANK));
2496 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2497 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2498 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2499 NUM_BANKS(ADDR_SURF_2_BANK));
2500 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2501 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2502 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2503 NUM_BANKS(ADDR_SURF_16_BANK));
2504 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2505 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2506 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2507 NUM_BANKS(ADDR_SURF_16_BANK));
2508 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2509 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2510 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2511 NUM_BANKS(ADDR_SURF_16_BANK));
2512 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2513 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2514 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2515 NUM_BANKS(ADDR_SURF_8_BANK));
2516 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2517 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2518 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2519 NUM_BANKS(ADDR_SURF_4_BANK));
2520 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2521 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2522 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2523 NUM_BANKS(ADDR_SURF_2_BANK));
2524 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2525 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2526 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2527 NUM_BANKS(ADDR_SURF_2_BANK));
2528
2529 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2530 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2531 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2532 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2533 break;
2534
2535 case 8:
2536 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2537 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2538 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2539 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2540 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2541 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2542 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2543 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2544 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2545 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2546 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2547 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2548 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2549 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2550 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2551 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2552 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2553 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2554 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2555 TILE_SPLIT(split_equal_to_row_size));
2556 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2557 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2558 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2559 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2560 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2561 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2562 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2563 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2564 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2566 TILE_SPLIT(split_equal_to_row_size));
2567 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2568 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2569 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2571 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2572 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2573 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2574 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2575 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2576 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2577 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2578 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2579 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2580 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2581 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2582 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2583 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2584 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2585 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2586 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2587 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2588 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2590 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2591 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2592 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2593 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2594 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2595 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2596 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2597 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2598 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2599 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2600 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2601 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2602 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2603 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2604 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2605 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2606 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2607 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2608 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2609 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2610 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2611 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2612 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2613 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2614
2615 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2616 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2617 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2618 NUM_BANKS(ADDR_SURF_16_BANK));
2619 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2620 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2621 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2622 NUM_BANKS(ADDR_SURF_16_BANK));
2623 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2624 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2625 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2626 NUM_BANKS(ADDR_SURF_16_BANK));
2627 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2629 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2630 NUM_BANKS(ADDR_SURF_16_BANK));
2631 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2632 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2633 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2634 NUM_BANKS(ADDR_SURF_8_BANK));
2635 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2636 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2637 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2638 NUM_BANKS(ADDR_SURF_4_BANK));
2639 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2640 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2641 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2642 NUM_BANKS(ADDR_SURF_2_BANK));
2643 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2644 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2645 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2646 NUM_BANKS(ADDR_SURF_16_BANK));
2647 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2648 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2649 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2650 NUM_BANKS(ADDR_SURF_16_BANK));
2651 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2652 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2653 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2654 NUM_BANKS(ADDR_SURF_16_BANK));
2655 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2656 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2657 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2658 NUM_BANKS(ADDR_SURF_16_BANK));
2659 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2660 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2661 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2662 NUM_BANKS(ADDR_SURF_8_BANK));
2663 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2664 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2665 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2666 NUM_BANKS(ADDR_SURF_4_BANK));
2667 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2668 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2669 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2670 NUM_BANKS(ADDR_SURF_2_BANK));
2671
2672 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2673 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2674 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2675 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2676 break;
2677
2678 case 4:
8cc1a532 2679 if (num_rbs == 4) {
f0e201f2
JP		 2680 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2681 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2682 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2683 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2684 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2685 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2686 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2687 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2688 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2689 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2690 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2691 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2692 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2693 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2694 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2695 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2696 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2697 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2698 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2699 TILE_SPLIT(split_equal_to_row_size));
2700 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2701 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2702 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2703 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2704 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2705 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2706 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2707 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2708 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2709 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2710 TILE_SPLIT(split_equal_to_row_size));
2711 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2712 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2713 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2714 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2715 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2716 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2717 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2718 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2719 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2720 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2721 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2722 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2723 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2724 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2725 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2726 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2727 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2728 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2729 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2730 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2731 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2732 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2733 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2734 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2735 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2736 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2737 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2738 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2739 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2740 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2741 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2742 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2743 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2744 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2745 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2746 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2747 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2748 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2749 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2750 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2751 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2752 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2753 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2754 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2755 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2756 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2757 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2758
8cc1a532 2759 } else if (num_rbs < 4) {
f0e201f2
JP		 2760 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2761 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2762 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2763 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2764 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2765 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2766 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2767 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2768 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2769 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2770 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2771 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2772 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2773 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2774 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2775 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2776 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2777 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2778 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2779 TILE_SPLIT(split_equal_to_row_size));
2780 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2781 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2782 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2783 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2784 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2785 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2786 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2787 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2788 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2789 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2790 TILE_SPLIT(split_equal_to_row_size));
2791 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2792 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2793 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2794 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2795 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2796 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2797 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2798 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2799 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2800 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2801 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2802 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2803 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2804 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2805 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2806 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2807 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2808 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2809 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2810 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2811 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2812 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2813 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2815 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2816 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2817 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2818 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2819 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2820 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2821 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2822 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2823 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2824 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2825 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2826 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2827 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2828 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2829 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2830 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2831 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2832 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2833 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2834 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2835 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2836 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2837 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
8cc1a532 2838 }
f0e201f2
JP		 2839
2840 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2841 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2842 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2843 NUM_BANKS(ADDR_SURF_16_BANK));
2844 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2845 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2846 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2847 NUM_BANKS(ADDR_SURF_16_BANK));
2848 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2849 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2850 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2851 NUM_BANKS(ADDR_SURF_16_BANK));
2852 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2853 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2854 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2855 NUM_BANKS(ADDR_SURF_16_BANK));
2856 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2857 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2858 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2859 NUM_BANKS(ADDR_SURF_16_BANK));
2860 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2861 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2862 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2863 NUM_BANKS(ADDR_SURF_8_BANK));
2864 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2865 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2866 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2867 NUM_BANKS(ADDR_SURF_4_BANK));
2868 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2869 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2870 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2871 NUM_BANKS(ADDR_SURF_16_BANK));
2872 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2873 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2874 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2875 NUM_BANKS(ADDR_SURF_16_BANK));
2876 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2877 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2878 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2879 NUM_BANKS(ADDR_SURF_16_BANK));
2880 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2881 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2882 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2883 NUM_BANKS(ADDR_SURF_16_BANK));
2884 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2885 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2886 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2887 NUM_BANKS(ADDR_SURF_16_BANK));
2888 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2889 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2890 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2891 NUM_BANKS(ADDR_SURF_8_BANK));
2892 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2893 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2894 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2895 NUM_BANKS(ADDR_SURF_4_BANK));
2896
2897 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2898 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2899 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2900 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2901 break;
2902
2903 case 2:
2904 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2905 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2906 PIPE_CONFIG(ADDR_SURF_P2) |
2907 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2908 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2909 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2910 PIPE_CONFIG(ADDR_SURF_P2) |
2911 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2912 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2913 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2914 PIPE_CONFIG(ADDR_SURF_P2) |
2915 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2916 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2917 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2918 PIPE_CONFIG(ADDR_SURF_P2) |
2919 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2920 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2921 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2922 PIPE_CONFIG(ADDR_SURF_P2) |
2923 TILE_SPLIT(split_equal_to_row_size));
2924 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2925 PIPE_CONFIG(ADDR_SURF_P2) |
2926 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2927 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2928 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2929 PIPE_CONFIG(ADDR_SURF_P2) |
2930 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2931 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2932 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2933 PIPE_CONFIG(ADDR_SURF_P2) |
2934 TILE_SPLIT(split_equal_to_row_size));
2935 tile[8] = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2936 PIPE_CONFIG(ADDR_SURF_P2);
2937 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2938 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2939 PIPE_CONFIG(ADDR_SURF_P2));
2940 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2941 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2942 PIPE_CONFIG(ADDR_SURF_P2) |
2943 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2944 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2945 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2946 PIPE_CONFIG(ADDR_SURF_P2) |
2947 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2948 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2949 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2950 PIPE_CONFIG(ADDR_SURF_P2) |
2951 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2952 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2953 PIPE_CONFIG(ADDR_SURF_P2) |
2954 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2955 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2956 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2957 PIPE_CONFIG(ADDR_SURF_P2) |
2958 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2959 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2960 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2961 PIPE_CONFIG(ADDR_SURF_P2) |
2962 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2963 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2964 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2965 PIPE_CONFIG(ADDR_SURF_P2) |
2966 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2967 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2968 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2969 PIPE_CONFIG(ADDR_SURF_P2));
2970 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2971 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2972 PIPE_CONFIG(ADDR_SURF_P2) |
2973 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2974 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2975 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2976 PIPE_CONFIG(ADDR_SURF_P2) |
2977 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2978 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2979 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2980 PIPE_CONFIG(ADDR_SURF_P2) |
2981 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2982
2983 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2984 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2985 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2986 NUM_BANKS(ADDR_SURF_16_BANK));
2987 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2988 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2989 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2990 NUM_BANKS(ADDR_SURF_16_BANK));
2991 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2992 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2993 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2994 NUM_BANKS(ADDR_SURF_16_BANK));
2995 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2996 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2997 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2998 NUM_BANKS(ADDR_SURF_16_BANK));
2999 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3000 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3001 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3002 NUM_BANKS(ADDR_SURF_16_BANK));
3003 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3004 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3005 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3006 NUM_BANKS(ADDR_SURF_16_BANK));
3007 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3008 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3009 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3010 NUM_BANKS(ADDR_SURF_8_BANK));
3011 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3012 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3013 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3014 NUM_BANKS(ADDR_SURF_16_BANK));
3015 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3016 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3017 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3018 NUM_BANKS(ADDR_SURF_16_BANK));
3019 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3020 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3021 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3022 NUM_BANKS(ADDR_SURF_16_BANK));
3023 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3024 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3025 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3026 NUM_BANKS(ADDR_SURF_16_BANK));
3027 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3028 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3029 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3030 NUM_BANKS(ADDR_SURF_16_BANK));
3031 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3032 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3033 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3034 NUM_BANKS(ADDR_SURF_16_BANK));
3035 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3036 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3037 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3038 NUM_BANKS(ADDR_SURF_8_BANK));
3039
3040 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3041 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
3042 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3043 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
3044 break;
3045
3046 default:
8cc1a532 3047 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
f0e201f2 3048 }
8cc1a532
AD		 3049}
3050
3051/**
3052 * cik_select_se_sh - select which SE, SH to address
3053 *
3054 * @rdev: radeon_device pointer
3055 * @se_num: shader engine to address
3056 * @sh_num: sh block to address
3057 *
3058 * Select which SE, SH combinations to address. Certain
3059 * registers are instanced per SE or SH. 0xffffffff means
3060 * broadcast to all SEs or SHs (CIK).
3061 */
3062static void cik_select_se_sh(struct radeon_device *rdev,
3063 u32 se_num, u32 sh_num)
3064{
3065 u32 data = INSTANCE_BROADCAST_WRITES;
3066
3067 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 3068 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 3069 else if (se_num == 0xffffffff)
3070 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3071 else if (sh_num == 0xffffffff)
3072 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3073 else
3074 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3075 WREG32(GRBM_GFX_INDEX, data);
3076}
3077
3078/**
3079 * cik_create_bitmask - create a bitmask
3080 *
3081 * @bit_width: length of the mask
3082 *
3083 * create a variable length bit mask (CIK).
3084 * Returns the bitmask.
3085 */
3086static u32 cik_create_bitmask(u32 bit_width)
3087{
3088 u32 i, mask = 0;
3089
3090 for (i = 0; i < bit_width; i++) {
3091 mask <<= 1;
3092 mask |= 1;
3093 }
3094 return mask;
3095}
3096
3097/**
972c5ddb 3098 * cik_get_rb_disabled - computes the mask of disabled RBs
8cc1a532
AD		 3099 *
3100 * @rdev: radeon_device pointer
3101 * @max_rb_num: max RBs (render backends) for the asic
3102 * @se_num: number of SEs (shader engines) for the asic
3103 * @sh_per_se: number of SH blocks per SE for the asic
3104 *
3105 * Calculates the bitmask of disabled RBs (CIK).
3106 * Returns the disabled RB bitmask.
3107 */
3108static u32 cik_get_rb_disabled(struct radeon_device *rdev,
9fadb352 3109 u32 max_rb_num_per_se,
8cc1a532
AD		 3110 u32 sh_per_se)
3111{
3112 u32 data, mask;
3113
3114 data = RREG32(CC_RB_BACKEND_DISABLE);
3115 if (data & 1)
3116 data &= BACKEND_DISABLE_MASK;
3117 else
3118 data = 0;
3119 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3120
3121 data >>= BACKEND_DISABLE_SHIFT;
3122
9fadb352 3123 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
8cc1a532
AD		 3124
3125 return data & mask;
3126}
3127
3128/**
3129 * cik_setup_rb - setup the RBs on the asic
3130 *
3131 * @rdev: radeon_device pointer
3132 * @se_num: number of SEs (shader engines) for the asic
3133 * @sh_per_se: number of SH blocks per SE for the asic
3134 * @max_rb_num: max RBs (render backends) for the asic
3135 *
3136 * Configures per-SE/SH RB registers (CIK).
3137 */
3138static void cik_setup_rb(struct radeon_device *rdev,
3139 u32 se_num, u32 sh_per_se,
9fadb352 3140 u32 max_rb_num_per_se)
8cc1a532
AD		 3141{
3142 int i, j;
3143 u32 data, mask;
3144 u32 disabled_rbs = 0;
3145 u32 enabled_rbs = 0;
3146
1c0a4625 3147 mutex_lock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3148 for (i = 0; i < se_num; i++) {
3149 for (j = 0; j < sh_per_se; j++) {
3150 cik_select_se_sh(rdev, i, j);
9fadb352 3151 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
fc821b70
AD		 3152 if (rdev->family == CHIP_HAWAII)
3153 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3154 else
3155 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
8cc1a532
AD		 3156 }
3157 }
3158 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 3159 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3160
3161 mask = 1;
9fadb352 3162 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
8cc1a532
AD		 3163 if (!(disabled_rbs & mask))
3164 enabled_rbs |= mask;
3165 mask <<= 1;
3166 }
3167
439a1cff
MO		 3168 rdev->config.cik.backend_enable_mask = enabled_rbs;
3169
1c0a4625 3170 mutex_lock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3171 for (i = 0; i < se_num; i++) {
3172 cik_select_se_sh(rdev, i, 0xffffffff);
3173 data = 0;
3174 for (j = 0; j < sh_per_se; j++) {
3175 switch (enabled_rbs & 3) {
fc821b70
AD		 3176 case 0:
3177 if (j == 0)
3178 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3179 else
3180 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3181 break;
8cc1a532
AD		 3182 case 1:
3183 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3184 break;
3185 case 2:
3186 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3187 break;
3188 case 3:
3189 default:
3190 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3191 break;
3192 }
3193 enabled_rbs >>= 2;
3194 }
3195 WREG32(PA_SC_RASTER_CONFIG, data);
3196 }
3197 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 3198 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3199}
3200
3201/**
3202 * cik_gpu_init - setup the 3D engine
3203 *
3204 * @rdev: radeon_device pointer
3205 *
3206 * Configures the 3D engine and tiling configuration
3207 * registers so that the 3D engine is usable.
3208 */
3209static void cik_gpu_init(struct radeon_device *rdev)
3210{
3211 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3212 u32 mc_shared_chmap, mc_arb_ramcfg;
3213 u32 hdp_host_path_cntl;
3214 u32 tmp;
6101b3ae 3215 int i, j;
8cc1a532
AD		 3216
3217 switch (rdev->family) {
3218 case CHIP_BONAIRE:
3219 rdev->config.cik.max_shader_engines = 2;
3220 rdev->config.cik.max_tile_pipes = 4;
3221 rdev->config.cik.max_cu_per_sh = 7;
3222 rdev->config.cik.max_sh_per_se = 1;
3223 rdev->config.cik.max_backends_per_se = 2;
3224 rdev->config.cik.max_texture_channel_caches = 4;
3225 rdev->config.cik.max_gprs = 256;
3226 rdev->config.cik.max_gs_threads = 32;
3227 rdev->config.cik.max_hw_contexts = 8;
3228
3229 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3230 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3231 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3232 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3233 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3234 break;
b496038b
AD		 3235 case CHIP_HAWAII:
3236 rdev->config.cik.max_shader_engines = 4;
3237 rdev->config.cik.max_tile_pipes = 16;
3238 rdev->config.cik.max_cu_per_sh = 11;
3239 rdev->config.cik.max_sh_per_se = 1;
3240 rdev->config.cik.max_backends_per_se = 4;
3241 rdev->config.cik.max_texture_channel_caches = 16;
3242 rdev->config.cik.max_gprs = 256;
3243 rdev->config.cik.max_gs_threads = 32;
3244 rdev->config.cik.max_hw_contexts = 8;
3245
3246 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3247 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3248 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3249 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3250 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3251 break;
8cc1a532 3252 case CHIP_KAVERI:
b2e4c70a
AD		 3253 rdev->config.cik.max_shader_engines = 1;
3254 rdev->config.cik.max_tile_pipes = 4;
3255 if ((rdev->pdev->device == 0x1304) ||
3256 (rdev->pdev->device == 0x1305) ||
3257 (rdev->pdev->device == 0x130C) ||
3258 (rdev->pdev->device == 0x130F) ||
3259 (rdev->pdev->device == 0x1310) ||
3260 (rdev->pdev->device == 0x1311) ||
3261 (rdev->pdev->device == 0x131C)) {
3262 rdev->config.cik.max_cu_per_sh = 8;
3263 rdev->config.cik.max_backends_per_se = 2;
3264 } else if ((rdev->pdev->device == 0x1309) ||
3265 (rdev->pdev->device == 0x130A) ||
3266 (rdev->pdev->device == 0x130D) ||
7c4622d5
AD		 3267 (rdev->pdev->device == 0x1313) ||
3268 (rdev->pdev->device == 0x131D)) {
b2e4c70a
AD		 3269 rdev->config.cik.max_cu_per_sh = 6;
3270 rdev->config.cik.max_backends_per_se = 2;
3271 } else if ((rdev->pdev->device == 0x1306) ||
3272 (rdev->pdev->device == 0x1307) ||
3273 (rdev->pdev->device == 0x130B) ||
3274 (rdev->pdev->device == 0x130E) ||
3275 (rdev->pdev->device == 0x1315) ||
6dc14baf 3276 (rdev->pdev->device == 0x1318) ||
b2e4c70a
AD		 3277 (rdev->pdev->device == 0x131B)) {
3278 rdev->config.cik.max_cu_per_sh = 4;
3279 rdev->config.cik.max_backends_per_se = 1;
3280 } else {
3281 rdev->config.cik.max_cu_per_sh = 3;
3282 rdev->config.cik.max_backends_per_se = 1;
3283 }
3284 rdev->config.cik.max_sh_per_se = 1;
3285 rdev->config.cik.max_texture_channel_caches = 4;
3286 rdev->config.cik.max_gprs = 256;
3287 rdev->config.cik.max_gs_threads = 16;
3288 rdev->config.cik.max_hw_contexts = 8;
3289
3290 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3291 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3292 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3293 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3294 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 3295 break;
3296 case CHIP_KABINI:
f73a9e83 3297 case CHIP_MULLINS:
8cc1a532
AD		 3298 default:
3299 rdev->config.cik.max_shader_engines = 1;
3300 rdev->config.cik.max_tile_pipes = 2;
3301 rdev->config.cik.max_cu_per_sh = 2;
3302 rdev->config.cik.max_sh_per_se = 1;
3303 rdev->config.cik.max_backends_per_se = 1;
3304 rdev->config.cik.max_texture_channel_caches = 2;
3305 rdev->config.cik.max_gprs = 256;
3306 rdev->config.cik.max_gs_threads = 16;
3307 rdev->config.cik.max_hw_contexts = 8;
3308
3309 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3310 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3311 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3312 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3313 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3314 break;
3315 }
3316
3317 /* Initialize HDP */
3318 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3319 WREG32((0x2c14 + j), 0x00000000);
3320 WREG32((0x2c18 + j), 0x00000000);
3321 WREG32((0x2c1c + j), 0x00000000);
3322 WREG32((0x2c20 + j), 0x00000000);
3323 WREG32((0x2c24 + j), 0x00000000);
3324 }
3325
3326 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
dc12a3ec
LL		 3327 WREG32(SRBM_INT_CNTL, 0x1);
3328 WREG32(SRBM_INT_ACK, 0x1);
8cc1a532
AD		 3329
3330 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3331
3332 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3333 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3334
3335 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3336 rdev->config.cik.mem_max_burst_length_bytes = 256;
3337 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3338 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3339 if (rdev->config.cik.mem_row_size_in_kb > 4)
3340 rdev->config.cik.mem_row_size_in_kb = 4;
3341 /* XXX use MC settings? */
3342 rdev->config.cik.shader_engine_tile_size = 32;
3343 rdev->config.cik.num_gpus = 1;
3344 rdev->config.cik.multi_gpu_tile_size = 64;
3345
3346 /* fix up row size */
3347 gb_addr_config &= ~ROW_SIZE_MASK;
3348 switch (rdev->config.cik.mem_row_size_in_kb) {
3349 case 1:
3350 default:
3351 gb_addr_config |= ROW_SIZE(0);
3352 break;
3353 case 2:
3354 gb_addr_config |= ROW_SIZE(1);
3355 break;
3356 case 4:
3357 gb_addr_config |= ROW_SIZE(2);
3358 break;
3359 }
3360
3361 /* setup tiling info dword. gb_addr_config is not adequate since it does
3362 * not have bank info, so create a custom tiling dword.
3363 * bits 3:0 num_pipes
3364 * bits 7:4 num_banks
3365 * bits 11:8 group_size
3366 * bits 15:12 row_size
3367 */
3368 rdev->config.cik.tile_config = 0;
3369 switch (rdev->config.cik.num_tile_pipes) {
3370 case 1:
3371 rdev->config.cik.tile_config |= (0 << 0);
3372 break;
3373 case 2:
3374 rdev->config.cik.tile_config |= (1 << 0);
3375 break;
3376 case 4:
3377 rdev->config.cik.tile_config |= (2 << 0);
3378 break;
3379 case 8:
3380 default:
3381 /* XXX what about 12? */
3382 rdev->config.cik.tile_config |= (3 << 0);
3383 break;
3384 }
a537314e
MD		 3385 rdev->config.cik.tile_config |=
3386 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 3387 rdev->config.cik.tile_config |=
3388 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3389 rdev->config.cik.tile_config |=
3390 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3391
3392 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3393 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3394 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 3395 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3396 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 3397 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3398 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3399 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 3400
3401 cik_tiling_mode_table_init(rdev);
3402
3403 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3404 rdev->config.cik.max_sh_per_se,
3405 rdev->config.cik.max_backends_per_se);
3406
52da51f0 3407 rdev->config.cik.active_cus = 0;
65fcf668
AD		 3408 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3409 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6101b3ae
AD		 3410 rdev->config.cik.active_cus +=
3411 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
65fcf668
AD		 3412 }
3413 }
3414
8cc1a532
AD		 3415 /* set HW defaults for 3D engine */
3416 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3417
1c0a4625
OG		 3418 mutex_lock(&rdev->grbm_idx_mutex);
3419 /*
3420 * making sure that the following register writes will be broadcasted
3421 * to all the shaders
3422 */
3423 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
8cc1a532
AD		 3424 WREG32(SX_DEBUG_1, 0x20);
3425
3426 WREG32(TA_CNTL_AUX, 0x00010000);
3427
3428 tmp = RREG32(SPI_CONFIG_CNTL);
3429 tmp |= 0x03000000;
3430 WREG32(SPI_CONFIG_CNTL, tmp);
3431
3432 WREG32(SQ_CONFIG, 1);
3433
3434 WREG32(DB_DEBUG, 0);
3435
3436 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3437 tmp |= 0x00000400;
3438 WREG32(DB_DEBUG2, tmp);
3439
3440 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3441 tmp |= 0x00020200;
3442 WREG32(DB_DEBUG3, tmp);
3443
3444 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3445 tmp |= 0x00018208;
3446 WREG32(CB_HW_CONTROL, tmp);
3447
3448 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3449
3450 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3451 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3452 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3453 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3454
3455 WREG32(VGT_NUM_INSTANCES, 1);
3456
3457 WREG32(CP_PERFMON_CNTL, 0);
3458
3459 WREG32(SQ_CONFIG, 0);
3460
3461 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3462 FORCE_EOV_MAX_REZ_CNT(255)));
3463
3464 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3465 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3466
3467 WREG32(VGT_GS_VERTEX_REUSE, 16);
3468 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3469
3470 tmp = RREG32(HDP_MISC_CNTL);
3471 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3472 WREG32(HDP_MISC_CNTL, tmp);
3473
3474 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3475 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3476
3477 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3478 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
1c0a4625 3479 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3480
3481 udelay(50);
3482}
3483
2cae3bc3
AD		 3484/*
3485 * GPU scratch registers helpers function.
3486 */
3487/**
3488 * cik_scratch_init - setup driver info for CP scratch regs
3489 *
3490 * @rdev: radeon_device pointer
3491 *
3492 * Set up the number and offset of the CP scratch registers.
3493 * NOTE: use of CP scratch registers is a legacy inferface and
3494 * is not used by default on newer asics (r6xx+). On newer asics,
3495 * memory buffers are used for fences rather than scratch regs.
3496 */
3497static void cik_scratch_init(struct radeon_device *rdev)
3498{
3499 int i;
3500
3501 rdev->scratch.num_reg = 7;
3502 rdev->scratch.reg_base = SCRATCH_REG0;
3503 for (i = 0; i < rdev->scratch.num_reg; i++) {
3504 rdev->scratch.free[i] = true;
3505 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3506 }
3507}
3508
fbc832c7
AD		 3509/**
3510 * cik_ring_test - basic gfx ring test
3511 *
3512 * @rdev: radeon_device pointer
3513 * @ring: radeon_ring structure holding ring information
3514 *
3515 * Allocate a scratch register and write to it using the gfx ring (CIK).
3516 * Provides a basic gfx ring test to verify that the ring is working.
3517 * Used by cik_cp_gfx_resume();
3518 * Returns 0 on success, error on failure.
3519 */
3520int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3521{
3522 uint32_t scratch;
3523 uint32_t tmp = 0;
3524 unsigned i;
3525 int r;
3526
3527 r = radeon_scratch_get(rdev, &scratch);
3528 if (r) {
3529 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3530 return r;
3531 }
3532 WREG32(scratch, 0xCAFEDEAD);
3533 r = radeon_ring_lock(rdev, ring, 3);
3534 if (r) {
3535 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3536 radeon_scratch_free(rdev, scratch);
3537 return r;
3538 }
3539 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3540 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3541 radeon_ring_write(ring, 0xDEADBEEF);
1538a9e0 3542 radeon_ring_unlock_commit(rdev, ring, false);
963e81f9 3543
fbc832c7
AD		 3544 for (i = 0; i < rdev->usec_timeout; i++) {
3545 tmp = RREG32(scratch);
3546 if (tmp == 0xDEADBEEF)
3547 break;
3548 DRM_UDELAY(1);
3549 }
3550 if (i < rdev->usec_timeout) {
3551 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3552 } else {
3553 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3554 ring->idx, scratch, tmp);
3555 r = -EINVAL;
3556 }
3557 radeon_scratch_free(rdev, scratch);
3558 return r;
3559}
3560
780f5ddd
AD		 3561/**
3562 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3563 *
3564 * @rdev: radeon_device pointer
3565 * @ridx: radeon ring index
3566 *
3567 * Emits an hdp flush on the cp.
3568 */
3569static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3570 int ridx)
3571{
3572 struct radeon_ring *ring = &rdev->ring[ridx];
5d259067 3573 u32 ref_and_mask;
780f5ddd 3574
5d259067
AD		 3575 switch (ring->idx) {
3576 case CAYMAN_RING_TYPE_CP1_INDEX:
3577 case CAYMAN_RING_TYPE_CP2_INDEX:
3578 default:
3579 switch (ring->me) {
3580 case 0:
3581 ref_and_mask = CP2 << ring->pipe;
3582 break;
3583 case 1:
3584 ref_and_mask = CP6 << ring->pipe;
3585 break;
3586 default:
3587 return;
3588 }
3589 break;
3590 case RADEON_RING_TYPE_GFX_INDEX:
3591 ref_and_mask = CP0;
3592 break;
3593 }
3594
3595 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3596 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3597 WAIT_REG_MEM_FUNCTION(3) | /* == */
3598 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3599 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3600 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3601 radeon_ring_write(ring, ref_and_mask);
3602 radeon_ring_write(ring, ref_and_mask);
3603 radeon_ring_write(ring, 0x20); /* poll interval */
780f5ddd
AD		 3604}
3605
2cae3bc3 3606/**
b07fdd38 3607 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3608 *
3609 * @rdev: radeon_device pointer
3610 * @fence: radeon fence object
3611 *
3612 * Emits a fence sequnce number on the gfx ring and flushes
3613 * GPU caches.
3614 */
b07fdd38
AD		 3615void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3616 struct radeon_fence *fence)
2cae3bc3
AD		 3617{
3618 struct radeon_ring *ring = &rdev->ring[fence->ring];
3619 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3620
a9c73a0e
CK		 3621 /* Workaround for cache flush problems. First send a dummy EOP
3622 * event down the pipe with seq one below.
3623 */
3624 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3625 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3626 EOP_TC_ACTION_EN |
3627 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3628 EVENT_INDEX(5)));
3629 radeon_ring_write(ring, addr & 0xfffffffc);
3630 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3631 DATA_SEL(1) | INT_SEL(0));
3632 radeon_ring_write(ring, fence->seq - 1);
3633 radeon_ring_write(ring, 0);
3634
3635 /* Then send the real EOP event down the pipe. */
2cae3bc3
AD		 3636 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3637 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3638 EOP_TC_ACTION_EN |
3639 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3640 EVENT_INDEX(5)));
3641 radeon_ring_write(ring, addr & 0xfffffffc);
3642 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3643 radeon_ring_write(ring, fence->seq);
3644 radeon_ring_write(ring, 0);
2cae3bc3
AD		 3645}
3646
b07fdd38
AD		 3647/**
3648 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3649 *
3650 * @rdev: radeon_device pointer
3651 * @fence: radeon fence object
3652 *
3653 * Emits a fence sequnce number on the compute ring and flushes
3654 * GPU caches.
3655 */
3656void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3657 struct radeon_fence *fence)
3658{
3659 struct radeon_ring *ring = &rdev->ring[fence->ring];
3660 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3661
3662 /* RELEASE_MEM - flush caches, send int */
3663 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3664 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3665 EOP_TC_ACTION_EN |
3666 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3667 EVENT_INDEX(5)));
3668 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3669 radeon_ring_write(ring, addr & 0xfffffffc);
3670 radeon_ring_write(ring, upper_32_bits(addr));
3671 radeon_ring_write(ring, fence->seq);
3672 radeon_ring_write(ring, 0);
b07fdd38
AD		 3673}
3674
86302eea
CK		 3675/**
3676 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3677 *
3678 * @rdev: radeon_device pointer
3679 * @ring: radeon ring buffer object
3680 * @semaphore: radeon semaphore object
3681 * @emit_wait: Is this a sempahore wait?
3682 *
3683 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3684 * from running ahead of semaphore waits.
3685 */
1654b817 3686bool cik_semaphore_ring_emit(struct radeon_device *rdev,
2cae3bc3
AD		 3687 struct radeon_ring *ring,
3688 struct radeon_semaphore *semaphore,
3689 bool emit_wait)
3690{
3691 uint64_t addr = semaphore->gpu_addr;
3692 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3693
3694 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
5e167cdb 3695 radeon_ring_write(ring, lower_32_bits(addr));
2cae3bc3 3696 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
1654b817 3697
86302eea
CK		 3698 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3699 /* Prevent the PFP from running ahead of the semaphore wait */
3700 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3701 radeon_ring_write(ring, 0x0);
3702 }
3703
1654b817 3704 return true;
2cae3bc3
AD		 3705}
3706
c9dbd705
AD		 3707/**
3708 * cik_copy_cpdma - copy pages using the CP DMA engine
3709 *
3710 * @rdev: radeon_device pointer
3711 * @src_offset: src GPU address
3712 * @dst_offset: dst GPU address
3713 * @num_gpu_pages: number of GPU pages to xfer
57d20a43 3714 * @resv: reservation object to sync to
c9dbd705
AD		 3715 *
3716 * Copy GPU paging using the CP DMA engine (CIK+).
3717 * Used by the radeon ttm implementation to move pages if
3718 * registered as the asic copy callback.
3719 */
57d20a43
CK		 3720struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3721 uint64_t src_offset, uint64_t dst_offset,
3722 unsigned num_gpu_pages,
3723 struct reservation_object *resv)
c9dbd705 3724{
57d20a43 3725 struct radeon_fence *fence;
975700d2 3726 struct radeon_sync sync;
c9dbd705
AD		 3727 int ring_index = rdev->asic->copy.blit_ring_index;
3728 struct radeon_ring *ring = &rdev->ring[ring_index];
3729 u32 size_in_bytes, cur_size_in_bytes, control;
3730 int i, num_loops;
3731 int r = 0;
3732
975700d2 3733 radeon_sync_create(&sync);
c9dbd705
AD		 3734
3735 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3736 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3737 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3738 if (r) {
3739 DRM_ERROR("radeon: moving bo (%d).\n", r);
975700d2 3740 radeon_sync_free(rdev, &sync, NULL);
57d20a43 3741 return ERR_PTR(r);
c9dbd705
AD		 3742 }
3743
975700d2
CK		 3744 radeon_sync_resv(rdev, &sync, resv, false);
3745 radeon_sync_rings(rdev, &sync, ring->idx);
c9dbd705
AD		 3746
3747 for (i = 0; i < num_loops; i++) {
3748 cur_size_in_bytes = size_in_bytes;
3749 if (cur_size_in_bytes > 0x1fffff)
3750 cur_size_in_bytes = 0x1fffff;
3751 size_in_bytes -= cur_size_in_bytes;
3752 control = 0;
3753 if (size_in_bytes == 0)
3754 control |= PACKET3_DMA_DATA_CP_SYNC;
3755 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3756 radeon_ring_write(ring, control);
3757 radeon_ring_write(ring, lower_32_bits(src_offset));
3758 radeon_ring_write(ring, upper_32_bits(src_offset));
3759 radeon_ring_write(ring, lower_32_bits(dst_offset));
3760 radeon_ring_write(ring, upper_32_bits(dst_offset));
3761 radeon_ring_write(ring, cur_size_in_bytes);
3762 src_offset += cur_size_in_bytes;
3763 dst_offset += cur_size_in_bytes;
3764 }
3765
57d20a43 3766 r = radeon_fence_emit(rdev, &fence, ring->idx);
c9dbd705
AD		 3767 if (r) {
3768 radeon_ring_unlock_undo(rdev, ring);
975700d2 3769 radeon_sync_free(rdev, &sync, NULL);
57d20a43 3770 return ERR_PTR(r);
c9dbd705
AD		 3771 }
3772
1538a9e0 3773 radeon_ring_unlock_commit(rdev, ring, false);
975700d2 3774 radeon_sync_free(rdev, &sync, fence);
c9dbd705 3775
57d20a43 3776 return fence;
c9dbd705
AD		 3777}
3778
2cae3bc3
AD		 3779/*
3780 * IB stuff
3781 */
3782/**
3783 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3784 *
3785 * @rdev: radeon_device pointer
3786 * @ib: radeon indirect buffer object
3787 *
5fc45397 3788 * Emits a DE (drawing engine) or CE (constant engine) IB
2cae3bc3
AD		 3789 * on the gfx ring. IBs are usually generated by userspace
3790 * acceleration drivers and submitted to the kernel for
5fc45397 3791 * scheduling on the ring. This function schedules the IB
2cae3bc3
AD		 3792 * on the gfx ring for execution by the GPU.
3793 */
3794void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3795{
3796 struct radeon_ring *ring = &rdev->ring[ib->ring];
7c42bc1a 3797 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
2cae3bc3
AD		 3798 u32 header, control = INDIRECT_BUFFER_VALID;
3799
3800 if (ib->is_const_ib) {
3801 /* set switch buffer packet before const IB */
3802 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3803 radeon_ring_write(ring, 0);
3804
3805 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3806 } else {
3807 u32 next_rptr;
3808 if (ring->rptr_save_reg) {
3809 next_rptr = ring->wptr + 3 + 4;
3810 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3811 radeon_ring_write(ring, ((ring->rptr_save_reg -
3812 PACKET3_SET_UCONFIG_REG_START) >> 2));
3813 radeon_ring_write(ring, next_rptr);
3814 } else if (rdev->wb.enabled) {
3815 next_rptr = ring->wptr + 5 + 4;
3816 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3817 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3818 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5e167cdb 3819 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
2cae3bc3
AD		 3820 radeon_ring_write(ring, next_rptr);
3821 }
3822
3823 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3824 }
3825
7c42bc1a 3826 control |= ib->length_dw | (vm_id << 24);
2cae3bc3
AD		 3827
3828 radeon_ring_write(ring, header);
5f3e226f 3829 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC));
2cae3bc3
AD		 3830 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3831 radeon_ring_write(ring, control);
3832}
3833
fbc832c7
AD		 3834/**
3835 * cik_ib_test - basic gfx ring IB test
3836 *
3837 * @rdev: radeon_device pointer
3838 * @ring: radeon_ring structure holding ring information
3839 *
3840 * Allocate an IB and execute it on the gfx ring (CIK).
3841 * Provides a basic gfx ring test to verify that IBs are working.
3842 * Returns 0 on success, error on failure.
3843 */
3844int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3845{
3846 struct radeon_ib ib;
3847 uint32_t scratch;
3848 uint32_t tmp = 0;
3849 unsigned i;
3850 int r;
3851
3852 r = radeon_scratch_get(rdev, &scratch);
3853 if (r) {
3854 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3855 return r;
3856 }
3857 WREG32(scratch, 0xCAFEDEAD);
3858 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3859 if (r) {
3860 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
5510f124 3861 radeon_scratch_free(rdev, scratch);
fbc832c7
AD		 3862 return r;
3863 }
3864 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3865 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3866 ib.ptr[2] = 0xDEADBEEF;
3867 ib.length_dw = 3;
1538a9e0 3868 r = radeon_ib_schedule(rdev, &ib, NULL, false);
fbc832c7
AD		 3869 if (r) {
3870 radeon_scratch_free(rdev, scratch);
3871 radeon_ib_free(rdev, &ib);
3872 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3873 return r;
3874 }
04db4caf
MD		 3875 r = radeon_fence_wait_timeout(ib.fence, false, usecs_to_jiffies(
3876 RADEON_USEC_IB_TEST_TIMEOUT));
3877 if (r < 0) {
fbc832c7 3878 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
5510f124
CK		 3879 radeon_scratch_free(rdev, scratch);
3880 radeon_ib_free(rdev, &ib);
fbc832c7 3881 return r;
04db4caf
MD		 3882 } else if (r == 0) {
3883 DRM_ERROR("radeon: fence wait timed out.\n");
3884 radeon_scratch_free(rdev, scratch);
3885 radeon_ib_free(rdev, &ib);
3886 return -ETIMEDOUT;
fbc832c7 3887 }
04db4caf 3888 r = 0;
fbc832c7
AD		 3889 for (i = 0; i < rdev->usec_timeout; i++) {
3890 tmp = RREG32(scratch);
3891 if (tmp == 0xDEADBEEF)
3892 break;
3893 DRM_UDELAY(1);
3894 }
3895 if (i < rdev->usec_timeout) {
3896 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3897 } else {
3898 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3899 scratch, tmp);
3900 r = -EINVAL;
3901 }
3902 radeon_scratch_free(rdev, scratch);
3903 radeon_ib_free(rdev, &ib);
3904 return r;
3905}
3906
841cf442
AD		 3907/*
3908 * CP.
3909 * On CIK, gfx and compute now have independant command processors.
3910 *
3911 * GFX
3912 * Gfx consists of a single ring and can process both gfx jobs and
3913 * compute jobs. The gfx CP consists of three microengines (ME):
3914 * PFP - Pre-Fetch Parser
3915 * ME - Micro Engine
3916 * CE - Constant Engine
3917 * The PFP and ME make up what is considered the Drawing Engine (DE).
3918 * The CE is an asynchronous engine used for updating buffer desciptors
3919 * used by the DE so that they can be loaded into cache in parallel
3920 * while the DE is processing state update packets.
3921 *
3922 * Compute
3923 * The compute CP consists of two microengines (ME):
3924 * MEC1 - Compute MicroEngine 1
3925 * MEC2 - Compute MicroEngine 2
3926 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3927 * The queues are exposed to userspace and are programmed directly
3928 * by the compute runtime.
3929 */
3930/**
3931 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3932 *
3933 * @rdev: radeon_device pointer
3934 * @enable: enable or disable the MEs
3935 *
3936 * Halts or unhalts the gfx MEs.
3937 */
3938static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3939{
3940 if (enable)
3941 WREG32(CP_ME_CNTL, 0);
3942 else {
50efa51a
AD		 3943 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3944 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
841cf442
AD		 3945 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3946 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3947 }
3948 udelay(50);
3949}
3950
3951/**
3952 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3953 *
3954 * @rdev: radeon_device pointer
3955 *
3956 * Loads the gfx PFP, ME, and CE ucode.
3957 * Returns 0 for success, -EINVAL if the ucode is not available.
3958 */
3959static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3960{
841cf442
AD		 3961 int i;
3962
3963 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3964 return -EINVAL;
3965
3966 cik_cp_gfx_enable(rdev, false);
3967
f2c6b0f4
AD		 3968 if (rdev->new_fw) {
3969 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3970 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3971 const struct gfx_firmware_header_v1_0 *ce_hdr =
3972 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3973 const struct gfx_firmware_header_v1_0 *me_hdr =
3974 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3975 const __le32 *fw_data;
3976 u32 fw_size;
3977
3978 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3979 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3980 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3981
3982 /* PFP */
3983 fw_data = (const __le32 *)
3984 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3985 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3986 WREG32(CP_PFP_UCODE_ADDR, 0);
3987 for (i = 0; i < fw_size; i++)
3988 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 3989 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
f2c6b0f4
AD		 3990
3991 /* CE */
3992 fw_data = (const __le32 *)
3993 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3994 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3995 WREG32(CP_CE_UCODE_ADDR, 0);
3996 for (i = 0; i < fw_size; i++)
3997 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 3998 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
f2c6b0f4
AD		 3999
4000 /* ME */
4001 fw_data = (const __be32 *)
4002 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4003 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4004 WREG32(CP_ME_RAM_WADDR, 0);
4005 for (i = 0; i < fw_size; i++)
4006 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
38aea071
AD		 4007 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
4008 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
f2c6b0f4
AD		 4009 } else {
4010 const __be32 *fw_data;
4011
4012 /* PFP */
4013 fw_data = (const __be32 *)rdev->pfp_fw->data;
4014 WREG32(CP_PFP_UCODE_ADDR, 0);
4015 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4016 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4017 WREG32(CP_PFP_UCODE_ADDR, 0);
4018
4019 /* CE */
4020 fw_data = (const __be32 *)rdev->ce_fw->data;
4021 WREG32(CP_CE_UCODE_ADDR, 0);
4022 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4023 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4024 WREG32(CP_CE_UCODE_ADDR, 0);
4025
4026 /* ME */
4027 fw_data = (const __be32 *)rdev->me_fw->data;
4028 WREG32(CP_ME_RAM_WADDR, 0);
4029 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4030 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4031 WREG32(CP_ME_RAM_WADDR, 0);
4032 }
841cf442 4033
841cf442
AD		 4034 return 0;
4035}
4036
4037/**
4038 * cik_cp_gfx_start - start the gfx ring
4039 *
4040 * @rdev: radeon_device pointer
4041 *
4042 * Enables the ring and loads the clear state context and other
4043 * packets required to init the ring.
4044 * Returns 0 for success, error for failure.
4045 */
4046static int cik_cp_gfx_start(struct radeon_device *rdev)
4047{
4048 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4049 int r, i;
4050
4051 /* init the CP */
4052 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4053 WREG32(CP_ENDIAN_SWAP, 0);
4054 WREG32(CP_DEVICE_ID, 1);
4055
4056 cik_cp_gfx_enable(rdev, true);
4057
4058 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4059 if (r) {
4060 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4061 return r;
4062 }
4063
4064 /* init the CE partitions. CE only used for gfx on CIK */
4065 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4066 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
dc4edad6
JZ		 4067 radeon_ring_write(ring, 0x8000);
4068 radeon_ring_write(ring, 0x8000);
841cf442
AD		 4069
4070 /* setup clear context state */
4071 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4072 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4073
4074 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4075 radeon_ring_write(ring, 0x80000000);
4076 radeon_ring_write(ring, 0x80000000);
4077
4078 for (i = 0; i < cik_default_size; i++)
4079 radeon_ring_write(ring, cik_default_state[i]);
4080
4081 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4082 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4083
4084 /* set clear context state */
4085 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4086 radeon_ring_write(ring, 0);
4087
4088 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4089 radeon_ring_write(ring, 0x00000316);
4090 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4091 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4092
1538a9e0 4093 radeon_ring_unlock_commit(rdev, ring, false);
841cf442
AD		 4094
4095 return 0;
4096}
4097
4098/**
4099 * cik_cp_gfx_fini - stop the gfx ring
4100 *
4101 * @rdev: radeon_device pointer
4102 *
4103 * Stop the gfx ring and tear down the driver ring
4104 * info.
4105 */
4106static void cik_cp_gfx_fini(struct radeon_device *rdev)
4107{
4108 cik_cp_gfx_enable(rdev, false);
4109 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4110}
4111
4112/**
4113 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4114 *
4115 * @rdev: radeon_device pointer
4116 *
4117 * Program the location and size of the gfx ring buffer
4118 * and test it to make sure it's working.
4119 * Returns 0 for success, error for failure.
4120 */
4121static int cik_cp_gfx_resume(struct radeon_device *rdev)
4122{
4123 struct radeon_ring *ring;
4124 u32 tmp;
4125 u32 rb_bufsz;
4126 u64 rb_addr;
4127 int r;
4128
4129 WREG32(CP_SEM_WAIT_TIMER, 0x0);
939c0d3c
AD		 4130 if (rdev->family != CHIP_HAWAII)
4131 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
841cf442
AD		 4132
4133 /* Set the write pointer delay */
4134 WREG32(CP_RB_WPTR_DELAY, 0);
4135
4136 /* set the RB to use vmid 0 */
4137 WREG32(CP_RB_VMID, 0);
4138
4139 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4140
4141 /* ring 0 - compute and gfx */
4142 /* Set ring buffer size */
4143 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 4144 rb_bufsz = order_base_2(ring->ring_size / 8);
4145 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 4146#ifdef __BIG_ENDIAN
4147 tmp |= BUF_SWAP_32BIT;
4148#endif
4149 WREG32(CP_RB0_CNTL, tmp);
4150
4151 /* Initialize the ring buffer's read and write pointers */
4152 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4153 ring->wptr = 0;
4154 WREG32(CP_RB0_WPTR, ring->wptr);
4155
4156 /* set the wb address wether it's enabled or not */
4157 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4158 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4159
4160 /* scratch register shadowing is no longer supported */
4161 WREG32(SCRATCH_UMSK, 0);
4162
4163 if (!rdev->wb.enabled)
4164 tmp |= RB_NO_UPDATE;
4165
4166 mdelay(1);
4167 WREG32(CP_RB0_CNTL, tmp);
4168
4169 rb_addr = ring->gpu_addr >> 8;
4170 WREG32(CP_RB0_BASE, rb_addr);
4171 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4172
841cf442
AD		 4173 /* start the ring */
4174 cik_cp_gfx_start(rdev);
4175 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4176 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4177 if (r) {
4178 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4179 return r;
4180 }
50efa51a
AD		 4181
4182 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4183 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4184
841cf442
AD		 4185 return 0;
4186}
4187
ea31bf69
AD		 4188u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4189 struct radeon_ring *ring)
963e81f9
AD		 4190{
4191 u32 rptr;
4192
ea31bf69
AD		 4193 if (rdev->wb.enabled)
4194 rptr = rdev->wb.wb[ring->rptr_offs/4];
4195 else
4196 rptr = RREG32(CP_RB0_RPTR);
4197
4198 return rptr;
4199}
4200
4201u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4202 struct radeon_ring *ring)
4203{
4204 u32 wptr;
4205
4206 wptr = RREG32(CP_RB0_WPTR);
963e81f9 4207
ea31bf69
AD		 4208 return wptr;
4209}
4210
4211void cik_gfx_set_wptr(struct radeon_device *rdev,
4212 struct radeon_ring *ring)
4213{
4214 WREG32(CP_RB0_WPTR, ring->wptr);
4215 (void)RREG32(CP_RB0_WPTR);
4216}
4217
4218u32 cik_compute_get_rptr(struct radeon_device *rdev,
4219 struct radeon_ring *ring)
4220{
4221 u32 rptr;
963e81f9
AD		 4222
4223 if (rdev->wb.enabled) {
ea31bf69 4224 rptr = rdev->wb.wb[ring->rptr_offs/4];
963e81f9 4225 } else {
f61d5b46 4226 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4227 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4228 rptr = RREG32(CP_HQD_PQ_RPTR);
4229 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4230 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4231 }
963e81f9
AD		 4232
4233 return rptr;
4234}
4235
ea31bf69
AD		 4236u32 cik_compute_get_wptr(struct radeon_device *rdev,
4237 struct radeon_ring *ring)
963e81f9
AD		 4238{
4239 u32 wptr;
4240
4241 if (rdev->wb.enabled) {
ea31bf69
AD		 4242 /* XXX check if swapping is necessary on BE */
4243 wptr = rdev->wb.wb[ring->wptr_offs/4];
963e81f9 4244 } else {
f61d5b46 4245 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4246 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4247 wptr = RREG32(CP_HQD_PQ_WPTR);
4248 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4249 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4250 }
963e81f9
AD		 4251
4252 return wptr;
4253}
4254
ea31bf69
AD		 4255void cik_compute_set_wptr(struct radeon_device *rdev,
4256 struct radeon_ring *ring)
963e81f9 4257{
ea31bf69
AD		 4258 /* XXX check if swapping is necessary on BE */
4259 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
d5754ab8 4260 WDOORBELL32(ring->doorbell_index, ring->wptr);
963e81f9
AD		 4261}
4262
161569de
JG		 4263static void cik_compute_stop(struct radeon_device *rdev,
4264 struct radeon_ring *ring)
4265{
4266 u32 j, tmp;
4267
4268 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4269 /* Disable wptr polling. */
4270 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4271 tmp &= ~WPTR_POLL_EN;
4272 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4273 /* Disable HQD. */
4274 if (RREG32(CP_HQD_ACTIVE) & 1) {
4275 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4276 for (j = 0; j < rdev->usec_timeout; j++) {
4277 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4278 break;
4279 udelay(1);
4280 }
4281 WREG32(CP_HQD_DEQUEUE_REQUEST, 0);
4282 WREG32(CP_HQD_PQ_RPTR, 0);
4283 WREG32(CP_HQD_PQ_WPTR, 0);
4284 }
4285 cik_srbm_select(rdev, 0, 0, 0, 0);
4286}
4287
841cf442
AD		 4288/**
4289 * cik_cp_compute_enable - enable/disable the compute CP MEs
4290 *
4291 * @rdev: radeon_device pointer
4292 * @enable: enable or disable the MEs
4293 *
4294 * Halts or unhalts the compute MEs.
4295 */
4296static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4297{
4298 if (enable)
4299 WREG32(CP_MEC_CNTL, 0);
b2b3d8d9 4300 else {
161569de
JG		 4301 /*
4302 * To make hibernation reliable we need to clear compute ring
4303 * configuration before halting the compute ring.
4304 */
4305 mutex_lock(&rdev->srbm_mutex);
4306 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
4307 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
4308 mutex_unlock(&rdev->srbm_mutex);
4309
841cf442 4310 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
b2b3d8d9
AD		 4311 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4312 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4313 }
841cf442
AD		 4314 udelay(50);
4315}
4316
4317/**
4318 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4319 *
4320 * @rdev: radeon_device pointer
4321 *
4322 * Loads the compute MEC1&2 ucode.
4323 * Returns 0 for success, -EINVAL if the ucode is not available.
4324 */
4325static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4326{
841cf442
AD		 4327 int i;
4328
4329 if (!rdev->mec_fw)
4330 return -EINVAL;
4331
4332 cik_cp_compute_enable(rdev, false);
4333
f2c6b0f4
AD		 4334 if (rdev->new_fw) {
4335 const struct gfx_firmware_header_v1_0 *mec_hdr =
4336 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4337 const __le32 *fw_data;
4338 u32 fw_size;
4339
4340 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4341
4342 /* MEC1 */
4343 fw_data = (const __le32 *)
4344 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4345 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4346 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4347 for (i = 0; i < fw_size; i++)
4348 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4349 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
841cf442 4350
841cf442 4351 /* MEC2 */
f2c6b0f4
AD		 4352 if (rdev->family == CHIP_KAVERI) {
4353 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4354 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4355
4356 fw_data = (const __le32 *)
4357 (rdev->mec2_fw->data +
4358 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4359 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4360 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4361 for (i = 0; i < fw_size; i++)
4362 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4363 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
f2c6b0f4
AD		 4364 }
4365 } else {
4366 const __be32 *fw_data;
4367
4368 /* MEC1 */
841cf442 4369 fw_data = (const __be32 *)rdev->mec_fw->data;
f2c6b0f4 4370 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
841cf442 4371 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
f2c6b0f4
AD		 4372 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4373 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4374
4375 if (rdev->family == CHIP_KAVERI) {
4376 /* MEC2 */
4377 fw_data = (const __be32 *)rdev->mec_fw->data;
4378 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4379 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4380 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4381 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4382 }
841cf442
AD		 4383 }
4384
4385 return 0;
4386}
4387
4388/**
4389 * cik_cp_compute_start - start the compute queues
4390 *
4391 * @rdev: radeon_device pointer
4392 *
4393 * Enable the compute queues.
4394 * Returns 0 for success, error for failure.
4395 */
4396static int cik_cp_compute_start(struct radeon_device *rdev)
4397{
963e81f9
AD		 4398 cik_cp_compute_enable(rdev, true);
4399
841cf442
AD		 4400 return 0;
4401}
4402
4403/**
4404 * cik_cp_compute_fini - stop the compute queues
4405 *
4406 * @rdev: radeon_device pointer
4407 *
4408 * Stop the compute queues and tear down the driver queue
4409 * info.
4410 */
4411static void cik_cp_compute_fini(struct radeon_device *rdev)
4412{
963e81f9
AD		 4413 int i, idx, r;
4414
841cf442 4415 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 4416
4417 for (i = 0; i < 2; i++) {
4418 if (i == 0)
4419 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4420 else
4421 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4422
4423 if (rdev->ring[idx].mqd_obj) {
4424 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4425 if (unlikely(r != 0))
4426 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4427
4428 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4429 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4430
4431 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4432 rdev->ring[idx].mqd_obj = NULL;
4433 }
4434 }
841cf442
AD		 4435}
4436
963e81f9
AD		 4437static void cik_mec_fini(struct radeon_device *rdev)
4438{
4439 int r;
4440
4441 if (rdev->mec.hpd_eop_obj) {
4442 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4443 if (unlikely(r != 0))
4444 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4445 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4446 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4447
4448 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4449 rdev->mec.hpd_eop_obj = NULL;
4450 }
4451}
4452
4453#define MEC_HPD_SIZE 2048
4454
4455static int cik_mec_init(struct radeon_device *rdev)
4456{
4457 int r;
4458 u32 *hpd;
4459
4460 /*
4461 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4462 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
62a7b7fb
OG		 4463 * Nonetheless, we assign only 1 pipe because all other pipes will
4464 * be handled by KFD
963e81f9 4465 */
62a7b7fb
OG		 4466 rdev->mec.num_mec = 1;
4467 rdev->mec.num_pipe = 1;
963e81f9
AD		 4468 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4469
4470 if (rdev->mec.hpd_eop_obj == NULL) {
4471 r = radeon_bo_create(rdev,
4472 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4473 PAGE_SIZE, true,
831b6966 4474 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
963e81f9
AD		 4475 &rdev->mec.hpd_eop_obj);
4476 if (r) {
4477 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4478 return r;
4479 }
4480 }
4481
4482 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4483 if (unlikely(r != 0)) {
4484 cik_mec_fini(rdev);
4485 return r;
4486 }
4487 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4488 &rdev->mec.hpd_eop_gpu_addr);
4489 if (r) {
4490 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4491 cik_mec_fini(rdev);
4492 return r;
4493 }
4494 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4495 if (r) {
4496 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4497 cik_mec_fini(rdev);
4498 return r;
4499 }
4500
4501 /* clear memory. Not sure if this is required or not */
4502 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4503
4504 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4505 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4506
4507 return 0;
4508}
4509
4510struct hqd_registers
4511{
4512 u32 cp_mqd_base_addr;
4513 u32 cp_mqd_base_addr_hi;
4514 u32 cp_hqd_active;
4515 u32 cp_hqd_vmid;
4516 u32 cp_hqd_persistent_state;
4517 u32 cp_hqd_pipe_priority;
4518 u32 cp_hqd_queue_priority;
4519 u32 cp_hqd_quantum;
4520 u32 cp_hqd_pq_base;
4521 u32 cp_hqd_pq_base_hi;
4522 u32 cp_hqd_pq_rptr;
4523 u32 cp_hqd_pq_rptr_report_addr;
4524 u32 cp_hqd_pq_rptr_report_addr_hi;
4525 u32 cp_hqd_pq_wptr_poll_addr;
4526 u32 cp_hqd_pq_wptr_poll_addr_hi;
4527 u32 cp_hqd_pq_doorbell_control;
4528 u32 cp_hqd_pq_wptr;
4529 u32 cp_hqd_pq_control;
4530 u32 cp_hqd_ib_base_addr;
4531 u32 cp_hqd_ib_base_addr_hi;
4532 u32 cp_hqd_ib_rptr;
4533 u32 cp_hqd_ib_control;
4534 u32 cp_hqd_iq_timer;
4535 u32 cp_hqd_iq_rptr;
4536 u32 cp_hqd_dequeue_request;
4537 u32 cp_hqd_dma_offload;
4538 u32 cp_hqd_sema_cmd;
4539 u32 cp_hqd_msg_type;
4540 u32 cp_hqd_atomic0_preop_lo;
4541 u32 cp_hqd_atomic0_preop_hi;
4542 u32 cp_hqd_atomic1_preop_lo;
4543 u32 cp_hqd_atomic1_preop_hi;
4544 u32 cp_hqd_hq_scheduler0;
4545 u32 cp_hqd_hq_scheduler1;
4546 u32 cp_mqd_control;
4547};
4548
4549struct bonaire_mqd
4550{
4551 u32 header;
4552 u32 dispatch_initiator;
4553 u32 dimensions[3];
4554 u32 start_idx[3];
4555 u32 num_threads[3];
4556 u32 pipeline_stat_enable;
4557 u32 perf_counter_enable;
4558 u32 pgm[2];
4559 u32 tba[2];
4560 u32 tma[2];
4561 u32 pgm_rsrc[2];
4562 u32 vmid;
4563 u32 resource_limits;
4564 u32 static_thread_mgmt01[2];
4565 u32 tmp_ring_size;
4566 u32 static_thread_mgmt23[2];
4567 u32 restart[3];
4568 u32 thread_trace_enable;
4569 u32 reserved1;
4570 u32 user_data[16];
4571 u32 vgtcs_invoke_count[2];
4572 struct hqd_registers queue_state;
4573 u32 dequeue_cntr;
4574 u32 interrupt_queue[64];
4575};
4576
841cf442
AD		 4577/**
4578 * cik_cp_compute_resume - setup the compute queue registers
4579 *
4580 * @rdev: radeon_device pointer
4581 *
4582 * Program the compute queues and test them to make sure they
4583 * are working.
4584 * Returns 0 for success, error for failure.
4585 */
4586static int cik_cp_compute_resume(struct radeon_device *rdev)
4587{
370ce45b 4588 int r, i, j, idx;
963e81f9
AD		 4589 u32 tmp;
4590 bool use_doorbell = true;
4591 u64 hqd_gpu_addr;
4592 u64 mqd_gpu_addr;
4593 u64 eop_gpu_addr;
4594 u64 wb_gpu_addr;
4595 u32 *buf;
4596 struct bonaire_mqd *mqd;
841cf442 4597
841cf442
AD		 4598 r = cik_cp_compute_start(rdev);
4599 if (r)
4600 return r;
963e81f9
AD		 4601
4602 /* fix up chicken bits */
4603 tmp = RREG32(CP_CPF_DEBUG);
4604 tmp |= (1 << 23);
4605 WREG32(CP_CPF_DEBUG, tmp);
4606
4607 /* init the pipes */
f61d5b46 4608 mutex_lock(&rdev->srbm_mutex);
963e81f9 4609
62a7b7fb 4610 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr;
963e81f9 4611
62a7b7fb 4612 cik_srbm_select(rdev, 0, 0, 0, 0);
963e81f9 4613
62a7b7fb
OG		 4614 /* write the EOP addr */
4615 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4616 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
963e81f9 4617
62a7b7fb
OG		 4618 /* set the VMID assigned */
4619 WREG32(CP_HPD_EOP_VMID, 0);
4620
4621 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4622 tmp = RREG32(CP_HPD_EOP_CONTROL);
4623 tmp &= ~EOP_SIZE_MASK;
4624 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4625 WREG32(CP_HPD_EOP_CONTROL, tmp);
963e81f9 4626
f61d5b46 4627 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4628
4629 /* init the queues. Just two for now. */
4630 for (i = 0; i < 2; i++) {
4631 if (i == 0)
4632 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4633 else
4634 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4635
4636 if (rdev->ring[idx].mqd_obj == NULL) {
4637 r = radeon_bo_create(rdev,
4638 sizeof(struct bonaire_mqd),
4639 PAGE_SIZE, true,
02376d82 4640 RADEON_GEM_DOMAIN_GTT, 0, NULL,
831b6966 4641 NULL, &rdev->ring[idx].mqd_obj);
963e81f9
AD		 4642 if (r) {
4643 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4644 return r;
4645 }
4646 }
4647
4648 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4649 if (unlikely(r != 0)) {
4650 cik_cp_compute_fini(rdev);
4651 return r;
4652 }
4653 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4654 &mqd_gpu_addr);
4655 if (r) {
4656 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4657 cik_cp_compute_fini(rdev);
4658 return r;
4659 }
4660 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4661 if (r) {
4662 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4663 cik_cp_compute_fini(rdev);
4664 return r;
4665 }
4666
963e81f9
AD		 4667 /* init the mqd struct */
4668 memset(buf, 0, sizeof(struct bonaire_mqd));
4669
4670 mqd = (struct bonaire_mqd *)buf;
4671 mqd->header = 0xC0310800;
4672 mqd->static_thread_mgmt01[0] = 0xffffffff;
4673 mqd->static_thread_mgmt01[1] = 0xffffffff;
4674 mqd->static_thread_mgmt23[0] = 0xffffffff;
4675 mqd->static_thread_mgmt23[1] = 0xffffffff;
4676
f61d5b46 4677 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4678 cik_srbm_select(rdev, rdev->ring[idx].me,
4679 rdev->ring[idx].pipe,
4680 rdev->ring[idx].queue, 0);
4681
4682 /* disable wptr polling */
4683 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4684 tmp &= ~WPTR_POLL_EN;
4685 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4686
4687 /* enable doorbell? */
4688 mqd->queue_state.cp_hqd_pq_doorbell_control =
4689 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4690 if (use_doorbell)
4691 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4692 else
4693 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4694 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4695 mqd->queue_state.cp_hqd_pq_doorbell_control);
4696
4697 /* disable the queue if it's active */
4698 mqd->queue_state.cp_hqd_dequeue_request = 0;
4699 mqd->queue_state.cp_hqd_pq_rptr = 0;
4700 mqd->queue_state.cp_hqd_pq_wptr= 0;
4701 if (RREG32(CP_HQD_ACTIVE) & 1) {
4702 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
370ce45b 4703 for (j = 0; j < rdev->usec_timeout; j++) {
963e81f9
AD		 4704 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4705 break;
4706 udelay(1);
4707 }
4708 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4709 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4710 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4711 }
4712
4713 /* set the pointer to the MQD */
4714 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4715 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4716 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4717 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4718 /* set MQD vmid to 0 */
4719 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4720 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4721 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4722
4723 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4724 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4725 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4726 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4727 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4728 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4729
4730 /* set up the HQD, this is similar to CP_RB0_CNTL */
4731 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4732 mqd->queue_state.cp_hqd_pq_control &=
4733 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4734
4735 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4736 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 4737 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4738 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 4739#ifdef __BIG_ENDIAN
4740 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4741#endif
4742 mqd->queue_state.cp_hqd_pq_control &=
4743 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4744 mqd->queue_state.cp_hqd_pq_control |=
4745 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4746 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4747
4748 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4749 if (i == 0)
4750 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4751 else
4752 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4753 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4754 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4755 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4756 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4757 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4758
4759 /* set the wb address wether it's enabled or not */
4760 if (i == 0)
4761 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4762 else
4763 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4764 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4765 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4766 upper_32_bits(wb_gpu_addr) & 0xffff;
4767 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4768 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4769 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4770 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4771
4772 /* enable the doorbell if requested */
4773 if (use_doorbell) {
4774 mqd->queue_state.cp_hqd_pq_doorbell_control =
4775 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4776 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4777 mqd->queue_state.cp_hqd_pq_doorbell_control |=
d5754ab8 4778 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
963e81f9
AD		 4779 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4780 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4781 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4782
4783 } else {
4784 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4785 }
4786 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4787 mqd->queue_state.cp_hqd_pq_doorbell_control);
4788
4789 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4790 rdev->ring[idx].wptr = 0;
4791 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4792 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
ff212f25 4793 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
963e81f9
AD		 4794
4795 /* set the vmid for the queue */
4796 mqd->queue_state.cp_hqd_vmid = 0;
4797 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4798
4799 /* activate the queue */
4800 mqd->queue_state.cp_hqd_active = 1;
4801 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4802
4803 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4804 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4805
4806 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4807 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4808
4809 rdev->ring[idx].ready = true;
4810 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4811 if (r)
4812 rdev->ring[idx].ready = false;
4813 }
4814
841cf442
AD		 4815 return 0;
4816}
4817
841cf442
AD		 4818static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4819{
4820 cik_cp_gfx_enable(rdev, enable);
4821 cik_cp_compute_enable(rdev, enable);
4822}
4823
841cf442
AD		 4824static int cik_cp_load_microcode(struct radeon_device *rdev)
4825{
4826 int r;
4827
4828 r = cik_cp_gfx_load_microcode(rdev);
4829 if (r)
4830 return r;
4831 r = cik_cp_compute_load_microcode(rdev);
4832 if (r)
4833 return r;
4834
4835 return 0;
4836}
4837
841cf442
AD		 4838static void cik_cp_fini(struct radeon_device *rdev)
4839{
4840 cik_cp_gfx_fini(rdev);
4841 cik_cp_compute_fini(rdev);
4842}
4843
841cf442
AD		 4844static int cik_cp_resume(struct radeon_device *rdev)
4845{
4846 int r;
4847
4214faf6
AD		 4848 cik_enable_gui_idle_interrupt(rdev, false);
4849
841cf442
AD		 4850 r = cik_cp_load_microcode(rdev);
4851 if (r)
4852 return r;
4853
4854 r = cik_cp_gfx_resume(rdev);
4855 if (r)
4856 return r;
4857 r = cik_cp_compute_resume(rdev);
4858 if (r)
4859 return r;
4860
4214faf6
AD		 4861 cik_enable_gui_idle_interrupt(rdev, true);
4862
841cf442
AD		 4863 return 0;
4864}
4865
cc066715 4866static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 4867{
6f2043ce
AD		 4868 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4869 RREG32(GRBM_STATUS));
4870 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4871 RREG32(GRBM_STATUS2));
4872 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4873 RREG32(GRBM_STATUS_SE0));
4874 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4875 RREG32(GRBM_STATUS_SE1));
4876 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4877 RREG32(GRBM_STATUS_SE2));
4878 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4879 RREG32(GRBM_STATUS_SE3));
4880 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4881 RREG32(SRBM_STATUS));
4882 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4883 RREG32(SRBM_STATUS2));
cc066715
AD		 4884 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4885 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4886 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4887 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 4888 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4889 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4890 RREG32(CP_STALLED_STAT1));
4891 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4892 RREG32(CP_STALLED_STAT2));
4893 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4894 RREG32(CP_STALLED_STAT3));
4895 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4896 RREG32(CP_CPF_BUSY_STAT));
4897 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4898 RREG32(CP_CPF_STALLED_STAT1));
4899 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4900 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4901 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4902 RREG32(CP_CPC_STALLED_STAT1));
4903 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 4904}
6f2043ce 4905
21a93e13 4906/**
cc066715 4907 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 4908 *
4909 * @rdev: radeon_device pointer
21a93e13 4910 *
cc066715
AD		 4911 * Check which blocks are busy and return the relevant reset
4912 * mask to be used by cik_gpu_soft_reset().
4913 * Returns a mask of the blocks to be reset.
21a93e13 4914 */
2483b4ea 4915u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 4916{
cc066715
AD		 4917 u32 reset_mask = 0;
4918 u32 tmp;
21a93e13 4919
cc066715
AD		 4920 /* GRBM_STATUS */
4921 tmp = RREG32(GRBM_STATUS);
4922 if (tmp & (PA_BUSY | SC_BUSY |
4923 BCI_BUSY | SX_BUSY |
4924 TA_BUSY | VGT_BUSY |
4925 DB_BUSY | CB_BUSY |
4926 GDS_BUSY | SPI_BUSY |
4927 IA_BUSY | IA_BUSY_NO_DMA))
4928 reset_mask |= RADEON_RESET_GFX;
21a93e13 4929
cc066715
AD		 4930 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4931 reset_mask |= RADEON_RESET_CP;
21a93e13 4932
cc066715
AD		 4933 /* GRBM_STATUS2 */
4934 tmp = RREG32(GRBM_STATUS2);
4935 if (tmp & RLC_BUSY)
4936 reset_mask |= RADEON_RESET_RLC;
21a93e13 4937
cc066715
AD		 4938 /* SDMA0_STATUS_REG */
4939 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4940 if (!(tmp & SDMA_IDLE))
4941 reset_mask |= RADEON_RESET_DMA;
21a93e13 4942
cc066715
AD		 4943 /* SDMA1_STATUS_REG */
4944 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4945 if (!(tmp & SDMA_IDLE))
4946 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4947
cc066715
AD		 4948 /* SRBM_STATUS2 */
4949 tmp = RREG32(SRBM_STATUS2);
4950 if (tmp & SDMA_BUSY)
4951 reset_mask |= RADEON_RESET_DMA;
21a93e13 4952
cc066715
AD		 4953 if (tmp & SDMA1_BUSY)
4954 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4955
cc066715
AD		 4956 /* SRBM_STATUS */
4957 tmp = RREG32(SRBM_STATUS);
21a93e13 4958
cc066715
AD		 4959 if (tmp & IH_BUSY)
4960 reset_mask |= RADEON_RESET_IH;
21a93e13 4961
cc066715
AD		 4962 if (tmp & SEM_BUSY)
4963 reset_mask |= RADEON_RESET_SEM;
21a93e13 4964
cc066715
AD		 4965 if (tmp & GRBM_RQ_PENDING)
4966 reset_mask |= RADEON_RESET_GRBM;
21a93e13 4967
cc066715
AD		 4968 if (tmp & VMC_BUSY)
4969 reset_mask |= RADEON_RESET_VMC;
21a93e13 4970
cc066715
AD		 4971 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4972 MCC_BUSY | MCD_BUSY))
4973 reset_mask |= RADEON_RESET_MC;
21a93e13 4974
cc066715
AD		 4975 if (evergreen_is_display_hung(rdev))
4976 reset_mask |= RADEON_RESET_DISPLAY;
4977
4978 /* Skip MC reset as it's mostly likely not hung, just busy */
4979 if (reset_mask & RADEON_RESET_MC) {
4980 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4981 reset_mask &= ~RADEON_RESET_MC;
21a93e13 4982 }
cc066715
AD		 4983
4984 return reset_mask;
21a93e13
AD		 4985}
4986
4987/**
cc066715 4988 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 4989 *
4990 * @rdev: radeon_device pointer
cc066715 4991 * @reset_mask: mask of which blocks to reset
21a93e13 4992 *
cc066715 4993 * Soft reset the blocks specified in @reset_mask.
21a93e13 4994 */
cc066715 4995static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 4996{
6f2043ce 4997 struct evergreen_mc_save save;
cc066715
AD		 4998 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4999 u32 tmp;
21a93e13 5000
cc066715
AD		 5001 if (reset_mask == 0)
5002 return;
21a93e13 5003
cc066715 5004 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 5005
cc066715
AD		 5006 cik_print_gpu_status_regs(rdev);
5007 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5008 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5009 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5010 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 5011
fb2c7f4d
AD		 5012 /* disable CG/PG */
5013 cik_fini_pg(rdev);
5014 cik_fini_cg(rdev);
5015
cc066715
AD		 5016 /* stop the rlc */
5017 cik_rlc_stop(rdev);
21a93e13 5018
cc066715
AD		 5019 /* Disable GFX parsing/prefetching */
5020 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 5021
cc066715
AD		 5022 /* Disable MEC parsing/prefetching */
5023 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 5024
cc066715
AD		 5025 if (reset_mask & RADEON_RESET_DMA) {
5026 /* sdma0 */
5027 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5028 tmp |= SDMA_HALT;
5029 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5030 }
5031 if (reset_mask & RADEON_RESET_DMA1) {
5032 /* sdma1 */
5033 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5034 tmp |= SDMA_HALT;
5035 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5036 }
21a93e13 5037
6f2043ce 5038 evergreen_mc_stop(rdev, &save);
cc066715 5039 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 5040 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5041 }
21a93e13 5042
cc066715
AD		 5043 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5044 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 5045
cc066715
AD		 5046 if (reset_mask & RADEON_RESET_CP) {
5047 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 5048
cc066715
AD		 5049 srbm_soft_reset |= SOFT_RESET_GRBM;
5050 }
21a93e13 5051
cc066715
AD		 5052 if (reset_mask & RADEON_RESET_DMA)
5053 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 5054
cc066715
AD		 5055 if (reset_mask & RADEON_RESET_DMA1)
5056 srbm_soft_reset |= SOFT_RESET_SDMA1;
5057
5058 if (reset_mask & RADEON_RESET_DISPLAY)
5059 srbm_soft_reset |= SOFT_RESET_DC;
5060
5061 if (reset_mask & RADEON_RESET_RLC)
5062 grbm_soft_reset |= SOFT_RESET_RLC;
5063
5064 if (reset_mask & RADEON_RESET_SEM)
5065 srbm_soft_reset |= SOFT_RESET_SEM;
5066
5067 if (reset_mask & RADEON_RESET_IH)
5068 srbm_soft_reset |= SOFT_RESET_IH;
5069
5070 if (reset_mask & RADEON_RESET_GRBM)
5071 srbm_soft_reset |= SOFT_RESET_GRBM;
5072
5073 if (reset_mask & RADEON_RESET_VMC)
5074 srbm_soft_reset |= SOFT_RESET_VMC;
5075
5076 if (!(rdev->flags & RADEON_IS_IGP)) {
5077 if (reset_mask & RADEON_RESET_MC)
5078 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 5079 }
5080
cc066715
AD		 5081 if (grbm_soft_reset) {
5082 tmp = RREG32(GRBM_SOFT_RESET);
5083 tmp |= grbm_soft_reset;
5084 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5085 WREG32(GRBM_SOFT_RESET, tmp);
5086 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 5087
cc066715 5088 udelay(50);
21a93e13 5089
cc066715
AD		 5090 tmp &= ~grbm_soft_reset;
5091 WREG32(GRBM_SOFT_RESET, tmp);
5092 tmp = RREG32(GRBM_SOFT_RESET);
5093 }
21a93e13 5094
cc066715
AD		 5095 if (srbm_soft_reset) {
5096 tmp = RREG32(SRBM_SOFT_RESET);
5097 tmp |= srbm_soft_reset;
5098 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5099 WREG32(SRBM_SOFT_RESET, tmp);
5100 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 5101
cc066715 5102 udelay(50);
21a93e13 5103
cc066715
AD		 5104 tmp &= ~srbm_soft_reset;
5105 WREG32(SRBM_SOFT_RESET, tmp);
5106 tmp = RREG32(SRBM_SOFT_RESET);
5107 }
21a93e13 5108
6f2043ce
AD		 5109 /* Wait a little for things to settle down */
5110 udelay(50);
21a93e13 5111
6f2043ce 5112 evergreen_mc_resume(rdev, &save);
cc066715
AD		 5113 udelay(50);
5114
5115 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 5116}
5117
0279ed19
AD		 5118struct kv_reset_save_regs {
5119 u32 gmcon_reng_execute;
5120 u32 gmcon_misc;
5121 u32 gmcon_misc3;
5122};
5123
5124static void kv_save_regs_for_reset(struct radeon_device *rdev,
5125 struct kv_reset_save_regs *save)
5126{
5127 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5128 save->gmcon_misc = RREG32(GMCON_MISC);
5129 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5130
5131 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5132 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5133 STCTRL_STUTTER_EN));
5134}
5135
5136static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5137 struct kv_reset_save_regs *save)
5138{
5139 int i;
5140
5141 WREG32(GMCON_PGFSM_WRITE, 0);
5142 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5143
5144 for (i = 0; i < 5; i++)
5145 WREG32(GMCON_PGFSM_WRITE, 0);
5146
5147 WREG32(GMCON_PGFSM_WRITE, 0);
5148 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5149
5150 for (i = 0; i < 5; i++)
5151 WREG32(GMCON_PGFSM_WRITE, 0);
5152
5153 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5154 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5155
5156 for (i = 0; i < 5; i++)
5157 WREG32(GMCON_PGFSM_WRITE, 0);
5158
5159 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5160 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5161
5162 for (i = 0; i < 5; i++)
5163 WREG32(GMCON_PGFSM_WRITE, 0);
5164
5165 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5166 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5167
5168 for (i = 0; i < 5; i++)
5169 WREG32(GMCON_PGFSM_WRITE, 0);
5170
5171 WREG32(GMCON_PGFSM_WRITE, 0);
5172 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5173
5174 for (i = 0; i < 5; i++)
5175 WREG32(GMCON_PGFSM_WRITE, 0);
5176
5177 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5178 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5179
5180 for (i = 0; i < 5; i++)
5181 WREG32(GMCON_PGFSM_WRITE, 0);
5182
5183 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5184 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5185
5186 for (i = 0; i < 5; i++)
5187 WREG32(GMCON_PGFSM_WRITE, 0);
5188
5189 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5190 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5191
5192 for (i = 0; i < 5; i++)
5193 WREG32(GMCON_PGFSM_WRITE, 0);
5194
5195 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5196 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5197
5198 for (i = 0; i < 5; i++)
5199 WREG32(GMCON_PGFSM_WRITE, 0);
5200
5201 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5202 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5203
5204 WREG32(GMCON_MISC3, save->gmcon_misc3);
5205 WREG32(GMCON_MISC, save->gmcon_misc);
5206 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5207}
5208
5209static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5210{
5211 struct evergreen_mc_save save;
5212 struct kv_reset_save_regs kv_save = { 0 };
5213 u32 tmp, i;
5214
5215 dev_info(rdev->dev, "GPU pci config reset\n");
5216
5217 /* disable dpm? */
5218
5219 /* disable cg/pg */
5220 cik_fini_pg(rdev);
5221 cik_fini_cg(rdev);
5222
5223 /* Disable GFX parsing/prefetching */
5224 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5225
5226 /* Disable MEC parsing/prefetching */
5227 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5228
5229 /* sdma0 */
5230 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5231 tmp |= SDMA_HALT;
5232 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5233 /* sdma1 */
5234 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5235 tmp |= SDMA_HALT;
5236 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5237 /* XXX other engines? */
5238
5239 /* halt the rlc, disable cp internal ints */
5240 cik_rlc_stop(rdev);
5241
5242 udelay(50);
5243
5244 /* disable mem access */
5245 evergreen_mc_stop(rdev, &save);
5246 if (evergreen_mc_wait_for_idle(rdev)) {
5247 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5248 }
5249
5250 if (rdev->flags & RADEON_IS_IGP)
5251 kv_save_regs_for_reset(rdev, &kv_save);
5252
5253 /* disable BM */
5254 pci_clear_master(rdev->pdev);
5255 /* reset */
5256 radeon_pci_config_reset(rdev);
5257
5258 udelay(100);
5259
5260 /* wait for asic to come out of reset */
5261 for (i = 0; i < rdev->usec_timeout; i++) {
5262 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5263 break;
5264 udelay(1);
5265 }
5266
5267 /* does asic init need to be run first??? */
5268 if (rdev->flags & RADEON_IS_IGP)
5269 kv_restore_regs_for_reset(rdev, &kv_save);
5270}
5271
21a93e13 5272/**
cc066715 5273 * cik_asic_reset - soft reset GPU
21a93e13
AD		 5274 *
5275 * @rdev: radeon_device pointer
71fe2899 5276 * @hard: force hard reset
21a93e13 5277 *
cc066715
AD		 5278 * Look up which blocks are hung and attempt
5279 * to reset them.
6f2043ce 5280 * Returns 0 for success.
21a93e13 5281 */
71fe2899 5282int cik_asic_reset(struct radeon_device *rdev, bool hard)
21a93e13 5283{
cc066715 5284 u32 reset_mask;
21a93e13 5285
71fe2899
JG		 5286 if (hard) {
5287 cik_gpu_pci_config_reset(rdev);
5288 return 0;
5289 }
5290
cc066715 5291 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5292
cc066715
AD		 5293 if (reset_mask)
5294 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 5295
0279ed19 5296 /* try soft reset */
cc066715 5297 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 5298
cc066715
AD		 5299 reset_mask = cik_gpu_check_soft_reset(rdev);
5300
0279ed19
AD		 5301 /* try pci config reset */
5302 if (reset_mask && radeon_hard_reset)
5303 cik_gpu_pci_config_reset(rdev);
5304
5305 reset_mask = cik_gpu_check_soft_reset(rdev);
5306
cc066715
AD		 5307 if (!reset_mask)
5308 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 5309
5310 return 0;
5311}
5312
5313/**
cc066715 5314 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 5315 *
5316 * @rdev: radeon_device pointer
cc066715 5317 * @ring: radeon_ring structure holding ring information
21a93e13 5318 *
cc066715
AD		 5319 * Check if the 3D engine is locked up (CIK).
5320 * Returns true if the engine is locked, false if not.
21a93e13 5321 */
cc066715 5322bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 5323{
cc066715 5324 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5325
cc066715
AD		 5326 if (!(reset_mask & (RADEON_RESET_GFX |
5327 RADEON_RESET_COMPUTE |
5328 RADEON_RESET_CP))) {
ff212f25 5329 radeon_ring_lockup_update(rdev, ring);
cc066715 5330 return false;
21a93e13 5331 }
cc066715 5332 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 5333}
5334
1c49165d 5335/* MC */
21a93e13 5336/**
1c49165d 5337 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 5338 *
5339 * @rdev: radeon_device pointer
21a93e13 5340 *
1c49165d
AD		 5341 * Set the location of vram, gart, and AGP in the GPU's
5342 * physical address space (CIK).
21a93e13 5343 */
1c49165d 5344static void cik_mc_program(struct radeon_device *rdev)
21a93e13 5345{
1c49165d 5346 struct evergreen_mc_save save;
21a93e13 5347 u32 tmp;
1c49165d 5348 int i, j;
21a93e13 5349
1c49165d
AD		 5350 /* Initialize HDP */
5351 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5352 WREG32((0x2c14 + j), 0x00000000);
5353 WREG32((0x2c18 + j), 0x00000000);
5354 WREG32((0x2c1c + j), 0x00000000);
5355 WREG32((0x2c20 + j), 0x00000000);
5356 WREG32((0x2c24 + j), 0x00000000);
21a93e13 5357 }
1c49165d 5358 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 5359
1c49165d
AD		 5360 evergreen_mc_stop(rdev, &save);
5361 if (radeon_mc_wait_for_idle(rdev)) {
5362 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5363 }
1c49165d
AD		 5364 /* Lockout access through VGA aperture*/
5365 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5366 /* Update configuration */
5367 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5368 rdev->mc.vram_start >> 12);
5369 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5370 rdev->mc.vram_end >> 12);
5371 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5372 rdev->vram_scratch.gpu_addr >> 12);
5373 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5374 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5375 WREG32(MC_VM_FB_LOCATION, tmp);
5376 /* XXX double check these! */
5377 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5378 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5379 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5380 WREG32(MC_VM_AGP_BASE, 0);
5381 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5382 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5383 if (radeon_mc_wait_for_idle(rdev)) {
5384 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5385 }
1c49165d
AD		 5386 evergreen_mc_resume(rdev, &save);
5387 /* we need to own VRAM, so turn off the VGA renderer here
5388 * to stop it overwriting our objects */
5389 rv515_vga_render_disable(rdev);
21a93e13
AD		 5390}
5391
5392/**
1c49165d 5393 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 5394 *
5395 * @rdev: radeon_device pointer
21a93e13 5396 *
1c49165d
AD		 5397 * Look up the amount of vram, vram width, and decide how to place
5398 * vram and gart within the GPU's physical address space (CIK).
5399 * Returns 0 for success.
21a93e13 5400 */
1c49165d 5401static int cik_mc_init(struct radeon_device *rdev)
21a93e13 5402{
1c49165d
AD		 5403 u32 tmp;
5404 int chansize, numchan;
21a93e13 5405
1c49165d
AD		 5406 /* Get VRAM informations */
5407 rdev->mc.vram_is_ddr = true;
5408 tmp = RREG32(MC_ARB_RAMCFG);
5409 if (tmp & CHANSIZE_MASK) {
5410 chansize = 64;
21a93e13 5411 } else {
1c49165d 5412 chansize = 32;
21a93e13 5413 }
1c49165d
AD		 5414 tmp = RREG32(MC_SHARED_CHMAP);
5415 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5416 case 0:
5417 default:
5418 numchan = 1;
5419 break;
5420 case 1:
5421 numchan = 2;
5422 break;
5423 case 2:
5424 numchan = 4;
5425 break;
5426 case 3:
5427 numchan = 8;
5428 break;
5429 case 4:
5430 numchan = 3;
5431 break;
5432 case 5:
5433 numchan = 6;
5434 break;
5435 case 6:
5436 numchan = 10;
5437 break;
5438 case 7:
5439 numchan = 12;
5440 break;
5441 case 8:
5442 numchan = 16;
5443 break;
5444 }
5445 rdev->mc.vram_width = numchan * chansize;
5446 /* Could aper size report 0 ? */
5447 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5448 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5449 /* size in MB on si */
13c5bfda
AD		 5450 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5451 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 5452 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5453 si_vram_gtt_location(rdev, &rdev->mc);
5454 radeon_update_bandwidth_info(rdev);
5455
5456 return 0;
5457}
5458
5459/*
5460 * GART
5461 * VMID 0 is the physical GPU addresses as used by the kernel.
5462 * VMIDs 1-15 are used for userspace clients and are handled
5463 * by the radeon vm/hsa code.
5464 */
5465/**
5466 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5467 *
5468 * @rdev: radeon_device pointer
5469 *
5470 * Flush the TLB for the VMID 0 page table (CIK).
5471 */
5472void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5473{
5474 /* flush hdp cache */
5475 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5476
5477 /* bits 0-15 are the VM contexts0-15 */
5478 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5479}
5480
08dcc57f
BG		 5481static void cik_pcie_init_compute_vmid(struct radeon_device *rdev)
5482{
5483 int i;
5484 uint32_t sh_mem_bases, sh_mem_config;
5485
5486 sh_mem_bases = 0x6000 | 0x6000 << 16;
5487 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
5488 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
5489
5490 mutex_lock(&rdev->srbm_mutex);
5491 for (i = 8; i < 16; i++) {
5492 cik_srbm_select(rdev, 0, 0, 0, i);
5493 /* CP and shaders */
5494 WREG32(SH_MEM_CONFIG, sh_mem_config);
5495 WREG32(SH_MEM_APE1_BASE, 1);
5496 WREG32(SH_MEM_APE1_LIMIT, 0);
5497 WREG32(SH_MEM_BASES, sh_mem_bases);
5498 }
5499 cik_srbm_select(rdev, 0, 0, 0, 0);
5500 mutex_unlock(&rdev->srbm_mutex);
5501}
5502
1c49165d
AD		 5503/**
5504 * cik_pcie_gart_enable - gart enable
5505 *
5506 * @rdev: radeon_device pointer
5507 *
5508 * This sets up the TLBs, programs the page tables for VMID0,
5509 * sets up the hw for VMIDs 1-15 which are allocated on
5510 * demand, and sets up the global locations for the LDS, GDS,
5511 * and GPUVM for FSA64 clients (CIK).
5512 * Returns 0 for success, errors for failure.
5513 */
5514static int cik_pcie_gart_enable(struct radeon_device *rdev)
5515{
5516 int r, i;
5517
5518 if (rdev->gart.robj == NULL) {
5519 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5520 return -EINVAL;
5521 }
5522 r = radeon_gart_table_vram_pin(rdev);
5523 if (r)
5524 return r;
1c49165d
AD		 5525 /* Setup TLB control */
5526 WREG32(MC_VM_MX_L1_TLB_CNTL,
5527 (0xA << 7) |
5528 ENABLE_L1_TLB |
ec3dbbcb 5529 ENABLE_L1_FRAGMENT_PROCESSING |
1c49165d
AD		 5530 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5531 ENABLE_ADVANCED_DRIVER_MODEL |
5532 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5533 /* Setup L2 cache */
5534 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5535 ENABLE_L2_FRAGMENT_PROCESSING |
5536 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5537 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5538 EFFECTIVE_L2_QUEUE_SIZE(7) |
5539 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5540 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5541 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
ec3dbbcb
CK		 5542 BANK_SELECT(4) |
5543 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
1c49165d
AD		 5544 /* setup context0 */
5545 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
7c0411d2 5546 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1c49165d
AD		 5547 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5548 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5549 (u32)(rdev->dummy_page.addr >> 12));
5550 WREG32(VM_CONTEXT0_CNTL2, 0);
5551 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5552 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5553
5554 WREG32(0x15D4, 0);
5555 WREG32(0x15D8, 0);
5556 WREG32(0x15DC, 0);
5557
054e01d6 5558 /* restore context1-15 */
1c49165d
AD		 5559 /* set vm size, must be a multiple of 4 */
5560 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
607d4806 5561 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
1c49165d
AD		 5562 for (i = 1; i < 16; i++) {
5563 if (i < 8)
5564 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
054e01d6 5565 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5566 else
5567 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
054e01d6 5568 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5569 }
5570
5571 /* enable context1-15 */
5572 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5573 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 5574 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 5575 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4510fb98 5576 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
a00024b0
AD		 5577 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5578 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5579 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5580 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5581 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5582 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5583 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5584 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5585 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5586 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5587 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5588 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d 5589
1c49165d
AD		 5590 if (rdev->family == CHIP_KAVERI) {
5591 u32 tmp = RREG32(CHUB_CONTROL);
5592 tmp &= ~BYPASS_VM;
5593 WREG32(CHUB_CONTROL, tmp);
5594 }
5595
5596 /* XXX SH_MEM regs */
5597 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 5598 mutex_lock(&rdev->srbm_mutex);
1c49165d 5599 for (i = 0; i < 16; i++) {
b556b12e 5600 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 5601 /* CP and shaders */
1c49165d
AD		 5602 WREG32(SH_MEM_CONFIG, 0);
5603 WREG32(SH_MEM_APE1_BASE, 1);
5604 WREG32(SH_MEM_APE1_LIMIT, 0);
5605 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 5606 /* SDMA GFX */
5607 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5608 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5609 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5610 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5611 /* XXX SDMA RLC - todo */
1c49165d 5612 }
b556b12e 5613 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5614 mutex_unlock(&rdev->srbm_mutex);
1c49165d 5615
08dcc57f
BG		 5616 cik_pcie_init_compute_vmid(rdev);
5617
1c49165d
AD		 5618 cik_pcie_gart_tlb_flush(rdev);
5619 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5620 (unsigned)(rdev->mc.gtt_size >> 20),
5621 (unsigned long long)rdev->gart.table_addr);
5622 rdev->gart.ready = true;
5623 return 0;
5624}
5625
5626/**
5627 * cik_pcie_gart_disable - gart disable
5628 *
5629 * @rdev: radeon_device pointer
5630 *
5631 * This disables all VM page table (CIK).
5632 */
5633static void cik_pcie_gart_disable(struct radeon_device *rdev)
5634{
054e01d6
CK		 5635 unsigned i;
5636
5637 for (i = 1; i < 16; ++i) {
5638 uint32_t reg;
5639 if (i < 8)
5640 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5641 else
5642 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5643 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5644 }
5645
1c49165d
AD		 5646 /* Disable all tables */
5647 WREG32(VM_CONTEXT0_CNTL, 0);
5648 WREG32(VM_CONTEXT1_CNTL, 0);
5649 /* Setup TLB control */
5650 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5651 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5652 /* Setup L2 cache */
5653 WREG32(VM_L2_CNTL,
5654 ENABLE_L2_FRAGMENT_PROCESSING |
5655 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5656 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5657 EFFECTIVE_L2_QUEUE_SIZE(7) |
5658 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5659 WREG32(VM_L2_CNTL2, 0);
5660 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5661 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5662 radeon_gart_table_vram_unpin(rdev);
5663}
5664
5665/**
5666 * cik_pcie_gart_fini - vm fini callback
5667 *
5668 * @rdev: radeon_device pointer
5669 *
5670 * Tears down the driver GART/VM setup (CIK).
5671 */
5672static void cik_pcie_gart_fini(struct radeon_device *rdev)
5673{
5674 cik_pcie_gart_disable(rdev);
5675 radeon_gart_table_vram_free(rdev);
5676 radeon_gart_fini(rdev);
5677}
5678
5679/* vm parser */
5680/**
5681 * cik_ib_parse - vm ib_parse callback
5682 *
5683 * @rdev: radeon_device pointer
5684 * @ib: indirect buffer pointer
5685 *
5686 * CIK uses hw IB checking so this is a nop (CIK).
5687 */
5688int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5689{
5690 return 0;
5691}
5692
5693/*
5694 * vm
5695 * VMID 0 is the physical GPU addresses as used by the kernel.
5696 * VMIDs 1-15 are used for userspace clients and are handled
5697 * by the radeon vm/hsa code.
5698 */
5699/**
5700 * cik_vm_init - cik vm init callback
5701 *
5702 * @rdev: radeon_device pointer
5703 *
5704 * Inits cik specific vm parameters (number of VMs, base of vram for
5705 * VMIDs 1-15) (CIK).
5706 * Returns 0 for success.
5707 */
5708int cik_vm_init(struct radeon_device *rdev)
5709{
62a7b7fb
OG		 5710 /*
5711 * number of VMs
5712 * VMID 0 is reserved for System
5713 * radeon graphics/compute will use VMIDs 1-7
5714 * amdkfd will use VMIDs 8-15
5715 */
5716 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;
1c49165d
AD		 5717 /* base offset of vram pages */
5718 if (rdev->flags & RADEON_IS_IGP) {
5719 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5720 tmp <<= 22;
5721 rdev->vm_manager.vram_base_offset = tmp;
5722 } else
5723 rdev->vm_manager.vram_base_offset = 0;
5724
5725 return 0;
5726}
5727
5728/**
5729 * cik_vm_fini - cik vm fini callback
5730 *
5731 * @rdev: radeon_device pointer
5732 *
5733 * Tear down any asic specific VM setup (CIK).
5734 */
5735void cik_vm_fini(struct radeon_device *rdev)
5736{
5737}
5738
3ec7d11b
AD		 5739/**
5740 * cik_vm_decode_fault - print human readable fault info
5741 *
5742 * @rdev: radeon_device pointer
5743 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5744 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5745 *
5746 * Print human readable fault information (CIK).
5747 */
5748static void cik_vm_decode_fault(struct radeon_device *rdev,
5749 u32 status, u32 addr, u32 mc_client)
5750{
939c0d3c 5751 u32 mc_id;
3ec7d11b
AD		 5752 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5753 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 5754 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5755 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 5756
939c0d3c
AD		 5757 if (rdev->family == CHIP_HAWAII)
5758 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5759 else
5760 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5761
328a50c7 5762 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 5763 protections, vmid, addr,
5764 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 5765 block, mc_client, mc_id);
3ec7d11b
AD		 5766}
5767
f96ab484
AD		 5768/**
5769 * cik_vm_flush - cik vm flush using the CP
5770 *
5771 * @rdev: radeon_device pointer
5772 *
5773 * Update the page table base and flush the VM TLB
5774 * using the CP (CIK).
5775 */
faffaf62
CK		 5776void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5777 unsigned vm_id, uint64_t pd_addr)
f96ab484 5778{
faffaf62 5779 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
f96ab484
AD		 5780
5781 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
f1d2a26b 5782 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484 5783 WRITE_DATA_DST_SEL(0)));
faffaf62 5784 if (vm_id < 8) {
f96ab484 5785 radeon_ring_write(ring,
faffaf62 5786 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
f96ab484
AD		 5787 } else {
5788 radeon_ring_write(ring,
faffaf62 5789 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
f96ab484
AD		 5790 }
5791 radeon_ring_write(ring, 0);
faffaf62 5792 radeon_ring_write(ring, pd_addr >> 12);
f96ab484
AD		 5793
5794 /* update SH_MEM_* regs */
5795 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5796 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5797 WRITE_DATA_DST_SEL(0)));
5798 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5799 radeon_ring_write(ring, 0);
faffaf62 5800 radeon_ring_write(ring, VMID(vm_id));
f96ab484
AD		 5801
5802 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4fb0bbd5 5803 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5804 WRITE_DATA_DST_SEL(0)));
5805 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5806 radeon_ring_write(ring, 0);
5807
5808 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5809 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5810 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5811 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 5812
f96ab484 5813 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5814 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5815 WRITE_DATA_DST_SEL(0)));
5816 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5817 radeon_ring_write(ring, 0);
5818 radeon_ring_write(ring, VMID(0));
6f2043ce 5819
f96ab484 5820 /* HDP flush */
faffaf62 5821 cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
f96ab484
AD		 5822
5823 /* bits 0-15 are the VM contexts0-15 */
5824 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5825 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5826 WRITE_DATA_DST_SEL(0)));
5827 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5828 radeon_ring_write(ring, 0);
faffaf62 5829 radeon_ring_write(ring, 1 << vm_id);
f96ab484 5830
3a01fd36
AD		 5831 /* wait for the invalidate to complete */
5832 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5833 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
5834 WAIT_REG_MEM_FUNCTION(0) | /* always */
5835 WAIT_REG_MEM_ENGINE(0))); /* me */
5836 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5837 radeon_ring_write(ring, 0);
5838 radeon_ring_write(ring, 0); /* ref */
5839 radeon_ring_write(ring, 0); /* mask */
5840 radeon_ring_write(ring, 0x20); /* poll interval */
5841
b07fdd38 5842 /* compute doesn't have PFP */
f1d2a26b 5843 if (usepfp) {
b07fdd38
AD		 5844 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5845 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5846 radeon_ring_write(ring, 0x0);
5847 }
cc066715 5848}
6f2043ce 5849
f6796cae
AD		 5850/*
5851 * RLC
5852 * The RLC is a multi-purpose microengine that handles a
5853 * variety of functions, the most important of which is
5854 * the interrupt controller.
5855 */
866d83de
AD		 5856static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5857 bool enable)
f6796cae 5858{
866d83de 5859 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 5860
866d83de
AD		 5861 if (enable)
5862 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5863 else
5864 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 5865 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 5866}
f6796cae 5867
866d83de 5868static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 5869{
cc066715 5870 u32 tmp;
6f2043ce 5871
866d83de
AD		 5872 tmp = RREG32(RLC_LB_CNTL);
5873 if (enable)
5874 tmp |= LOAD_BALANCE_ENABLE;
5875 else
5876 tmp &= ~LOAD_BALANCE_ENABLE;
5877 WREG32(RLC_LB_CNTL, tmp);
5878}
cc066715 5879
866d83de
AD		 5880static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5881{
5882 u32 i, j, k;
5883 u32 mask;
cc066715 5884
1c0a4625 5885 mutex_lock(&rdev->grbm_idx_mutex);
f6796cae
AD		 5886 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5887 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5888 cik_select_se_sh(rdev, i, j);
5889 for (k = 0; k < rdev->usec_timeout; k++) {
5890 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5891 break;
5892 udelay(1);
5893 }
5894 }
5895 }
5896 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 5897 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 5898
f6796cae
AD		 5899 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5900 for (k = 0; k < rdev->usec_timeout; k++) {
5901 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5902 break;
5903 udelay(1);
5904 }
5905}
cc066715 5906
22c775ce
AD		 5907static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5908{
5909 u32 tmp;
cc066715 5910
22c775ce
AD		 5911 tmp = RREG32(RLC_CNTL);
5912 if (tmp != rlc)
5913 WREG32(RLC_CNTL, rlc);
5914}
cc066715 5915
22c775ce
AD		 5916static u32 cik_halt_rlc(struct radeon_device *rdev)
5917{
5918 u32 data, orig;
cc066715 5919
22c775ce 5920 orig = data = RREG32(RLC_CNTL);
cc066715 5921
22c775ce
AD		 5922 if (data & RLC_ENABLE) {
5923 u32 i;
cc066715 5924
22c775ce
AD		 5925 data &= ~RLC_ENABLE;
5926 WREG32(RLC_CNTL, data);
cc066715 5927
22c775ce
AD		 5928 for (i = 0; i < rdev->usec_timeout; i++) {
5929 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5930 break;
5931 udelay(1);
5932 }
cc066715 5933
22c775ce
AD		 5934 cik_wait_for_rlc_serdes(rdev);
5935 }
cc066715 5936
22c775ce
AD		 5937 return orig;
5938}
cc066715 5939
a412fce0
AD		 5940void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5941{
5942 u32 tmp, i, mask;
5943
5944 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5945 WREG32(RLC_GPR_REG2, tmp);
5946
5947 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5948 for (i = 0; i < rdev->usec_timeout; i++) {
5949 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5950 break;
5951 udelay(1);
5952 }
5953
5954 for (i = 0; i < rdev->usec_timeout; i++) {
5955 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5956 break;
5957 udelay(1);
5958 }
5959}
5960
5961void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5962{
5963 u32 tmp;
5964
5965 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5966 WREG32(RLC_GPR_REG2, tmp);
5967}
5968
866d83de
AD		 5969/**
5970 * cik_rlc_stop - stop the RLC ME
5971 *
5972 * @rdev: radeon_device pointer
5973 *
5974 * Halt the RLC ME (MicroEngine) (CIK).
5975 */
5976static void cik_rlc_stop(struct radeon_device *rdev)
5977{
22c775ce 5978 WREG32(RLC_CNTL, 0);
866d83de
AD		 5979
5980 cik_enable_gui_idle_interrupt(rdev, false);
5981
866d83de
AD		 5982 cik_wait_for_rlc_serdes(rdev);
5983}
5984
f6796cae
AD		 5985/**
5986 * cik_rlc_start - start the RLC ME
5987 *
5988 * @rdev: radeon_device pointer
5989 *
5990 * Unhalt the RLC ME (MicroEngine) (CIK).
5991 */
5992static void cik_rlc_start(struct radeon_device *rdev)
5993{
f6796cae 5994 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 5995
866d83de 5996 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 5997
f6796cae 5998 udelay(50);
6f2043ce
AD		 5999}
6000
6001/**
f6796cae 6002 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 6003 *
6004 * @rdev: radeon_device pointer
6005 *
f6796cae
AD		 6006 * Initialize the RLC registers, load the ucode,
6007 * and start the RLC (CIK).
6008 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 6009 */
f6796cae 6010static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 6011{
22c775ce 6012 u32 i, size, tmp;
cc066715 6013
f6796cae
AD		 6014 if (!rdev->rlc_fw)
6015 return -EINVAL;
cc066715 6016
cc066715
AD		 6017 cik_rlc_stop(rdev);
6018
22c775ce
AD		 6019 /* disable CG */
6020 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6021 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 6022
866d83de 6023 si_rlc_reset(rdev);
6f2043ce 6024
22c775ce 6025 cik_init_pg(rdev);
6f2043ce 6026
22c775ce 6027 cik_init_cg(rdev);
cc066715 6028
f6796cae
AD		 6029 WREG32(RLC_LB_CNTR_INIT, 0);
6030 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 6031
1c0a4625 6032 mutex_lock(&rdev->grbm_idx_mutex);
f6796cae
AD		 6033 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6034 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6035 WREG32(RLC_LB_PARAMS, 0x00600408);
6036 WREG32(RLC_LB_CNTL, 0x80000004);
1c0a4625 6037 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6038
f6796cae
AD		 6039 WREG32(RLC_MC_CNTL, 0);
6040 WREG32(RLC_UCODE_CNTL, 0);
cc066715 6041
f2c6b0f4
AD		 6042 if (rdev->new_fw) {
6043 const struct rlc_firmware_header_v1_0 *hdr =
6044 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6045 const __le32 *fw_data = (const __le32 *)
6046 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6047
6048 radeon_ucode_print_rlc_hdr(&hdr->header);
6049
6050 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6051 WREG32(RLC_GPM_UCODE_ADDR, 0);
6052 for (i = 0; i < size; i++)
6053 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 6054 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
f2c6b0f4
AD		 6055 } else {
6056 const __be32 *fw_data;
6057
6058 switch (rdev->family) {
6059 case CHIP_BONAIRE:
6060 case CHIP_HAWAII:
6061 default:
6062 size = BONAIRE_RLC_UCODE_SIZE;
6063 break;
6064 case CHIP_KAVERI:
6065 size = KV_RLC_UCODE_SIZE;
6066 break;
6067 case CHIP_KABINI:
6068 size = KB_RLC_UCODE_SIZE;
6069 break;
6070 case CHIP_MULLINS:
6071 size = ML_RLC_UCODE_SIZE;
6072 break;
6073 }
6074
6075 fw_data = (const __be32 *)rdev->rlc_fw->data;
6076 WREG32(RLC_GPM_UCODE_ADDR, 0);
6077 for (i = 0; i < size; i++)
6078 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6079 WREG32(RLC_GPM_UCODE_ADDR, 0);
6080 }
cc066715 6081
866d83de
AD		 6082 /* XXX - find out what chips support lbpw */
6083 cik_enable_lbpw(rdev, false);
cc066715 6084
22c775ce
AD		 6085 if (rdev->family == CHIP_BONAIRE)
6086 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 6087
f6796cae 6088 cik_rlc_start(rdev);
cc066715 6089
f6796cae
AD		 6090 return 0;
6091}
cc066715 6092
22c775ce
AD		 6093static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6094{
6095 u32 data, orig, tmp, tmp2;
cc066715 6096
22c775ce 6097 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 6098
473359bc 6099 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 6100 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6101
22c775ce 6102 tmp = cik_halt_rlc(rdev);
cc066715 6103
1c0a4625 6104 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6105 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6106 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6107 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6108 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6109 WREG32(RLC_SERDES_WR_CTRL, tmp2);
1c0a4625 6110 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6111
22c775ce 6112 cik_update_rlc(rdev, tmp);
cc066715 6113
22c775ce
AD		 6114 data |= CGCG_EN | CGLS_EN;
6115 } else {
ddc76ff6 6116 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 6117
22c775ce
AD		 6118 RREG32(CB_CGTT_SCLK_CTRL);
6119 RREG32(CB_CGTT_SCLK_CTRL);
6120 RREG32(CB_CGTT_SCLK_CTRL);
6121 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 6122
22c775ce 6123 data &= ~(CGCG_EN | CGLS_EN);
cc066715 6124 }
6f2043ce 6125
22c775ce
AD		 6126 if (orig != data)
6127 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 6128
6f2043ce
AD		 6129}
6130
22c775ce 6131static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 6132{
22c775ce
AD		 6133 u32 data, orig, tmp = 0;
6134
473359bc
AD		 6135 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6136 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6137 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6138 orig = data = RREG32(CP_MEM_SLP_CNTL);
6139 data |= CP_MEM_LS_EN;
6140 if (orig != data)
6141 WREG32(CP_MEM_SLP_CNTL, data);
6142 }
6143 }
cc066715 6144
22c775ce 6145 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4bb62c95 6146 data |= 0x00000001;
22c775ce
AD		 6147 data &= 0xfffffffd;
6148 if (orig != data)
6149 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6150
6151 tmp = cik_halt_rlc(rdev);
6152
1c0a4625 6153 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6154 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6155 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6156 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6157 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6158 WREG32(RLC_SERDES_WR_CTRL, data);
1c0a4625 6159 mutex_unlock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6160
6161 cik_update_rlc(rdev, tmp);
6162
473359bc
AD		 6163 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6164 orig = data = RREG32(CGTS_SM_CTRL_REG);
6165 data &= ~SM_MODE_MASK;
6166 data |= SM_MODE(0x2);
6167 data |= SM_MODE_ENABLE;
6168 data &= ~CGTS_OVERRIDE;
6169 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6170 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6171 data &= ~CGTS_LS_OVERRIDE;
6172 data &= ~ON_MONITOR_ADD_MASK;
6173 data |= ON_MONITOR_ADD_EN;
6174 data |= ON_MONITOR_ADD(0x96);
6175 if (orig != data)
6176 WREG32(CGTS_SM_CTRL_REG, data);
6177 }
22c775ce
AD		 6178 } else {
6179 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4bb62c95 6180 data |= 0x00000003;
22c775ce
AD		 6181 if (orig != data)
6182 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6183
6184 data = RREG32(RLC_MEM_SLP_CNTL);
6185 if (data & RLC_MEM_LS_EN) {
6186 data &= ~RLC_MEM_LS_EN;
6187 WREG32(RLC_MEM_SLP_CNTL, data);
6188 }
6f2043ce 6189
22c775ce
AD		 6190 data = RREG32(CP_MEM_SLP_CNTL);
6191 if (data & CP_MEM_LS_EN) {
6192 data &= ~CP_MEM_LS_EN;
6193 WREG32(CP_MEM_SLP_CNTL, data);
6194 }
cc066715 6195
22c775ce
AD		 6196 orig = data = RREG32(CGTS_SM_CTRL_REG);
6197 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6198 if (orig != data)
6199 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 6200
22c775ce 6201 tmp = cik_halt_rlc(rdev);
cc066715 6202
1c0a4625 6203 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6204 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6205 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6206 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6207 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6208 WREG32(RLC_SERDES_WR_CTRL, data);
1c0a4625 6209 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6210
22c775ce 6211 cik_update_rlc(rdev, tmp);
cc066715 6212 }
6f2043ce 6213}
1c49165d 6214
22c775ce 6215static const u32 mc_cg_registers[] =
21a93e13 6216{
22c775ce
AD		 6217 MC_HUB_MISC_HUB_CG,
6218 MC_HUB_MISC_SIP_CG,
6219 MC_HUB_MISC_VM_CG,
6220 MC_XPB_CLK_GAT,
6221 ATC_MISC_CG,
6222 MC_CITF_MISC_WR_CG,
6223 MC_CITF_MISC_RD_CG,
6224 MC_CITF_MISC_VM_CG,
6225 VM_L2_CG,
6226};
21a93e13 6227
22c775ce
AD		 6228static void cik_enable_mc_ls(struct radeon_device *rdev,
6229 bool enable)
1c49165d 6230{
22c775ce
AD		 6231 int i;
6232 u32 orig, data;
1c49165d 6233
22c775ce
AD		 6234 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6235 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6236 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 6237 data |= MC_LS_ENABLE;
6238 else
6239 data &= ~MC_LS_ENABLE;
6240 if (data != orig)
6241 WREG32(mc_cg_registers[i], data);
1c49165d 6242 }
22c775ce 6243}
1c49165d 6244
22c775ce
AD		 6245static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6246 bool enable)
6247{
6248 int i;
6249 u32 orig, data;
6250
6251 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6252 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6253 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 6254 data |= MC_CG_ENABLE;
6255 else
6256 data &= ~MC_CG_ENABLE;
6257 if (data != orig)
6258 WREG32(mc_cg_registers[i], data);
1c49165d 6259 }
1c49165d
AD		 6260}
6261
22c775ce
AD		 6262static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6263 bool enable)
1c49165d 6264{
22c775ce 6265 u32 orig, data;
1c49165d 6266
473359bc 6267 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 6268 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6269 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 6270 } else {
22c775ce
AD		 6271 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6272 data |= 0xff000000;
6273 if (data != orig)
6274 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6275
22c775ce
AD		 6276 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6277 data |= 0xff000000;
6278 if (data != orig)
6279 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6280 }
1c49165d
AD		 6281}
6282
22c775ce
AD		 6283static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6284 bool enable)
1c49165d 6285{
22c775ce
AD		 6286 u32 orig, data;
6287
473359bc 6288 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 6289 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6290 data |= 0x100;
6291 if (orig != data)
6292 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6293
6294 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6295 data |= 0x100;
6296 if (orig != data)
6297 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6298 } else {
6299 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6300 data &= ~0x100;
6301 if (orig != data)
6302 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6303
22c775ce
AD		 6304 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6305 data &= ~0x100;
6306 if (orig != data)
6307 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6308 }
1c49165d
AD		 6309}
6310
22c775ce
AD		 6311static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6312 bool enable)
1c49165d 6313{
22c775ce 6314 u32 orig, data;
1c49165d 6315
473359bc 6316 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 6317 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6318 data = 0xfff;
6319 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6320
22c775ce
AD		 6321 orig = data = RREG32(UVD_CGC_CTRL);
6322 data |= DCM;
6323 if (orig != data)
6324 WREG32(UVD_CGC_CTRL, data);
6325 } else {
6326 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6327 data &= ~0xfff;
6328 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6329
22c775ce
AD		 6330 orig = data = RREG32(UVD_CGC_CTRL);
6331 data &= ~DCM;
6332 if (orig != data)
6333 WREG32(UVD_CGC_CTRL, data);
1c49165d 6334 }
22c775ce 6335}
1c49165d 6336
473359bc
AD		 6337static void cik_enable_bif_mgls(struct radeon_device *rdev,
6338 bool enable)
6339{
6340 u32 orig, data;
1c49165d 6341
473359bc 6342 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 6343
473359bc
AD		 6344 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6345 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6346 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6347 else
6348 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6349 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 6350
473359bc
AD		 6351 if (orig != data)
6352 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6353}
1c49165d 6354
22c775ce
AD		 6355static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6356 bool enable)
6357{
6358 u32 orig, data;
1c49165d 6359
22c775ce 6360 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 6361
473359bc 6362 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 6363 data &= ~CLOCK_GATING_DIS;
6364 else
6365 data |= CLOCK_GATING_DIS;
6366
6367 if (orig != data)
6368 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 6369}
6370
22c775ce
AD		 6371static void cik_enable_hdp_ls(struct radeon_device *rdev,
6372 bool enable)
1c49165d 6373{
22c775ce
AD		 6374 u32 orig, data;
6375
6376 orig = data = RREG32(HDP_MEM_POWER_LS);
6377
473359bc 6378 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 6379 data |= HDP_LS_ENABLE;
6380 else
6381 data &= ~HDP_LS_ENABLE;
6382
6383 if (orig != data)
6384 WREG32(HDP_MEM_POWER_LS, data);
6385}
6386
6387void cik_update_cg(struct radeon_device *rdev,
6388 u32 block, bool enable)
6389{
4214faf6 6390
22c775ce 6391 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 6392 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 6393 /* order matters! */
6394 if (enable) {
6395 cik_enable_mgcg(rdev, true);
6396 cik_enable_cgcg(rdev, true);
6397 } else {
6398 cik_enable_cgcg(rdev, false);
6399 cik_enable_mgcg(rdev, false);
6400 }
4214faf6 6401 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 6402 }
6403
6404 if (block & RADEON_CG_BLOCK_MC) {
6405 if (!(rdev->flags & RADEON_IS_IGP)) {
6406 cik_enable_mc_mgcg(rdev, enable);
6407 cik_enable_mc_ls(rdev, enable);
6408 }
6409 }
6410
6411 if (block & RADEON_CG_BLOCK_SDMA) {
6412 cik_enable_sdma_mgcg(rdev, enable);
6413 cik_enable_sdma_mgls(rdev, enable);
6414 }
6415
473359bc
AD		 6416 if (block & RADEON_CG_BLOCK_BIF) {
6417 cik_enable_bif_mgls(rdev, enable);
6418 }
6419
22c775ce
AD		 6420 if (block & RADEON_CG_BLOCK_UVD) {
6421 if (rdev->has_uvd)
6422 cik_enable_uvd_mgcg(rdev, enable);
6423 }
6424
6425 if (block & RADEON_CG_BLOCK_HDP) {
6426 cik_enable_hdp_mgcg(rdev, enable);
6427 cik_enable_hdp_ls(rdev, enable);
6428 }
a1d6f97c
AD		 6429
6430 if (block & RADEON_CG_BLOCK_VCE) {
6431 vce_v2_0_enable_mgcg(rdev, enable);
6432 }
1c49165d
AD		 6433}
6434
22c775ce 6435static void cik_init_cg(struct radeon_device *rdev)
1c49165d 6436{
22c775ce 6437
ddc76ff6 6438 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 6439
6440 if (rdev->has_uvd)
6441 si_init_uvd_internal_cg(rdev);
6442
6443 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6444 RADEON_CG_BLOCK_SDMA |
473359bc 6445 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 6446 RADEON_CG_BLOCK_UVD |
6447 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 6448}
6449
473359bc 6450static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 6451{
473359bc
AD		 6452 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6453 RADEON_CG_BLOCK_SDMA |
6454 RADEON_CG_BLOCK_BIF |
6455 RADEON_CG_BLOCK_UVD |
6456 RADEON_CG_BLOCK_HDP), false);
6457
6458 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 6459}
6460
22c775ce
AD		 6461static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6462 bool enable)
1c49165d 6463{
22c775ce 6464 u32 data, orig;
1c49165d 6465
22c775ce 6466 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6467 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6468 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6469 else
6470 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6471 if (orig != data)
6472 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6473}
6474
22c775ce
AD		 6475static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6476 bool enable)
1c49165d 6477{
22c775ce
AD		 6478 u32 data, orig;
6479
6480 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6481 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6482 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6483 else
6484 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6485 if (orig != data)
6486 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6487}
6488
22c775ce 6489static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 6490{
22c775ce 6491 u32 data, orig;
3ec7d11b 6492
22c775ce 6493 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6494 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 6495 data &= ~DISABLE_CP_PG;
6496 else
6497 data |= DISABLE_CP_PG;
6498 if (orig != data)
6499 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 6500}
6501
22c775ce 6502static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 6503{
22c775ce 6504 u32 data, orig;
f96ab484 6505
22c775ce 6506 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6507 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 6508 data &= ~DISABLE_GDS_PG;
6509 else
6510 data |= DISABLE_GDS_PG;
6511 if (orig != data)
6512 WREG32(RLC_PG_CNTL, data);
6513}
6514
6515#define CP_ME_TABLE_SIZE 96
6516#define CP_ME_TABLE_OFFSET 2048
6517#define CP_MEC_TABLE_OFFSET 4096
6518
6519void cik_init_cp_pg_table(struct radeon_device *rdev)
6520{
22c775ce
AD		 6521 volatile u32 *dst_ptr;
6522 int me, i, max_me = 4;
6523 u32 bo_offset = 0;
f2c6b0f4 6524 u32 table_offset, table_size;
22c775ce
AD		 6525
6526 if (rdev->family == CHIP_KAVERI)
6527 max_me = 5;
6528
6529 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 6530 return;
6531
22c775ce
AD		 6532 /* write the cp table buffer */
6533 dst_ptr = rdev->rlc.cp_table_ptr;
6534 for (me = 0; me < max_me; me++) {
f2c6b0f4
AD		 6535 if (rdev->new_fw) {
6536 const __le32 *fw_data;
6537 const struct gfx_firmware_header_v1_0 *hdr;
6538
6539 if (me == 0) {
6540 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6541 fw_data = (const __le32 *)
6542 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6543 table_offset = le32_to_cpu(hdr->jt_offset);
6544 table_size = le32_to_cpu(hdr->jt_size);
6545 } else if (me == 1) {
6546 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6547 fw_data = (const __le32 *)
6548 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6549 table_offset = le32_to_cpu(hdr->jt_offset);
6550 table_size = le32_to_cpu(hdr->jt_size);
6551 } else if (me == 2) {
6552 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6553 fw_data = (const __le32 *)
6554 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6555 table_offset = le32_to_cpu(hdr->jt_offset);
6556 table_size = le32_to_cpu(hdr->jt_size);
6557 } else if (me == 3) {
6558 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6559 fw_data = (const __le32 *)
6560 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6561 table_offset = le32_to_cpu(hdr->jt_offset);
6562 table_size = le32_to_cpu(hdr->jt_size);
6563 } else {
6564 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6565 fw_data = (const __le32 *)
6566 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6567 table_offset = le32_to_cpu(hdr->jt_offset);
6568 table_size = le32_to_cpu(hdr->jt_size);
6569 }
6570
6571 for (i = 0; i < table_size; i ++) {
6572 dst_ptr[bo_offset + i] =
6573 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6574 }
6575 bo_offset += table_size;
22c775ce 6576 } else {
f2c6b0f4
AD		 6577 const __be32 *fw_data;
6578 table_size = CP_ME_TABLE_SIZE;
6579
6580 if (me == 0) {
6581 fw_data = (const __be32 *)rdev->ce_fw->data;
6582 table_offset = CP_ME_TABLE_OFFSET;
6583 } else if (me == 1) {
6584 fw_data = (const __be32 *)rdev->pfp_fw->data;
6585 table_offset = CP_ME_TABLE_OFFSET;
6586 } else if (me == 2) {
6587 fw_data = (const __be32 *)rdev->me_fw->data;
6588 table_offset = CP_ME_TABLE_OFFSET;
6589 } else {
6590 fw_data = (const __be32 *)rdev->mec_fw->data;
6591 table_offset = CP_MEC_TABLE_OFFSET;
6592 }
22c775ce 6593
f2c6b0f4
AD		 6594 for (i = 0; i < table_size; i ++) {
6595 dst_ptr[bo_offset + i] =
6596 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6597 }
6598 bo_offset += table_size;
22c775ce 6599 }
f96ab484 6600 }
22c775ce 6601}
f96ab484 6602
22c775ce
AD		 6603static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6604 bool enable)
6605{
6606 u32 data, orig;
6607
2b19d17f 6608 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 6609 orig = data = RREG32(RLC_PG_CNTL);
6610 data |= GFX_PG_ENABLE;
6611 if (orig != data)
6612 WREG32(RLC_PG_CNTL, data);
6613
6614 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6615 data |= AUTO_PG_EN;
6616 if (orig != data)
6617 WREG32(RLC_AUTO_PG_CTRL, data);
6618 } else {
6619 orig = data = RREG32(RLC_PG_CNTL);
6620 data &= ~GFX_PG_ENABLE;
6621 if (orig != data)
6622 WREG32(RLC_PG_CNTL, data);
f96ab484 6623
22c775ce
AD		 6624 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6625 data &= ~AUTO_PG_EN;
6626 if (orig != data)
6627 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 6628
22c775ce
AD		 6629 data = RREG32(DB_RENDER_CONTROL);
6630 }
6631}
f96ab484 6632
22c775ce
AD		 6633static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6634{
6635 u32 mask = 0, tmp, tmp1;
6636 int i;
f96ab484 6637
1c0a4625 6638 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6639 cik_select_se_sh(rdev, se, sh);
6640 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6641 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6642 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 6643 mutex_unlock(&rdev->grbm_idx_mutex);
f96ab484 6644
22c775ce 6645 tmp &= 0xffff0000;
f96ab484 6646
22c775ce
AD		 6647 tmp |= tmp1;
6648 tmp >>= 16;
6649
6650 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6651 mask <<= 1;
6652 mask |= 1;
b07fdd38 6653 }
22c775ce
AD		 6654
6655 return (~tmp) & mask;
f96ab484
AD		 6656}
6657
22c775ce 6658static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 6659{
22c775ce
AD		 6660 u32 i, j, k, active_cu_number = 0;
6661 u32 mask, counter, cu_bitmap;
6662 u32 tmp = 0;
d0e092d9 6663
22c775ce
AD		 6664 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6665 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6666 mask = 1;
6667 cu_bitmap = 0;
6668 counter = 0;
6669 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6670 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6671 if (counter < 2)
6672 cu_bitmap |= mask;
6673 counter ++;
d0e092d9 6674 }
22c775ce 6675 mask <<= 1;
d0e092d9 6676 }
d0e092d9 6677
22c775ce
AD		 6678 active_cu_number += counter;
6679 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 6680 }
d0e092d9 6681 }
22c775ce
AD		 6682
6683 WREG32(RLC_PG_AO_CU_MASK, tmp);
6684
6685 tmp = RREG32(RLC_MAX_PG_CU);
6686 tmp &= ~MAX_PU_CU_MASK;
6687 tmp |= MAX_PU_CU(active_cu_number);
6688 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 6689}
6690
22c775ce
AD		 6691static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6692 bool enable)
605de6b9 6693{
22c775ce 6694 u32 data, orig;
605de6b9 6695
22c775ce 6696 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6697 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 6698 data |= STATIC_PER_CU_PG_ENABLE;
6699 else
6700 data &= ~STATIC_PER_CU_PG_ENABLE;
6701 if (orig != data)
6702 WREG32(RLC_PG_CNTL, data);
6703}
6704
6705static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6706 bool enable)
6707{
6708 u32 data, orig;
605de6b9 6709
22c775ce 6710 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6711 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 6712 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 6713 else
22c775ce
AD		 6714 data &= ~DYN_PER_CU_PG_ENABLE;
6715 if (orig != data)
6716 WREG32(RLC_PG_CNTL, data);
6717}
605de6b9 6718
22c775ce
AD		 6719#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6720#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6721
6722static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6723{
6724 u32 data, orig;
6725 u32 i;
6726
6727 if (rdev->rlc.cs_data) {
6728 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6729 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 6730 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 6731 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 6732 } else {
22c775ce
AD		 6733 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6734 for (i = 0; i < 3; i++)
6735 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6736 }
6737 if (rdev->rlc.reg_list) {
6738 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6739 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6740 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 6741 }
605de6b9 6742
22c775ce
AD		 6743 orig = data = RREG32(RLC_PG_CNTL);
6744 data |= GFX_PG_SRC;
6745 if (orig != data)
6746 WREG32(RLC_PG_CNTL, data);
605de6b9 6747
22c775ce
AD		 6748 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6749 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 6750
22c775ce
AD		 6751 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6752 data &= ~IDLE_POLL_COUNT_MASK;
6753 data |= IDLE_POLL_COUNT(0x60);
6754 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 6755
22c775ce
AD		 6756 data = 0x10101010;
6757 WREG32(RLC_PG_DELAY, data);
605de6b9 6758
22c775ce
AD		 6759 data = RREG32(RLC_PG_DELAY_2);
6760 data &= ~0xff;
6761 data |= 0x3;
6762 WREG32(RLC_PG_DELAY_2, data);
605de6b9 6763
22c775ce
AD		 6764 data = RREG32(RLC_AUTO_PG_CTRL);
6765 data &= ~GRBM_REG_SGIT_MASK;
6766 data |= GRBM_REG_SGIT(0x700);
6767 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 6768
605de6b9
AD		 6769}
6770
22c775ce 6771static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 6772{
473359bc
AD		 6773 cik_enable_gfx_cgpg(rdev, enable);
6774 cik_enable_gfx_static_mgpg(rdev, enable);
6775 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 6776}
f6796cae 6777
a0f38609
AD		 6778u32 cik_get_csb_size(struct radeon_device *rdev)
6779{
6780 u32 count = 0;
6781 const struct cs_section_def *sect = NULL;
6782 const struct cs_extent_def *ext = NULL;
f6796cae 6783
a0f38609
AD		 6784 if (rdev->rlc.cs_data == NULL)
6785 return 0;
f6796cae 6786
a0f38609
AD		 6787 /* begin clear state */
6788 count += 2;
6789 /* context control state */
6790 count += 3;
6791
6792 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6793 for (ext = sect->section; ext->extent != NULL; ++ext) {
6794 if (sect->id == SECT_CONTEXT)
6795 count += 2 + ext->reg_count;
6796 else
6797 return 0;
f6796cae
AD		 6798 }
6799 }
a0f38609
AD		 6800 /* pa_sc_raster_config/pa_sc_raster_config1 */
6801 count += 4;
6802 /* end clear state */
6803 count += 2;
6804 /* clear state */
6805 count += 2;
f6796cae 6806
a0f38609 6807 return count;
f6796cae
AD		 6808}
6809
a0f38609 6810void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 6811{
a0f38609
AD		 6812 u32 count = 0, i;
6813 const struct cs_section_def *sect = NULL;
6814 const struct cs_extent_def *ext = NULL;
f6796cae 6815
a0f38609
AD		 6816 if (rdev->rlc.cs_data == NULL)
6817 return;
6818 if (buffer == NULL)
6819 return;
f6796cae 6820
6ba81e53
AD		 6821 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6822 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
a0f38609 6823
6ba81e53
AD		 6824 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6825 buffer[count++] = cpu_to_le32(0x80000000);
6826 buffer[count++] = cpu_to_le32(0x80000000);
a0f38609
AD		 6827
6828 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6829 for (ext = sect->section; ext->extent != NULL; ++ext) {
6830 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 6831 buffer[count++] =
6832 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6833 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
a0f38609 6834 for (i = 0; i < ext->reg_count; i++)
6ba81e53 6835 buffer[count++] = cpu_to_le32(ext->extent[i]);
a0f38609
AD		 6836 } else {
6837 return;
6838 }
6839 }
6840 }
f6796cae 6841
6ba81e53
AD		 6842 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6843 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
f6796cae
AD		 6844 switch (rdev->family) {
6845 case CHIP_BONAIRE:
6ba81e53
AD		 6846 buffer[count++] = cpu_to_le32(0x16000012);
6847 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6848 break;
6849 case CHIP_KAVERI:
6ba81e53
AD		 6850 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6851 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6852 break;
6853 case CHIP_KABINI:
f73a9e83 6854 case CHIP_MULLINS:
6ba81e53
AD		 6855 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6856 buffer[count++] = cpu_to_le32(0x00000000);
a0f38609 6857 break;
bbfe90bd 6858 case CHIP_HAWAII:
a8947f57
AD		 6859 buffer[count++] = cpu_to_le32(0x3a00161a);
6860 buffer[count++] = cpu_to_le32(0x0000002e);
bbfe90bd 6861 break;
a0f38609 6862 default:
6ba81e53
AD		 6863 buffer[count++] = cpu_to_le32(0x00000000);
6864 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6865 break;
6866 }
6867
6ba81e53
AD		 6868 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6869 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
f6796cae 6870
6ba81e53
AD		 6871 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6872 buffer[count++] = cpu_to_le32(0);
a0f38609 6873}
f6796cae 6874
473359bc 6875static void cik_init_pg(struct radeon_device *rdev)
22c775ce 6876{
473359bc 6877 if (rdev->pg_flags) {
22c775ce
AD		 6878 cik_enable_sck_slowdown_on_pu(rdev, true);
6879 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 6880 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 6881 cik_init_gfx_cgpg(rdev);
6882 cik_enable_cp_pg(rdev, true);
6883 cik_enable_gds_pg(rdev, true);
6884 }
22c775ce
AD		 6885 cik_init_ao_cu_mask(rdev);
6886 cik_update_gfx_pg(rdev, true);
6887 }
6888}
f6796cae 6889
473359bc
AD		 6890static void cik_fini_pg(struct radeon_device *rdev)
6891{
6892 if (rdev->pg_flags) {
6893 cik_update_gfx_pg(rdev, false);
2b19d17f 6894 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 6895 cik_enable_cp_pg(rdev, false);
6896 cik_enable_gds_pg(rdev, false);
6897 }
6898 }
f6796cae 6899}
a59781bb
AD		 6900
6901/*
6902 * Interrupts
6903 * Starting with r6xx, interrupts are handled via a ring buffer.
6904 * Ring buffers are areas of GPU accessible memory that the GPU
6905 * writes interrupt vectors into and the host reads vectors out of.
6906 * There is a rptr (read pointer) that determines where the
6907 * host is currently reading, and a wptr (write pointer)
6908 * which determines where the GPU has written. When the
6909 * pointers are equal, the ring is idle. When the GPU
6910 * writes vectors to the ring buffer, it increments the
6911 * wptr. When there is an interrupt, the host then starts
6912 * fetching commands and processing them until the pointers are
6913 * equal again at which point it updates the rptr.
6914 */
6915
6916/**
6917 * cik_enable_interrupts - Enable the interrupt ring buffer
6918 *
6919 * @rdev: radeon_device pointer
6920 *
6921 * Enable the interrupt ring buffer (CIK).
6922 */
6923static void cik_enable_interrupts(struct radeon_device *rdev)
6924{
6925 u32 ih_cntl = RREG32(IH_CNTL);
6926 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6927
6928 ih_cntl |= ENABLE_INTR;
6929 ih_rb_cntl |= IH_RB_ENABLE;
6930 WREG32(IH_CNTL, ih_cntl);
6931 WREG32(IH_RB_CNTL, ih_rb_cntl);
6932 rdev->ih.enabled = true;
6933}
6934
6935/**
6936 * cik_disable_interrupts - Disable the interrupt ring buffer
6937 *
6938 * @rdev: radeon_device pointer
6939 *
6940 * Disable the interrupt ring buffer (CIK).
6941 */
6942static void cik_disable_interrupts(struct radeon_device *rdev)
6943{
6944 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6945 u32 ih_cntl = RREG32(IH_CNTL);
6946
6947 ih_rb_cntl &= ~IH_RB_ENABLE;
6948 ih_cntl &= ~ENABLE_INTR;
6949 WREG32(IH_RB_CNTL, ih_rb_cntl);
6950 WREG32(IH_CNTL, ih_cntl);
6951 /* set rptr, wptr to 0 */
6952 WREG32(IH_RB_RPTR, 0);
6953 WREG32(IH_RB_WPTR, 0);
6954 rdev->ih.enabled = false;
6955 rdev->ih.rptr = 0;
6956}
6957
6958/**
6959 * cik_disable_interrupt_state - Disable all interrupt sources
6960 *
6961 * @rdev: radeon_device pointer
6962 *
6963 * Clear all interrupt enable bits used by the driver (CIK).
6964 */
6965static void cik_disable_interrupt_state(struct radeon_device *rdev)
6966{
6967 u32 tmp;
6968
6969 /* gfx ring */
4214faf6
AD		 6970 tmp = RREG32(CP_INT_CNTL_RING0) &
6971 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6972 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 6973 /* sdma */
6974 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6975 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6976 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6977 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 6978 /* compute queues */
6979 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6980 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6981 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6982 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6983 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6984 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6985 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6986 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6987 /* grbm */
6988 WREG32(GRBM_INT_CNTL, 0);
dc12a3ec
LL		 6989 /* SRBM */
6990 WREG32(SRBM_INT_CNTL, 0);
a59781bb
AD		 6991 /* vline/vblank, etc. */
6992 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6993 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6994 if (rdev->num_crtc >= 4) {
6995 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6996 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6997 }
6998 if (rdev->num_crtc >= 6) {
6999 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7000 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7001 }
f5d636d2
CK		 7002 /* pflip */
7003 if (rdev->num_crtc >= 2) {
7004 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7005 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7006 }
7007 if (rdev->num_crtc >= 4) {
7008 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7009 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7010 }
7011 if (rdev->num_crtc >= 6) {
7012 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7013 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7014 }
a59781bb
AD		 7015
7016 /* dac hotplug */
7017 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7018
7019 /* digital hotplug */
7020 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7021 WREG32(DC_HPD1_INT_CONTROL, tmp);
7022 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7023 WREG32(DC_HPD2_INT_CONTROL, tmp);
7024 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7025 WREG32(DC_HPD3_INT_CONTROL, tmp);
7026 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7027 WREG32(DC_HPD4_INT_CONTROL, tmp);
7028 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7029 WREG32(DC_HPD5_INT_CONTROL, tmp);
7030 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7031 WREG32(DC_HPD6_INT_CONTROL, tmp);
7032
7033}
7034
7035/**
7036 * cik_irq_init - init and enable the interrupt ring
7037 *
7038 * @rdev: radeon_device pointer
7039 *
7040 * Allocate a ring buffer for the interrupt controller,
7041 * enable the RLC, disable interrupts, enable the IH
7042 * ring buffer and enable it (CIK).
7043 * Called at device load and reume.
7044 * Returns 0 for success, errors for failure.
7045 */
7046static int cik_irq_init(struct radeon_device *rdev)
7047{
7048 int ret = 0;
7049 int rb_bufsz;
7050 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7051
7052 /* allocate ring */
7053 ret = r600_ih_ring_alloc(rdev);
7054 if (ret)
7055 return ret;
7056
7057 /* disable irqs */
7058 cik_disable_interrupts(rdev);
7059
7060 /* init rlc */
7061 ret = cik_rlc_resume(rdev);
7062 if (ret) {
7063 r600_ih_ring_fini(rdev);
7064 return ret;
7065 }
7066
7067 /* setup interrupt control */
7068 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7069 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7070 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7071 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7072 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7073 */
7074 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7075 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7076 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7077 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7078
7079 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 7080 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 7081
7082 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7083 IH_WPTR_OVERFLOW_CLEAR |
7084 (rb_bufsz << 1));
7085
7086 if (rdev->wb.enabled)
7087 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7088
7089 /* set the writeback address whether it's enabled or not */
7090 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7091 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7092
7093 WREG32(IH_RB_CNTL, ih_rb_cntl);
7094
7095 /* set rptr, wptr to 0 */
7096 WREG32(IH_RB_RPTR, 0);
7097 WREG32(IH_RB_WPTR, 0);
7098
7099 /* Default settings for IH_CNTL (disabled at first) */
7100 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7101 /* RPTR_REARM only works if msi's are enabled */
7102 if (rdev->msi_enabled)
7103 ih_cntl |= RPTR_REARM;
7104 WREG32(IH_CNTL, ih_cntl);
7105
7106 /* force the active interrupt state to all disabled */
7107 cik_disable_interrupt_state(rdev);
7108
7109 pci_set_master(rdev->pdev);
7110
7111 /* enable irqs */
7112 cik_enable_interrupts(rdev);
7113
7114 return ret;
7115}
7116
7117/**
7118 * cik_irq_set - enable/disable interrupt sources
7119 *
7120 * @rdev: radeon_device pointer
7121 *
7122 * Enable interrupt sources on the GPU (vblanks, hpd,
7123 * etc.) (CIK).
7124 * Returns 0 for success, errors for failure.
7125 */
7126int cik_irq_set(struct radeon_device *rdev)
7127{
4214faf6 7128 u32 cp_int_cntl;
28b57b85 7129 u32 cp_m1p0;
a59781bb
AD		 7130 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7131 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7132 u32 grbm_int_cntl = 0;
21a93e13 7133 u32 dma_cntl, dma_cntl1;
a59781bb
AD		 7134
7135 if (!rdev->irq.installed) {
7136 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7137 return -EINVAL;
7138 }
7139 /* don't enable anything if the ih is disabled */
7140 if (!rdev->ih.enabled) {
7141 cik_disable_interrupts(rdev);
7142 /* force the active interrupt state to all disabled */
7143 cik_disable_interrupt_state(rdev);
7144 return 0;
7145 }
7146
4214faf6
AD		 7147 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7148 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7149 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7150
f6b355dd
AD		 7151 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7152 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7153 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7154 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7155 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7156 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
a59781bb 7157
21a93e13
AD		 7158 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7159 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7160
2b0781a6 7161 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
2b0781a6 7162
a59781bb
AD		 7163 /* enable CP interrupts on all rings */
7164 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7165 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7166 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7167 }
2b0781a6
AD		 7168 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7169 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7170 DRM_DEBUG("si_irq_set: sw int cp1\n");
7171 if (ring->me == 1) {
7172 switch (ring->pipe) {
7173 case 0:
7174 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7175 break;
2b0781a6
AD		 7176 default:
7177 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7178 break;
7179 }
7180 } else {
7181 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7182 }
7183 }
7184 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7185 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7186 DRM_DEBUG("si_irq_set: sw int cp2\n");
7187 if (ring->me == 1) {
7188 switch (ring->pipe) {
7189 case 0:
7190 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7191 break;
2b0781a6
AD		 7192 default:
7193 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7194 break;
7195 }
7196 } else {
7197 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7198 }
7199 }
a59781bb 7200
21a93e13
AD		 7201 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7202 DRM_DEBUG("cik_irq_set: sw int dma\n");
7203 dma_cntl |= TRAP_ENABLE;
7204 }
7205
7206 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7207 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7208 dma_cntl1 |= TRAP_ENABLE;
7209 }
7210
a59781bb
AD		 7211 if (rdev->irq.crtc_vblank_int[0] ||
7212 atomic_read(&rdev->irq.pflip[0])) {
7213 DRM_DEBUG("cik_irq_set: vblank 0\n");
7214 crtc1 |= VBLANK_INTERRUPT_MASK;
7215 }
7216 if (rdev->irq.crtc_vblank_int[1] ||
7217 atomic_read(&rdev->irq.pflip[1])) {
7218 DRM_DEBUG("cik_irq_set: vblank 1\n");
7219 crtc2 |= VBLANK_INTERRUPT_MASK;
7220 }
7221 if (rdev->irq.crtc_vblank_int[2] ||
7222 atomic_read(&rdev->irq.pflip[2])) {
7223 DRM_DEBUG("cik_irq_set: vblank 2\n");
7224 crtc3 |= VBLANK_INTERRUPT_MASK;
7225 }
7226 if (rdev->irq.crtc_vblank_int[3] ||
7227 atomic_read(&rdev->irq.pflip[3])) {
7228 DRM_DEBUG("cik_irq_set: vblank 3\n");
7229 crtc4 |= VBLANK_INTERRUPT_MASK;
7230 }
7231 if (rdev->irq.crtc_vblank_int[4] ||
7232 atomic_read(&rdev->irq.pflip[4])) {
7233 DRM_DEBUG("cik_irq_set: vblank 4\n");
7234 crtc5 |= VBLANK_INTERRUPT_MASK;
7235 }
7236 if (rdev->irq.crtc_vblank_int[5] ||
7237 atomic_read(&rdev->irq.pflip[5])) {
7238 DRM_DEBUG("cik_irq_set: vblank 5\n");
7239 crtc6 |= VBLANK_INTERRUPT_MASK;
7240 }
7241 if (rdev->irq.hpd[0]) {
7242 DRM_DEBUG("cik_irq_set: hpd 1\n");
f6b355dd 7243 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7244 }
7245 if (rdev->irq.hpd[1]) {
7246 DRM_DEBUG("cik_irq_set: hpd 2\n");
f6b355dd 7247 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7248 }
7249 if (rdev->irq.hpd[2]) {
7250 DRM_DEBUG("cik_irq_set: hpd 3\n");
f6b355dd 7251 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7252 }
7253 if (rdev->irq.hpd[3]) {
7254 DRM_DEBUG("cik_irq_set: hpd 4\n");
f6b355dd 7255 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7256 }
7257 if (rdev->irq.hpd[4]) {
7258 DRM_DEBUG("cik_irq_set: hpd 5\n");
f6b355dd 7259 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7260 }
7261 if (rdev->irq.hpd[5]) {
7262 DRM_DEBUG("cik_irq_set: hpd 6\n");
f6b355dd 7263 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7264 }
7265
7266 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7267
21a93e13
AD		 7268 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7269 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7270
2b0781a6 7271 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
2b0781a6 7272
a59781bb
AD		 7273 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7274
7275 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7276 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7277 if (rdev->num_crtc >= 4) {
7278 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7279 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7280 }
7281 if (rdev->num_crtc >= 6) {
7282 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7283 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7284 }
7285
f5d636d2
CK		 7286 if (rdev->num_crtc >= 2) {
7287 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7288 GRPH_PFLIP_INT_MASK);
7289 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7290 GRPH_PFLIP_INT_MASK);
7291 }
7292 if (rdev->num_crtc >= 4) {
7293 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7294 GRPH_PFLIP_INT_MASK);
7295 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7296 GRPH_PFLIP_INT_MASK);
7297 }
7298 if (rdev->num_crtc >= 6) {
7299 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7300 GRPH_PFLIP_INT_MASK);
7301 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7302 GRPH_PFLIP_INT_MASK);
7303 }
7304
a59781bb
AD		 7305 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7306 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7307 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7308 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7309 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7310 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7311
cffefd9b
AD		 7312 /* posting read */
7313 RREG32(SRBM_STATUS);
7314
a59781bb
AD		 7315 return 0;
7316}
7317
7318/**
7319 * cik_irq_ack - ack interrupt sources
7320 *
7321 * @rdev: radeon_device pointer
7322 *
7323 * Ack interrupt sources on the GPU (vblanks, hpd,
7324 * etc.) (CIK). Certain interrupts sources are sw
7325 * generated and do not require an explicit ack.
7326 */
7327static inline void cik_irq_ack(struct radeon_device *rdev)
7328{
7329 u32 tmp;
7330
7331 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7332 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7333 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7334 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7335 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7336 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7337 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7338
f5d636d2
CK		 7339 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7340 EVERGREEN_CRTC0_REGISTER_OFFSET);
7341 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7342 EVERGREEN_CRTC1_REGISTER_OFFSET);
7343 if (rdev->num_crtc >= 4) {
7344 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7345 EVERGREEN_CRTC2_REGISTER_OFFSET);
7346 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7347 EVERGREEN_CRTC3_REGISTER_OFFSET);
7348 }
7349 if (rdev->num_crtc >= 6) {
7350 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7351 EVERGREEN_CRTC4_REGISTER_OFFSET);
7352 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7353 EVERGREEN_CRTC5_REGISTER_OFFSET);
7354 }
7355
7356 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7357 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7358 GRPH_PFLIP_INT_CLEAR);
7359 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7360 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7361 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7362 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7363 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7364 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7365 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7366 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7367 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7368 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7369 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7370
7371 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 7372 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7373 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7374 GRPH_PFLIP_INT_CLEAR);
7375 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7376 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7377 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7378 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7379 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7380 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7381 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7382 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7383 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7384 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7385 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7386 }
7387
7388 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 7389 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7390 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7391 GRPH_PFLIP_INT_CLEAR);
7392 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7393 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7394 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7395 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7396 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7397 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7398 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7399 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7400 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7401 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7402 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7403 }
7404
7405 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7406 tmp = RREG32(DC_HPD1_INT_CONTROL);
7407 tmp |= DC_HPDx_INT_ACK;
7408 WREG32(DC_HPD1_INT_CONTROL, tmp);
7409 }
7410 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7411 tmp = RREG32(DC_HPD2_INT_CONTROL);
7412 tmp |= DC_HPDx_INT_ACK;
7413 WREG32(DC_HPD2_INT_CONTROL, tmp);
7414 }
7415 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7416 tmp = RREG32(DC_HPD3_INT_CONTROL);
7417 tmp |= DC_HPDx_INT_ACK;
7418 WREG32(DC_HPD3_INT_CONTROL, tmp);
7419 }
7420 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7421 tmp = RREG32(DC_HPD4_INT_CONTROL);
7422 tmp |= DC_HPDx_INT_ACK;
7423 WREG32(DC_HPD4_INT_CONTROL, tmp);
7424 }
7425 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7426 tmp = RREG32(DC_HPD5_INT_CONTROL);
7427 tmp |= DC_HPDx_INT_ACK;
7428 WREG32(DC_HPD5_INT_CONTROL, tmp);
7429 }
7430 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7431 tmp = RREG32(DC_HPD5_INT_CONTROL);
7432 tmp |= DC_HPDx_INT_ACK;
7433 WREG32(DC_HPD6_INT_CONTROL, tmp);
7434 }
f6b355dd
AD		 7435 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {
7436 tmp = RREG32(DC_HPD1_INT_CONTROL);
7437 tmp |= DC_HPDx_RX_INT_ACK;
7438 WREG32(DC_HPD1_INT_CONTROL, tmp);
7439 }
7440 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
7441 tmp = RREG32(DC_HPD2_INT_CONTROL);
7442 tmp |= DC_HPDx_RX_INT_ACK;
7443 WREG32(DC_HPD2_INT_CONTROL, tmp);
7444 }
7445 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
7446 tmp = RREG32(DC_HPD3_INT_CONTROL);
7447 tmp |= DC_HPDx_RX_INT_ACK;
7448 WREG32(DC_HPD3_INT_CONTROL, tmp);
7449 }
7450 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
7451 tmp = RREG32(DC_HPD4_INT_CONTROL);
7452 tmp |= DC_HPDx_RX_INT_ACK;
7453 WREG32(DC_HPD4_INT_CONTROL, tmp);
7454 }
7455 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
7456 tmp = RREG32(DC_HPD5_INT_CONTROL);
7457 tmp |= DC_HPDx_RX_INT_ACK;
7458 WREG32(DC_HPD5_INT_CONTROL, tmp);
7459 }
7460 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
7461 tmp = RREG32(DC_HPD5_INT_CONTROL);
7462 tmp |= DC_HPDx_RX_INT_ACK;
7463 WREG32(DC_HPD6_INT_CONTROL, tmp);
7464 }
a59781bb
AD		 7465}
7466
7467/**
7468 * cik_irq_disable - disable interrupts
7469 *
7470 * @rdev: radeon_device pointer
7471 *
7472 * Disable interrupts on the hw (CIK).
7473 */
7474static void cik_irq_disable(struct radeon_device *rdev)
7475{
7476 cik_disable_interrupts(rdev);
7477 /* Wait and acknowledge irq */
7478 mdelay(1);
7479 cik_irq_ack(rdev);
7480 cik_disable_interrupt_state(rdev);
7481}
7482
7483/**
7484 * cik_irq_disable - disable interrupts for suspend
7485 *
7486 * @rdev: radeon_device pointer
7487 *
7488 * Disable interrupts and stop the RLC (CIK).
7489 * Used for suspend.
7490 */
7491static void cik_irq_suspend(struct radeon_device *rdev)
7492{
7493 cik_irq_disable(rdev);
7494 cik_rlc_stop(rdev);
7495}
7496
7497/**
7498 * cik_irq_fini - tear down interrupt support
7499 *
7500 * @rdev: radeon_device pointer
7501 *
7502 * Disable interrupts on the hw and free the IH ring
7503 * buffer (CIK).
7504 * Used for driver unload.
7505 */
7506static void cik_irq_fini(struct radeon_device *rdev)
7507{
7508 cik_irq_suspend(rdev);
7509 r600_ih_ring_fini(rdev);
7510}
7511
7512/**
7513 * cik_get_ih_wptr - get the IH ring buffer wptr
7514 *
7515 * @rdev: radeon_device pointer
7516 *
7517 * Get the IH ring buffer wptr from either the register
7518 * or the writeback memory buffer (CIK). Also check for
7519 * ring buffer overflow and deal with it.
7520 * Used by cik_irq_process().
7521 * Returns the value of the wptr.
7522 */
7523static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7524{
7525 u32 wptr, tmp;
7526
7527 if (rdev->wb.enabled)
7528 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7529 else
7530 wptr = RREG32(IH_RB_WPTR);
7531
7532 if (wptr & RB_OVERFLOW) {
11bab0ae 7533 wptr &= ~RB_OVERFLOW;
a59781bb
AD		 7534 /* When a ring buffer overflow happen start parsing interrupt
7535 * from the last not overwritten vector (wptr + 16). Hopefully
7536 * this should allow us to catchup.
7537 */
6cc2fda2
MD		 7538 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7539 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
a59781bb
AD		 7540 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7541 tmp = RREG32(IH_RB_CNTL);
7542 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7543 WREG32(IH_RB_CNTL, tmp);
7544 }
7545 return (wptr & rdev->ih.ptr_mask);
7546}
7547
7548/* CIK IV Ring
7549 * Each IV ring entry is 128 bits:
7550 * [7:0] - interrupt source id
7551 * [31:8] - reserved
7552 * [59:32] - interrupt source data
7553 * [63:60] - reserved
21a93e13
AD		 7554 * [71:64] - RINGID
7555 * CP:
7556 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 7557 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7558 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7559 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7560 * PIPE_ID - ME0 0=3D
7561 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 7562 * SDMA:
7563 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7564 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7565 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 7566 * [79:72] - VMID
7567 * [95:80] - PASID
7568 * [127:96] - reserved
7569 */
7570/**
7571 * cik_irq_process - interrupt handler
7572 *
7573 * @rdev: radeon_device pointer
7574 *
7575 * Interrupt hander (CIK). Walk the IH ring,
7576 * ack interrupts and schedule work to handle
7577 * interrupt events.
7578 * Returns irq process return code.
7579 */
7580int cik_irq_process(struct radeon_device *rdev)
7581{
2b0781a6
AD		 7582 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7583 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 7584 u32 wptr;
7585 u32 rptr;
7586 u32 src_id, src_data, ring_id;
7587 u8 me_id, pipe_id, queue_id;
7588 u32 ring_index;
7589 bool queue_hotplug = false;
f6b355dd 7590 bool queue_dp = false;
a59781bb 7591 bool queue_reset = false;
3ec7d11b 7592 u32 addr, status, mc_client;
41a524ab 7593 bool queue_thermal = false;
a59781bb
AD		 7594
7595 if (!rdev->ih.enabled || rdev->shutdown)
7596 return IRQ_NONE;
7597
7598 wptr = cik_get_ih_wptr(rdev);
7599
7600restart_ih:
7601 /* is somebody else already processing irqs? */
7602 if (atomic_xchg(&rdev->ih.lock, 1))
7603 return IRQ_NONE;
7604
7605 rptr = rdev->ih.rptr;
7606 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7607
7608 /* Order reading of wptr vs. reading of IH ring data */
7609 rmb();
7610
7611 /* display interrupts */
7612 cik_irq_ack(rdev);
7613
7614 while (rptr != wptr) {
7615 /* wptr/rptr are in bytes! */
7616 ring_index = rptr / 4;
e28740ec
OG		 7617
7618 radeon_kfd_interrupt(rdev,
7619 (const void *) &rdev->ih.ring[ring_index]);
7620
a59781bb
AD		 7621 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7622 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7623 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 7624
7625 switch (src_id) {
7626 case 1: /* D1 vblank/vline */
7627 switch (src_data) {
7628 case 0: /* D1 vblank */
07f18f0b
MK		 7629 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT))
7630 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7631
7632 if (rdev->irq.crtc_vblank_int[0]) {
7633 drm_handle_vblank(rdev->ddev, 0);
7634 rdev->pm.vblank_sync = true;
7635 wake_up(&rdev->irq.vblank_queue);
a59781bb 7636 }
07f18f0b
MK		 7637 if (atomic_read(&rdev->irq.pflip[0]))
7638 radeon_crtc_handle_vblank(rdev, 0);
7639 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7640 DRM_DEBUG("IH: D1 vblank\n");
7641
a59781bb
AD		 7642 break;
7643 case 1: /* D1 vline */
07f18f0b
MK		 7644 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT))
7645 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7646
7647 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7648 DRM_DEBUG("IH: D1 vline\n");
7649
a59781bb
AD		 7650 break;
7651 default:
7652 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7653 break;
7654 }
7655 break;
7656 case 2: /* D2 vblank/vline */
7657 switch (src_data) {
7658 case 0: /* D2 vblank */
07f18f0b
MK		 7659 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
7660 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7661
7662 if (rdev->irq.crtc_vblank_int[1]) {
7663 drm_handle_vblank(rdev->ddev, 1);
7664 rdev->pm.vblank_sync = true;
7665 wake_up(&rdev->irq.vblank_queue);
a59781bb 7666 }
07f18f0b
MK		 7667 if (atomic_read(&rdev->irq.pflip[1]))
7668 radeon_crtc_handle_vblank(rdev, 1);
7669 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7670 DRM_DEBUG("IH: D2 vblank\n");
7671
a59781bb
AD		 7672 break;
7673 case 1: /* D2 vline */
07f18f0b
MK		 7674 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT))
7675 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7676
7677 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7678 DRM_DEBUG("IH: D2 vline\n");
7679
a59781bb
AD		 7680 break;
7681 default:
7682 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7683 break;
7684 }
7685 break;
7686 case 3: /* D3 vblank/vline */
7687 switch (src_data) {
7688 case 0: /* D3 vblank */
07f18f0b
MK		 7689 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
7690 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7691
7692 if (rdev->irq.crtc_vblank_int[2]) {
7693 drm_handle_vblank(rdev->ddev, 2);
7694 rdev->pm.vblank_sync = true;
7695 wake_up(&rdev->irq.vblank_queue);
a59781bb 7696 }
07f18f0b
MK		 7697 if (atomic_read(&rdev->irq.pflip[2]))
7698 radeon_crtc_handle_vblank(rdev, 2);
7699 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7700 DRM_DEBUG("IH: D3 vblank\n");
7701
a59781bb
AD		 7702 break;
7703 case 1: /* D3 vline */
07f18f0b
MK		 7704 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
7705 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7706
7707 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7708 DRM_DEBUG("IH: D3 vline\n");
7709
a59781bb
AD		 7710 break;
7711 default:
7712 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7713 break;
7714 }
7715 break;
7716 case 4: /* D4 vblank/vline */
7717 switch (src_data) {
7718 case 0: /* D4 vblank */
07f18f0b
MK		 7719 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
7720 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7721
7722 if (rdev->irq.crtc_vblank_int[3]) {
7723 drm_handle_vblank(rdev->ddev, 3);
7724 rdev->pm.vblank_sync = true;
7725 wake_up(&rdev->irq.vblank_queue);
a59781bb 7726 }
07f18f0b
MK		 7727 if (atomic_read(&rdev->irq.pflip[3]))
7728 radeon_crtc_handle_vblank(rdev, 3);
7729 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7730 DRM_DEBUG("IH: D4 vblank\n");
7731
a59781bb
AD		 7732 break;
7733 case 1: /* D4 vline */
07f18f0b
MK		 7734 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
7735 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7736
7737 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7738 DRM_DEBUG("IH: D4 vline\n");
7739
a59781bb
AD		 7740 break;
7741 default:
7742 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7743 break;
7744 }
7745 break;
7746 case 5: /* D5 vblank/vline */
7747 switch (src_data) {
7748 case 0: /* D5 vblank */
07f18f0b
MK		 7749 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
7750 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7751
7752 if (rdev->irq.crtc_vblank_int[4]) {
7753 drm_handle_vblank(rdev->ddev, 4);
7754 rdev->pm.vblank_sync = true;
7755 wake_up(&rdev->irq.vblank_queue);
a59781bb 7756 }
07f18f0b
MK		 7757 if (atomic_read(&rdev->irq.pflip[4]))
7758 radeon_crtc_handle_vblank(rdev, 4);
7759 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7760 DRM_DEBUG("IH: D5 vblank\n");
7761
a59781bb
AD		 7762 break;
7763 case 1: /* D5 vline */
07f18f0b
MK		 7764 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
7765 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7766
7767 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7768 DRM_DEBUG("IH: D5 vline\n");
7769
a59781bb
AD		 7770 break;
7771 default:
7772 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7773 break;
7774 }
7775 break;
7776 case 6: /* D6 vblank/vline */
7777 switch (src_data) {
7778 case 0: /* D6 vblank */
07f18f0b
MK		 7779 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
7780 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7781
7782 if (rdev->irq.crtc_vblank_int[5]) {
7783 drm_handle_vblank(rdev->ddev, 5);
7784 rdev->pm.vblank_sync = true;
7785 wake_up(&rdev->irq.vblank_queue);
a59781bb 7786 }
07f18f0b
MK		 7787 if (atomic_read(&rdev->irq.pflip[5]))
7788 radeon_crtc_handle_vblank(rdev, 5);
7789 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7790 DRM_DEBUG("IH: D6 vblank\n");
7791
a59781bb
AD		 7792 break;
7793 case 1: /* D6 vline */
07f18f0b
MK		 7794 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
7795 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7796
7797 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7798 DRM_DEBUG("IH: D6 vline\n");
7799
a59781bb
AD		 7800 break;
7801 default:
7802 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7803 break;
7804 }
7805 break;
f5d636d2
CK		 7806 case 8: /* D1 page flip */
7807 case 10: /* D2 page flip */
7808 case 12: /* D3 page flip */
7809 case 14: /* D4 page flip */
7810 case 16: /* D5 page flip */
7811 case 18: /* D6 page flip */
7812 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 7813 if (radeon_use_pflipirq > 0)
7814 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 7815 break;
a59781bb
AD		 7816 case 42: /* HPD hotplug */
7817 switch (src_data) {
7818 case 0:
07f18f0b
MK		 7819 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT))
7820 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7821
7822 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7823 queue_hotplug = true;
7824 DRM_DEBUG("IH: HPD1\n");
7825
a59781bb
AD		 7826 break;
7827 case 1:
07f18f0b
MK		 7828 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT))
7829 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7830
7831 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7832 queue_hotplug = true;
7833 DRM_DEBUG("IH: HPD2\n");
7834
a59781bb
AD		 7835 break;
7836 case 2:
07f18f0b
MK		 7837 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT))
7838 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7839
7840 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7841 queue_hotplug = true;
7842 DRM_DEBUG("IH: HPD3\n");
7843
a59781bb
AD		 7844 break;
7845 case 3:
07f18f0b
MK		 7846 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT))
7847 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7848
7849 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7850 queue_hotplug = true;
7851 DRM_DEBUG("IH: HPD4\n");
7852
a59781bb
AD		 7853 break;
7854 case 4:
07f18f0b
MK		 7855 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT))
7856 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7857
7858 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7859 queue_hotplug = true;
7860 DRM_DEBUG("IH: HPD5\n");
7861
a59781bb
AD		 7862 break;
7863 case 5:
07f18f0b
MK		 7864 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT))
7865 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7866
7867 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7868 queue_hotplug = true;
7869 DRM_DEBUG("IH: HPD6\n");
7870
a59781bb 7871 break;
f6b355dd 7872 case 6:
07f18f0b
MK		 7873 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT))
7874 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7875
7876 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;
7877 queue_dp = true;
7878 DRM_DEBUG("IH: HPD_RX 1\n");
7879
f6b355dd
AD		 7880 break;
7881 case 7:
07f18f0b
MK		 7882 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT))
7883 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7884
7885 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
7886 queue_dp = true;
7887 DRM_DEBUG("IH: HPD_RX 2\n");
7888
f6b355dd
AD		 7889 break;
7890 case 8:
07f18f0b
MK		 7891 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
7892 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7893
7894 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
7895 queue_dp = true;
7896 DRM_DEBUG("IH: HPD_RX 3\n");
7897
f6b355dd
AD		 7898 break;
7899 case 9:
07f18f0b
MK		 7900 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
7901 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7902
7903 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
7904 queue_dp = true;
7905 DRM_DEBUG("IH: HPD_RX 4\n");
7906
f6b355dd
AD		 7907 break;
7908 case 10:
07f18f0b
MK		 7909 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
7910 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7911
7912 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
7913 queue_dp = true;
7914 DRM_DEBUG("IH: HPD_RX 5\n");
7915
f6b355dd
AD		 7916 break;
7917 case 11:
07f18f0b
MK		 7918 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
7919 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7920
7921 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
7922 queue_dp = true;
7923 DRM_DEBUG("IH: HPD_RX 6\n");
7924
f6b355dd 7925 break;
a59781bb
AD		 7926 default:
7927 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7928 break;
7929 }
7930 break;
dc12a3ec
LL		 7931 case 96:
7932 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
7933 WREG32(SRBM_INT_ACK, 0x1);
7934 break;
6a3808b8
CK		 7935 case 124: /* UVD */
7936 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7937 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7938 break;
9d97c99b
AD		 7939 case 146:
7940 case 147:
3ec7d11b
AD		 7941 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7942 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7943 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9b7d786b
CK		 7944 /* reset addr and status */
7945 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7946 if (addr == 0x0 && status == 0x0)
7947 break;
9d97c99b
AD		 7948 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7949 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 7950 addr);
9d97c99b 7951 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 7952 status);
7953 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b 7954 break;
d93f7937
CK		 7955 case 167: /* VCE */
7956 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7957 switch (src_data) {
7958 case 0:
7959 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7960 break;
7961 case 1:
7962 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7963 break;
7964 default:
7965 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7966 break;
7967 }
7968 break;
a59781bb
AD		 7969 case 176: /* GFX RB CP_INT */
7970 case 177: /* GFX IB CP_INT */
7971 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7972 break;
7973 case 181: /* CP EOP event */
7974 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 7975 /* XXX check the bitfield order! */
7976 me_id = (ring_id & 0x60) >> 5;
7977 pipe_id = (ring_id & 0x18) >> 3;
7978 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 7979 switch (me_id) {
7980 case 0:
7981 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7982 break;
7983 case 1:
a59781bb 7984 case 2:
2b0781a6
AD		 7985 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7986 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7987 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7988 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 7989 break;
7990 }
7991 break;
7992 case 184: /* CP Privileged reg access */
7993 DRM_ERROR("Illegal register access in command stream\n");
7994 /* XXX check the bitfield order! */
7995 me_id = (ring_id & 0x60) >> 5;
7996 pipe_id = (ring_id & 0x18) >> 3;
7997 queue_id = (ring_id & 0x7) >> 0;
7998 switch (me_id) {
7999 case 0:
8000 /* This results in a full GPU reset, but all we need to do is soft
8001 * reset the CP for gfx
8002 */
8003 queue_reset = true;
8004 break;
8005 case 1:
8006 /* XXX compute */
2b0781a6 8007 queue_reset = true;
a59781bb
AD		 8008 break;
8009 case 2:
8010 /* XXX compute */
2b0781a6 8011 queue_reset = true;
a59781bb
AD		 8012 break;
8013 }
8014 break;
8015 case 185: /* CP Privileged inst */
8016 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 8017 /* XXX check the bitfield order! */
8018 me_id = (ring_id & 0x60) >> 5;
8019 pipe_id = (ring_id & 0x18) >> 3;
8020 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 8021 switch (me_id) {
8022 case 0:
8023 /* This results in a full GPU reset, but all we need to do is soft
8024 * reset the CP for gfx
8025 */
8026 queue_reset = true;
8027 break;
8028 case 1:
8029 /* XXX compute */
2b0781a6 8030 queue_reset = true;
a59781bb
AD		 8031 break;
8032 case 2:
8033 /* XXX compute */
2b0781a6 8034 queue_reset = true;
a59781bb
AD		 8035 break;
8036 }
8037 break;
21a93e13
AD		 8038 case 224: /* SDMA trap event */
8039 /* XXX check the bitfield order! */
8040 me_id = (ring_id & 0x3) >> 0;
8041 queue_id = (ring_id & 0xc) >> 2;
8042 DRM_DEBUG("IH: SDMA trap\n");
8043 switch (me_id) {
8044 case 0:
8045 switch (queue_id) {
8046 case 0:
8047 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8048 break;
8049 case 1:
8050 /* XXX compute */
8051 break;
8052 case 2:
8053 /* XXX compute */
8054 break;
8055 }
8056 break;
8057 case 1:
8058 switch (queue_id) {
8059 case 0:
8060 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8061 break;
8062 case 1:
8063 /* XXX compute */
8064 break;
8065 case 2:
8066 /* XXX compute */
8067 break;
8068 }
8069 break;
8070 }
8071 break;
41a524ab
AD		 8072 case 230: /* thermal low to high */
8073 DRM_DEBUG("IH: thermal low to high\n");
8074 rdev->pm.dpm.thermal.high_to_low = false;
8075 queue_thermal = true;
8076 break;
8077 case 231: /* thermal high to low */
8078 DRM_DEBUG("IH: thermal high to low\n");
8079 rdev->pm.dpm.thermal.high_to_low = true;
8080 queue_thermal = true;
8081 break;
8082 case 233: /* GUI IDLE */
8083 DRM_DEBUG("IH: GUI idle\n");
8084 break;
21a93e13
AD		 8085 case 241: /* SDMA Privileged inst */
8086 case 247: /* SDMA Privileged inst */
8087 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8088 /* XXX check the bitfield order! */
8089 me_id = (ring_id & 0x3) >> 0;
8090 queue_id = (ring_id & 0xc) >> 2;
8091 switch (me_id) {
8092 case 0:
8093 switch (queue_id) {
8094 case 0:
8095 queue_reset = true;
8096 break;
8097 case 1:
8098 /* XXX compute */
8099 queue_reset = true;
8100 break;
8101 case 2:
8102 /* XXX compute */
8103 queue_reset = true;
8104 break;
8105 }
8106 break;
8107 case 1:
8108 switch (queue_id) {
8109 case 0:
8110 queue_reset = true;
8111 break;
8112 case 1:
8113 /* XXX compute */
8114 queue_reset = true;
8115 break;
8116 case 2:
8117 /* XXX compute */
8118 queue_reset = true;
8119 break;
8120 }
8121 break;
8122 }
8123 break;
a59781bb
AD		 8124 default:
8125 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8126 break;
8127 }
8128
8129 /* wptr/rptr are in bytes! */
8130 rptr += 16;
8131 rptr &= rdev->ih.ptr_mask;
f55e03b9 8132 WREG32(IH_RB_RPTR, rptr);
a59781bb 8133 }
f6b355dd
AD		 8134 if (queue_dp)
8135 schedule_work(&rdev->dp_work);
a59781bb 8136 if (queue_hotplug)
cb5d4166 8137 schedule_delayed_work(&rdev->hotplug_work, 0);
3c036389
CK		 8138 if (queue_reset) {
8139 rdev->needs_reset = true;
8140 wake_up_all(&rdev->fence_queue);
8141 }
41a524ab
AD		 8142 if (queue_thermal)
8143 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb 8144 rdev->ih.rptr = rptr;
a59781bb
AD		 8145 atomic_set(&rdev->ih.lock, 0);
8146
8147 /* make sure wptr hasn't changed while processing */
8148 wptr = cik_get_ih_wptr(rdev);
8149 if (wptr != rptr)
8150 goto restart_ih;
8151
8152 return IRQ_HANDLED;
8153}
7bf94a2c
AD		 8154
8155/*
8156 * startup/shutdown callbacks
8157 */
bc48a15a
JG		 8158static void cik_uvd_init(struct radeon_device *rdev)
8159{
8160 int r;
8161
8162 if (!rdev->has_uvd)
8163 return;
8164
8165 r = radeon_uvd_init(rdev);
8166 if (r) {
8167 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
8168 /*
8169 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
8170 * to early fails cik_uvd_start() and thus nothing happens
8171 * there. So it is pointless to try to go through that code
8172 * hence why we disable uvd here.
8173 */
8174 rdev->has_uvd = 0;
8175 return;
8176 }
8177 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
8178 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
8179}
8180
8181static void cik_uvd_start(struct radeon_device *rdev)
8182{
8183 int r;
8184
8185 if (!rdev->has_uvd)
8186 return;
8187
8188 r = radeon_uvd_resume(rdev);
8189 if (r) {
8190 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
8191 goto error;
8192 }
8193 r = uvd_v4_2_resume(rdev);
8194 if (r) {
8195 dev_err(rdev->dev, "failed UVD 4.2 resume (%d).\n", r);
8196 goto error;
8197 }
8198 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
8199 if (r) {
8200 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
8201 goto error;
8202 }
8203 return;
8204
8205error:
8206 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8207}
8208
8209static void cik_uvd_resume(struct radeon_device *rdev)
8210{
8211 struct radeon_ring *ring;
8212 int r;
8213
8214 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
8215 return;
8216
8217 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8218 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, RADEON_CP_PACKET2);
8219 if (r) {
8220 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
8221 return;
8222 }
8223 r = uvd_v1_0_init(rdev);
8224 if (r) {
8225 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
8226 return;
8227 }
8228}
8229
cb25f7e0
JG		 8230static void cik_vce_init(struct radeon_device *rdev)
8231{
8232 int r;
8233
8234 if (!rdev->has_vce)
8235 return;
8236
8237 r = radeon_vce_init(rdev);
8238 if (r) {
8239 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
8240 /*
8241 * At this point rdev->vce.vcpu_bo is NULL which trickles down
8242 * to early fails cik_vce_start() and thus nothing happens
8243 * there. So it is pointless to try to go through that code
8244 * hence why we disable vce here.
8245 */
8246 rdev->has_vce = 0;
8247 return;
8248 }
8249 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
8250 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
8251 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
8252 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
8253}
8254
8255static void cik_vce_start(struct radeon_device *rdev)
8256{
8257 int r;
8258
8259 if (!rdev->has_vce)
8260 return;
8261
8262 r = radeon_vce_resume(rdev);
8263 if (r) {
8264 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8265 goto error;
8266 }
8267 r = vce_v2_0_resume(rdev);
8268 if (r) {
8269 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8270 goto error;
8271 }
8272 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
8273 if (r) {
8274 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
8275 goto error;
8276 }
8277 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
8278 if (r) {
8279 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
8280 goto error;
8281 }
8282 return;
8283
8284error:
8285 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8286 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8287}
8288
8289static void cik_vce_resume(struct radeon_device *rdev)
8290{
8291 struct radeon_ring *ring;
8292 int r;
8293
8294 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
8295 return;
8296
8297 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8298 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8299 if (r) {
8300 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8301 return;
8302 }
8303 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8304 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8305 if (r) {
8306 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8307 return;
8308 }
8309 r = vce_v1_0_init(rdev);
8310 if (r) {
8311 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
8312 return;
8313 }
8314}
8315
7bf94a2c
AD		 8316/**
8317 * cik_startup - program the asic to a functional state
8318 *
8319 * @rdev: radeon_device pointer
8320 *
8321 * Programs the asic to a functional state (CIK).
8322 * Called by cik_init() and cik_resume().
8323 * Returns 0 for success, error for failure.
8324 */
8325static int cik_startup(struct radeon_device *rdev)
8326{
8327 struct radeon_ring *ring;
0e16e4cf 8328 u32 nop;
7bf94a2c
AD		 8329 int r;
8330
8a7cd276
AD		 8331 /* enable pcie gen2/3 link */
8332 cik_pcie_gen3_enable(rdev);
7235711a
AD		 8333 /* enable aspm */
8334 cik_program_aspm(rdev);
8a7cd276 8335
e5903d39
AD		 8336 /* scratch needs to be initialized before MC */
8337 r = r600_vram_scratch_init(rdev);
8338 if (r)
8339 return r;
8340
6fab3feb
AD		 8341 cik_mc_program(rdev);
8342
6c7bccea 8343 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7bf94a2c
AD		 8344 r = ci_mc_load_microcode(rdev);
8345 if (r) {
8346 DRM_ERROR("Failed to load MC firmware!\n");
8347 return r;
8348 }
8349 }
8350
7bf94a2c
AD		 8351 r = cik_pcie_gart_enable(rdev);
8352 if (r)
8353 return r;
8354 cik_gpu_init(rdev);
8355
8356 /* allocate rlc buffers */
22c775ce
AD		 8357 if (rdev->flags & RADEON_IS_IGP) {
8358 if (rdev->family == CHIP_KAVERI) {
8359 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8360 rdev->rlc.reg_list_size =
8361 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8362 } else {
8363 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8364 rdev->rlc.reg_list_size =
8365 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8366 }
8367 }
8368 rdev->rlc.cs_data = ci_cs_data;
e70a15f5 8369 rdev->rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */
9406d216 8370 rdev->rlc.cp_table_size += 64 * 1024; /* GDS */
1fd11777 8371 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 8372 if (r) {
8373 DRM_ERROR("Failed to init rlc BOs!\n");
8374 return r;
8375 }
8376
8377 /* allocate wb buffer */
8378 r = radeon_wb_init(rdev);
8379 if (r)
8380 return r;
8381
963e81f9
AD		 8382 /* allocate mec buffers */
8383 r = cik_mec_init(rdev);
8384 if (r) {
8385 DRM_ERROR("Failed to init MEC BOs!\n");
8386 return r;
8387 }
8388
7bf94a2c
AD		 8389 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8390 if (r) {
8391 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8392 return r;
8393 }
8394
963e81f9
AD		 8395 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8396 if (r) {
8397 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8398 return r;
8399 }
8400
8401 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8402 if (r) {
8403 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8404 return r;
8405 }
8406
7bf94a2c
AD		 8407 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8408 if (r) {
8409 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8410 return r;
8411 }
8412
8413 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8414 if (r) {
8415 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8416 return r;
8417 }
8418
bc48a15a 8419 cik_uvd_start(rdev);
cb25f7e0 8420 cik_vce_start(rdev);
d93f7937 8421
7bf94a2c
AD		 8422 /* Enable IRQ */
8423 if (!rdev->irq.installed) {
8424 r = radeon_irq_kms_init(rdev);
8425 if (r)
8426 return r;
8427 }
8428
8429 r = cik_irq_init(rdev);
8430 if (r) {
8431 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8432 radeon_irq_kms_fini(rdev);
8433 return r;
8434 }
8435 cik_irq_set(rdev);
8436
0e16e4cf 8437 if (rdev->family == CHIP_HAWAII) {
78cd3661
AD		 8438 if (rdev->new_fw)
8439 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8440 else
8441 nop = RADEON_CP_PACKET2;
0e16e4cf
AD		 8442 } else {
8443 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8444 }
8445
7bf94a2c
AD		 8446 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8447 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
0e16e4cf 8448 nop);
7bf94a2c
AD		 8449 if (r)
8450 return r;
8451
963e81f9 8452 /* set up the compute queues */
2615b53a 8453 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8454 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8455 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
0e16e4cf 8456 nop);
963e81f9
AD		 8457 if (r)
8458 return r;
8459 ring->me = 1; /* first MEC */
8460 ring->pipe = 0; /* first pipe */
8461 ring->queue = 0; /* first queue */
8462 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8463
2615b53a 8464 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8465 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8466 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
0e16e4cf 8467 nop);
963e81f9
AD		 8468 if (r)
8469 return r;
8470 /* dGPU only have 1 MEC */
8471 ring->me = 1; /* first MEC */
8472 ring->pipe = 0; /* first pipe */
8473 ring->queue = 1; /* second queue */
8474 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8475
7bf94a2c
AD		 8476 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8477 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 8478 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8479 if (r)
8480 return r;
8481
8482 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8483 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 8484 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8485 if (r)
8486 return r;
8487
8488 r = cik_cp_resume(rdev);
8489 if (r)
8490 return r;
8491
8492 r = cik_sdma_resume(rdev);
8493 if (r)
8494 return r;
8495
bc48a15a 8496 cik_uvd_resume(rdev);
cb25f7e0 8497 cik_vce_resume(rdev);
d93f7937 8498
7bf94a2c
AD		 8499 r = radeon_ib_pool_init(rdev);
8500 if (r) {
8501 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8502 return r;
8503 }
8504
8505 r = radeon_vm_manager_init(rdev);
8506 if (r) {
8507 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8508 return r;
8509 }
8510
bfc1f97d 8511 r = radeon_audio_init(rdev);
b530602f
AD		 8512 if (r)
8513 return r;
8514
e28740ec
OG		 8515 r = radeon_kfd_resume(rdev);
8516 if (r)
8517 return r;
8518
7bf94a2c
AD		 8519 return 0;
8520}
8521
8522/**
8523 * cik_resume - resume the asic to a functional state
8524 *
8525 * @rdev: radeon_device pointer
8526 *
8527 * Programs the asic to a functional state (CIK).
8528 * Called at resume.
8529 * Returns 0 for success, error for failure.
8530 */
8531int cik_resume(struct radeon_device *rdev)
8532{
8533 int r;
8534
8535 /* post card */
8536 atom_asic_init(rdev->mode_info.atom_context);
8537
0aafd313
AD		 8538 /* init golden registers */
8539 cik_init_golden_registers(rdev);
8540
bc6a6295
AD		 8541 if (rdev->pm.pm_method == PM_METHOD_DPM)
8542 radeon_pm_resume(rdev);
6c7bccea 8543
7bf94a2c
AD		 8544 rdev->accel_working = true;
8545 r = cik_startup(rdev);
8546 if (r) {
8547 DRM_ERROR("cik startup failed on resume\n");
8548 rdev->accel_working = false;
8549 return r;
8550 }
8551
8552 return r;
8553
8554}
8555
8556/**
8557 * cik_suspend - suspend the asic
8558 *
8559 * @rdev: radeon_device pointer
8560 *
8561 * Bring the chip into a state suitable for suspend (CIK).
8562 * Called at suspend.
8563 * Returns 0 for success.
8564 */
8565int cik_suspend(struct radeon_device *rdev)
8566{
e28740ec 8567 radeon_kfd_suspend(rdev);
6c7bccea 8568 radeon_pm_suspend(rdev);
7991d665 8569 radeon_audio_fini(rdev);
7bf94a2c
AD		 8570 radeon_vm_manager_fini(rdev);
8571 cik_cp_enable(rdev, false);
8572 cik_sdma_enable(rdev, false);
bc48a15a
JG		 8573 if (rdev->has_uvd) {
8574 uvd_v1_0_fini(rdev);
8575 radeon_uvd_suspend(rdev);
8576 }
cb25f7e0
JG		 8577 if (rdev->has_vce)
8578 radeon_vce_suspend(rdev);
473359bc
AD		 8579 cik_fini_pg(rdev);
8580 cik_fini_cg(rdev);
7bf94a2c
AD		 8581 cik_irq_suspend(rdev);
8582 radeon_wb_disable(rdev);
8583 cik_pcie_gart_disable(rdev);
8584 return 0;
8585}
8586
8587/* Plan is to move initialization in that function and use
8588 * helper function so that radeon_device_init pretty much
8589 * do nothing more than calling asic specific function. This
8590 * should also allow to remove a bunch of callback function
8591 * like vram_info.
8592 */
8593/**
8594 * cik_init - asic specific driver and hw init
8595 *
8596 * @rdev: radeon_device pointer
8597 *
8598 * Setup asic specific driver variables and program the hw
8599 * to a functional state (CIK).
8600 * Called at driver startup.
8601 * Returns 0 for success, errors for failure.
8602 */
8603int cik_init(struct radeon_device *rdev)
8604{
8605 struct radeon_ring *ring;
8606 int r;
8607
8608 /* Read BIOS */
8609 if (!radeon_get_bios(rdev)) {
8610 if (ASIC_IS_AVIVO(rdev))
8611 return -EINVAL;
8612 }
8613 /* Must be an ATOMBIOS */
8614 if (!rdev->is_atom_bios) {
8615 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8616 return -EINVAL;
8617 }
8618 r = radeon_atombios_init(rdev);
8619 if (r)
8620 return r;
8621
8622 /* Post card if necessary */
8623 if (!radeon_card_posted(rdev)) {
8624 if (!rdev->bios) {
8625 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8626 return -EINVAL;
8627 }
8628 DRM_INFO("GPU not posted. posting now...\n");
8629 atom_asic_init(rdev->mode_info.atom_context);
8630 }
0aafd313
AD		 8631 /* init golden registers */
8632 cik_init_golden_registers(rdev);
7bf94a2c
AD		 8633 /* Initialize scratch registers */
8634 cik_scratch_init(rdev);
8635 /* Initialize surface registers */
8636 radeon_surface_init(rdev);
8637 /* Initialize clocks */
8638 radeon_get_clock_info(rdev->ddev);
8639
8640 /* Fence driver */
8641 r = radeon_fence_driver_init(rdev);
8642 if (r)
8643 return r;
8644
8645 /* initialize memory controller */
8646 r = cik_mc_init(rdev);
8647 if (r)
8648 return r;
8649 /* Memory manager */
8650 r = radeon_bo_init(rdev);
8651 if (r)
8652 return r;
8653
01ac8794
AD		 8654 if (rdev->flags & RADEON_IS_IGP) {
8655 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8656 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8657 r = cik_init_microcode(rdev);
8658 if (r) {
8659 DRM_ERROR("Failed to load firmware!\n");
8660 return r;
8661 }
8662 }
8663 } else {
8664 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8665 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8666 !rdev->mc_fw) {
8667 r = cik_init_microcode(rdev);
8668 if (r) {
8669 DRM_ERROR("Failed to load firmware!\n");
8670 return r;
8671 }
8672 }
8673 }
8674
6c7bccea
AD		 8675 /* Initialize power management */
8676 radeon_pm_init(rdev);
8677
7bf94a2c
AD		 8678 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8679 ring->ring_obj = NULL;
8680 r600_ring_init(rdev, ring, 1024 * 1024);
8681
963e81f9
AD		 8682 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8683 ring->ring_obj = NULL;
8684 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8685 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8686 if (r)
8687 return r;
8688
8689 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8690 ring->ring_obj = NULL;
8691 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8692 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8693 if (r)
8694 return r;
8695
7bf94a2c
AD		 8696 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8697 ring->ring_obj = NULL;
8698 r600_ring_init(rdev, ring, 256 * 1024);
8699
8700 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8701 ring->ring_obj = NULL;
8702 r600_ring_init(rdev, ring, 256 * 1024);
8703
bc48a15a 8704 cik_uvd_init(rdev);
cb25f7e0 8705 cik_vce_init(rdev);
d93f7937 8706
7bf94a2c
AD		 8707 rdev->ih.ring_obj = NULL;
8708 r600_ih_ring_init(rdev, 64 * 1024);
8709
8710 r = r600_pcie_gart_init(rdev);
8711 if (r)
8712 return r;
8713
8714 rdev->accel_working = true;
8715 r = cik_startup(rdev);
8716 if (r) {
8717 dev_err(rdev->dev, "disabling GPU acceleration\n");
8718 cik_cp_fini(rdev);
8719 cik_sdma_fini(rdev);
8720 cik_irq_fini(rdev);
1fd11777 8721 sumo_rlc_fini(rdev);
963e81f9 8722 cik_mec_fini(rdev);
7bf94a2c
AD		 8723 radeon_wb_fini(rdev);
8724 radeon_ib_pool_fini(rdev);
8725 radeon_vm_manager_fini(rdev);
8726 radeon_irq_kms_fini(rdev);
8727 cik_pcie_gart_fini(rdev);
8728 rdev->accel_working = false;
8729 }
8730
8731 /* Don't start up if the MC ucode is missing.
8732 * The default clocks and voltages before the MC ucode
8733 * is loaded are not suffient for advanced operations.
8734 */
8735 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8736 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8737 return -EINVAL;
8738 }
8739
8740 return 0;
8741}
8742
8743/**
8744 * cik_fini - asic specific driver and hw fini
8745 *
8746 * @rdev: radeon_device pointer
8747 *
8748 * Tear down the asic specific driver variables and program the hw
8749 * to an idle state (CIK).
8750 * Called at driver unload.
8751 */
8752void cik_fini(struct radeon_device *rdev)
8753{
6c7bccea 8754 radeon_pm_fini(rdev);
7bf94a2c
AD		 8755 cik_cp_fini(rdev);
8756 cik_sdma_fini(rdev);
473359bc
AD		 8757 cik_fini_pg(rdev);
8758 cik_fini_cg(rdev);
7bf94a2c 8759 cik_irq_fini(rdev);
1fd11777 8760 sumo_rlc_fini(rdev);
963e81f9 8761 cik_mec_fini(rdev);
7bf94a2c
AD		 8762 radeon_wb_fini(rdev);
8763 radeon_vm_manager_fini(rdev);
8764 radeon_ib_pool_fini(rdev);
8765 radeon_irq_kms_fini(rdev);
e409b128 8766 uvd_v1_0_fini(rdev);
87167bb1 8767 radeon_uvd_fini(rdev);
d93f7937 8768 radeon_vce_fini(rdev);
7bf94a2c
AD		 8769 cik_pcie_gart_fini(rdev);
8770 r600_vram_scratch_fini(rdev);
8771 radeon_gem_fini(rdev);
8772 radeon_fence_driver_fini(rdev);
8773 radeon_bo_fini(rdev);
8774 radeon_atombios_fini(rdev);
8775 kfree(rdev->bios);
8776 rdev->bios = NULL;
8777}
cd84a27d 8778
134b480f
AD		 8779void dce8_program_fmt(struct drm_encoder *encoder)
8780{
8781 struct drm_device *dev = encoder->dev;
8782 struct radeon_device *rdev = dev->dev_private;
8783 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8784 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8785 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8786 int bpc = 0;
8787 u32 tmp = 0;
6214bb74 8788 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 8789
6214bb74
AD		 8790 if (connector) {
8791 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 8792 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 8793 dither = radeon_connector->dither;
8794 }
134b480f
AD		 8795
8796 /* LVDS/eDP FMT is set up by atom */
8797 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8798 return;
8799
8800 /* not needed for analog */
8801 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8802 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8803 return;
8804
8805 if (bpc == 0)
8806 return;
8807
8808 switch (bpc) {
8809 case 6:
6214bb74 8810 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8811 /* XXX sort out optimal dither settings */
8812 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8813 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8814 else
8815 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8816 break;
8817 case 8:
6214bb74 8818 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8819 /* XXX sort out optimal dither settings */
8820 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8821 FMT_RGB_RANDOM_ENABLE |
8822 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8823 else
8824 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8825 break;
8826 case 10:
6214bb74 8827 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8828 /* XXX sort out optimal dither settings */
8829 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8830 FMT_RGB_RANDOM_ENABLE |
8831 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8832 else
8833 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8834 break;
8835 default:
8836 /* not needed */
8837 break;
8838 }
8839
8840 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8841}
8842
cd84a27d
AD		 8843/* display watermark setup */
8844/**
8845 * dce8_line_buffer_adjust - Set up the line buffer
8846 *
8847 * @rdev: radeon_device pointer
8848 * @radeon_crtc: the selected display controller
8849 * @mode: the current display mode on the selected display
8850 * controller
8851 *
8852 * Setup up the line buffer allocation for
8853 * the selected display controller (CIK).
8854 * Returns the line buffer size in pixels.
8855 */
8856static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8857 struct radeon_crtc *radeon_crtc,
8858 struct drm_display_mode *mode)
8859{
bc01a8c7
AD		 8860 u32 tmp, buffer_alloc, i;
8861 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 8862 /*
8863 * Line Buffer Setup
8864 * There are 6 line buffers, one for each display controllers.
8865 * There are 3 partitions per LB. Select the number of partitions
8866 * to enable based on the display width. For display widths larger
8867 * than 4096, you need use to use 2 display controllers and combine
8868 * them using the stereo blender.
8869 */
8870 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 8871 if (mode->crtc_hdisplay < 1920) {
cd84a27d 8872 tmp = 1;
bc01a8c7
AD		 8873 buffer_alloc = 2;
8874 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 8875 tmp = 2;
bc01a8c7
AD		 8876 buffer_alloc = 2;
8877 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 8878 tmp = 0;
bc01a8c7
AD		 8879 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8880 } else {
cd84a27d
AD		 8881 DRM_DEBUG_KMS("Mode too big for LB!\n");
8882 tmp = 0;
bc01a8c7 8883 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 8884 }
bc01a8c7 8885 } else {
cd84a27d 8886 tmp = 1;
bc01a8c7
AD		 8887 buffer_alloc = 0;
8888 }
cd84a27d
AD		 8889
8890 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8891 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8892
bc01a8c7
AD		 8893 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8894 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8895 for (i = 0; i < rdev->usec_timeout; i++) {
8896 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8897 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8898 break;
8899 udelay(1);
8900 }
8901
cd84a27d
AD		 8902 if (radeon_crtc->base.enabled && mode) {
8903 switch (tmp) {
8904 case 0:
8905 default:
8906 return 4096 * 2;
8907 case 1:
8908 return 1920 * 2;
8909 case 2:
8910 return 2560 * 2;
8911 }
8912 }
8913
8914 /* controller not enabled, so no lb used */
8915 return 0;
8916}
8917
8918/**
8919 * cik_get_number_of_dram_channels - get the number of dram channels
8920 *
8921 * @rdev: radeon_device pointer
8922 *
8923 * Look up the number of video ram channels (CIK).
8924 * Used for display watermark bandwidth calculations
8925 * Returns the number of dram channels
8926 */
8927static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8928{
8929 u32 tmp = RREG32(MC_SHARED_CHMAP);
8930
8931 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8932 case 0:
8933 default:
8934 return 1;
8935 case 1:
8936 return 2;
8937 case 2:
8938 return 4;
8939 case 3:
8940 return 8;
8941 case 4:
8942 return 3;
8943 case 5:
8944 return 6;
8945 case 6:
8946 return 10;
8947 case 7:
8948 return 12;
8949 case 8:
8950 return 16;
8951 }
8952}
8953
8954struct dce8_wm_params {
8955 u32 dram_channels; /* number of dram channels */
8956 u32 yclk; /* bandwidth per dram data pin in kHz */
8957 u32 sclk; /* engine clock in kHz */
8958 u32 disp_clk; /* display clock in kHz */
8959 u32 src_width; /* viewport width */
8960 u32 active_time; /* active display time in ns */
8961 u32 blank_time; /* blank time in ns */
8962 bool interlaced; /* mode is interlaced */
8963 fixed20_12 vsc; /* vertical scale ratio */
8964 u32 num_heads; /* number of active crtcs */
8965 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8966 u32 lb_size; /* line buffer allocated to pipe */
8967 u32 vtaps; /* vertical scaler taps */
8968};
8969
8970/**
8971 * dce8_dram_bandwidth - get the dram bandwidth
8972 *
8973 * @wm: watermark calculation data
8974 *
8975 * Calculate the raw dram bandwidth (CIK).
8976 * Used for display watermark bandwidth calculations
8977 * Returns the dram bandwidth in MBytes/s
8978 */
8979static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8980{
8981 /* Calculate raw DRAM Bandwidth */
8982 fixed20_12 dram_efficiency; /* 0.7 */
8983 fixed20_12 yclk, dram_channels, bandwidth;
8984 fixed20_12 a;
8985
8986 a.full = dfixed_const(1000);
8987 yclk.full = dfixed_const(wm->yclk);
8988 yclk.full = dfixed_div(yclk, a);
8989 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8990 a.full = dfixed_const(10);
8991 dram_efficiency.full = dfixed_const(7);
8992 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8993 bandwidth.full = dfixed_mul(dram_channels, yclk);
8994 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8995
8996 return dfixed_trunc(bandwidth);
8997}
8998
8999/**
9000 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
9001 *
9002 * @wm: watermark calculation data
9003 *
9004 * Calculate the dram bandwidth used for display (CIK).
9005 * Used for display watermark bandwidth calculations
9006 * Returns the dram bandwidth for display in MBytes/s
9007 */
9008static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9009{
9010 /* Calculate DRAM Bandwidth and the part allocated to display. */
9011 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9012 fixed20_12 yclk, dram_channels, bandwidth;
9013 fixed20_12 a;
9014
9015 a.full = dfixed_const(1000);
9016 yclk.full = dfixed_const(wm->yclk);
9017 yclk.full = dfixed_div(yclk, a);
9018 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9019 a.full = dfixed_const(10);
9020 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9021 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9022 bandwidth.full = dfixed_mul(dram_channels, yclk);
9023 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9024
9025 return dfixed_trunc(bandwidth);
9026}
9027
9028/**
9029 * dce8_data_return_bandwidth - get the data return bandwidth
9030 *
9031 * @wm: watermark calculation data
9032 *
9033 * Calculate the data return bandwidth used for display (CIK).
9034 * Used for display watermark bandwidth calculations
9035 * Returns the data return bandwidth in MBytes/s
9036 */
9037static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9038{
9039 /* Calculate the display Data return Bandwidth */
9040 fixed20_12 return_efficiency; /* 0.8 */
9041 fixed20_12 sclk, bandwidth;
9042 fixed20_12 a;
9043
9044 a.full = dfixed_const(1000);
9045 sclk.full = dfixed_const(wm->sclk);
9046 sclk.full = dfixed_div(sclk, a);
9047 a.full = dfixed_const(10);
9048 return_efficiency.full = dfixed_const(8);
9049 return_efficiency.full = dfixed_div(return_efficiency, a);
9050 a.full = dfixed_const(32);
9051 bandwidth.full = dfixed_mul(a, sclk);
9052 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9053
9054 return dfixed_trunc(bandwidth);
9055}
9056
9057/**
9058 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9059 *
9060 * @wm: watermark calculation data
9061 *
9062 * Calculate the dmif bandwidth used for display (CIK).
9063 * Used for display watermark bandwidth calculations
9064 * Returns the dmif bandwidth in MBytes/s
9065 */
9066static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9067{
9068 /* Calculate the DMIF Request Bandwidth */
9069 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9070 fixed20_12 disp_clk, bandwidth;
9071 fixed20_12 a, b;
9072
9073 a.full = dfixed_const(1000);
9074 disp_clk.full = dfixed_const(wm->disp_clk);
9075 disp_clk.full = dfixed_div(disp_clk, a);
9076 a.full = dfixed_const(32);
9077 b.full = dfixed_mul(a, disp_clk);
9078
9079 a.full = dfixed_const(10);
9080 disp_clk_request_efficiency.full = dfixed_const(8);
9081 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9082
9083 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9084
9085 return dfixed_trunc(bandwidth);
9086}
9087
9088/**
9089 * dce8_available_bandwidth - get the min available bandwidth
9090 *
9091 * @wm: watermark calculation data
9092 *
9093 * Calculate the min available bandwidth used for display (CIK).
9094 * Used for display watermark bandwidth calculations
9095 * Returns the min available bandwidth in MBytes/s
9096 */
9097static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9098{
9099 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9100 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9101 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9102 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9103
9104 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9105}
9106
9107/**
9108 * dce8_average_bandwidth - get the average available bandwidth
9109 *
9110 * @wm: watermark calculation data
9111 *
9112 * Calculate the average available bandwidth used for display (CIK).
9113 * Used for display watermark bandwidth calculations
9114 * Returns the average available bandwidth in MBytes/s
9115 */
9116static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9117{
9118 /* Calculate the display mode Average Bandwidth
9119 * DisplayMode should contain the source and destination dimensions,
9120 * timing, etc.
9121 */
9122 fixed20_12 bpp;
9123 fixed20_12 line_time;
9124 fixed20_12 src_width;
9125 fixed20_12 bandwidth;
9126 fixed20_12 a;
9127
9128 a.full = dfixed_const(1000);
9129 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9130 line_time.full = dfixed_div(line_time, a);
9131 bpp.full = dfixed_const(wm->bytes_per_pixel);
9132 src_width.full = dfixed_const(wm->src_width);
9133 bandwidth.full = dfixed_mul(src_width, bpp);
9134 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9135 bandwidth.full = dfixed_div(bandwidth, line_time);
9136
9137 return dfixed_trunc(bandwidth);
9138}
9139
9140/**
9141 * dce8_latency_watermark - get the latency watermark
9142 *
9143 * @wm: watermark calculation data
9144 *
9145 * Calculate the latency watermark (CIK).
9146 * Used for display watermark bandwidth calculations
9147 * Returns the latency watermark in ns
9148 */
9149static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9150{
9151 /* First calculate the latency in ns */
9152 u32 mc_latency = 2000; /* 2000 ns. */
9153 u32 available_bandwidth = dce8_available_bandwidth(wm);
9154 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9155 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9156 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9157 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9158 (wm->num_heads * cursor_line_pair_return_time);
9159 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9160 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9161 u32 tmp, dmif_size = 12288;
9162 fixed20_12 a, b, c;
9163
9164 if (wm->num_heads == 0)
9165 return 0;
9166
9167 a.full = dfixed_const(2);
9168 b.full = dfixed_const(1);
9169 if ((wm->vsc.full > a.full) ||
9170 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9171 (wm->vtaps >= 5) ||
9172 ((wm->vsc.full >= a.full) && wm->interlaced))
9173 max_src_lines_per_dst_line = 4;
9174 else
9175 max_src_lines_per_dst_line = 2;
9176
9177 a.full = dfixed_const(available_bandwidth);
9178 b.full = dfixed_const(wm->num_heads);
9179 a.full = dfixed_div(a, b);
9180
9181 b.full = dfixed_const(mc_latency + 512);
9182 c.full = dfixed_const(wm->disp_clk);
9183 b.full = dfixed_div(b, c);
9184
9185 c.full = dfixed_const(dmif_size);
9186 b.full = dfixed_div(c, b);
9187
9188 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9189
9190 b.full = dfixed_const(1000);
9191 c.full = dfixed_const(wm->disp_clk);
9192 b.full = dfixed_div(c, b);
9193 c.full = dfixed_const(wm->bytes_per_pixel);
9194 b.full = dfixed_mul(b, c);
9195
9196 lb_fill_bw = min(tmp, dfixed_trunc(b));
9197
9198 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9199 b.full = dfixed_const(1000);
9200 c.full = dfixed_const(lb_fill_bw);
9201 b.full = dfixed_div(c, b);
9202 a.full = dfixed_div(a, b);
9203 line_fill_time = dfixed_trunc(a);
9204
9205 if (line_fill_time < wm->active_time)
9206 return latency;
9207 else
9208 return latency + (line_fill_time - wm->active_time);
9209
9210}
9211
9212/**
9213 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9214 * average and available dram bandwidth
9215 *
9216 * @wm: watermark calculation data
9217 *
9218 * Check if the display average bandwidth fits in the display
9219 * dram bandwidth (CIK).
9220 * Used for display watermark bandwidth calculations
9221 * Returns true if the display fits, false if not.
9222 */
9223static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9224{
9225 if (dce8_average_bandwidth(wm) <=
9226 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9227 return true;
9228 else
9229 return false;
9230}
9231
9232/**
9233 * dce8_average_bandwidth_vs_available_bandwidth - check
9234 * average and available bandwidth
9235 *
9236 * @wm: watermark calculation data
9237 *
9238 * Check if the display average bandwidth fits in the display
9239 * available bandwidth (CIK).
9240 * Used for display watermark bandwidth calculations
9241 * Returns true if the display fits, false if not.
9242 */
9243static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9244{
9245 if (dce8_average_bandwidth(wm) <=
9246 (dce8_available_bandwidth(wm) / wm->num_heads))
9247 return true;
9248 else
9249 return false;
9250}
9251
9252/**
9253 * dce8_check_latency_hiding - check latency hiding
9254 *
9255 * @wm: watermark calculation data
9256 *
9257 * Check latency hiding (CIK).
9258 * Used for display watermark bandwidth calculations
9259 * Returns true if the display fits, false if not.
9260 */
9261static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9262{
9263 u32 lb_partitions = wm->lb_size / wm->src_width;
9264 u32 line_time = wm->active_time + wm->blank_time;
9265 u32 latency_tolerant_lines;
9266 u32 latency_hiding;
9267 fixed20_12 a;
9268
9269 a.full = dfixed_const(1);
9270 if (wm->vsc.full > a.full)
9271 latency_tolerant_lines = 1;
9272 else {
9273 if (lb_partitions <= (wm->vtaps + 1))
9274 latency_tolerant_lines = 1;
9275 else
9276 latency_tolerant_lines = 2;
9277 }
9278
9279 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9280
9281 if (dce8_latency_watermark(wm) <= latency_hiding)
9282 return true;
9283 else
9284 return false;
9285}
9286
9287/**
9288 * dce8_program_watermarks - program display watermarks
9289 *
9290 * @rdev: radeon_device pointer
9291 * @radeon_crtc: the selected display controller
9292 * @lb_size: line buffer size
9293 * @num_heads: number of display controllers in use
9294 *
9295 * Calculate and program the display watermarks for the
9296 * selected display controller (CIK).
9297 */
9298static void dce8_program_watermarks(struct radeon_device *rdev,
9299 struct radeon_crtc *radeon_crtc,
9300 u32 lb_size, u32 num_heads)
9301{
9302 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 9303 struct dce8_wm_params wm_low, wm_high;
cd84a27d
AD		 9304 u32 pixel_period;
9305 u32 line_time = 0;
9306 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9307 u32 tmp, wm_mask;
9308
9309 if (radeon_crtc->base.enabled && num_heads && mode) {
9310 pixel_period = 1000000 / (u32)mode->clock;
9311 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9312
58ea2dea
AD		 9313 /* watermark for high clocks */
9314 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9315 rdev->pm.dpm_enabled) {
9316 wm_high.yclk =
9317 radeon_dpm_get_mclk(rdev, false) * 10;
9318 wm_high.sclk =
9319 radeon_dpm_get_sclk(rdev, false) * 10;
9320 } else {
9321 wm_high.yclk = rdev->pm.current_mclk * 10;
9322 wm_high.sclk = rdev->pm.current_sclk * 10;
9323 }
9324
9325 wm_high.disp_clk = mode->clock;
9326 wm_high.src_width = mode->crtc_hdisplay;
9327 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9328 wm_high.blank_time = line_time - wm_high.active_time;
9329 wm_high.interlaced = false;
cd84a27d 9330 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 9331 wm_high.interlaced = true;
9332 wm_high.vsc = radeon_crtc->vsc;
9333 wm_high.vtaps = 1;
cd84a27d 9334 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 9335 wm_high.vtaps = 2;
9336 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9337 wm_high.lb_size = lb_size;
9338 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9339 wm_high.num_heads = num_heads;
cd84a27d
AD		 9340
9341 /* set for high clocks */
58ea2dea
AD		 9342 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9343
9344 /* possibly force display priority to high */
9345 /* should really do this at mode validation time... */
9346 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9347 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9348 !dce8_check_latency_hiding(&wm_high) ||
9349 (rdev->disp_priority == 2)) {
9350 DRM_DEBUG_KMS("force priority to high\n");
9351 }
9352
9353 /* watermark for low clocks */
9354 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9355 rdev->pm.dpm_enabled) {
9356 wm_low.yclk =
9357 radeon_dpm_get_mclk(rdev, true) * 10;
9358 wm_low.sclk =
9359 radeon_dpm_get_sclk(rdev, true) * 10;
9360 } else {
9361 wm_low.yclk = rdev->pm.current_mclk * 10;
9362 wm_low.sclk = rdev->pm.current_sclk * 10;
9363 }
9364
9365 wm_low.disp_clk = mode->clock;
9366 wm_low.src_width = mode->crtc_hdisplay;
9367 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9368 wm_low.blank_time = line_time - wm_low.active_time;
9369 wm_low.interlaced = false;
9370 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9371 wm_low.interlaced = true;
9372 wm_low.vsc = radeon_crtc->vsc;
9373 wm_low.vtaps = 1;
9374 if (radeon_crtc->rmx_type != RMX_OFF)
9375 wm_low.vtaps = 2;
9376 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9377 wm_low.lb_size = lb_size;
9378 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9379 wm_low.num_heads = num_heads;
9380
cd84a27d 9381 /* set for low clocks */
58ea2dea 9382 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 9383
9384 /* possibly force display priority to high */
9385 /* should really do this at mode validation time... */
58ea2dea
AD		 9386 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9387 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9388 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 9389 (rdev->disp_priority == 2)) {
9390 DRM_DEBUG_KMS("force priority to high\n");
9391 }
5b5561b3
MK		 9392
9393 /* Save number of lines the linebuffer leads before the scanout */
9394 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
cd84a27d
AD		 9395 }
9396
9397 /* select wm A */
9398 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9399 tmp = wm_mask;
9400 tmp &= ~LATENCY_WATERMARK_MASK(3);
9401 tmp |= LATENCY_WATERMARK_MASK(1);
9402 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9403 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9404 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9405 LATENCY_HIGH_WATERMARK(line_time)));
9406 /* select wm B */
9407 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9408 tmp &= ~LATENCY_WATERMARK_MASK(3);
9409 tmp |= LATENCY_WATERMARK_MASK(2);
9410 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9411 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9412 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9413 LATENCY_HIGH_WATERMARK(line_time)));
9414 /* restore original selection */
9415 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 9416
9417 /* save values for DPM */
9418 radeon_crtc->line_time = line_time;
9419 radeon_crtc->wm_high = latency_watermark_a;
9420 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 9421}
9422
9423/**
9424 * dce8_bandwidth_update - program display watermarks
9425 *
9426 * @rdev: radeon_device pointer
9427 *
9428 * Calculate and program the display watermarks and line
9429 * buffer allocation (CIK).
9430 */
9431void dce8_bandwidth_update(struct radeon_device *rdev)
9432{
9433 struct drm_display_mode *mode = NULL;
9434 u32 num_heads = 0, lb_size;
9435 int i;
9436
8efe82ca
AD		 9437 if (!rdev->mode_info.mode_config_initialized)
9438 return;
9439
cd84a27d
AD		 9440 radeon_update_display_priority(rdev);
9441
9442 for (i = 0; i < rdev->num_crtc; i++) {
9443 if (rdev->mode_info.crtcs[i]->base.enabled)
9444 num_heads++;
9445 }
9446 for (i = 0; i < rdev->num_crtc; i++) {
9447 mode = &rdev->mode_info.crtcs[i]->base.mode;
9448 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9449 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9450 }
9451}
44fa346f
AD		 9452
9453/**
9454 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9455 *
9456 * @rdev: radeon_device pointer
9457 *
9458 * Fetches a GPU clock counter snapshot (SI).
9459 * Returns the 64 bit clock counter snapshot.
9460 */
9461uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9462{
9463 uint64_t clock;
9464
9465 mutex_lock(&rdev->gpu_clock_mutex);
9466 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9467 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
3cf8bb1a 9468 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
44fa346f
AD		 9469 mutex_unlock(&rdev->gpu_clock_mutex);
9470 return clock;
9471}
9472
87167bb1 9473static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
3cf8bb1a 9474 u32 cntl_reg, u32 status_reg)
87167bb1
CK		 9475{
9476 int r, i;
9477 struct atom_clock_dividers dividers;
9478 uint32_t tmp;
9479
9480 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9481 clock, false, &dividers);
9482 if (r)
9483 return r;
9484
9485 tmp = RREG32_SMC(cntl_reg);
9486 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9487 tmp |= dividers.post_divider;
9488 WREG32_SMC(cntl_reg, tmp);
9489
9490 for (i = 0; i < 100; i++) {
9491 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9492 break;
9493 mdelay(10);
9494 }
9495 if (i == 100)
9496 return -ETIMEDOUT;
9497
9498 return 0;
9499}
9500
9501int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9502{
9503 int r = 0;
9504
9505 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9506 if (r)
9507 return r;
9508
9509 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9510 return r;
9511}
9512
5ad6bf91
AD		 9513int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9514{
9515 int r, i;
9516 struct atom_clock_dividers dividers;
9517 u32 tmp;
9518
9519 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9520 ecclk, false, &dividers);
9521 if (r)
9522 return r;
9523
9524 for (i = 0; i < 100; i++) {
9525 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9526 break;
9527 mdelay(10);
9528 }
9529 if (i == 100)
9530 return -ETIMEDOUT;
9531
9532 tmp = RREG32_SMC(CG_ECLK_CNTL);
9533 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9534 tmp |= dividers.post_divider;
9535 WREG32_SMC(CG_ECLK_CNTL, tmp);
9536
9537 for (i = 0; i < 100; i++) {
9538 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9539 break;
9540 mdelay(10);
9541 }
9542 if (i == 100)
9543 return -ETIMEDOUT;
9544
9545 return 0;
9546}
9547
8a7cd276 9548static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 9549{
8a7cd276
AD		 9550 struct pci_dev *root = rdev->pdev->bus->self;
9551 int bridge_pos, gpu_pos;
9552 u32 speed_cntl, mask, current_data_rate;
9553 int ret, i;
9554 u16 tmp16;
87167bb1 9555
0bd252de
AW		 9556 if (pci_is_root_bus(rdev->pdev->bus))
9557 return;
9558
8a7cd276
AD		 9559 if (radeon_pcie_gen2 == 0)
9560 return;
87167bb1 9561
8a7cd276
AD		 9562 if (rdev->flags & RADEON_IS_IGP)
9563 return;
87167bb1 9564
8a7cd276
AD		 9565 if (!(rdev->flags & RADEON_IS_PCIE))
9566 return;
87167bb1 9567
8a7cd276
AD		 9568 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9569 if (ret != 0)
9570 return;
87167bb1 9571
8a7cd276
AD		 9572 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9573 return;
87167bb1 9574
8a7cd276
AD		 9575 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9576 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9577 LC_CURRENT_DATA_RATE_SHIFT;
9578 if (mask & DRM_PCIE_SPEED_80) {
9579 if (current_data_rate == 2) {
9580 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9581 return;
9582 }
9583 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9584 } else if (mask & DRM_PCIE_SPEED_50) {
9585 if (current_data_rate == 1) {
9586 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9587 return;
9588 }
9589 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9590 }
87167bb1 9591
8a7cd276
AD		 9592 bridge_pos = pci_pcie_cap(root);
9593 if (!bridge_pos)
9594 return;
9595
9596 gpu_pos = pci_pcie_cap(rdev->pdev);
9597 if (!gpu_pos)
9598 return;
9599
9600 if (mask & DRM_PCIE_SPEED_80) {
9601 /* re-try equalization if gen3 is not already enabled */
9602 if (current_data_rate != 2) {
9603 u16 bridge_cfg, gpu_cfg;
9604 u16 bridge_cfg2, gpu_cfg2;
9605 u32 max_lw, current_lw, tmp;
9606
9607 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9608 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9609
9610 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9611 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9612
9613 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9614 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9615
9616 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9617 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9618 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9619
9620 if (current_lw < max_lw) {
9621 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9622 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9623 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9624 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9625 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9626 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9627 }
9628 }
9629
9630 for (i = 0; i < 10; i++) {
9631 /* check status */
9632 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9633 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9634 break;
9635
9636 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9637 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9638
9639 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9640 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9641
9642 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9643 tmp |= LC_SET_QUIESCE;
9644 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9645
9646 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9647 tmp |= LC_REDO_EQ;
9648 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9649
9650 mdelay(100);
9651
9652 /* linkctl */
9653 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9654 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9655 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9656 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9657
9658 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9659 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9660 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9661 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9662
9663 /* linkctl2 */
9664 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9665 tmp16 &= ~((1 << 4) | (7 << 9));
9666 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9667 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9668
9669 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9670 tmp16 &= ~((1 << 4) | (7 << 9));
9671 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9672 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9673
9674 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9675 tmp &= ~LC_SET_QUIESCE;
9676 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9677 }
9678 }
9679 }
9680
9681 /* set the link speed */
9682 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9683 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9684 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9685
9686 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9687 tmp16 &= ~0xf;
9688 if (mask & DRM_PCIE_SPEED_80)
9689 tmp16 |= 3; /* gen3 */
9690 else if (mask & DRM_PCIE_SPEED_50)
9691 tmp16 |= 2; /* gen2 */
9692 else
9693 tmp16 |= 1; /* gen1 */
9694 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9695
9696 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9697 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9698 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9699
9700 for (i = 0; i < rdev->usec_timeout; i++) {
9701 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9702 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9703 break;
9704 udelay(1);
9705 }
9706}
7235711a
AD		 9707
9708static void cik_program_aspm(struct radeon_device *rdev)
9709{
9710 u32 data, orig;
9711 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9712 bool disable_clkreq = false;
9713
9714 if (radeon_aspm == 0)
9715 return;
9716
9717 /* XXX double check IGPs */
9718 if (rdev->flags & RADEON_IS_IGP)
9719 return;
9720
9721 if (!(rdev->flags & RADEON_IS_PCIE))
9722 return;
9723
9724 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9725 data &= ~LC_XMIT_N_FTS_MASK;
9726 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9727 if (orig != data)
9728 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9729
9730 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9731 data |= LC_GO_TO_RECOVERY;
9732 if (orig != data)
9733 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9734
9735 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9736 data |= P_IGNORE_EDB_ERR;
9737 if (orig != data)
9738 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9739
9740 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9741 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9742 data |= LC_PMI_TO_L1_DIS;
9743 if (!disable_l0s)
9744 data |= LC_L0S_INACTIVITY(7);
9745
9746 if (!disable_l1) {
9747 data |= LC_L1_INACTIVITY(7);
9748 data &= ~LC_PMI_TO_L1_DIS;
9749 if (orig != data)
9750 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9751
9752 if (!disable_plloff_in_l1) {
9753 bool clk_req_support;
9754
9755 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9756 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9757 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9758 if (orig != data)
9759 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9760
9761 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9762 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9763 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9764 if (orig != data)
9765 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9766
9767 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9768 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9769 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9770 if (orig != data)
9771 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9772
9773 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9774 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9775 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9776 if (orig != data)
9777 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9778
9779 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9780 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9781 data |= LC_DYN_LANES_PWR_STATE(3);
9782 if (orig != data)
9783 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9784
0bd252de
AW		 9785 if (!disable_clkreq &&
9786 !pci_is_root_bus(rdev->pdev->bus)) {
7235711a
AD		 9787 struct pci_dev *root = rdev->pdev->bus->self;
9788 u32 lnkcap;
9789
9790 clk_req_support = false;
9791 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9792 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9793 clk_req_support = true;
9794 } else {
9795 clk_req_support = false;
9796 }
9797
9798 if (clk_req_support) {
9799 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9800 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9801 if (orig != data)
9802 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9803
9804 orig = data = RREG32_SMC(THM_CLK_CNTL);
9805 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9806 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9807 if (orig != data)
9808 WREG32_SMC(THM_CLK_CNTL, data);
9809
9810 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9811 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9812 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9813 if (orig != data)
9814 WREG32_SMC(MISC_CLK_CTRL, data);
9815
9816 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9817 data &= ~BCLK_AS_XCLK;
9818 if (orig != data)
9819 WREG32_SMC(CG_CLKPIN_CNTL, data);
9820
9821 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9822 data &= ~FORCE_BIF_REFCLK_EN;
9823 if (orig != data)
9824 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9825
9826 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9827 data &= ~MPLL_CLKOUT_SEL_MASK;
9828 data |= MPLL_CLKOUT_SEL(4);
9829 if (orig != data)
9830 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9831 }
9832 }
9833 } else {
9834 if (orig != data)
9835 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9836 }
9837
9838 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9839 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9840 if (orig != data)
9841 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9842
9843 if (!disable_l0s) {
9844 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9845 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9846 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9847 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9848 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9849 data &= ~LC_L0S_INACTIVITY_MASK;
9850 if (orig != data)
9851 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9852 }
9853 }
9854 }
87167bb1 9855}
Linux 6.x block layer and io_uring tree(s)