projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/radeon/r600_dma: Move 'r600_gpu_check_soft_reset()'s prototype to shared location
[linux-2.6-block.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
c182615f 24
8cc1a532 25#include <linux/firmware.h>
8cc1a532 26#include <linux/module.h>
2ef79416
TZ		 27#include <linux/pci.h>
28#include <linux/slab.h>
c182615f 29
c182615f
SR		 30#include <drm/drm_vblank.h>
31
32#include "atom.h"
4fe1999e 33#include "evergreen.h"
c182615f
SR		 34#include "cik_blit_shaders.h"
35#include "cikd.h"
36#include "clearstate_ci.h"
8cc1a532 37#include "radeon.h"
6f2043ce 38#include "radeon_asic.h"
bfc1f97d 39#include "radeon_audio.h"
8c68e393 40#include "radeon_ucode.h"
02c81327 41
75cb00dc
MO		 42#define SH_MEM_CONFIG_GFX_DEFAULT \
43 ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
44
02c81327
AD		 45MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
46MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
47MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
48MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
49MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
277babc3 50MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
02c81327 51MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 52MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 53MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
f2c6b0f4
AD		 54
55MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
56MODULE_FIRMWARE("radeon/bonaire_me.bin");
57MODULE_FIRMWARE("radeon/bonaire_ce.bin");
58MODULE_FIRMWARE("radeon/bonaire_mec.bin");
59MODULE_FIRMWARE("radeon/bonaire_mc.bin");
60MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
61MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
62MODULE_FIRMWARE("radeon/bonaire_smc.bin");
b2ea0dcd 63MODULE_FIRMWARE("radeon/bonaire_k_smc.bin");
f2c6b0f4 64
d4775655
AD		 65MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
66MODULE_FIRMWARE("radeon/HAWAII_me.bin");
67MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
68MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
69MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
277babc3 70MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
d4775655
AD		 71MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
72MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
73MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
f2c6b0f4
AD		 74
75MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
76MODULE_FIRMWARE("radeon/hawaii_me.bin");
77MODULE_FIRMWARE("radeon/hawaii_ce.bin");
78MODULE_FIRMWARE("radeon/hawaii_mec.bin");
79MODULE_FIRMWARE("radeon/hawaii_mc.bin");
80MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
81MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
82MODULE_FIRMWARE("radeon/hawaii_smc.bin");
b2ea0dcd 83MODULE_FIRMWARE("radeon/hawaii_k_smc.bin");
f2c6b0f4 84
02c81327
AD		 85MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
86MODULE_FIRMWARE("radeon/KAVERI_me.bin");
87MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
88MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
89MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 90MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
f2c6b0f4
AD		 91
92MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
93MODULE_FIRMWARE("radeon/kaveri_me.bin");
94MODULE_FIRMWARE("radeon/kaveri_ce.bin");
95MODULE_FIRMWARE("radeon/kaveri_mec.bin");
96MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
97MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
98MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
99
02c81327
AD		 100MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
101MODULE_FIRMWARE("radeon/KABINI_me.bin");
102MODULE_FIRMWARE("radeon/KABINI_ce.bin");
103MODULE_FIRMWARE("radeon/KABINI_mec.bin");
104MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 105MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
f2c6b0f4
AD		 106
107MODULE_FIRMWARE("radeon/kabini_pfp.bin");
108MODULE_FIRMWARE("radeon/kabini_me.bin");
109MODULE_FIRMWARE("radeon/kabini_ce.bin");
110MODULE_FIRMWARE("radeon/kabini_mec.bin");
111MODULE_FIRMWARE("radeon/kabini_rlc.bin");
112MODULE_FIRMWARE("radeon/kabini_sdma.bin");
113
f73a9e83
SL		 114MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
115MODULE_FIRMWARE("radeon/MULLINS_me.bin");
116MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
117MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
118MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
119MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
02c81327 120
f2c6b0f4
AD		 121MODULE_FIRMWARE("radeon/mullins_pfp.bin");
122MODULE_FIRMWARE("radeon/mullins_me.bin");
123MODULE_FIRMWARE("radeon/mullins_ce.bin");
124MODULE_FIRMWARE("radeon/mullins_mec.bin");
125MODULE_FIRMWARE("radeon/mullins_rlc.bin");
126MODULE_FIRMWARE("radeon/mullins_sdma.bin");
127
a59781bb
AD		 128extern int r600_ih_ring_alloc(struct radeon_device *rdev);
129extern void r600_ih_ring_fini(struct radeon_device *rdev);
1c49165d 130extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 131extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 132extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
65fcf668 133static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
2483b4ea
CK		 134extern int cik_sdma_resume(struct radeon_device *rdev);
135extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
136extern void cik_sdma_fini(struct radeon_device *rdev);
a1d6f97c 137extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
cc066715 138static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 139static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 140static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 141static void cik_init_pg(struct radeon_device *rdev);
142static void cik_init_cg(struct radeon_device *rdev);
fb2c7f4d
AD		 143static void cik_fini_pg(struct radeon_device *rdev);
144static void cik_fini_cg(struct radeon_device *rdev);
4214faf6
AD		 145static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
146 bool enable);
6f2043ce 147
353eec2a
AD		 148/**
149 * cik_get_allowed_info_register - fetch the register for the info ioctl
150 *
151 * @rdev: radeon_device pointer
152 * @reg: register offset in bytes
153 * @val: register value
154 *
155 * Returns 0 for success or -EINVAL for an invalid register
156 *
157 */
158int cik_get_allowed_info_register(struct radeon_device *rdev,
159 u32 reg, u32 *val)
160{
161 switch (reg) {
162 case GRBM_STATUS:
163 case GRBM_STATUS2:
164 case GRBM_STATUS_SE0:
165 case GRBM_STATUS_SE1:
166 case GRBM_STATUS_SE2:
167 case GRBM_STATUS_SE3:
168 case SRBM_STATUS:
169 case SRBM_STATUS2:
170 case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET):
171 case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET):
172 case UVD_STATUS:
173 /* TODO VCE */
174 *val = RREG32(reg);
175 return 0;
176 default:
177 return -EINVAL;
178 }
179}
180
9e5acbc2
DV		 181/*
182 * Indirect registers accessor
183 */
184u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)
185{
186 unsigned long flags;
187 u32 r;
188
189 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
190 WREG32(CIK_DIDT_IND_INDEX, (reg));
191 r = RREG32(CIK_DIDT_IND_DATA);
192 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
193 return r;
194}
195
196void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)
197{
198 unsigned long flags;
199
200 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
201 WREG32(CIK_DIDT_IND_INDEX, (reg));
202 WREG32(CIK_DIDT_IND_DATA, (v));
203 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
204}
205
286d9cc6
AD		 206/* get temperature in millidegrees */
207int ci_get_temp(struct radeon_device *rdev)
208{
209 u32 temp;
210 int actual_temp = 0;
211
212 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
213 CTF_TEMP_SHIFT;
214
215 if (temp & 0x200)
216 actual_temp = 255;
217 else
218 actual_temp = temp & 0x1ff;
219
847a75fb 220 return actual_temp * 1000;
286d9cc6
AD		 221}
222
223/* get temperature in millidegrees */
224int kv_get_temp(struct radeon_device *rdev)
225{
226 u32 temp;
227 int actual_temp = 0;
228
229 temp = RREG32_SMC(0xC0300E0C);
230
231 if (temp)
232 actual_temp = (temp / 8) - 49;
233 else
234 actual_temp = 0;
235
847a75fb 236 return actual_temp * 1000;
286d9cc6 237}
6f2043ce 238
6e2c3c0a
AD		 239/*
240 * Indirect registers accessor
241 */
242u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
243{
0a5b7b0b 244 unsigned long flags;
6e2c3c0a
AD		 245 u32 r;
246
0a5b7b0b 247 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 248 WREG32(PCIE_INDEX, reg);
249 (void)RREG32(PCIE_INDEX);
250 r = RREG32(PCIE_DATA);
0a5b7b0b 251 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 252 return r;
253}
254
255void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
256{
0a5b7b0b
AD		 257 unsigned long flags;
258
259 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 260 WREG32(PCIE_INDEX, reg);
261 (void)RREG32(PCIE_INDEX);
262 WREG32(PCIE_DATA, v);
263 (void)RREG32(PCIE_DATA);
0a5b7b0b 264 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 265}
266
22c775ce
AD		 267static const u32 spectre_rlc_save_restore_register_list[] =
268{
269 (0x0e00 << 16) | (0xc12c >> 2),
270 0x00000000,
271 (0x0e00 << 16) | (0xc140 >> 2),
272 0x00000000,
273 (0x0e00 << 16) | (0xc150 >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0xc15c >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0xc168 >> 2),
278 0x00000000,
279 (0x0e00 << 16) | (0xc170 >> 2),
280 0x00000000,
281 (0x0e00 << 16) | (0xc178 >> 2),
282 0x00000000,
283 (0x0e00 << 16) | (0xc204 >> 2),
284 0x00000000,
285 (0x0e00 << 16) | (0xc2b4 >> 2),
286 0x00000000,
287 (0x0e00 << 16) | (0xc2b8 >> 2),
288 0x00000000,
289 (0x0e00 << 16) | (0xc2bc >> 2),
290 0x00000000,
291 (0x0e00 << 16) | (0xc2c0 >> 2),
292 0x00000000,
293 (0x0e00 << 16) | (0x8228 >> 2),
294 0x00000000,
295 (0x0e00 << 16) | (0x829c >> 2),
296 0x00000000,
297 (0x0e00 << 16) | (0x869c >> 2),
298 0x00000000,
299 (0x0600 << 16) | (0x98f4 >> 2),
300 0x00000000,
301 (0x0e00 << 16) | (0x98f8 >> 2),
302 0x00000000,
303 (0x0e00 << 16) | (0x9900 >> 2),
304 0x00000000,
305 (0x0e00 << 16) | (0xc260 >> 2),
306 0x00000000,
307 (0x0e00 << 16) | (0x90e8 >> 2),
308 0x00000000,
309 (0x0e00 << 16) | (0x3c000 >> 2),
310 0x00000000,
311 (0x0e00 << 16) | (0x3c00c >> 2),
312 0x00000000,
313 (0x0e00 << 16) | (0x8c1c >> 2),
314 0x00000000,
315 (0x0e00 << 16) | (0x9700 >> 2),
316 0x00000000,
317 (0x0e00 << 16) | (0xcd20 >> 2),
318 0x00000000,
319 (0x4e00 << 16) | (0xcd20 >> 2),
320 0x00000000,
321 (0x5e00 << 16) | (0xcd20 >> 2),
322 0x00000000,
323 (0x6e00 << 16) | (0xcd20 >> 2),
324 0x00000000,
325 (0x7e00 << 16) | (0xcd20 >> 2),
326 0x00000000,
327 (0x8e00 << 16) | (0xcd20 >> 2),
328 0x00000000,
329 (0x9e00 << 16) | (0xcd20 >> 2),
330 0x00000000,
331 (0xae00 << 16) | (0xcd20 >> 2),
332 0x00000000,
333 (0xbe00 << 16) | (0xcd20 >> 2),
334 0x00000000,
335 (0x0e00 << 16) | (0x89bc >> 2),
336 0x00000000,
337 (0x0e00 << 16) | (0x8900 >> 2),
338 0x00000000,
339 0x3,
340 (0x0e00 << 16) | (0xc130 >> 2),
341 0x00000000,
342 (0x0e00 << 16) | (0xc134 >> 2),
343 0x00000000,
344 (0x0e00 << 16) | (0xc1fc >> 2),
345 0x00000000,
346 (0x0e00 << 16) | (0xc208 >> 2),
347 0x00000000,
348 (0x0e00 << 16) | (0xc264 >> 2),
349 0x00000000,
350 (0x0e00 << 16) | (0xc268 >> 2),
351 0x00000000,
352 (0x0e00 << 16) | (0xc26c >> 2),
353 0x00000000,
354 (0x0e00 << 16) | (0xc270 >> 2),
355 0x00000000,
356 (0x0e00 << 16) | (0xc274 >> 2),
357 0x00000000,
358 (0x0e00 << 16) | (0xc278 >> 2),
359 0x00000000,
360 (0x0e00 << 16) | (0xc27c >> 2),
361 0x00000000,
362 (0x0e00 << 16) | (0xc280 >> 2),
363 0x00000000,
364 (0x0e00 << 16) | (0xc284 >> 2),
365 0x00000000,
366 (0x0e00 << 16) | (0xc288 >> 2),
367 0x00000000,
368 (0x0e00 << 16) | (0xc28c >> 2),
369 0x00000000,
370 (0x0e00 << 16) | (0xc290 >> 2),
371 0x00000000,
372 (0x0e00 << 16) | (0xc294 >> 2),
373 0x00000000,
374 (0x0e00 << 16) | (0xc298 >> 2),
375 0x00000000,
376 (0x0e00 << 16) | (0xc29c >> 2),
377 0x00000000,
378 (0x0e00 << 16) | (0xc2a0 >> 2),
379 0x00000000,
380 (0x0e00 << 16) | (0xc2a4 >> 2),
381 0x00000000,
382 (0x0e00 << 16) | (0xc2a8 >> 2),
383 0x00000000,
384 (0x0e00 << 16) | (0xc2ac >> 2),
385 0x00000000,
386 (0x0e00 << 16) | (0xc2b0 >> 2),
387 0x00000000,
388 (0x0e00 << 16) | (0x301d0 >> 2),
389 0x00000000,
390 (0x0e00 << 16) | (0x30238 >> 2),
391 0x00000000,
392 (0x0e00 << 16) | (0x30250 >> 2),
393 0x00000000,
394 (0x0e00 << 16) | (0x30254 >> 2),
395 0x00000000,
396 (0x0e00 << 16) | (0x30258 >> 2),
397 0x00000000,
398 (0x0e00 << 16) | (0x3025c >> 2),
399 0x00000000,
400 (0x4e00 << 16) | (0xc900 >> 2),
401 0x00000000,
402 (0x5e00 << 16) | (0xc900 >> 2),
403 0x00000000,
404 (0x6e00 << 16) | (0xc900 >> 2),
405 0x00000000,
406 (0x7e00 << 16) | (0xc900 >> 2),
407 0x00000000,
408 (0x8e00 << 16) | (0xc900 >> 2),
409 0x00000000,
410 (0x9e00 << 16) | (0xc900 >> 2),
411 0x00000000,
412 (0xae00 << 16) | (0xc900 >> 2),
413 0x00000000,
414 (0xbe00 << 16) | (0xc900 >> 2),
415 0x00000000,
416 (0x4e00 << 16) | (0xc904 >> 2),
417 0x00000000,
418 (0x5e00 << 16) | (0xc904 >> 2),
419 0x00000000,
420 (0x6e00 << 16) | (0xc904 >> 2),
421 0x00000000,
422 (0x7e00 << 16) | (0xc904 >> 2),
423 0x00000000,
424 (0x8e00 << 16) | (0xc904 >> 2),
425 0x00000000,
426 (0x9e00 << 16) | (0xc904 >> 2),
427 0x00000000,
428 (0xae00 << 16) | (0xc904 >> 2),
429 0x00000000,
430 (0xbe00 << 16) | (0xc904 >> 2),
431 0x00000000,
432 (0x4e00 << 16) | (0xc908 >> 2),
433 0x00000000,
434 (0x5e00 << 16) | (0xc908 >> 2),
435 0x00000000,
436 (0x6e00 << 16) | (0xc908 >> 2),
437 0x00000000,
438 (0x7e00 << 16) | (0xc908 >> 2),
439 0x00000000,
440 (0x8e00 << 16) | (0xc908 >> 2),
441 0x00000000,
442 (0x9e00 << 16) | (0xc908 >> 2),
443 0x00000000,
444 (0xae00 << 16) | (0xc908 >> 2),
445 0x00000000,
446 (0xbe00 << 16) | (0xc908 >> 2),
447 0x00000000,
448 (0x4e00 << 16) | (0xc90c >> 2),
449 0x00000000,
450 (0x5e00 << 16) | (0xc90c >> 2),
451 0x00000000,
452 (0x6e00 << 16) | (0xc90c >> 2),
453 0x00000000,
454 (0x7e00 << 16) | (0xc90c >> 2),
455 0x00000000,
456 (0x8e00 << 16) | (0xc90c >> 2),
457 0x00000000,
458 (0x9e00 << 16) | (0xc90c >> 2),
459 0x00000000,
460 (0xae00 << 16) | (0xc90c >> 2),
461 0x00000000,
462 (0xbe00 << 16) | (0xc90c >> 2),
463 0x00000000,
464 (0x4e00 << 16) | (0xc910 >> 2),
465 0x00000000,
466 (0x5e00 << 16) | (0xc910 >> 2),
467 0x00000000,
468 (0x6e00 << 16) | (0xc910 >> 2),
469 0x00000000,
470 (0x7e00 << 16) | (0xc910 >> 2),
471 0x00000000,
472 (0x8e00 << 16) | (0xc910 >> 2),
473 0x00000000,
474 (0x9e00 << 16) | (0xc910 >> 2),
475 0x00000000,
476 (0xae00 << 16) | (0xc910 >> 2),
477 0x00000000,
478 (0xbe00 << 16) | (0xc910 >> 2),
479 0x00000000,
480 (0x0e00 << 16) | (0xc99c >> 2),
481 0x00000000,
482 (0x0e00 << 16) | (0x9834 >> 2),
483 0x00000000,
484 (0x0000 << 16) | (0x30f00 >> 2),
485 0x00000000,
486 (0x0001 << 16) | (0x30f00 >> 2),
487 0x00000000,
488 (0x0000 << 16) | (0x30f04 >> 2),
489 0x00000000,
490 (0x0001 << 16) | (0x30f04 >> 2),
491 0x00000000,
492 (0x0000 << 16) | (0x30f08 >> 2),
493 0x00000000,
494 (0x0001 << 16) | (0x30f08 >> 2),
495 0x00000000,
496 (0x0000 << 16) | (0x30f0c >> 2),
497 0x00000000,
498 (0x0001 << 16) | (0x30f0c >> 2),
499 0x00000000,
500 (0x0600 << 16) | (0x9b7c >> 2),
501 0x00000000,
502 (0x0e00 << 16) | (0x8a14 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0x8a18 >> 2),
505 0x00000000,
506 (0x0600 << 16) | (0x30a00 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0x8bf0 >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x8bcc >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x8b24 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x30a04 >> 2),
515 0x00000000,
516 (0x0600 << 16) | (0x30a10 >> 2),
517 0x00000000,
518 (0x0600 << 16) | (0x30a14 >> 2),
519 0x00000000,
520 (0x0600 << 16) | (0x30a18 >> 2),
521 0x00000000,
522 (0x0600 << 16) | (0x30a2c >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0xc700 >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0xc704 >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0xc708 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0xc768 >> 2),
531 0x00000000,
532 (0x0400 << 16) | (0xc770 >> 2),
533 0x00000000,
534 (0x0400 << 16) | (0xc774 >> 2),
535 0x00000000,
536 (0x0400 << 16) | (0xc778 >> 2),
537 0x00000000,
538 (0x0400 << 16) | (0xc77c >> 2),
539 0x00000000,
540 (0x0400 << 16) | (0xc780 >> 2),
541 0x00000000,
542 (0x0400 << 16) | (0xc784 >> 2),
543 0x00000000,
544 (0x0400 << 16) | (0xc788 >> 2),
545 0x00000000,
546 (0x0400 << 16) | (0xc78c >> 2),
547 0x00000000,
548 (0x0400 << 16) | (0xc798 >> 2),
549 0x00000000,
550 (0x0400 << 16) | (0xc79c >> 2),
551 0x00000000,
552 (0x0400 << 16) | (0xc7a0 >> 2),
553 0x00000000,
554 (0x0400 << 16) | (0xc7a4 >> 2),
555 0x00000000,
556 (0x0400 << 16) | (0xc7a8 >> 2),
557 0x00000000,
558 (0x0400 << 16) | (0xc7ac >> 2),
559 0x00000000,
560 (0x0400 << 16) | (0xc7b0 >> 2),
561 0x00000000,
562 (0x0400 << 16) | (0xc7b4 >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0x9100 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0x3c010 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0x92a8 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0x92ac >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x92b4 >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x92b8 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x92bc >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x92c0 >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x92c4 >> 2),
581 0x00000000,
582 (0x0e00 << 16) | (0x92c8 >> 2),
583 0x00000000,
584 (0x0e00 << 16) | (0x92cc >> 2),
585 0x00000000,
586 (0x0e00 << 16) | (0x92d0 >> 2),
587 0x00000000,
588 (0x0e00 << 16) | (0x8c00 >> 2),
589 0x00000000,
590 (0x0e00 << 16) | (0x8c04 >> 2),
591 0x00000000,
592 (0x0e00 << 16) | (0x8c20 >> 2),
593 0x00000000,
594 (0x0e00 << 16) | (0x8c38 >> 2),
595 0x00000000,
596 (0x0e00 << 16) | (0x8c3c >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0xae00 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0x9604 >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0xac08 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0xac0c >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0xac10 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0xac14 >> 2),
609 0x00000000,
610 (0x0e00 << 16) | (0xac58 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0xac68 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0xac6c >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0xac70 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0xac74 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0xac78 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0xac7c >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0xac80 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0xac84 >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0xac88 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0xac8c >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x970c >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x9714 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x9718 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x971c >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x31068 >> 2),
641 0x00000000,
642 (0x4e00 << 16) | (0x31068 >> 2),
643 0x00000000,
644 (0x5e00 << 16) | (0x31068 >> 2),
645 0x00000000,
646 (0x6e00 << 16) | (0x31068 >> 2),
647 0x00000000,
648 (0x7e00 << 16) | (0x31068 >> 2),
649 0x00000000,
650 (0x8e00 << 16) | (0x31068 >> 2),
651 0x00000000,
652 (0x9e00 << 16) | (0x31068 >> 2),
653 0x00000000,
654 (0xae00 << 16) | (0x31068 >> 2),
655 0x00000000,
656 (0xbe00 << 16) | (0x31068 >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0xcd10 >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0xcd14 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0x88b0 >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0x88b4 >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0x88b8 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0x88bc >> 2),
669 0x00000000,
670 (0x0400 << 16) | (0x89c0 >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0x88c4 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0x88c8 >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0x88d0 >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0x88d4 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0x88d8 >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0x8980 >> 2),
683 0x00000000,
684 (0x0e00 << 16) | (0x30938 >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0x3093c >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0x30940 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0x89a0 >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0x30900 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x30904 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x89b4 >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x3c210 >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x3c214 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0x3c218 >> 2),
703 0x00000000,
704 (0x0e00 << 16) | (0x8904 >> 2),
705 0x00000000,
706 0x5,
707 (0x0e00 << 16) | (0x8c28 >> 2),
708 (0x0e00 << 16) | (0x8c2c >> 2),
709 (0x0e00 << 16) | (0x8c30 >> 2),
710 (0x0e00 << 16) | (0x8c34 >> 2),
711 (0x0e00 << 16) | (0x9600 >> 2),
712};
713
714static const u32 kalindi_rlc_save_restore_register_list[] =
715{
716 (0x0e00 << 16) | (0xc12c >> 2),
717 0x00000000,
718 (0x0e00 << 16) | (0xc140 >> 2),
719 0x00000000,
720 (0x0e00 << 16) | (0xc150 >> 2),
721 0x00000000,
722 (0x0e00 << 16) | (0xc15c >> 2),
723 0x00000000,
724 (0x0e00 << 16) | (0xc168 >> 2),
725 0x00000000,
726 (0x0e00 << 16) | (0xc170 >> 2),
727 0x00000000,
728 (0x0e00 << 16) | (0xc204 >> 2),
729 0x00000000,
730 (0x0e00 << 16) | (0xc2b4 >> 2),
731 0x00000000,
732 (0x0e00 << 16) | (0xc2b8 >> 2),
733 0x00000000,
734 (0x0e00 << 16) | (0xc2bc >> 2),
735 0x00000000,
736 (0x0e00 << 16) | (0xc2c0 >> 2),
737 0x00000000,
738 (0x0e00 << 16) | (0x8228 >> 2),
739 0x00000000,
740 (0x0e00 << 16) | (0x829c >> 2),
741 0x00000000,
742 (0x0e00 << 16) | (0x869c >> 2),
743 0x00000000,
744 (0x0600 << 16) | (0x98f4 >> 2),
745 0x00000000,
746 (0x0e00 << 16) | (0x98f8 >> 2),
747 0x00000000,
748 (0x0e00 << 16) | (0x9900 >> 2),
749 0x00000000,
750 (0x0e00 << 16) | (0xc260 >> 2),
751 0x00000000,
752 (0x0e00 << 16) | (0x90e8 >> 2),
753 0x00000000,
754 (0x0e00 << 16) | (0x3c000 >> 2),
755 0x00000000,
756 (0x0e00 << 16) | (0x3c00c >> 2),
757 0x00000000,
758 (0x0e00 << 16) | (0x8c1c >> 2),
759 0x00000000,
760 (0x0e00 << 16) | (0x9700 >> 2),
761 0x00000000,
762 (0x0e00 << 16) | (0xcd20 >> 2),
763 0x00000000,
764 (0x4e00 << 16) | (0xcd20 >> 2),
765 0x00000000,
766 (0x5e00 << 16) | (0xcd20 >> 2),
767 0x00000000,
768 (0x6e00 << 16) | (0xcd20 >> 2),
769 0x00000000,
770 (0x7e00 << 16) | (0xcd20 >> 2),
771 0x00000000,
772 (0x0e00 << 16) | (0x89bc >> 2),
773 0x00000000,
774 (0x0e00 << 16) | (0x8900 >> 2),
775 0x00000000,
776 0x3,
777 (0x0e00 << 16) | (0xc130 >> 2),
778 0x00000000,
779 (0x0e00 << 16) | (0xc134 >> 2),
780 0x00000000,
781 (0x0e00 << 16) | (0xc1fc >> 2),
782 0x00000000,
783 (0x0e00 << 16) | (0xc208 >> 2),
784 0x00000000,
785 (0x0e00 << 16) | (0xc264 >> 2),
786 0x00000000,
787 (0x0e00 << 16) | (0xc268 >> 2),
788 0x00000000,
789 (0x0e00 << 16) | (0xc26c >> 2),
790 0x00000000,
791 (0x0e00 << 16) | (0xc270 >> 2),
792 0x00000000,
793 (0x0e00 << 16) | (0xc274 >> 2),
794 0x00000000,
795 (0x0e00 << 16) | (0xc28c >> 2),
796 0x00000000,
797 (0x0e00 << 16) | (0xc290 >> 2),
798 0x00000000,
799 (0x0e00 << 16) | (0xc294 >> 2),
800 0x00000000,
801 (0x0e00 << 16) | (0xc298 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0xc2a0 >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0xc2a4 >> 2),
806 0x00000000,
807 (0x0e00 << 16) | (0xc2a8 >> 2),
808 0x00000000,
809 (0x0e00 << 16) | (0xc2ac >> 2),
810 0x00000000,
811 (0x0e00 << 16) | (0x301d0 >> 2),
812 0x00000000,
813 (0x0e00 << 16) | (0x30238 >> 2),
814 0x00000000,
815 (0x0e00 << 16) | (0x30250 >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x30254 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0x30258 >> 2),
820 0x00000000,
821 (0x0e00 << 16) | (0x3025c >> 2),
822 0x00000000,
823 (0x4e00 << 16) | (0xc900 >> 2),
824 0x00000000,
825 (0x5e00 << 16) | (0xc900 >> 2),
826 0x00000000,
827 (0x6e00 << 16) | (0xc900 >> 2),
828 0x00000000,
829 (0x7e00 << 16) | (0xc900 >> 2),
830 0x00000000,
831 (0x4e00 << 16) | (0xc904 >> 2),
832 0x00000000,
833 (0x5e00 << 16) | (0xc904 >> 2),
834 0x00000000,
835 (0x6e00 << 16) | (0xc904 >> 2),
836 0x00000000,
837 (0x7e00 << 16) | (0xc904 >> 2),
838 0x00000000,
839 (0x4e00 << 16) | (0xc908 >> 2),
840 0x00000000,
841 (0x5e00 << 16) | (0xc908 >> 2),
842 0x00000000,
843 (0x6e00 << 16) | (0xc908 >> 2),
844 0x00000000,
845 (0x7e00 << 16) | (0xc908 >> 2),
846 0x00000000,
847 (0x4e00 << 16) | (0xc90c >> 2),
848 0x00000000,
849 (0x5e00 << 16) | (0xc90c >> 2),
850 0x00000000,
851 (0x6e00 << 16) | (0xc90c >> 2),
852 0x00000000,
853 (0x7e00 << 16) | (0xc90c >> 2),
854 0x00000000,
855 (0x4e00 << 16) | (0xc910 >> 2),
856 0x00000000,
857 (0x5e00 << 16) | (0xc910 >> 2),
858 0x00000000,
859 (0x6e00 << 16) | (0xc910 >> 2),
860 0x00000000,
861 (0x7e00 << 16) | (0xc910 >> 2),
862 0x00000000,
863 (0x0e00 << 16) | (0xc99c >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0x9834 >> 2),
866 0x00000000,
867 (0x0000 << 16) | (0x30f00 >> 2),
868 0x00000000,
869 (0x0000 << 16) | (0x30f04 >> 2),
870 0x00000000,
871 (0x0000 << 16) | (0x30f08 >> 2),
872 0x00000000,
873 (0x0000 << 16) | (0x30f0c >> 2),
874 0x00000000,
875 (0x0600 << 16) | (0x9b7c >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0x8a14 >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0x8a18 >> 2),
880 0x00000000,
881 (0x0600 << 16) | (0x30a00 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0x8bf0 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0x8bcc >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0x8b24 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0x30a04 >> 2),
890 0x00000000,
891 (0x0600 << 16) | (0x30a10 >> 2),
892 0x00000000,
893 (0x0600 << 16) | (0x30a14 >> 2),
894 0x00000000,
895 (0x0600 << 16) | (0x30a18 >> 2),
896 0x00000000,
897 (0x0600 << 16) | (0x30a2c >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0xc700 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0xc704 >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0xc708 >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0xc768 >> 2),
906 0x00000000,
907 (0x0400 << 16) | (0xc770 >> 2),
908 0x00000000,
909 (0x0400 << 16) | (0xc774 >> 2),
910 0x00000000,
911 (0x0400 << 16) | (0xc798 >> 2),
912 0x00000000,
913 (0x0400 << 16) | (0xc79c >> 2),
914 0x00000000,
915 (0x0e00 << 16) | (0x9100 >> 2),
916 0x00000000,
917 (0x0e00 << 16) | (0x3c010 >> 2),
918 0x00000000,
919 (0x0e00 << 16) | (0x8c00 >> 2),
920 0x00000000,
921 (0x0e00 << 16) | (0x8c04 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0x8c20 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0x8c38 >> 2),
926 0x00000000,
927 (0x0e00 << 16) | (0x8c3c >> 2),
928 0x00000000,
929 (0x0e00 << 16) | (0xae00 >> 2),
930 0x00000000,
931 (0x0e00 << 16) | (0x9604 >> 2),
932 0x00000000,
933 (0x0e00 << 16) | (0xac08 >> 2),
934 0x00000000,
935 (0x0e00 << 16) | (0xac0c >> 2),
936 0x00000000,
937 (0x0e00 << 16) | (0xac10 >> 2),
938 0x00000000,
939 (0x0e00 << 16) | (0xac14 >> 2),
940 0x00000000,
941 (0x0e00 << 16) | (0xac58 >> 2),
942 0x00000000,
943 (0x0e00 << 16) | (0xac68 >> 2),
944 0x00000000,
945 (0x0e00 << 16) | (0xac6c >> 2),
946 0x00000000,
947 (0x0e00 << 16) | (0xac70 >> 2),
948 0x00000000,
949 (0x0e00 << 16) | (0xac74 >> 2),
950 0x00000000,
951 (0x0e00 << 16) | (0xac78 >> 2),
952 0x00000000,
953 (0x0e00 << 16) | (0xac7c >> 2),
954 0x00000000,
955 (0x0e00 << 16) | (0xac80 >> 2),
956 0x00000000,
957 (0x0e00 << 16) | (0xac84 >> 2),
958 0x00000000,
959 (0x0e00 << 16) | (0xac88 >> 2),
960 0x00000000,
961 (0x0e00 << 16) | (0xac8c >> 2),
962 0x00000000,
963 (0x0e00 << 16) | (0x970c >> 2),
964 0x00000000,
965 (0x0e00 << 16) | (0x9714 >> 2),
966 0x00000000,
967 (0x0e00 << 16) | (0x9718 >> 2),
968 0x00000000,
969 (0x0e00 << 16) | (0x971c >> 2),
970 0x00000000,
971 (0x0e00 << 16) | (0x31068 >> 2),
972 0x00000000,
973 (0x4e00 << 16) | (0x31068 >> 2),
974 0x00000000,
975 (0x5e00 << 16) | (0x31068 >> 2),
976 0x00000000,
977 (0x6e00 << 16) | (0x31068 >> 2),
978 0x00000000,
979 (0x7e00 << 16) | (0x31068 >> 2),
980 0x00000000,
981 (0x0e00 << 16) | (0xcd10 >> 2),
982 0x00000000,
983 (0x0e00 << 16) | (0xcd14 >> 2),
984 0x00000000,
985 (0x0e00 << 16) | (0x88b0 >> 2),
986 0x00000000,
987 (0x0e00 << 16) | (0x88b4 >> 2),
988 0x00000000,
989 (0x0e00 << 16) | (0x88b8 >> 2),
990 0x00000000,
991 (0x0e00 << 16) | (0x88bc >> 2),
992 0x00000000,
993 (0x0400 << 16) | (0x89c0 >> 2),
994 0x00000000,
995 (0x0e00 << 16) | (0x88c4 >> 2),
996 0x00000000,
997 (0x0e00 << 16) | (0x88c8 >> 2),
998 0x00000000,
999 (0x0e00 << 16) | (0x88d0 >> 2),
1000 0x00000000,
1001 (0x0e00 << 16) | (0x88d4 >> 2),
1002 0x00000000,
1003 (0x0e00 << 16) | (0x88d8 >> 2),
1004 0x00000000,
1005 (0x0e00 << 16) | (0x8980 >> 2),
1006 0x00000000,
1007 (0x0e00 << 16) | (0x30938 >> 2),
1008 0x00000000,
1009 (0x0e00 << 16) | (0x3093c >> 2),
1010 0x00000000,
1011 (0x0e00 << 16) | (0x30940 >> 2),
1012 0x00000000,
1013 (0x0e00 << 16) | (0x89a0 >> 2),
1014 0x00000000,
1015 (0x0e00 << 16) | (0x30900 >> 2),
1016 0x00000000,
1017 (0x0e00 << 16) | (0x30904 >> 2),
1018 0x00000000,
1019 (0x0e00 << 16) | (0x89b4 >> 2),
1020 0x00000000,
1021 (0x0e00 << 16) | (0x3e1fc >> 2),
1022 0x00000000,
1023 (0x0e00 << 16) | (0x3c210 >> 2),
1024 0x00000000,
1025 (0x0e00 << 16) | (0x3c214 >> 2),
1026 0x00000000,
1027 (0x0e00 << 16) | (0x3c218 >> 2),
1028 0x00000000,
1029 (0x0e00 << 16) | (0x8904 >> 2),
1030 0x00000000,
1031 0x5,
1032 (0x0e00 << 16) | (0x8c28 >> 2),
1033 (0x0e00 << 16) | (0x8c2c >> 2),
1034 (0x0e00 << 16) | (0x8c30 >> 2),
1035 (0x0e00 << 16) | (0x8c34 >> 2),
1036 (0x0e00 << 16) | (0x9600 >> 2),
1037};
1038
0aafd313
AD		 1039static const u32 bonaire_golden_spm_registers[] =
1040{
1041 0x30800, 0xe0ffffff, 0xe0000000
1042};
1043
1044static const u32 bonaire_golden_common_registers[] =
1045{
1046 0xc770, 0xffffffff, 0x00000800,
1047 0xc774, 0xffffffff, 0x00000800,
1048 0xc798, 0xffffffff, 0x00007fbf,
1049 0xc79c, 0xffffffff, 0x00007faf
1050};
1051
1052static const u32 bonaire_golden_registers[] =
1053{
1054 0x3354, 0x00000333, 0x00000333,
1055 0x3350, 0x000c0fc0, 0x00040200,
1056 0x9a10, 0x00010000, 0x00058208,
1057 0x3c000, 0xffff1fff, 0x00140000,
1058 0x3c200, 0xfdfc0fff, 0x00000100,
1059 0x3c234, 0x40000000, 0x40000200,
1060 0x9830, 0xffffffff, 0x00000000,
1061 0x9834, 0xf00fffff, 0x00000400,
1062 0x9838, 0x0002021c, 0x00020200,
1063 0xc78, 0x00000080, 0x00000000,
1064 0x5bb0, 0x000000f0, 0x00000070,
1065 0x5bc0, 0xf0311fff, 0x80300000,
1066 0x98f8, 0x73773777, 0x12010001,
1067 0x350c, 0x00810000, 0x408af000,
1068 0x7030, 0x31000111, 0x00000011,
1069 0x2f48, 0x73773777, 0x12010001,
1070 0x220c, 0x00007fb6, 0x0021a1b1,
1071 0x2210, 0x00007fb6, 0x002021b1,
1072 0x2180, 0x00007fb6, 0x00002191,
1073 0x2218, 0x00007fb6, 0x002121b1,
1074 0x221c, 0x00007fb6, 0x002021b1,
1075 0x21dc, 0x00007fb6, 0x00002191,
1076 0x21e0, 0x00007fb6, 0x00002191,
1077 0x3628, 0x0000003f, 0x0000000a,
1078 0x362c, 0x0000003f, 0x0000000a,
1079 0x2ae4, 0x00073ffe, 0x000022a2,
1080 0x240c, 0x000007ff, 0x00000000,
1081 0x8a14, 0xf000003f, 0x00000007,
1082 0x8bf0, 0x00002001, 0x00000001,
1083 0x8b24, 0xffffffff, 0x00ffffff,
1084 0x30a04, 0x0000ff0f, 0x00000000,
1085 0x28a4c, 0x07ffffff, 0x06000000,
1086 0x4d8, 0x00000fff, 0x00000100,
1087 0x3e78, 0x00000001, 0x00000002,
1088 0x9100, 0x03000000, 0x0362c688,
1089 0x8c00, 0x000000ff, 0x00000001,
1090 0xe40, 0x00001fff, 0x00001fff,
1091 0x9060, 0x0000007f, 0x00000020,
1092 0x9508, 0x00010000, 0x00010000,
1093 0xac14, 0x000003ff, 0x000000f3,
1094 0xac0c, 0xffffffff, 0x00001032
1095};
1096
1097static const u32 bonaire_mgcg_cgcg_init[] =
1098{
1099 0xc420, 0xffffffff, 0xfffffffc,
1100 0x30800, 0xffffffff, 0xe0000000,
1101 0x3c2a0, 0xffffffff, 0x00000100,
1102 0x3c208, 0xffffffff, 0x00000100,
1103 0x3c2c0, 0xffffffff, 0xc0000100,
1104 0x3c2c8, 0xffffffff, 0xc0000100,
1105 0x3c2c4, 0xffffffff, 0xc0000100,
1106 0x55e4, 0xffffffff, 0x00600100,
1107 0x3c280, 0xffffffff, 0x00000100,
1108 0x3c214, 0xffffffff, 0x06000100,
1109 0x3c220, 0xffffffff, 0x00000100,
1110 0x3c218, 0xffffffff, 0x06000100,
1111 0x3c204, 0xffffffff, 0x00000100,
1112 0x3c2e0, 0xffffffff, 0x00000100,
1113 0x3c224, 0xffffffff, 0x00000100,
1114 0x3c200, 0xffffffff, 0x00000100,
1115 0x3c230, 0xffffffff, 0x00000100,
1116 0x3c234, 0xffffffff, 0x00000100,
1117 0x3c250, 0xffffffff, 0x00000100,
1118 0x3c254, 0xffffffff, 0x00000100,
1119 0x3c258, 0xffffffff, 0x00000100,
1120 0x3c25c, 0xffffffff, 0x00000100,
1121 0x3c260, 0xffffffff, 0x00000100,
1122 0x3c27c, 0xffffffff, 0x00000100,
1123 0x3c278, 0xffffffff, 0x00000100,
1124 0x3c210, 0xffffffff, 0x06000100,
1125 0x3c290, 0xffffffff, 0x00000100,
1126 0x3c274, 0xffffffff, 0x00000100,
1127 0x3c2b4, 0xffffffff, 0x00000100,
1128 0x3c2b0, 0xffffffff, 0x00000100,
1129 0x3c270, 0xffffffff, 0x00000100,
1130 0x30800, 0xffffffff, 0xe0000000,
1131 0x3c020, 0xffffffff, 0x00010000,
1132 0x3c024, 0xffffffff, 0x00030002,
1133 0x3c028, 0xffffffff, 0x00040007,
1134 0x3c02c, 0xffffffff, 0x00060005,
1135 0x3c030, 0xffffffff, 0x00090008,
1136 0x3c034, 0xffffffff, 0x00010000,
1137 0x3c038, 0xffffffff, 0x00030002,
1138 0x3c03c, 0xffffffff, 0x00040007,
1139 0x3c040, 0xffffffff, 0x00060005,
1140 0x3c044, 0xffffffff, 0x00090008,
1141 0x3c048, 0xffffffff, 0x00010000,
1142 0x3c04c, 0xffffffff, 0x00030002,
1143 0x3c050, 0xffffffff, 0x00040007,
1144 0x3c054, 0xffffffff, 0x00060005,
1145 0x3c058, 0xffffffff, 0x00090008,
1146 0x3c05c, 0xffffffff, 0x00010000,
1147 0x3c060, 0xffffffff, 0x00030002,
1148 0x3c064, 0xffffffff, 0x00040007,
1149 0x3c068, 0xffffffff, 0x00060005,
1150 0x3c06c, 0xffffffff, 0x00090008,
1151 0x3c070, 0xffffffff, 0x00010000,
1152 0x3c074, 0xffffffff, 0x00030002,
1153 0x3c078, 0xffffffff, 0x00040007,
1154 0x3c07c, 0xffffffff, 0x00060005,
1155 0x3c080, 0xffffffff, 0x00090008,
1156 0x3c084, 0xffffffff, 0x00010000,
1157 0x3c088, 0xffffffff, 0x00030002,
1158 0x3c08c, 0xffffffff, 0x00040007,
1159 0x3c090, 0xffffffff, 0x00060005,
1160 0x3c094, 0xffffffff, 0x00090008,
1161 0x3c098, 0xffffffff, 0x00010000,
1162 0x3c09c, 0xffffffff, 0x00030002,
1163 0x3c0a0, 0xffffffff, 0x00040007,
1164 0x3c0a4, 0xffffffff, 0x00060005,
1165 0x3c0a8, 0xffffffff, 0x00090008,
1166 0x3c000, 0xffffffff, 0x96e00200,
1167 0x8708, 0xffffffff, 0x00900100,
1168 0xc424, 0xffffffff, 0x0020003f,
1169 0x38, 0xffffffff, 0x0140001c,
1170 0x3c, 0x000f0000, 0x000f0000,
1171 0x220, 0xffffffff, 0xC060000C,
1172 0x224, 0xc0000fff, 0x00000100,
1173 0xf90, 0xffffffff, 0x00000100,
1174 0xf98, 0x00000101, 0x00000000,
1175 0x20a8, 0xffffffff, 0x00000104,
1176 0x55e4, 0xff000fff, 0x00000100,
1177 0x30cc, 0xc0000fff, 0x00000104,
1178 0xc1e4, 0x00000001, 0x00000001,
1179 0xd00c, 0xff000ff0, 0x00000100,
1180 0xd80c, 0xff000ff0, 0x00000100
1181};
1182
1183static const u32 spectre_golden_spm_registers[] =
1184{
1185 0x30800, 0xe0ffffff, 0xe0000000
1186};
1187
1188static const u32 spectre_golden_common_registers[] =
1189{
1190 0xc770, 0xffffffff, 0x00000800,
1191 0xc774, 0xffffffff, 0x00000800,
1192 0xc798, 0xffffffff, 0x00007fbf,
1193 0xc79c, 0xffffffff, 0x00007faf
1194};
1195
1196static const u32 spectre_golden_registers[] =
1197{
1198 0x3c000, 0xffff1fff, 0x96940200,
1199 0x3c00c, 0xffff0001, 0xff000000,
1200 0x3c200, 0xfffc0fff, 0x00000100,
1201 0x6ed8, 0x00010101, 0x00010000,
1202 0x9834, 0xf00fffff, 0x00000400,
1203 0x9838, 0xfffffffc, 0x00020200,
1204 0x5bb0, 0x000000f0, 0x00000070,
1205 0x5bc0, 0xf0311fff, 0x80300000,
1206 0x98f8, 0x73773777, 0x12010001,
1207 0x9b7c, 0x00ff0000, 0x00fc0000,
1208 0x2f48, 0x73773777, 0x12010001,
1209 0x8a14, 0xf000003f, 0x00000007,
1210 0x8b24, 0xffffffff, 0x00ffffff,
1211 0x28350, 0x3f3f3fff, 0x00000082,
f1553174 1212 0x28354, 0x0000003f, 0x00000000,
0aafd313
AD		 1213 0x3e78, 0x00000001, 0x00000002,
1214 0x913c, 0xffff03df, 0x00000004,
1215 0xc768, 0x00000008, 0x00000008,
1216 0x8c00, 0x000008ff, 0x00000800,
1217 0x9508, 0x00010000, 0x00010000,
1218 0xac0c, 0xffffffff, 0x54763210,
1219 0x214f8, 0x01ff01ff, 0x00000002,
1220 0x21498, 0x007ff800, 0x00200000,
1221 0x2015c, 0xffffffff, 0x00000f40,
1222 0x30934, 0xffffffff, 0x00000001
1223};
1224
1225static const u32 spectre_mgcg_cgcg_init[] =
1226{
1227 0xc420, 0xffffffff, 0xfffffffc,
1228 0x30800, 0xffffffff, 0xe0000000,
1229 0x3c2a0, 0xffffffff, 0x00000100,
1230 0x3c208, 0xffffffff, 0x00000100,
1231 0x3c2c0, 0xffffffff, 0x00000100,
1232 0x3c2c8, 0xffffffff, 0x00000100,
1233 0x3c2c4, 0xffffffff, 0x00000100,
1234 0x55e4, 0xffffffff, 0x00600100,
1235 0x3c280, 0xffffffff, 0x00000100,
1236 0x3c214, 0xffffffff, 0x06000100,
1237 0x3c220, 0xffffffff, 0x00000100,
1238 0x3c218, 0xffffffff, 0x06000100,
1239 0x3c204, 0xffffffff, 0x00000100,
1240 0x3c2e0, 0xffffffff, 0x00000100,
1241 0x3c224, 0xffffffff, 0x00000100,
1242 0x3c200, 0xffffffff, 0x00000100,
1243 0x3c230, 0xffffffff, 0x00000100,
1244 0x3c234, 0xffffffff, 0x00000100,
1245 0x3c250, 0xffffffff, 0x00000100,
1246 0x3c254, 0xffffffff, 0x00000100,
1247 0x3c258, 0xffffffff, 0x00000100,
1248 0x3c25c, 0xffffffff, 0x00000100,
1249 0x3c260, 0xffffffff, 0x00000100,
1250 0x3c27c, 0xffffffff, 0x00000100,
1251 0x3c278, 0xffffffff, 0x00000100,
1252 0x3c210, 0xffffffff, 0x06000100,
1253 0x3c290, 0xffffffff, 0x00000100,
1254 0x3c274, 0xffffffff, 0x00000100,
1255 0x3c2b4, 0xffffffff, 0x00000100,
1256 0x3c2b0, 0xffffffff, 0x00000100,
1257 0x3c270, 0xffffffff, 0x00000100,
1258 0x30800, 0xffffffff, 0xe0000000,
1259 0x3c020, 0xffffffff, 0x00010000,
1260 0x3c024, 0xffffffff, 0x00030002,
1261 0x3c028, 0xffffffff, 0x00040007,
1262 0x3c02c, 0xffffffff, 0x00060005,
1263 0x3c030, 0xffffffff, 0x00090008,
1264 0x3c034, 0xffffffff, 0x00010000,
1265 0x3c038, 0xffffffff, 0x00030002,
1266 0x3c03c, 0xffffffff, 0x00040007,
1267 0x3c040, 0xffffffff, 0x00060005,
1268 0x3c044, 0xffffffff, 0x00090008,
1269 0x3c048, 0xffffffff, 0x00010000,
1270 0x3c04c, 0xffffffff, 0x00030002,
1271 0x3c050, 0xffffffff, 0x00040007,
1272 0x3c054, 0xffffffff, 0x00060005,
1273 0x3c058, 0xffffffff, 0x00090008,
1274 0x3c05c, 0xffffffff, 0x00010000,
1275 0x3c060, 0xffffffff, 0x00030002,
1276 0x3c064, 0xffffffff, 0x00040007,
1277 0x3c068, 0xffffffff, 0x00060005,
1278 0x3c06c, 0xffffffff, 0x00090008,
1279 0x3c070, 0xffffffff, 0x00010000,
1280 0x3c074, 0xffffffff, 0x00030002,
1281 0x3c078, 0xffffffff, 0x00040007,
1282 0x3c07c, 0xffffffff, 0x00060005,
1283 0x3c080, 0xffffffff, 0x00090008,
1284 0x3c084, 0xffffffff, 0x00010000,
1285 0x3c088, 0xffffffff, 0x00030002,
1286 0x3c08c, 0xffffffff, 0x00040007,
1287 0x3c090, 0xffffffff, 0x00060005,
1288 0x3c094, 0xffffffff, 0x00090008,
1289 0x3c098, 0xffffffff, 0x00010000,
1290 0x3c09c, 0xffffffff, 0x00030002,
1291 0x3c0a0, 0xffffffff, 0x00040007,
1292 0x3c0a4, 0xffffffff, 0x00060005,
1293 0x3c0a8, 0xffffffff, 0x00090008,
1294 0x3c0ac, 0xffffffff, 0x00010000,
1295 0x3c0b0, 0xffffffff, 0x00030002,
1296 0x3c0b4, 0xffffffff, 0x00040007,
1297 0x3c0b8, 0xffffffff, 0x00060005,
1298 0x3c0bc, 0xffffffff, 0x00090008,
1299 0x3c000, 0xffffffff, 0x96e00200,
1300 0x8708, 0xffffffff, 0x00900100,
1301 0xc424, 0xffffffff, 0x0020003f,
1302 0x38, 0xffffffff, 0x0140001c,
1303 0x3c, 0x000f0000, 0x000f0000,
1304 0x220, 0xffffffff, 0xC060000C,
1305 0x224, 0xc0000fff, 0x00000100,
1306 0xf90, 0xffffffff, 0x00000100,
1307 0xf98, 0x00000101, 0x00000000,
1308 0x20a8, 0xffffffff, 0x00000104,
1309 0x55e4, 0xff000fff, 0x00000100,
1310 0x30cc, 0xc0000fff, 0x00000104,
1311 0xc1e4, 0x00000001, 0x00000001,
1312 0xd00c, 0xff000ff0, 0x00000100,
1313 0xd80c, 0xff000ff0, 0x00000100
1314};
1315
1316static const u32 kalindi_golden_spm_registers[] =
1317{
1318 0x30800, 0xe0ffffff, 0xe0000000
1319};
1320
1321static const u32 kalindi_golden_common_registers[] =
1322{
1323 0xc770, 0xffffffff, 0x00000800,
1324 0xc774, 0xffffffff, 0x00000800,
1325 0xc798, 0xffffffff, 0x00007fbf,
1326 0xc79c, 0xffffffff, 0x00007faf
1327};
1328
1329static const u32 kalindi_golden_registers[] =
1330{
1331 0x3c000, 0xffffdfff, 0x6e944040,
1332 0x55e4, 0xff607fff, 0xfc000100,
1333 0x3c220, 0xff000fff, 0x00000100,
1334 0x3c224, 0xff000fff, 0x00000100,
1335 0x3c200, 0xfffc0fff, 0x00000100,
1336 0x6ed8, 0x00010101, 0x00010000,
1337 0x9830, 0xffffffff, 0x00000000,
1338 0x9834, 0xf00fffff, 0x00000400,
1339 0x5bb0, 0x000000f0, 0x00000070,
1340 0x5bc0, 0xf0311fff, 0x80300000,
1341 0x98f8, 0x73773777, 0x12010001,
1342 0x98fc, 0xffffffff, 0x00000010,
1343 0x9b7c, 0x00ff0000, 0x00fc0000,
1344 0x8030, 0x00001f0f, 0x0000100a,
1345 0x2f48, 0x73773777, 0x12010001,
1346 0x2408, 0x000fffff, 0x000c007f,
1347 0x8a14, 0xf000003f, 0x00000007,
1348 0x8b24, 0x3fff3fff, 0x00ffcfff,
1349 0x30a04, 0x0000ff0f, 0x00000000,
1350 0x28a4c, 0x07ffffff, 0x06000000,
1351 0x4d8, 0x00000fff, 0x00000100,
1352 0x3e78, 0x00000001, 0x00000002,
1353 0xc768, 0x00000008, 0x00000008,
1354 0x8c00, 0x000000ff, 0x00000003,
1355 0x214f8, 0x01ff01ff, 0x00000002,
1356 0x21498, 0x007ff800, 0x00200000,
1357 0x2015c, 0xffffffff, 0x00000f40,
1358 0x88c4, 0x001f3ae3, 0x00000082,
1359 0x88d4, 0x0000001f, 0x00000010,
1360 0x30934, 0xffffffff, 0x00000000
1361};
1362
1363static const u32 kalindi_mgcg_cgcg_init[] =
1364{
1365 0xc420, 0xffffffff, 0xfffffffc,
1366 0x30800, 0xffffffff, 0xe0000000,
1367 0x3c2a0, 0xffffffff, 0x00000100,
1368 0x3c208, 0xffffffff, 0x00000100,
1369 0x3c2c0, 0xffffffff, 0x00000100,
1370 0x3c2c8, 0xffffffff, 0x00000100,
1371 0x3c2c4, 0xffffffff, 0x00000100,
1372 0x55e4, 0xffffffff, 0x00600100,
1373 0x3c280, 0xffffffff, 0x00000100,
1374 0x3c214, 0xffffffff, 0x06000100,
1375 0x3c220, 0xffffffff, 0x00000100,
1376 0x3c218, 0xffffffff, 0x06000100,
1377 0x3c204, 0xffffffff, 0x00000100,
1378 0x3c2e0, 0xffffffff, 0x00000100,
1379 0x3c224, 0xffffffff, 0x00000100,
1380 0x3c200, 0xffffffff, 0x00000100,
1381 0x3c230, 0xffffffff, 0x00000100,
1382 0x3c234, 0xffffffff, 0x00000100,
1383 0x3c250, 0xffffffff, 0x00000100,
1384 0x3c254, 0xffffffff, 0x00000100,
1385 0x3c258, 0xffffffff, 0x00000100,
1386 0x3c25c, 0xffffffff, 0x00000100,
1387 0x3c260, 0xffffffff, 0x00000100,
1388 0x3c27c, 0xffffffff, 0x00000100,
1389 0x3c278, 0xffffffff, 0x00000100,
1390 0x3c210, 0xffffffff, 0x06000100,
1391 0x3c290, 0xffffffff, 0x00000100,
1392 0x3c274, 0xffffffff, 0x00000100,
1393 0x3c2b4, 0xffffffff, 0x00000100,
1394 0x3c2b0, 0xffffffff, 0x00000100,
1395 0x3c270, 0xffffffff, 0x00000100,
1396 0x30800, 0xffffffff, 0xe0000000,
1397 0x3c020, 0xffffffff, 0x00010000,
1398 0x3c024, 0xffffffff, 0x00030002,
1399 0x3c028, 0xffffffff, 0x00040007,
1400 0x3c02c, 0xffffffff, 0x00060005,
1401 0x3c030, 0xffffffff, 0x00090008,
1402 0x3c034, 0xffffffff, 0x00010000,
1403 0x3c038, 0xffffffff, 0x00030002,
1404 0x3c03c, 0xffffffff, 0x00040007,
1405 0x3c040, 0xffffffff, 0x00060005,
1406 0x3c044, 0xffffffff, 0x00090008,
1407 0x3c000, 0xffffffff, 0x96e00200,
1408 0x8708, 0xffffffff, 0x00900100,
1409 0xc424, 0xffffffff, 0x0020003f,
1410 0x38, 0xffffffff, 0x0140001c,
1411 0x3c, 0x000f0000, 0x000f0000,
1412 0x220, 0xffffffff, 0xC060000C,
1413 0x224, 0xc0000fff, 0x00000100,
1414 0x20a8, 0xffffffff, 0x00000104,
1415 0x55e4, 0xff000fff, 0x00000100,
1416 0x30cc, 0xc0000fff, 0x00000104,
1417 0xc1e4, 0x00000001, 0x00000001,
1418 0xd00c, 0xff000ff0, 0x00000100,
1419 0xd80c, 0xff000ff0, 0x00000100
1420};
1421
8efff337
AD		 1422static const u32 hawaii_golden_spm_registers[] =
1423{
1424 0x30800, 0xe0ffffff, 0xe0000000
1425};
1426
1427static const u32 hawaii_golden_common_registers[] =
1428{
1429 0x30800, 0xffffffff, 0xe0000000,
1430 0x28350, 0xffffffff, 0x3a00161a,
1431 0x28354, 0xffffffff, 0x0000002e,
1432 0x9a10, 0xffffffff, 0x00018208,
1433 0x98f8, 0xffffffff, 0x12011003
1434};
1435
1436static const u32 hawaii_golden_registers[] =
1437{
1438 0x3354, 0x00000333, 0x00000333,
1439 0x9a10, 0x00010000, 0x00058208,
1440 0x9830, 0xffffffff, 0x00000000,
1441 0x9834, 0xf00fffff, 0x00000400,
1442 0x9838, 0x0002021c, 0x00020200,
1443 0xc78, 0x00000080, 0x00000000,
1444 0x5bb0, 0x000000f0, 0x00000070,
1445 0x5bc0, 0xf0311fff, 0x80300000,
1446 0x350c, 0x00810000, 0x408af000,
1447 0x7030, 0x31000111, 0x00000011,
1448 0x2f48, 0x73773777, 0x12010001,
1449 0x2120, 0x0000007f, 0x0000001b,
1450 0x21dc, 0x00007fb6, 0x00002191,
1451 0x3628, 0x0000003f, 0x0000000a,
1452 0x362c, 0x0000003f, 0x0000000a,
1453 0x2ae4, 0x00073ffe, 0x000022a2,
1454 0x240c, 0x000007ff, 0x00000000,
1455 0x8bf0, 0x00002001, 0x00000001,
1456 0x8b24, 0xffffffff, 0x00ffffff,
1457 0x30a04, 0x0000ff0f, 0x00000000,
1458 0x28a4c, 0x07ffffff, 0x06000000,
1459 0x3e78, 0x00000001, 0x00000002,
1460 0xc768, 0x00000008, 0x00000008,
1461 0xc770, 0x00000f00, 0x00000800,
1462 0xc774, 0x00000f00, 0x00000800,
1463 0xc798, 0x00ffffff, 0x00ff7fbf,
1464 0xc79c, 0x00ffffff, 0x00ff7faf,
1465 0x8c00, 0x000000ff, 0x00000800,
1466 0xe40, 0x00001fff, 0x00001fff,
1467 0x9060, 0x0000007f, 0x00000020,
1468 0x9508, 0x00010000, 0x00010000,
1469 0xae00, 0x00100000, 0x000ff07c,
1470 0xac14, 0x000003ff, 0x0000000f,
1471 0xac10, 0xffffffff, 0x7564fdec,
1472 0xac0c, 0xffffffff, 0x3120b9a8,
1473 0xac08, 0x20000000, 0x0f9c0000
1474};
1475
1476static const u32 hawaii_mgcg_cgcg_init[] =
1477{
1478 0xc420, 0xffffffff, 0xfffffffd,
1479 0x30800, 0xffffffff, 0xe0000000,
1480 0x3c2a0, 0xffffffff, 0x00000100,
1481 0x3c208, 0xffffffff, 0x00000100,
1482 0x3c2c0, 0xffffffff, 0x00000100,
1483 0x3c2c8, 0xffffffff, 0x00000100,
1484 0x3c2c4, 0xffffffff, 0x00000100,
1485 0x55e4, 0xffffffff, 0x00200100,
1486 0x3c280, 0xffffffff, 0x00000100,
1487 0x3c214, 0xffffffff, 0x06000100,
1488 0x3c220, 0xffffffff, 0x00000100,
1489 0x3c218, 0xffffffff, 0x06000100,
1490 0x3c204, 0xffffffff, 0x00000100,
1491 0x3c2e0, 0xffffffff, 0x00000100,
1492 0x3c224, 0xffffffff, 0x00000100,
1493 0x3c200, 0xffffffff, 0x00000100,
1494 0x3c230, 0xffffffff, 0x00000100,
1495 0x3c234, 0xffffffff, 0x00000100,
1496 0x3c250, 0xffffffff, 0x00000100,
1497 0x3c254, 0xffffffff, 0x00000100,
1498 0x3c258, 0xffffffff, 0x00000100,
1499 0x3c25c, 0xffffffff, 0x00000100,
1500 0x3c260, 0xffffffff, 0x00000100,
1501 0x3c27c, 0xffffffff, 0x00000100,
1502 0x3c278, 0xffffffff, 0x00000100,
1503 0x3c210, 0xffffffff, 0x06000100,
1504 0x3c290, 0xffffffff, 0x00000100,
1505 0x3c274, 0xffffffff, 0x00000100,
1506 0x3c2b4, 0xffffffff, 0x00000100,
1507 0x3c2b0, 0xffffffff, 0x00000100,
1508 0x3c270, 0xffffffff, 0x00000100,
1509 0x30800, 0xffffffff, 0xe0000000,
1510 0x3c020, 0xffffffff, 0x00010000,
1511 0x3c024, 0xffffffff, 0x00030002,
1512 0x3c028, 0xffffffff, 0x00040007,
1513 0x3c02c, 0xffffffff, 0x00060005,
1514 0x3c030, 0xffffffff, 0x00090008,
1515 0x3c034, 0xffffffff, 0x00010000,
1516 0x3c038, 0xffffffff, 0x00030002,
1517 0x3c03c, 0xffffffff, 0x00040007,
1518 0x3c040, 0xffffffff, 0x00060005,
1519 0x3c044, 0xffffffff, 0x00090008,
1520 0x3c048, 0xffffffff, 0x00010000,
1521 0x3c04c, 0xffffffff, 0x00030002,
1522 0x3c050, 0xffffffff, 0x00040007,
1523 0x3c054, 0xffffffff, 0x00060005,
1524 0x3c058, 0xffffffff, 0x00090008,
1525 0x3c05c, 0xffffffff, 0x00010000,
1526 0x3c060, 0xffffffff, 0x00030002,
1527 0x3c064, 0xffffffff, 0x00040007,
1528 0x3c068, 0xffffffff, 0x00060005,
1529 0x3c06c, 0xffffffff, 0x00090008,
1530 0x3c070, 0xffffffff, 0x00010000,
1531 0x3c074, 0xffffffff, 0x00030002,
1532 0x3c078, 0xffffffff, 0x00040007,
1533 0x3c07c, 0xffffffff, 0x00060005,
1534 0x3c080, 0xffffffff, 0x00090008,
1535 0x3c084, 0xffffffff, 0x00010000,
1536 0x3c088, 0xffffffff, 0x00030002,
1537 0x3c08c, 0xffffffff, 0x00040007,
1538 0x3c090, 0xffffffff, 0x00060005,
1539 0x3c094, 0xffffffff, 0x00090008,
1540 0x3c098, 0xffffffff, 0x00010000,
1541 0x3c09c, 0xffffffff, 0x00030002,
1542 0x3c0a0, 0xffffffff, 0x00040007,
1543 0x3c0a4, 0xffffffff, 0x00060005,
1544 0x3c0a8, 0xffffffff, 0x00090008,
1545 0x3c0ac, 0xffffffff, 0x00010000,
1546 0x3c0b0, 0xffffffff, 0x00030002,
1547 0x3c0b4, 0xffffffff, 0x00040007,
1548 0x3c0b8, 0xffffffff, 0x00060005,
1549 0x3c0bc, 0xffffffff, 0x00090008,
1550 0x3c0c0, 0xffffffff, 0x00010000,
1551 0x3c0c4, 0xffffffff, 0x00030002,
1552 0x3c0c8, 0xffffffff, 0x00040007,
1553 0x3c0cc, 0xffffffff, 0x00060005,
1554 0x3c0d0, 0xffffffff, 0x00090008,
1555 0x3c0d4, 0xffffffff, 0x00010000,
1556 0x3c0d8, 0xffffffff, 0x00030002,
1557 0x3c0dc, 0xffffffff, 0x00040007,
1558 0x3c0e0, 0xffffffff, 0x00060005,
1559 0x3c0e4, 0xffffffff, 0x00090008,
1560 0x3c0e8, 0xffffffff, 0x00010000,
1561 0x3c0ec, 0xffffffff, 0x00030002,
1562 0x3c0f0, 0xffffffff, 0x00040007,
1563 0x3c0f4, 0xffffffff, 0x00060005,
1564 0x3c0f8, 0xffffffff, 0x00090008,
1565 0xc318, 0xffffffff, 0x00020200,
1566 0x3350, 0xffffffff, 0x00000200,
1567 0x15c0, 0xffffffff, 0x00000400,
1568 0x55e8, 0xffffffff, 0x00000000,
1569 0x2f50, 0xffffffff, 0x00000902,
1570 0x3c000, 0xffffffff, 0x96940200,
1571 0x8708, 0xffffffff, 0x00900100,
1572 0xc424, 0xffffffff, 0x0020003f,
1573 0x38, 0xffffffff, 0x0140001c,
1574 0x3c, 0x000f0000, 0x000f0000,
1575 0x220, 0xffffffff, 0xc060000c,
1576 0x224, 0xc0000fff, 0x00000100,
1577 0xf90, 0xffffffff, 0x00000100,
1578 0xf98, 0x00000101, 0x00000000,
1579 0x20a8, 0xffffffff, 0x00000104,
1580 0x55e4, 0xff000fff, 0x00000100,
1581 0x30cc, 0xc0000fff, 0x00000104,
1582 0xc1e4, 0x00000001, 0x00000001,
1583 0xd00c, 0xff000ff0, 0x00000100,
1584 0xd80c, 0xff000ff0, 0x00000100
1585};
1586
f73a9e83
SL		 1587static const u32 godavari_golden_registers[] =
1588{
1589 0x55e4, 0xff607fff, 0xfc000100,
1590 0x6ed8, 0x00010101, 0x00010000,
1591 0x9830, 0xffffffff, 0x00000000,
1592 0x98302, 0xf00fffff, 0x00000400,
1593 0x6130, 0xffffffff, 0x00010000,
1594 0x5bb0, 0x000000f0, 0x00000070,
1595 0x5bc0, 0xf0311fff, 0x80300000,
1596 0x98f8, 0x73773777, 0x12010001,
1597 0x98fc, 0xffffffff, 0x00000010,
1598 0x8030, 0x00001f0f, 0x0000100a,
1599 0x2f48, 0x73773777, 0x12010001,
1600 0x2408, 0x000fffff, 0x000c007f,
1601 0x8a14, 0xf000003f, 0x00000007,
1602 0x8b24, 0xffffffff, 0x00ff0fff,
1603 0x30a04, 0x0000ff0f, 0x00000000,
1604 0x28a4c, 0x07ffffff, 0x06000000,
1605 0x4d8, 0x00000fff, 0x00000100,
1606 0xd014, 0x00010000, 0x00810001,
1607 0xd814, 0x00010000, 0x00810001,
1608 0x3e78, 0x00000001, 0x00000002,
1609 0xc768, 0x00000008, 0x00000008,
1610 0xc770, 0x00000f00, 0x00000800,
1611 0xc774, 0x00000f00, 0x00000800,
1612 0xc798, 0x00ffffff, 0x00ff7fbf,
1613 0xc79c, 0x00ffffff, 0x00ff7faf,
1614 0x8c00, 0x000000ff, 0x00000001,
1615 0x214f8, 0x01ff01ff, 0x00000002,
1616 0x21498, 0x007ff800, 0x00200000,
1617 0x2015c, 0xffffffff, 0x00000f40,
1618 0x88c4, 0x001f3ae3, 0x00000082,
1619 0x88d4, 0x0000001f, 0x00000010,
1620 0x30934, 0xffffffff, 0x00000000
1621};
1622
1623
0aafd313
AD		 1624static void cik_init_golden_registers(struct radeon_device *rdev)
1625{
1626 switch (rdev->family) {
1627 case CHIP_BONAIRE:
1628 radeon_program_register_sequence(rdev,
1629 bonaire_mgcg_cgcg_init,
1630 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1631 radeon_program_register_sequence(rdev,
1632 bonaire_golden_registers,
1633 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1634 radeon_program_register_sequence(rdev,
1635 bonaire_golden_common_registers,
1636 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1637 radeon_program_register_sequence(rdev,
1638 bonaire_golden_spm_registers,
1639 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1640 break;
1641 case CHIP_KABINI:
1642 radeon_program_register_sequence(rdev,
1643 kalindi_mgcg_cgcg_init,
1644 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1645 radeon_program_register_sequence(rdev,
1646 kalindi_golden_registers,
1647 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1648 radeon_program_register_sequence(rdev,
1649 kalindi_golden_common_registers,
1650 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1651 radeon_program_register_sequence(rdev,
1652 kalindi_golden_spm_registers,
1653 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1654 break;
f73a9e83
SL		 1655 case CHIP_MULLINS:
1656 radeon_program_register_sequence(rdev,
1657 kalindi_mgcg_cgcg_init,
1658 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1659 radeon_program_register_sequence(rdev,
1660 godavari_golden_registers,
1661 (const u32)ARRAY_SIZE(godavari_golden_registers));
1662 radeon_program_register_sequence(rdev,
1663 kalindi_golden_common_registers,
1664 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1665 radeon_program_register_sequence(rdev,
1666 kalindi_golden_spm_registers,
1667 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1668 break;
0aafd313
AD		 1669 case CHIP_KAVERI:
1670 radeon_program_register_sequence(rdev,
1671 spectre_mgcg_cgcg_init,
1672 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1673 radeon_program_register_sequence(rdev,
1674 spectre_golden_registers,
1675 (const u32)ARRAY_SIZE(spectre_golden_registers));
1676 radeon_program_register_sequence(rdev,
1677 spectre_golden_common_registers,
1678 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1679 radeon_program_register_sequence(rdev,
1680 spectre_golden_spm_registers,
1681 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1682 break;
8efff337
AD		 1683 case CHIP_HAWAII:
1684 radeon_program_register_sequence(rdev,
1685 hawaii_mgcg_cgcg_init,
1686 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1687 radeon_program_register_sequence(rdev,
1688 hawaii_golden_registers,
1689 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1690 radeon_program_register_sequence(rdev,
1691 hawaii_golden_common_registers,
1692 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1693 radeon_program_register_sequence(rdev,
1694 hawaii_golden_spm_registers,
1695 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1696 break;
0aafd313
AD		 1697 default:
1698 break;
1699 }
1700}
1701
2c67912c
AD		 1702/**
1703 * cik_get_xclk - get the xclk
1704 *
1705 * @rdev: radeon_device pointer
1706 *
1707 * Returns the reference clock used by the gfx engine
1708 * (CIK).
1709 */
1710u32 cik_get_xclk(struct radeon_device *rdev)
1711{
3cf8bb1a 1712 u32 reference_clock = rdev->clock.spll.reference_freq;
2c67912c
AD		 1713
1714 if (rdev->flags & RADEON_IS_IGP) {
1715 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1716 return reference_clock / 2;
1717 } else {
1718 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1719 return reference_clock / 4;
1720 }
1721 return reference_clock;
1722}
1723
75efdee1
AD		 1724/**
1725 * cik_mm_rdoorbell - read a doorbell dword
1726 *
1727 * @rdev: radeon_device pointer
d5754ab8 1728 * @index: doorbell index
75efdee1
AD		 1729 *
1730 * Returns the value in the doorbell aperture at the
d5754ab8 1731 * requested doorbell index (CIK).
75efdee1 1732 */
d5754ab8 1733u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
75efdee1 1734{
d5754ab8
AL		 1735 if (index < rdev->doorbell.num_doorbells) {
1736 return readl(rdev->doorbell.ptr + index);
75efdee1 1737 } else {
d5754ab8 1738 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1739 return 0;
1740 }
1741}
1742
1743/**
1744 * cik_mm_wdoorbell - write a doorbell dword
1745 *
1746 * @rdev: radeon_device pointer
d5754ab8 1747 * @index: doorbell index
75efdee1
AD		 1748 * @v: value to write
1749 *
1750 * Writes @v to the doorbell aperture at the
d5754ab8 1751 * requested doorbell index (CIK).
75efdee1 1752 */
d5754ab8 1753void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
75efdee1 1754{
d5754ab8
AL		 1755 if (index < rdev->doorbell.num_doorbells) {
1756 writel(v, rdev->doorbell.ptr + index);
75efdee1 1757 } else {
d5754ab8 1758 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1759 }
1760}
1761
bc8273fe
AD		 1762#define BONAIRE_IO_MC_REGS_SIZE 36
1763
1764static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1765{
1766 {0x00000070, 0x04400000},
1767 {0x00000071, 0x80c01803},
1768 {0x00000072, 0x00004004},
1769 {0x00000073, 0x00000100},
1770 {0x00000074, 0x00ff0000},
1771 {0x00000075, 0x34000000},
1772 {0x00000076, 0x08000014},
1773 {0x00000077, 0x00cc08ec},
1774 {0x00000078, 0x00000400},
1775 {0x00000079, 0x00000000},
1776 {0x0000007a, 0x04090000},
1777 {0x0000007c, 0x00000000},
1778 {0x0000007e, 0x4408a8e8},
1779 {0x0000007f, 0x00000304},
1780 {0x00000080, 0x00000000},
1781 {0x00000082, 0x00000001},
1782 {0x00000083, 0x00000002},
1783 {0x00000084, 0xf3e4f400},
1784 {0x00000085, 0x052024e3},
1785 {0x00000087, 0x00000000},
1786 {0x00000088, 0x01000000},
1787 {0x0000008a, 0x1c0a0000},
1788 {0x0000008b, 0xff010000},
1789 {0x0000008d, 0xffffefff},
1790 {0x0000008e, 0xfff3efff},
1791 {0x0000008f, 0xfff3efbf},
1792 {0x00000092, 0xf7ffffff},
1793 {0x00000093, 0xffffff7f},
1794 {0x00000095, 0x00101101},
1795 {0x00000096, 0x00000fff},
1796 {0x00000097, 0x00116fff},
1797 {0x00000098, 0x60010000},
1798 {0x00000099, 0x10010000},
1799 {0x0000009a, 0x00006000},
1800 {0x0000009b, 0x00001000},
1801 {0x0000009f, 0x00b48000}
1802};
1803
d4775655
AD		 1804#define HAWAII_IO_MC_REGS_SIZE 22
1805
1806static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1807{
1808 {0x0000007d, 0x40000000},
1809 {0x0000007e, 0x40180304},
1810 {0x0000007f, 0x0000ff00},
1811 {0x00000081, 0x00000000},
1812 {0x00000083, 0x00000800},
1813 {0x00000086, 0x00000000},
1814 {0x00000087, 0x00000100},
1815 {0x00000088, 0x00020100},
1816 {0x00000089, 0x00000000},
1817 {0x0000008b, 0x00040000},
1818 {0x0000008c, 0x00000100},
1819 {0x0000008e, 0xff010000},
1820 {0x00000090, 0xffffefff},
1821 {0x00000091, 0xfff3efff},
1822 {0x00000092, 0xfff3efbf},
1823 {0x00000093, 0xf7ffffff},
1824 {0x00000094, 0xffffff7f},
1825 {0x00000095, 0x00000fff},
1826 {0x00000096, 0x00116fff},
1827 {0x00000097, 0x60010000},
1828 {0x00000098, 0x10010000},
1829 {0x0000009f, 0x00c79000}
1830};
1831
1832
b556b12e
AD		 1833/**
1834 * cik_srbm_select - select specific register instances
1835 *
1836 * @rdev: radeon_device pointer
1837 * @me: selected ME (micro engine)
1838 * @pipe: pipe
1839 * @queue: queue
1840 * @vmid: VMID
1841 *
1842 * Switches the currently active registers instances. Some
1843 * registers are instanced per VMID, others are instanced per
1844 * me/pipe/queue combination.
1845 */
1846static void cik_srbm_select(struct radeon_device *rdev,
1847 u32 me, u32 pipe, u32 queue, u32 vmid)
1848{
1849 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1850 MEID(me & 0x3) |
1851 VMID(vmid & 0xf) |
1852 QUEUEID(queue & 0x7));
1853 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1854}
1855
bc8273fe
AD		 1856/* ucode loading */
1857/**
1858 * ci_mc_load_microcode - load MC ucode into the hw
1859 *
1860 * @rdev: radeon_device pointer
1861 *
1862 * Load the GDDR MC ucode into the hw (CIK).
1863 * Returns 0 on success, error on failure.
1864 */
6c7bccea 1865int ci_mc_load_microcode(struct radeon_device *rdev)
bc8273fe 1866{
f2c6b0f4
AD		 1867 const __be32 *fw_data = NULL;
1868 const __le32 *new_fw_data = NULL;
6e4b070e 1869 u32 running, tmp;
f2c6b0f4
AD		 1870 u32 *io_mc_regs = NULL;
1871 const __le32 *new_io_mc_regs = NULL;
bcddee29 1872 int i, regs_size, ucode_size;
bc8273fe
AD		 1873
1874 if (!rdev->mc_fw)
1875 return -EINVAL;
1876
f2c6b0f4
AD		 1877 if (rdev->new_fw) {
1878 const struct mc_firmware_header_v1_0 *hdr =
1879 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
bcddee29 1880
f2c6b0f4
AD		 1881 radeon_ucode_print_mc_hdr(&hdr->header);
1882
1883 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1884 new_io_mc_regs = (const __le32 *)
1885 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1886 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1887 new_fw_data = (const __le32 *)
1888 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1889 } else {
1890 ucode_size = rdev->mc_fw->size / 4;
1891
1892 switch (rdev->family) {
1893 case CHIP_BONAIRE:
1894 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1895 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1896 break;
1897 case CHIP_HAWAII:
1898 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1899 regs_size = HAWAII_IO_MC_REGS_SIZE;
1900 break;
1901 default:
1902 return -EINVAL;
1903 }
1904 fw_data = (const __be32 *)rdev->mc_fw->data;
bc8273fe
AD		 1905 }
1906
1907 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1908
1909 if (running == 0) {
bc8273fe
AD		 1910 /* reset the engine and set to writable */
1911 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1912 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1913
1914 /* load mc io regs */
1915 for (i = 0; i < regs_size; i++) {
f2c6b0f4
AD		 1916 if (rdev->new_fw) {
1917 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1918 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1919 } else {
1920 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1921 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1922 }
bc8273fe 1923 }
9feb3dda
AD		 1924
1925 tmp = RREG32(MC_SEQ_MISC0);
1926 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1927 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1928 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1929 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1930 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1931 }
1932
bc8273fe 1933 /* load the MC ucode */
f2c6b0f4
AD		 1934 for (i = 0; i < ucode_size; i++) {
1935 if (rdev->new_fw)
1936 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1937 else
1938 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1939 }
bc8273fe
AD		 1940
1941 /* put the engine back into the active state */
1942 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1943 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1944 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1945
1946 /* wait for training to complete */
1947 for (i = 0; i < rdev->usec_timeout; i++) {
1948 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1949 break;
1950 udelay(1);
1951 }
1952 for (i = 0; i < rdev->usec_timeout; i++) {
1953 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1954 break;
1955 udelay(1);
1956 }
bc8273fe
AD		 1957 }
1958
1959 return 0;
1960}
1961
02c81327
AD		 1962/**
1963 * cik_init_microcode - load ucode images from disk
1964 *
1965 * @rdev: radeon_device pointer
1966 *
1967 * Use the firmware interface to load the ucode images into
1968 * the driver (not loaded into hw).
1969 * Returns 0 on success, error on failure.
1970 */
1971static int cik_init_microcode(struct radeon_device *rdev)
1972{
02c81327 1973 const char *chip_name;
f2c6b0f4 1974 const char *new_chip_name;
02c81327 1975 size_t pfp_req_size, me_req_size, ce_req_size,
d4775655 1976 mec_req_size, rlc_req_size, mc_req_size = 0,
277babc3 1977 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
02c81327 1978 char fw_name[30];
f2c6b0f4 1979 int new_fw = 0;
02c81327 1980 int err;
f2c6b0f4 1981 int num_fw;
b2ea0dcd 1982 bool new_smc = false;
02c81327
AD		 1983
1984 DRM_DEBUG("\n");
1985
02c81327
AD		 1986 switch (rdev->family) {
1987 case CHIP_BONAIRE:
1988 chip_name = "BONAIRE";
b2ea0dcd
AD		 1989 if ((rdev->pdev->revision == 0x80) ||
1990 (rdev->pdev->revision == 0x81) ||
1991 (rdev->pdev->device == 0x665f))
1992 new_smc = true;
f2c6b0f4 1993 new_chip_name = "bonaire";
02c81327
AD		 1994 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1995 me_req_size = CIK_ME_UCODE_SIZE * 4;
1996 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1997 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1998 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
277babc3
AD		 1999 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
2000 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
21a93e13 2001 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 2002 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
f2c6b0f4 2003 num_fw = 8;
02c81327 2004 break;
d4775655
AD		 2005 case CHIP_HAWAII:
2006 chip_name = "HAWAII";
b2ea0dcd
AD		 2007 if (rdev->pdev->revision == 0x80)
2008 new_smc = true;
f2c6b0f4 2009 new_chip_name = "hawaii";
d4775655
AD		 2010 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2011 me_req_size = CIK_ME_UCODE_SIZE * 4;
2012 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2013 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2014 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2015 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
277babc3 2016 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
d4775655
AD		 2017 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2018 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
f2c6b0f4 2019 num_fw = 8;
d4775655 2020 break;
02c81327
AD		 2021 case CHIP_KAVERI:
2022 chip_name = "KAVERI";
f2c6b0f4 2023 new_chip_name = "kaveri";
02c81327
AD		 2024 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2025 me_req_size = CIK_ME_UCODE_SIZE * 4;
2026 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2027 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2028 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 2029 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2030 num_fw = 7;
02c81327
AD		 2031 break;
2032 case CHIP_KABINI:
2033 chip_name = "KABINI";
f2c6b0f4 2034 new_chip_name = "kabini";
02c81327
AD		 2035 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2036 me_req_size = CIK_ME_UCODE_SIZE * 4;
2037 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2038 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2039 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 2040 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2041 num_fw = 6;
02c81327 2042 break;
f73a9e83
SL		 2043 case CHIP_MULLINS:
2044 chip_name = "MULLINS";
f2c6b0f4 2045 new_chip_name = "mullins";
f73a9e83
SL		 2046 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2047 me_req_size = CIK_ME_UCODE_SIZE * 4;
2048 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2049 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2050 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
2051 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2052 num_fw = 6;
f73a9e83 2053 break;
02c81327
AD		 2054 default: BUG();
2055 }
2056
f2c6b0f4 2057 DRM_INFO("Loading %s Microcode\n", new_chip_name);
02c81327 2058
f2c6b0f4 2059 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
0a168933 2060 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2061 if (err) {
2062 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2063 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2064 if (err)
2065 goto out;
2066 if (rdev->pfp_fw->size != pfp_req_size) {
7ca85295 2067 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2068 rdev->pfp_fw->size, fw_name);
2069 err = -EINVAL;
2070 goto out;
2071 }
2072 } else {
2073 err = radeon_ucode_validate(rdev->pfp_fw);
2074 if (err) {
7ca85295 2075 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4
AD		 2076 fw_name);
2077 goto out;
2078 } else {
2079 new_fw++;
2080 }
02c81327
AD		 2081 }
2082
f2c6b0f4 2083 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
0a168933 2084 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2085 if (err) {
2086 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2087 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2088 if (err)
2089 goto out;
2090 if (rdev->me_fw->size != me_req_size) {
7ca85295 2091 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2092 rdev->me_fw->size, fw_name);
2093 err = -EINVAL;
2094 }
2095 } else {
2096 err = radeon_ucode_validate(rdev->me_fw);
2097 if (err) {
7ca85295 2098 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4
AD		 2099 fw_name);
2100 goto out;
2101 } else {
2102 new_fw++;
2103 }
02c81327
AD		 2104 }
2105
f2c6b0f4 2106 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
0a168933 2107 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2108 if (err) {
2109 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2110 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2111 if (err)
2112 goto out;
2113 if (rdev->ce_fw->size != ce_req_size) {
7ca85295 2114 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2115 rdev->ce_fw->size, fw_name);
2116 err = -EINVAL;
2117 }
2118 } else {
2119 err = radeon_ucode_validate(rdev->ce_fw);
2120 if (err) {
7ca85295 2121 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4
AD		 2122 fw_name);
2123 goto out;
2124 } else {
2125 new_fw++;
2126 }
02c81327
AD		 2127 }
2128
f2c6b0f4 2129 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
0a168933 2130 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2131 if (err) {
2132 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2133 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2134 if (err)
2135 goto out;
2136 if (rdev->mec_fw->size != mec_req_size) {
7ca85295 2137 pr_err("cik_cp: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2138 rdev->mec_fw->size, fw_name);
2139 err = -EINVAL;
2140 }
2141 } else {
2142 err = radeon_ucode_validate(rdev->mec_fw);
2143 if (err) {
7ca85295 2144 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4
AD		 2145 fw_name);
2146 goto out;
2147 } else {
2148 new_fw++;
2149 }
2150 }
2151
2152 if (rdev->family == CHIP_KAVERI) {
2153 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2154 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2155 if (err) {
2156 goto out;
2157 } else {
2158 err = radeon_ucode_validate(rdev->mec2_fw);
2159 if (err) {
2160 goto out;
2161 } else {
2162 new_fw++;
2163 }
2164 }
02c81327
AD		 2165 }
2166
f2c6b0f4 2167 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
0a168933 2168 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2169 if (err) {
2170 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2171 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2172 if (err)
2173 goto out;
2174 if (rdev->rlc_fw->size != rlc_req_size) {
7ca85295 2175 pr_err("cik_rlc: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2176 rdev->rlc_fw->size, fw_name);
2177 err = -EINVAL;
2178 }
2179 } else {
2180 err = radeon_ucode_validate(rdev->rlc_fw);
2181 if (err) {
7ca85295 2182 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4
AD		 2183 fw_name);
2184 goto out;
2185 } else {
2186 new_fw++;
2187 }
02c81327
AD		 2188 }
2189
f2c6b0f4 2190 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
0a168933 2191 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2192 if (err) {
2193 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2194 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2195 if (err)
2196 goto out;
2197 if (rdev->sdma_fw->size != sdma_req_size) {
7ca85295 2198 pr_err("cik_sdma: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2199 rdev->sdma_fw->size, fw_name);
2200 err = -EINVAL;
2201 }
2202 } else {
2203 err = radeon_ucode_validate(rdev->sdma_fw);
2204 if (err) {
7ca85295 2205 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4
AD		 2206 fw_name);
2207 goto out;
2208 } else {
2209 new_fw++;
2210 }
21a93e13
AD		 2211 }
2212
cc8dbbb4 2213 /* No SMC, MC ucode on APUs */
02c81327 2214 if (!(rdev->flags & RADEON_IS_IGP)) {
f2c6b0f4 2215 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
0a168933 2216 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
277babc3 2217 if (err) {
f2c6b0f4 2218 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
277babc3 2219 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2220 if (err) {
2221 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2222 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2223 if (err)
2224 goto out;
2225 }
2226 if ((rdev->mc_fw->size != mc_req_size) &&
2227 (rdev->mc_fw->size != mc2_req_size)){
7ca85295 2228 pr_err("cik_mc: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2229 rdev->mc_fw->size, fw_name);
2230 err = -EINVAL;
2231 }
2232 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2233 } else {
2234 err = radeon_ucode_validate(rdev->mc_fw);
2235 if (err) {
7ca85295 2236 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4 2237 fw_name);
277babc3 2238 goto out;
f2c6b0f4
AD		 2239 } else {
2240 new_fw++;
2241 }
277babc3 2242 }
cc8dbbb4 2243
b2ea0dcd
AD		 2244 if (new_smc)
2245 snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
2246 else
2247 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
cc8dbbb4
AD		 2248 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2249 if (err) {
f2c6b0f4
AD		 2250 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2251 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2252 if (err) {
7ca85295 2253 pr_err("smc: error loading firmware \"%s\"\n",
f2c6b0f4
AD		 2254 fw_name);
2255 release_firmware(rdev->smc_fw);
2256 rdev->smc_fw = NULL;
2257 err = 0;
2258 } else if (rdev->smc_fw->size != smc_req_size) {
7ca85295 2259 pr_err("cik_smc: Bogus length %zu in firmware \"%s\"\n",
f2c6b0f4
AD		 2260 rdev->smc_fw->size, fw_name);
2261 err = -EINVAL;
2262 }
2263 } else {
2264 err = radeon_ucode_validate(rdev->smc_fw);
2265 if (err) {
7ca85295 2266 pr_err("cik_fw: validation failed for firmware \"%s\"\n",
f2c6b0f4
AD		 2267 fw_name);
2268 goto out;
2269 } else {
2270 new_fw++;
2271 }
cc8dbbb4 2272 }
02c81327
AD		 2273 }
2274
f2c6b0f4
AD		 2275 if (new_fw == 0) {
2276 rdev->new_fw = false;
2277 } else if (new_fw < num_fw) {
7ca85295 2278 pr_err("ci_fw: mixing new and old firmware!\n");
f2c6b0f4
AD		 2279 err = -EINVAL;
2280 } else {
2281 rdev->new_fw = true;
2282 }
2283
02c81327 2284out:
02c81327
AD		 2285 if (err) {
2286 if (err != -EINVAL)
7ca85295 2287 pr_err("cik_cp: Failed to load firmware \"%s\"\n",
02c81327
AD		 2288 fw_name);
2289 release_firmware(rdev->pfp_fw);
2290 rdev->pfp_fw = NULL;
2291 release_firmware(rdev->me_fw);
2292 rdev->me_fw = NULL;
2293 release_firmware(rdev->ce_fw);
2294 rdev->ce_fw = NULL;
f2c6b0f4
AD		 2295 release_firmware(rdev->mec_fw);
2296 rdev->mec_fw = NULL;
2297 release_firmware(rdev->mec2_fw);
2298 rdev->mec2_fw = NULL;
02c81327
AD		 2299 release_firmware(rdev->rlc_fw);
2300 rdev->rlc_fw = NULL;
f2c6b0f4
AD		 2301 release_firmware(rdev->sdma_fw);
2302 rdev->sdma_fw = NULL;
02c81327
AD		 2303 release_firmware(rdev->mc_fw);
2304 rdev->mc_fw = NULL;
cc8dbbb4
AD		 2305 release_firmware(rdev->smc_fw);
2306 rdev->smc_fw = NULL;
02c81327
AD		 2307 }
2308 return err;
2309}
2310
8cc1a532
AD		 2311/*
2312 * Core functions
2313 */
2314/**
2315 * cik_tiling_mode_table_init - init the hw tiling table
2316 *
2317 * @rdev: radeon_device pointer
2318 *
2319 * Starting with SI, the tiling setup is done globally in a
2320 * set of 32 tiling modes. Rather than selecting each set of
2321 * parameters per surface as on older asics, we just select
2322 * which index in the tiling table we want to use, and the
2323 * surface uses those parameters (CIK).
2324 */
2325static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2326{
f0e201f2
JP		 2327 u32 *tile = rdev->config.cik.tile_mode_array;
2328 u32 *macrotile = rdev->config.cik.macrotile_mode_array;
2329 const u32 num_tile_mode_states =
2330 ARRAY_SIZE(rdev->config.cik.tile_mode_array);
2331 const u32 num_secondary_tile_mode_states =
2332 ARRAY_SIZE(rdev->config.cik.macrotile_mode_array);
2333 u32 reg_offset, split_equal_to_row_size;
8cc1a532
AD		 2334 u32 num_pipe_configs;
2335 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2336 rdev->config.cik.max_shader_engines;
2337
2338 switch (rdev->config.cik.mem_row_size_in_kb) {
2339 case 1:
2340 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2341 break;
2342 case 2:
2343 default:
2344 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2345 break;
2346 case 4:
2347 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2348 break;
2349 }
2350
2351 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2352 if (num_pipe_configs > 8)
21e438af 2353 num_pipe_configs = 16;
8cc1a532 2354
f0e201f2
JP		 2355 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2356 tile[reg_offset] = 0;
2357 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2358 macrotile[reg_offset] = 0;
2359
2360 switch(num_pipe_configs) {
2361 case 16:
2362 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2363 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2364 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2365 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2366 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2367 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2368 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2369 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2370 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2371 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2372 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2373 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2374 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2375 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2376 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2377 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2378 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2379 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2380 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2381 TILE_SPLIT(split_equal_to_row_size));
2382 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2383 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2384 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2385 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2386 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2387 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2388 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2389 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2390 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2391 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2392 TILE_SPLIT(split_equal_to_row_size));
2393 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2394 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2395 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2396 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2397 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2398 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2399 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2400 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2401 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2402 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2403 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2404 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2405 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2406 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2407 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2408 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2409 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2410 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2411 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2412 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2413 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2414 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2415 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2416 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2417 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2418 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2419 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2420 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2421 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2422 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2423 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2424 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2425 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2426 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2427 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2428 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2429 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2430 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2431 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2432 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2433 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2434 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2435 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2436 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2437 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2438 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2439 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2440
2441 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2442 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2443 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2444 NUM_BANKS(ADDR_SURF_16_BANK));
2445 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2446 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2447 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2448 NUM_BANKS(ADDR_SURF_16_BANK));
2449 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2450 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2451 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2452 NUM_BANKS(ADDR_SURF_16_BANK));
2453 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2454 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2455 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2456 NUM_BANKS(ADDR_SURF_16_BANK));
2457 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2458 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2459 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2460 NUM_BANKS(ADDR_SURF_8_BANK));
2461 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2462 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2463 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2464 NUM_BANKS(ADDR_SURF_4_BANK));
2465 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2466 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2467 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2468 NUM_BANKS(ADDR_SURF_2_BANK));
2469 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2470 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2471 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2472 NUM_BANKS(ADDR_SURF_16_BANK));
2473 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2474 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2475 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2476 NUM_BANKS(ADDR_SURF_16_BANK));
2477 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2478 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2479 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2480 NUM_BANKS(ADDR_SURF_16_BANK));
2481 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2482 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2483 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2484 NUM_BANKS(ADDR_SURF_8_BANK));
2485 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2488 NUM_BANKS(ADDR_SURF_4_BANK));
2489 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2490 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2491 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2492 NUM_BANKS(ADDR_SURF_2_BANK));
2493 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2494 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2495 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2496 NUM_BANKS(ADDR_SURF_2_BANK));
2497
2498 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2499 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2500 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2501 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2502 break;
2503
2504 case 8:
2505 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2506 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2507 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2508 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2509 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2510 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2511 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2512 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2513 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2515 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2516 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2517 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2518 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2519 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2520 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2521 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2522 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2523 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2524 TILE_SPLIT(split_equal_to_row_size));
2525 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2526 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2527 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2528 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2529 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2530 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2531 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2532 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2533 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2534 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2535 TILE_SPLIT(split_equal_to_row_size));
2536 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2537 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2538 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2539 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2540 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2541 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2542 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2543 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2544 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2545 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2546 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2547 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2548 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2549 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2550 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2551 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2552 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2553 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2554 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2555 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2556 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2557 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2558 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2559 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2560 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2561 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2562 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2563 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2564 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2565 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2566 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2567 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2568 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2569 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2570 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2571 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2572 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2573 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2574 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2575 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2576 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2577 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2578 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2579 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2580 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2581 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2582 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2583
2584 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2585 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2586 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2587 NUM_BANKS(ADDR_SURF_16_BANK));
2588 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2589 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2590 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2591 NUM_BANKS(ADDR_SURF_16_BANK));
2592 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2593 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2594 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2595 NUM_BANKS(ADDR_SURF_16_BANK));
2596 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2597 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2598 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2599 NUM_BANKS(ADDR_SURF_16_BANK));
2600 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2601 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2602 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2603 NUM_BANKS(ADDR_SURF_8_BANK));
2604 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2605 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2606 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2607 NUM_BANKS(ADDR_SURF_4_BANK));
2608 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2609 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2610 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2611 NUM_BANKS(ADDR_SURF_2_BANK));
2612 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2613 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2614 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2615 NUM_BANKS(ADDR_SURF_16_BANK));
2616 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2617 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2618 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2619 NUM_BANKS(ADDR_SURF_16_BANK));
2620 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2621 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2622 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2623 NUM_BANKS(ADDR_SURF_16_BANK));
2624 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2625 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2626 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2627 NUM_BANKS(ADDR_SURF_16_BANK));
2628 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2629 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2630 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2631 NUM_BANKS(ADDR_SURF_8_BANK));
2632 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2633 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2634 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2635 NUM_BANKS(ADDR_SURF_4_BANK));
2636 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2637 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2638 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2639 NUM_BANKS(ADDR_SURF_2_BANK));
2640
2641 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2642 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2643 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2644 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2645 break;
2646
2647 case 4:
8cc1a532 2648 if (num_rbs == 4) {
f0e201f2
JP		 2649 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2650 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2651 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2652 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2653 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2654 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2655 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2656 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2657 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2658 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2659 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2660 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2661 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2662 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2663 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2664 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2665 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2666 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2667 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2668 TILE_SPLIT(split_equal_to_row_size));
2669 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2670 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2671 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2672 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2673 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2674 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2675 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2676 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2677 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2678 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2679 TILE_SPLIT(split_equal_to_row_size));
2680 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2681 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2682 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2683 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2684 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2685 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2686 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2687 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2688 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2689 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2690 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2691 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2692 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2693 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2694 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2695 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2696 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2697 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2698 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2699 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2700 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2701 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2702 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2703 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2704 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2705 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2706 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2707 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2708 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2709 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2710 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2711 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2712 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2713 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2714 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2715 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2716 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2717 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2718 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2719 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2720 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2721 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2722 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2723 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2724 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2725 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2726 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2727
8cc1a532 2728 } else if (num_rbs < 4) {
f0e201f2
JP		 2729 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2730 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2731 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2732 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2733 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2734 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2735 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2736 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2737 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2738 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2739 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2740 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2741 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2742 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2743 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2744 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2745 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2746 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2747 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2748 TILE_SPLIT(split_equal_to_row_size));
2749 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2750 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2751 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2752 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2753 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2754 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2755 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2756 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2757 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2758 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2759 TILE_SPLIT(split_equal_to_row_size));
2760 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2761 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2762 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2763 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2764 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2765 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2766 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2767 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2768 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2769 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2770 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2771 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2772 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2773 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2774 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2775 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2776 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2777 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2778 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2779 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2780 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2781 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2782 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2783 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2784 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2785 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2786 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2787 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2788 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2789 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2790 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2791 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2792 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2793 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2794 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2795 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2796 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2797 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2798 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2799 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2800 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2801 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2802 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2803 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2804 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2805 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2806 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
8cc1a532 2807 }
f0e201f2
JP		 2808
2809 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2810 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2811 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2812 NUM_BANKS(ADDR_SURF_16_BANK));
2813 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2814 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2815 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2816 NUM_BANKS(ADDR_SURF_16_BANK));
2817 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2818 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2819 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2820 NUM_BANKS(ADDR_SURF_16_BANK));
2821 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2822 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2823 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2824 NUM_BANKS(ADDR_SURF_16_BANK));
2825 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2826 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2827 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2828 NUM_BANKS(ADDR_SURF_16_BANK));
2829 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2830 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2831 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2832 NUM_BANKS(ADDR_SURF_8_BANK));
2833 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2834 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2835 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2836 NUM_BANKS(ADDR_SURF_4_BANK));
2837 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2838 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2839 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2840 NUM_BANKS(ADDR_SURF_16_BANK));
2841 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2842 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2843 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2844 NUM_BANKS(ADDR_SURF_16_BANK));
2845 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2846 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2847 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2848 NUM_BANKS(ADDR_SURF_16_BANK));
2849 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2850 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2851 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2852 NUM_BANKS(ADDR_SURF_16_BANK));
2853 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2854 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2855 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2856 NUM_BANKS(ADDR_SURF_16_BANK));
2857 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2858 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2859 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2860 NUM_BANKS(ADDR_SURF_8_BANK));
2861 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2862 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2863 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2864 NUM_BANKS(ADDR_SURF_4_BANK));
2865
2866 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2867 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2868 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2869 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2870 break;
2871
2872 case 2:
2873 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2874 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2875 PIPE_CONFIG(ADDR_SURF_P2) |
2876 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2877 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2878 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2879 PIPE_CONFIG(ADDR_SURF_P2) |
2880 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2881 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2882 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2883 PIPE_CONFIG(ADDR_SURF_P2) |
2884 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2885 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2886 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2887 PIPE_CONFIG(ADDR_SURF_P2) |
2888 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2889 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2890 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2891 PIPE_CONFIG(ADDR_SURF_P2) |
2892 TILE_SPLIT(split_equal_to_row_size));
2893 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2894 PIPE_CONFIG(ADDR_SURF_P2) |
2895 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2896 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2897 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2898 PIPE_CONFIG(ADDR_SURF_P2) |
2899 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2900 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2901 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2902 PIPE_CONFIG(ADDR_SURF_P2) |
2903 TILE_SPLIT(split_equal_to_row_size));
2904 tile[8] = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2905 PIPE_CONFIG(ADDR_SURF_P2);
2906 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2907 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2908 PIPE_CONFIG(ADDR_SURF_P2));
2909 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2910 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2911 PIPE_CONFIG(ADDR_SURF_P2) |
2912 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2913 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2914 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2915 PIPE_CONFIG(ADDR_SURF_P2) |
2916 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2917 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2918 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2919 PIPE_CONFIG(ADDR_SURF_P2) |
2920 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2921 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2922 PIPE_CONFIG(ADDR_SURF_P2) |
2923 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2924 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2925 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2926 PIPE_CONFIG(ADDR_SURF_P2) |
2927 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2928 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2929 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2930 PIPE_CONFIG(ADDR_SURF_P2) |
2931 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2932 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2933 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2934 PIPE_CONFIG(ADDR_SURF_P2) |
2935 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2936 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2937 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2938 PIPE_CONFIG(ADDR_SURF_P2));
2939 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2940 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2941 PIPE_CONFIG(ADDR_SURF_P2) |
2942 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2943 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2944 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2945 PIPE_CONFIG(ADDR_SURF_P2) |
2946 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2947 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2948 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2949 PIPE_CONFIG(ADDR_SURF_P2) |
2950 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2951
2952 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2953 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2954 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2955 NUM_BANKS(ADDR_SURF_16_BANK));
2956 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2957 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2958 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2959 NUM_BANKS(ADDR_SURF_16_BANK));
2960 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2961 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2962 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2963 NUM_BANKS(ADDR_SURF_16_BANK));
2964 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2965 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2966 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2967 NUM_BANKS(ADDR_SURF_16_BANK));
2968 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2969 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2970 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2971 NUM_BANKS(ADDR_SURF_16_BANK));
2972 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2973 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2974 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2975 NUM_BANKS(ADDR_SURF_16_BANK));
2976 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2977 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2978 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2979 NUM_BANKS(ADDR_SURF_8_BANK));
2980 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2981 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2982 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2983 NUM_BANKS(ADDR_SURF_16_BANK));
2984 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2985 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2986 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2987 NUM_BANKS(ADDR_SURF_16_BANK));
2988 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2989 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2990 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2991 NUM_BANKS(ADDR_SURF_16_BANK));
2992 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2993 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2994 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2995 NUM_BANKS(ADDR_SURF_16_BANK));
2996 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2997 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2998 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2999 NUM_BANKS(ADDR_SURF_16_BANK));
3000 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3001 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3002 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3003 NUM_BANKS(ADDR_SURF_16_BANK));
3004 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3005 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3006 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3007 NUM_BANKS(ADDR_SURF_8_BANK));
3008
3009 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3010 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
3011 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3012 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
3013 break;
3014
3015 default:
8cc1a532 3016 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
f0e201f2 3017 }
8cc1a532
AD		 3018}
3019
3020/**
3021 * cik_select_se_sh - select which SE, SH to address
3022 *
3023 * @rdev: radeon_device pointer
3024 * @se_num: shader engine to address
3025 * @sh_num: sh block to address
3026 *
3027 * Select which SE, SH combinations to address. Certain
3028 * registers are instanced per SE or SH. 0xffffffff means
3029 * broadcast to all SEs or SHs (CIK).
3030 */
3031static void cik_select_se_sh(struct radeon_device *rdev,
3032 u32 se_num, u32 sh_num)
3033{
3034 u32 data = INSTANCE_BROADCAST_WRITES;
3035
3036 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 3037 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 3038 else if (se_num == 0xffffffff)
3039 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3040 else if (sh_num == 0xffffffff)
3041 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3042 else
3043 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3044 WREG32(GRBM_GFX_INDEX, data);
3045}
3046
3047/**
3048 * cik_create_bitmask - create a bitmask
3049 *
3050 * @bit_width: length of the mask
3051 *
3052 * create a variable length bit mask (CIK).
3053 * Returns the bitmask.
3054 */
3055static u32 cik_create_bitmask(u32 bit_width)
3056{
3057 u32 i, mask = 0;
3058
3059 for (i = 0; i < bit_width; i++) {
3060 mask <<= 1;
3061 mask |= 1;
3062 }
3063 return mask;
3064}
3065
3066/**
972c5ddb 3067 * cik_get_rb_disabled - computes the mask of disabled RBs
8cc1a532
AD		 3068 *
3069 * @rdev: radeon_device pointer
28ae8ea4 3070 * @max_rb_num_per_se: max RBs (render backends) per SE (shader engine) for the asic
8cc1a532
AD		 3071 * @sh_per_se: number of SH blocks per SE for the asic
3072 *
3073 * Calculates the bitmask of disabled RBs (CIK).
3074 * Returns the disabled RB bitmask.
3075 */
3076static u32 cik_get_rb_disabled(struct radeon_device *rdev,
9fadb352 3077 u32 max_rb_num_per_se,
8cc1a532
AD		 3078 u32 sh_per_se)
3079{
3080 u32 data, mask;
3081
3082 data = RREG32(CC_RB_BACKEND_DISABLE);
3083 if (data & 1)
3084 data &= BACKEND_DISABLE_MASK;
3085 else
3086 data = 0;
3087 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3088
3089 data >>= BACKEND_DISABLE_SHIFT;
3090
9fadb352 3091 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
8cc1a532
AD		 3092
3093 return data & mask;
3094}
3095
3096/**
3097 * cik_setup_rb - setup the RBs on the asic
3098 *
3099 * @rdev: radeon_device pointer
3100 * @se_num: number of SEs (shader engines) for the asic
3101 * @sh_per_se: number of SH blocks per SE for the asic
28ae8ea4 3102 * @max_rb_num_per_se: max RBs (render backends) per SE for the asic
8cc1a532
AD		 3103 *
3104 * Configures per-SE/SH RB registers (CIK).
3105 */
3106static void cik_setup_rb(struct radeon_device *rdev,
3107 u32 se_num, u32 sh_per_se,
9fadb352 3108 u32 max_rb_num_per_se)
8cc1a532
AD		 3109{
3110 int i, j;
3111 u32 data, mask;
3112 u32 disabled_rbs = 0;
3113 u32 enabled_rbs = 0;
3114
3115 for (i = 0; i < se_num; i++) {
3116 for (j = 0; j < sh_per_se; j++) {
3117 cik_select_se_sh(rdev, i, j);
9fadb352 3118 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
fc821b70
AD		 3119 if (rdev->family == CHIP_HAWAII)
3120 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3121 else
3122 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
8cc1a532
AD		 3123 }
3124 }
3125 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3126
3127 mask = 1;
9fadb352 3128 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
8cc1a532
AD		 3129 if (!(disabled_rbs & mask))
3130 enabled_rbs |= mask;
3131 mask <<= 1;
3132 }
3133
439a1cff
MO		 3134 rdev->config.cik.backend_enable_mask = enabled_rbs;
3135
8cc1a532
AD		 3136 for (i = 0; i < se_num; i++) {
3137 cik_select_se_sh(rdev, i, 0xffffffff);
3138 data = 0;
3139 for (j = 0; j < sh_per_se; j++) {
3140 switch (enabled_rbs & 3) {
fc821b70
AD		 3141 case 0:
3142 if (j == 0)
3143 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3144 else
3145 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3146 break;
8cc1a532
AD		 3147 case 1:
3148 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3149 break;
3150 case 2:
3151 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3152 break;
3153 case 3:
3154 default:
3155 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3156 break;
3157 }
3158 enabled_rbs >>= 2;
3159 }
3160 WREG32(PA_SC_RASTER_CONFIG, data);
3161 }
3162 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3163}
3164
3165/**
3166 * cik_gpu_init - setup the 3D engine
3167 *
3168 * @rdev: radeon_device pointer
3169 *
3170 * Configures the 3D engine and tiling configuration
3171 * registers so that the 3D engine is usable.
3172 */
3173static void cik_gpu_init(struct radeon_device *rdev)
3174{
3175 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
81fa5cc3 3176 u32 mc_arb_ramcfg;
8cc1a532
AD		 3177 u32 hdp_host_path_cntl;
3178 u32 tmp;
6101b3ae 3179 int i, j;
8cc1a532
AD		 3180
3181 switch (rdev->family) {
3182 case CHIP_BONAIRE:
3183 rdev->config.cik.max_shader_engines = 2;
3184 rdev->config.cik.max_tile_pipes = 4;
3185 rdev->config.cik.max_cu_per_sh = 7;
3186 rdev->config.cik.max_sh_per_se = 1;
3187 rdev->config.cik.max_backends_per_se = 2;
3188 rdev->config.cik.max_texture_channel_caches = 4;
3189 rdev->config.cik.max_gprs = 256;
3190 rdev->config.cik.max_gs_threads = 32;
3191 rdev->config.cik.max_hw_contexts = 8;
3192
3193 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3194 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3195 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3196 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3197 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3198 break;
b496038b
AD		 3199 case CHIP_HAWAII:
3200 rdev->config.cik.max_shader_engines = 4;
3201 rdev->config.cik.max_tile_pipes = 16;
3202 rdev->config.cik.max_cu_per_sh = 11;
3203 rdev->config.cik.max_sh_per_se = 1;
3204 rdev->config.cik.max_backends_per_se = 4;
3205 rdev->config.cik.max_texture_channel_caches = 16;
3206 rdev->config.cik.max_gprs = 256;
3207 rdev->config.cik.max_gs_threads = 32;
3208 rdev->config.cik.max_hw_contexts = 8;
3209
3210 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3211 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3212 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3213 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3214 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3215 break;
8cc1a532 3216 case CHIP_KAVERI:
b2e4c70a
AD		 3217 rdev->config.cik.max_shader_engines = 1;
3218 rdev->config.cik.max_tile_pipes = 4;
0b58d90f
AD		 3219 rdev->config.cik.max_cu_per_sh = 8;
3220 rdev->config.cik.max_backends_per_se = 2;
b2e4c70a
AD		 3221 rdev->config.cik.max_sh_per_se = 1;
3222 rdev->config.cik.max_texture_channel_caches = 4;
3223 rdev->config.cik.max_gprs = 256;
3224 rdev->config.cik.max_gs_threads = 16;
3225 rdev->config.cik.max_hw_contexts = 8;
3226
3227 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3228 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3229 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3230 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3231 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 3232 break;
3233 case CHIP_KABINI:
f73a9e83 3234 case CHIP_MULLINS:
8cc1a532
AD		 3235 default:
3236 rdev->config.cik.max_shader_engines = 1;
3237 rdev->config.cik.max_tile_pipes = 2;
3238 rdev->config.cik.max_cu_per_sh = 2;
3239 rdev->config.cik.max_sh_per_se = 1;
3240 rdev->config.cik.max_backends_per_se = 1;
3241 rdev->config.cik.max_texture_channel_caches = 2;
3242 rdev->config.cik.max_gprs = 256;
3243 rdev->config.cik.max_gs_threads = 16;
3244 rdev->config.cik.max_hw_contexts = 8;
3245
3246 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3247 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3248 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3249 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3250 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3251 break;
3252 }
3253
3254 /* Initialize HDP */
3255 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3256 WREG32((0x2c14 + j), 0x00000000);
3257 WREG32((0x2c18 + j), 0x00000000);
3258 WREG32((0x2c1c + j), 0x00000000);
3259 WREG32((0x2c20 + j), 0x00000000);
3260 WREG32((0x2c24 + j), 0x00000000);
3261 }
3262
3263 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
dc12a3ec
LL		 3264 WREG32(SRBM_INT_CNTL, 0x1);
3265 WREG32(SRBM_INT_ACK, 0x1);
8cc1a532
AD		 3266
3267 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3268
81fa5cc3 3269 RREG32(MC_SHARED_CHMAP);
8cc1a532
AD		 3270 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3271
3272 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3273 rdev->config.cik.mem_max_burst_length_bytes = 256;
3274 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3275 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3276 if (rdev->config.cik.mem_row_size_in_kb > 4)
3277 rdev->config.cik.mem_row_size_in_kb = 4;
3278 /* XXX use MC settings? */
3279 rdev->config.cik.shader_engine_tile_size = 32;
3280 rdev->config.cik.num_gpus = 1;
3281 rdev->config.cik.multi_gpu_tile_size = 64;
3282
3283 /* fix up row size */
3284 gb_addr_config &= ~ROW_SIZE_MASK;
3285 switch (rdev->config.cik.mem_row_size_in_kb) {
3286 case 1:
3287 default:
3288 gb_addr_config |= ROW_SIZE(0);
3289 break;
3290 case 2:
3291 gb_addr_config |= ROW_SIZE(1);
3292 break;
3293 case 4:
3294 gb_addr_config |= ROW_SIZE(2);
3295 break;
3296 }
3297
3298 /* setup tiling info dword. gb_addr_config is not adequate since it does
3299 * not have bank info, so create a custom tiling dword.
3300 * bits 3:0 num_pipes
3301 * bits 7:4 num_banks
3302 * bits 11:8 group_size
3303 * bits 15:12 row_size
3304 */
3305 rdev->config.cik.tile_config = 0;
3306 switch (rdev->config.cik.num_tile_pipes) {
3307 case 1:
3308 rdev->config.cik.tile_config |= (0 << 0);
3309 break;
3310 case 2:
3311 rdev->config.cik.tile_config |= (1 << 0);
3312 break;
3313 case 4:
3314 rdev->config.cik.tile_config |= (2 << 0);
3315 break;
3316 case 8:
3317 default:
3318 /* XXX what about 12? */
3319 rdev->config.cik.tile_config |= (3 << 0);
3320 break;
3321 }
a537314e
MD		 3322 rdev->config.cik.tile_config |=
3323 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 3324 rdev->config.cik.tile_config |=
3325 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3326 rdev->config.cik.tile_config |=
3327 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3328
3329 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3330 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3331 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 3332 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3333 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 3334 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3335 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3336 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 3337
3338 cik_tiling_mode_table_init(rdev);
3339
3340 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3341 rdev->config.cik.max_sh_per_se,
3342 rdev->config.cik.max_backends_per_se);
3343
52da51f0 3344 rdev->config.cik.active_cus = 0;
65fcf668
AD		 3345 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3346 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6101b3ae
AD		 3347 rdev->config.cik.active_cus +=
3348 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
65fcf668
AD		 3349 }
3350 }
3351
8cc1a532
AD		 3352 /* set HW defaults for 3D engine */
3353 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3354
3355 WREG32(SX_DEBUG_1, 0x20);
3356
3357 WREG32(TA_CNTL_AUX, 0x00010000);
3358
3359 tmp = RREG32(SPI_CONFIG_CNTL);
3360 tmp |= 0x03000000;
3361 WREG32(SPI_CONFIG_CNTL, tmp);
3362
3363 WREG32(SQ_CONFIG, 1);
3364
3365 WREG32(DB_DEBUG, 0);
3366
3367 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3368 tmp |= 0x00000400;
3369 WREG32(DB_DEBUG2, tmp);
3370
3371 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3372 tmp |= 0x00020200;
3373 WREG32(DB_DEBUG3, tmp);
3374
3375 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3376 tmp |= 0x00018208;
3377 WREG32(CB_HW_CONTROL, tmp);
3378
3379 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3380
3381 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3382 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3383 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3384 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3385
3386 WREG32(VGT_NUM_INSTANCES, 1);
3387
3388 WREG32(CP_PERFMON_CNTL, 0);
3389
3390 WREG32(SQ_CONFIG, 0);
3391
3392 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3393 FORCE_EOV_MAX_REZ_CNT(255)));
3394
3395 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3396 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3397
3398 WREG32(VGT_GS_VERTEX_REUSE, 16);
3399 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3400
3401 tmp = RREG32(HDP_MISC_CNTL);
3402 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3403 WREG32(HDP_MISC_CNTL, tmp);
3404
3405 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3406 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3407
3408 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3409 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3410
3411 udelay(50);
3412}
3413
2cae3bc3
AD		 3414/*
3415 * GPU scratch registers helpers function.
3416 */
3417/**
3418 * cik_scratch_init - setup driver info for CP scratch regs
3419 *
3420 * @rdev: radeon_device pointer
3421 *
3422 * Set up the number and offset of the CP scratch registers.
3423 * NOTE: use of CP scratch registers is a legacy inferface and
3424 * is not used by default on newer asics (r6xx+). On newer asics,
3425 * memory buffers are used for fences rather than scratch regs.
3426 */
3427static void cik_scratch_init(struct radeon_device *rdev)
3428{
3429 int i;
3430
3431 rdev->scratch.num_reg = 7;
3432 rdev->scratch.reg_base = SCRATCH_REG0;
3433 for (i = 0; i < rdev->scratch.num_reg; i++) {
3434 rdev->scratch.free[i] = true;
3435 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3436 }
3437}
3438
fbc832c7
AD		 3439/**
3440 * cik_ring_test - basic gfx ring test
3441 *
3442 * @rdev: radeon_device pointer
3443 * @ring: radeon_ring structure holding ring information
3444 *
3445 * Allocate a scratch register and write to it using the gfx ring (CIK).
3446 * Provides a basic gfx ring test to verify that the ring is working.
3447 * Used by cik_cp_gfx_resume();
3448 * Returns 0 on success, error on failure.
3449 */
3450int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3451{
3452 uint32_t scratch;
3453 uint32_t tmp = 0;
3454 unsigned i;
3455 int r;
3456
3457 r = radeon_scratch_get(rdev, &scratch);
3458 if (r) {
3459 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3460 return r;
3461 }
3462 WREG32(scratch, 0xCAFEDEAD);
3463 r = radeon_ring_lock(rdev, ring, 3);
3464 if (r) {
3465 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3466 radeon_scratch_free(rdev, scratch);
3467 return r;
3468 }
3469 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3470 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3471 radeon_ring_write(ring, 0xDEADBEEF);
1538a9e0 3472 radeon_ring_unlock_commit(rdev, ring, false);
963e81f9 3473
fbc832c7
AD		 3474 for (i = 0; i < rdev->usec_timeout; i++) {
3475 tmp = RREG32(scratch);
3476 if (tmp == 0xDEADBEEF)
3477 break;
0e1a351d 3478 udelay(1);
fbc832c7
AD		 3479 }
3480 if (i < rdev->usec_timeout) {
3481 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3482 } else {
3483 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3484 ring->idx, scratch, tmp);
3485 r = -EINVAL;
3486 }
3487 radeon_scratch_free(rdev, scratch);
3488 return r;
3489}
3490
780f5ddd
AD		 3491/**
3492 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3493 *
3494 * @rdev: radeon_device pointer
3495 * @ridx: radeon ring index
3496 *
3497 * Emits an hdp flush on the cp.
3498 */
3499static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3500 int ridx)
3501{
3502 struct radeon_ring *ring = &rdev->ring[ridx];
5d259067 3503 u32 ref_and_mask;
780f5ddd 3504
5d259067
AD		 3505 switch (ring->idx) {
3506 case CAYMAN_RING_TYPE_CP1_INDEX:
3507 case CAYMAN_RING_TYPE_CP2_INDEX:
3508 default:
3509 switch (ring->me) {
3510 case 0:
3511 ref_and_mask = CP2 << ring->pipe;
3512 break;
3513 case 1:
3514 ref_and_mask = CP6 << ring->pipe;
3515 break;
3516 default:
3517 return;
3518 }
3519 break;
3520 case RADEON_RING_TYPE_GFX_INDEX:
3521 ref_and_mask = CP0;
3522 break;
3523 }
3524
3525 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3526 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3527 WAIT_REG_MEM_FUNCTION(3) | /* == */
3528 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3529 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3530 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3531 radeon_ring_write(ring, ref_and_mask);
3532 radeon_ring_write(ring, ref_and_mask);
3533 radeon_ring_write(ring, 0x20); /* poll interval */
780f5ddd
AD		 3534}
3535
2cae3bc3 3536/**
b07fdd38 3537 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3538 *
3539 * @rdev: radeon_device pointer
3540 * @fence: radeon fence object
3541 *
3542 * Emits a fence sequnce number on the gfx ring and flushes
3543 * GPU caches.
3544 */
b07fdd38
AD		 3545void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3546 struct radeon_fence *fence)
2cae3bc3
AD		 3547{
3548 struct radeon_ring *ring = &rdev->ring[fence->ring];
3549 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3550
a9c73a0e
CK		 3551 /* Workaround for cache flush problems. First send a dummy EOP
3552 * event down the pipe with seq one below.
3553 */
3554 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3555 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3556 EOP_TC_ACTION_EN |
3557 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3558 EVENT_INDEX(5)));
3559 radeon_ring_write(ring, addr & 0xfffffffc);
3560 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3561 DATA_SEL(1) | INT_SEL(0));
3562 radeon_ring_write(ring, fence->seq - 1);
3563 radeon_ring_write(ring, 0);
3564
3565 /* Then send the real EOP event down the pipe. */
2cae3bc3
AD		 3566 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3567 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3568 EOP_TC_ACTION_EN |
3569 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3570 EVENT_INDEX(5)));
3571 radeon_ring_write(ring, addr & 0xfffffffc);
3572 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3573 radeon_ring_write(ring, fence->seq);
3574 radeon_ring_write(ring, 0);
2cae3bc3
AD		 3575}
3576
b07fdd38
AD		 3577/**
3578 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3579 *
3580 * @rdev: radeon_device pointer
3581 * @fence: radeon fence object
3582 *
3583 * Emits a fence sequnce number on the compute ring and flushes
3584 * GPU caches.
3585 */
3586void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3587 struct radeon_fence *fence)
3588{
3589 struct radeon_ring *ring = &rdev->ring[fence->ring];
3590 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3591
3592 /* RELEASE_MEM - flush caches, send int */
3593 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3594 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3595 EOP_TC_ACTION_EN |
3596 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3597 EVENT_INDEX(5)));
3598 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3599 radeon_ring_write(ring, addr & 0xfffffffc);
3600 radeon_ring_write(ring, upper_32_bits(addr));
3601 radeon_ring_write(ring, fence->seq);
3602 radeon_ring_write(ring, 0);
b07fdd38
AD		 3603}
3604
86302eea
CK		 3605/**
3606 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3607 *
3608 * @rdev: radeon_device pointer
3609 * @ring: radeon ring buffer object
3610 * @semaphore: radeon semaphore object
3611 * @emit_wait: Is this a sempahore wait?
3612 *
3613 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3614 * from running ahead of semaphore waits.
3615 */
1654b817 3616bool cik_semaphore_ring_emit(struct radeon_device *rdev,
2cae3bc3
AD		 3617 struct radeon_ring *ring,
3618 struct radeon_semaphore *semaphore,
3619 bool emit_wait)
3620{
3621 uint64_t addr = semaphore->gpu_addr;
3622 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3623
3624 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
5e167cdb 3625 radeon_ring_write(ring, lower_32_bits(addr));
2cae3bc3 3626 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
1654b817 3627
86302eea
CK		 3628 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3629 /* Prevent the PFP from running ahead of the semaphore wait */
3630 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3631 radeon_ring_write(ring, 0x0);
3632 }
3633
1654b817 3634 return true;
2cae3bc3
AD		 3635}
3636
c9dbd705
AD		 3637/**
3638 * cik_copy_cpdma - copy pages using the CP DMA engine
3639 *
3640 * @rdev: radeon_device pointer
3641 * @src_offset: src GPU address
3642 * @dst_offset: dst GPU address
3643 * @num_gpu_pages: number of GPU pages to xfer
57d20a43 3644 * @resv: reservation object to sync to
c9dbd705
AD		 3645 *
3646 * Copy GPU paging using the CP DMA engine (CIK+).
3647 * Used by the radeon ttm implementation to move pages if
3648 * registered as the asic copy callback.
3649 */
57d20a43
CK		 3650struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3651 uint64_t src_offset, uint64_t dst_offset,
3652 unsigned num_gpu_pages,
52791eee 3653 struct dma_resv *resv)
c9dbd705 3654{
57d20a43 3655 struct radeon_fence *fence;
975700d2 3656 struct radeon_sync sync;
c9dbd705
AD		 3657 int ring_index = rdev->asic->copy.blit_ring_index;
3658 struct radeon_ring *ring = &rdev->ring[ring_index];
3659 u32 size_in_bytes, cur_size_in_bytes, control;
3660 int i, num_loops;
3661 int r = 0;
3662
975700d2 3663 radeon_sync_create(&sync);
c9dbd705
AD		 3664
3665 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3666 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3667 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3668 if (r) {
3669 DRM_ERROR("radeon: moving bo (%d).\n", r);
975700d2 3670 radeon_sync_free(rdev, &sync, NULL);
57d20a43 3671 return ERR_PTR(r);
c9dbd705
AD		 3672 }
3673
975700d2
CK		 3674 radeon_sync_resv(rdev, &sync, resv, false);
3675 radeon_sync_rings(rdev, &sync, ring->idx);
c9dbd705
AD		 3676
3677 for (i = 0; i < num_loops; i++) {
3678 cur_size_in_bytes = size_in_bytes;
3679 if (cur_size_in_bytes > 0x1fffff)
3680 cur_size_in_bytes = 0x1fffff;
3681 size_in_bytes -= cur_size_in_bytes;
3682 control = 0;
3683 if (size_in_bytes == 0)
3684 control |= PACKET3_DMA_DATA_CP_SYNC;
3685 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3686 radeon_ring_write(ring, control);
3687 radeon_ring_write(ring, lower_32_bits(src_offset));
3688 radeon_ring_write(ring, upper_32_bits(src_offset));
3689 radeon_ring_write(ring, lower_32_bits(dst_offset));
3690 radeon_ring_write(ring, upper_32_bits(dst_offset));
3691 radeon_ring_write(ring, cur_size_in_bytes);
3692 src_offset += cur_size_in_bytes;
3693 dst_offset += cur_size_in_bytes;
3694 }
3695
57d20a43 3696 r = radeon_fence_emit(rdev, &fence, ring->idx);
c9dbd705
AD		 3697 if (r) {
3698 radeon_ring_unlock_undo(rdev, ring);
975700d2 3699 radeon_sync_free(rdev, &sync, NULL);
57d20a43 3700 return ERR_PTR(r);
c9dbd705
AD		 3701 }
3702
1538a9e0 3703 radeon_ring_unlock_commit(rdev, ring, false);
975700d2 3704 radeon_sync_free(rdev, &sync, fence);
c9dbd705 3705
57d20a43 3706 return fence;
c9dbd705
AD		 3707}
3708
2cae3bc3
AD		 3709/*
3710 * IB stuff
3711 */
3712/**
3713 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3714 *
3715 * @rdev: radeon_device pointer
3716 * @ib: radeon indirect buffer object
3717 *
5fc45397 3718 * Emits a DE (drawing engine) or CE (constant engine) IB
2cae3bc3
AD		 3719 * on the gfx ring. IBs are usually generated by userspace
3720 * acceleration drivers and submitted to the kernel for
5fc45397 3721 * scheduling on the ring. This function schedules the IB
2cae3bc3
AD		 3722 * on the gfx ring for execution by the GPU.
3723 */
3724void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3725{
3726 struct radeon_ring *ring = &rdev->ring[ib->ring];
7c42bc1a 3727 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
2cae3bc3
AD		 3728 u32 header, control = INDIRECT_BUFFER_VALID;
3729
3730 if (ib->is_const_ib) {
3731 /* set switch buffer packet before const IB */
3732 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3733 radeon_ring_write(ring, 0);
3734
3735 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3736 } else {
3737 u32 next_rptr;
3738 if (ring->rptr_save_reg) {
3739 next_rptr = ring->wptr + 3 + 4;
3740 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3741 radeon_ring_write(ring, ((ring->rptr_save_reg -
3742 PACKET3_SET_UCONFIG_REG_START) >> 2));
3743 radeon_ring_write(ring, next_rptr);
3744 } else if (rdev->wb.enabled) {
3745 next_rptr = ring->wptr + 5 + 4;
3746 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3747 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3748 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5e167cdb 3749 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
2cae3bc3
AD		 3750 radeon_ring_write(ring, next_rptr);
3751 }
3752
3753 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3754 }
3755
7c42bc1a 3756 control |= ib->length_dw | (vm_id << 24);
2cae3bc3
AD		 3757
3758 radeon_ring_write(ring, header);
5f3e226f 3759 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC));
2cae3bc3
AD		 3760 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3761 radeon_ring_write(ring, control);
3762}
3763
fbc832c7
AD		 3764/**
3765 * cik_ib_test - basic gfx ring IB test
3766 *
3767 * @rdev: radeon_device pointer
3768 * @ring: radeon_ring structure holding ring information
3769 *
3770 * Allocate an IB and execute it on the gfx ring (CIK).
3771 * Provides a basic gfx ring test to verify that IBs are working.
3772 * Returns 0 on success, error on failure.
3773 */
3774int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3775{
3776 struct radeon_ib ib;
3777 uint32_t scratch;
3778 uint32_t tmp = 0;
3779 unsigned i;
3780 int r;
3781
3782 r = radeon_scratch_get(rdev, &scratch);
3783 if (r) {
3784 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3785 return r;
3786 }
3787 WREG32(scratch, 0xCAFEDEAD);
3788 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3789 if (r) {
3790 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
5510f124 3791 radeon_scratch_free(rdev, scratch);
fbc832c7
AD		 3792 return r;
3793 }
3794 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3795 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3796 ib.ptr[2] = 0xDEADBEEF;
3797 ib.length_dw = 3;
1538a9e0 3798 r = radeon_ib_schedule(rdev, &ib, NULL, false);
fbc832c7
AD		 3799 if (r) {
3800 radeon_scratch_free(rdev, scratch);
3801 radeon_ib_free(rdev, &ib);
3802 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3803 return r;
3804 }
04db4caf
MD		 3805 r = radeon_fence_wait_timeout(ib.fence, false, usecs_to_jiffies(
3806 RADEON_USEC_IB_TEST_TIMEOUT));
3807 if (r < 0) {
fbc832c7 3808 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
5510f124
CK		 3809 radeon_scratch_free(rdev, scratch);
3810 radeon_ib_free(rdev, &ib);
fbc832c7 3811 return r;
04db4caf
MD		 3812 } else if (r == 0) {
3813 DRM_ERROR("radeon: fence wait timed out.\n");
3814 radeon_scratch_free(rdev, scratch);
3815 radeon_ib_free(rdev, &ib);
3816 return -ETIMEDOUT;
fbc832c7 3817 }
04db4caf 3818 r = 0;
fbc832c7
AD		 3819 for (i = 0; i < rdev->usec_timeout; i++) {
3820 tmp = RREG32(scratch);
3821 if (tmp == 0xDEADBEEF)
3822 break;
0e1a351d 3823 udelay(1);
fbc832c7
AD		 3824 }
3825 if (i < rdev->usec_timeout) {
3826 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3827 } else {
3828 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3829 scratch, tmp);
3830 r = -EINVAL;
3831 }
3832 radeon_scratch_free(rdev, scratch);
3833 radeon_ib_free(rdev, &ib);
3834 return r;
3835}
3836
841cf442
AD		 3837/*
3838 * CP.
3839 * On CIK, gfx and compute now have independant command processors.
3840 *
3841 * GFX
3842 * Gfx consists of a single ring and can process both gfx jobs and
3843 * compute jobs. The gfx CP consists of three microengines (ME):
3844 * PFP - Pre-Fetch Parser
3845 * ME - Micro Engine
3846 * CE - Constant Engine
3847 * The PFP and ME make up what is considered the Drawing Engine (DE).
3848 * The CE is an asynchronous engine used for updating buffer desciptors
3849 * used by the DE so that they can be loaded into cache in parallel
3850 * while the DE is processing state update packets.
3851 *
3852 * Compute
3853 * The compute CP consists of two microengines (ME):
3854 * MEC1 - Compute MicroEngine 1
3855 * MEC2 - Compute MicroEngine 2
3856 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3857 * The queues are exposed to userspace and are programmed directly
3858 * by the compute runtime.
3859 */
3860/**
3861 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3862 *
3863 * @rdev: radeon_device pointer
3864 * @enable: enable or disable the MEs
3865 *
3866 * Halts or unhalts the gfx MEs.
3867 */
3868static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3869{
3870 if (enable)
3871 WREG32(CP_ME_CNTL, 0);
3872 else {
50efa51a
AD		 3873 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3874 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
841cf442
AD		 3875 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3876 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3877 }
3878 udelay(50);
3879}
3880
3881/**
3882 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3883 *
3884 * @rdev: radeon_device pointer
3885 *
3886 * Loads the gfx PFP, ME, and CE ucode.
3887 * Returns 0 for success, -EINVAL if the ucode is not available.
3888 */
3889static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3890{
841cf442
AD		 3891 int i;
3892
3893 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3894 return -EINVAL;
3895
3896 cik_cp_gfx_enable(rdev, false);
3897
f2c6b0f4
AD		 3898 if (rdev->new_fw) {
3899 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3900 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3901 const struct gfx_firmware_header_v1_0 *ce_hdr =
3902 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3903 const struct gfx_firmware_header_v1_0 *me_hdr =
3904 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3905 const __le32 *fw_data;
3906 u32 fw_size;
3907
3908 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3909 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3910 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3911
3912 /* PFP */
3913 fw_data = (const __le32 *)
3914 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3915 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3916 WREG32(CP_PFP_UCODE_ADDR, 0);
3917 for (i = 0; i < fw_size; i++)
3918 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 3919 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
f2c6b0f4
AD		 3920
3921 /* CE */
3922 fw_data = (const __le32 *)
3923 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3924 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3925 WREG32(CP_CE_UCODE_ADDR, 0);
3926 for (i = 0; i < fw_size; i++)
3927 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 3928 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
f2c6b0f4
AD		 3929
3930 /* ME */
3931 fw_data = (const __be32 *)
3932 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3933 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3934 WREG32(CP_ME_RAM_WADDR, 0);
3935 for (i = 0; i < fw_size; i++)
3936 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
38aea071
AD		 3937 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
3938 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
f2c6b0f4
AD		 3939 } else {
3940 const __be32 *fw_data;
3941
3942 /* PFP */
3943 fw_data = (const __be32 *)rdev->pfp_fw->data;
3944 WREG32(CP_PFP_UCODE_ADDR, 0);
3945 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3946 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3947 WREG32(CP_PFP_UCODE_ADDR, 0);
3948
3949 /* CE */
3950 fw_data = (const __be32 *)rdev->ce_fw->data;
3951 WREG32(CP_CE_UCODE_ADDR, 0);
3952 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3953 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3954 WREG32(CP_CE_UCODE_ADDR, 0);
3955
3956 /* ME */
3957 fw_data = (const __be32 *)rdev->me_fw->data;
3958 WREG32(CP_ME_RAM_WADDR, 0);
3959 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3960 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3961 WREG32(CP_ME_RAM_WADDR, 0);
3962 }
841cf442 3963
841cf442
AD		 3964 return 0;
3965}
3966
3967/**
3968 * cik_cp_gfx_start - start the gfx ring
3969 *
3970 * @rdev: radeon_device pointer
3971 *
3972 * Enables the ring and loads the clear state context and other
3973 * packets required to init the ring.
3974 * Returns 0 for success, error for failure.
3975 */
3976static int cik_cp_gfx_start(struct radeon_device *rdev)
3977{
3978 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3979 int r, i;
3980
3981 /* init the CP */
3982 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3983 WREG32(CP_ENDIAN_SWAP, 0);
3984 WREG32(CP_DEVICE_ID, 1);
3985
3986 cik_cp_gfx_enable(rdev, true);
3987
3988 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3989 if (r) {
3990 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3991 return r;
3992 }
3993
3994 /* init the CE partitions. CE only used for gfx on CIK */
3995 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3996 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
dc4edad6
JZ		 3997 radeon_ring_write(ring, 0x8000);
3998 radeon_ring_write(ring, 0x8000);
841cf442
AD		 3999
4000 /* setup clear context state */
4001 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4002 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4003
4004 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4005 radeon_ring_write(ring, 0x80000000);
4006 radeon_ring_write(ring, 0x80000000);
4007
4008 for (i = 0; i < cik_default_size; i++)
4009 radeon_ring_write(ring, cik_default_state[i]);
4010
4011 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4012 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4013
4014 /* set clear context state */
4015 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4016 radeon_ring_write(ring, 0);
4017
4018 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4019 radeon_ring_write(ring, 0x00000316);
4020 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4021 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4022
1538a9e0 4023 radeon_ring_unlock_commit(rdev, ring, false);
841cf442
AD		 4024
4025 return 0;
4026}
4027
4028/**
4029 * cik_cp_gfx_fini - stop the gfx ring
4030 *
4031 * @rdev: radeon_device pointer
4032 *
4033 * Stop the gfx ring and tear down the driver ring
4034 * info.
4035 */
4036static void cik_cp_gfx_fini(struct radeon_device *rdev)
4037{
4038 cik_cp_gfx_enable(rdev, false);
4039 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4040}
4041
4042/**
4043 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4044 *
4045 * @rdev: radeon_device pointer
4046 *
4047 * Program the location and size of the gfx ring buffer
4048 * and test it to make sure it's working.
4049 * Returns 0 for success, error for failure.
4050 */
4051static int cik_cp_gfx_resume(struct radeon_device *rdev)
4052{
4053 struct radeon_ring *ring;
4054 u32 tmp;
4055 u32 rb_bufsz;
4056 u64 rb_addr;
4057 int r;
4058
4059 WREG32(CP_SEM_WAIT_TIMER, 0x0);
939c0d3c
AD		 4060 if (rdev->family != CHIP_HAWAII)
4061 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
841cf442
AD		 4062
4063 /* Set the write pointer delay */
4064 WREG32(CP_RB_WPTR_DELAY, 0);
4065
4066 /* set the RB to use vmid 0 */
4067 WREG32(CP_RB_VMID, 0);
4068
4069 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4070
4071 /* ring 0 - compute and gfx */
4072 /* Set ring buffer size */
4073 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 4074 rb_bufsz = order_base_2(ring->ring_size / 8);
4075 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 4076#ifdef __BIG_ENDIAN
4077 tmp |= BUF_SWAP_32BIT;
4078#endif
4079 WREG32(CP_RB0_CNTL, tmp);
4080
4081 /* Initialize the ring buffer's read and write pointers */
4082 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4083 ring->wptr = 0;
4084 WREG32(CP_RB0_WPTR, ring->wptr);
4085
4086 /* set the wb address wether it's enabled or not */
4087 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4088 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4089
4090 /* scratch register shadowing is no longer supported */
4091 WREG32(SCRATCH_UMSK, 0);
4092
4093 if (!rdev->wb.enabled)
4094 tmp |= RB_NO_UPDATE;
4095
4096 mdelay(1);
4097 WREG32(CP_RB0_CNTL, tmp);
4098
4099 rb_addr = ring->gpu_addr >> 8;
4100 WREG32(CP_RB0_BASE, rb_addr);
4101 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4102
841cf442
AD		 4103 /* start the ring */
4104 cik_cp_gfx_start(rdev);
4105 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4106 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4107 if (r) {
4108 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4109 return r;
4110 }
50efa51a
AD		 4111
4112 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4113 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4114
841cf442
AD		 4115 return 0;
4116}
4117
ea31bf69
AD		 4118u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4119 struct radeon_ring *ring)
963e81f9
AD		 4120{
4121 u32 rptr;
4122
ea31bf69
AD		 4123 if (rdev->wb.enabled)
4124 rptr = rdev->wb.wb[ring->rptr_offs/4];
4125 else
4126 rptr = RREG32(CP_RB0_RPTR);
4127
4128 return rptr;
4129}
4130
4131u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4132 struct radeon_ring *ring)
4133{
0003b8d2 4134 return RREG32(CP_RB0_WPTR);
ea31bf69
AD		 4135}
4136
4137void cik_gfx_set_wptr(struct radeon_device *rdev,
4138 struct radeon_ring *ring)
4139{
4140 WREG32(CP_RB0_WPTR, ring->wptr);
4141 (void)RREG32(CP_RB0_WPTR);
4142}
4143
4144u32 cik_compute_get_rptr(struct radeon_device *rdev,
4145 struct radeon_ring *ring)
4146{
4147 u32 rptr;
963e81f9
AD		 4148
4149 if (rdev->wb.enabled) {
ea31bf69 4150 rptr = rdev->wb.wb[ring->rptr_offs/4];
963e81f9 4151 } else {
f61d5b46 4152 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4153 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4154 rptr = RREG32(CP_HQD_PQ_RPTR);
4155 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4156 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4157 }
963e81f9
AD		 4158
4159 return rptr;
4160}
4161
ea31bf69
AD		 4162u32 cik_compute_get_wptr(struct radeon_device *rdev,
4163 struct radeon_ring *ring)
963e81f9
AD		 4164{
4165 u32 wptr;
4166
4167 if (rdev->wb.enabled) {
ea31bf69
AD		 4168 /* XXX check if swapping is necessary on BE */
4169 wptr = rdev->wb.wb[ring->wptr_offs/4];
963e81f9 4170 } else {
f61d5b46 4171 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4172 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4173 wptr = RREG32(CP_HQD_PQ_WPTR);
4174 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4175 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4176 }
963e81f9
AD		 4177
4178 return wptr;
4179}
4180
ea31bf69
AD		 4181void cik_compute_set_wptr(struct radeon_device *rdev,
4182 struct radeon_ring *ring)
963e81f9 4183{
ea31bf69
AD		 4184 /* XXX check if swapping is necessary on BE */
4185 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
d5754ab8 4186 WDOORBELL32(ring->doorbell_index, ring->wptr);
963e81f9
AD		 4187}
4188
161569de
JG		 4189static void cik_compute_stop(struct radeon_device *rdev,
4190 struct radeon_ring *ring)
4191{
4192 u32 j, tmp;
4193
4194 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4195 /* Disable wptr polling. */
4196 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4197 tmp &= ~WPTR_POLL_EN;
4198 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4199 /* Disable HQD. */
4200 if (RREG32(CP_HQD_ACTIVE) & 1) {
4201 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4202 for (j = 0; j < rdev->usec_timeout; j++) {
4203 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4204 break;
4205 udelay(1);
4206 }
4207 WREG32(CP_HQD_DEQUEUE_REQUEST, 0);
4208 WREG32(CP_HQD_PQ_RPTR, 0);
4209 WREG32(CP_HQD_PQ_WPTR, 0);
4210 }
4211 cik_srbm_select(rdev, 0, 0, 0, 0);
4212}
4213
841cf442
AD		 4214/**
4215 * cik_cp_compute_enable - enable/disable the compute CP MEs
4216 *
4217 * @rdev: radeon_device pointer
4218 * @enable: enable or disable the MEs
4219 *
4220 * Halts or unhalts the compute MEs.
4221 */
4222static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4223{
4224 if (enable)
4225 WREG32(CP_MEC_CNTL, 0);
b2b3d8d9 4226 else {
161569de
JG		 4227 /*
4228 * To make hibernation reliable we need to clear compute ring
4229 * configuration before halting the compute ring.
4230 */
4231 mutex_lock(&rdev->srbm_mutex);
4232 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
4233 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
4234 mutex_unlock(&rdev->srbm_mutex);
4235
841cf442 4236 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
b2b3d8d9
AD		 4237 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4238 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4239 }
841cf442
AD		 4240 udelay(50);
4241}
4242
4243/**
4244 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4245 *
4246 * @rdev: radeon_device pointer
4247 *
4248 * Loads the compute MEC1&2 ucode.
4249 * Returns 0 for success, -EINVAL if the ucode is not available.
4250 */
4251static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4252{
841cf442
AD		 4253 int i;
4254
4255 if (!rdev->mec_fw)
4256 return -EINVAL;
4257
4258 cik_cp_compute_enable(rdev, false);
4259
f2c6b0f4
AD		 4260 if (rdev->new_fw) {
4261 const struct gfx_firmware_header_v1_0 *mec_hdr =
4262 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4263 const __le32 *fw_data;
4264 u32 fw_size;
4265
4266 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4267
4268 /* MEC1 */
4269 fw_data = (const __le32 *)
4270 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4271 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4272 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4273 for (i = 0; i < fw_size; i++)
4274 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4275 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
841cf442 4276
841cf442 4277 /* MEC2 */
f2c6b0f4
AD		 4278 if (rdev->family == CHIP_KAVERI) {
4279 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4280 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4281
4282 fw_data = (const __le32 *)
4283 (rdev->mec2_fw->data +
4284 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4285 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4286 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4287 for (i = 0; i < fw_size; i++)
4288 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4289 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
f2c6b0f4
AD		 4290 }
4291 } else {
4292 const __be32 *fw_data;
4293
4294 /* MEC1 */
841cf442 4295 fw_data = (const __be32 *)rdev->mec_fw->data;
f2c6b0f4 4296 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
841cf442 4297 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
f2c6b0f4
AD		 4298 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4299 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4300
4301 if (rdev->family == CHIP_KAVERI) {
4302 /* MEC2 */
4303 fw_data = (const __be32 *)rdev->mec_fw->data;
4304 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4305 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4306 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4307 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4308 }
841cf442
AD		 4309 }
4310
4311 return 0;
4312}
4313
4314/**
4315 * cik_cp_compute_start - start the compute queues
4316 *
4317 * @rdev: radeon_device pointer
4318 *
4319 * Enable the compute queues.
4320 * Returns 0 for success, error for failure.
4321 */
4322static int cik_cp_compute_start(struct radeon_device *rdev)
4323{
963e81f9
AD		 4324 cik_cp_compute_enable(rdev, true);
4325
841cf442
AD		 4326 return 0;
4327}
4328
4329/**
4330 * cik_cp_compute_fini - stop the compute queues
4331 *
4332 * @rdev: radeon_device pointer
4333 *
4334 * Stop the compute queues and tear down the driver queue
4335 * info.
4336 */
4337static void cik_cp_compute_fini(struct radeon_device *rdev)
4338{
963e81f9
AD		 4339 int i, idx, r;
4340
841cf442 4341 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 4342
4343 for (i = 0; i < 2; i++) {
4344 if (i == 0)
4345 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4346 else
4347 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4348
4349 if (rdev->ring[idx].mqd_obj) {
4350 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4351 if (unlikely(r != 0))
4352 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4353
4354 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4355 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4356
4357 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4358 rdev->ring[idx].mqd_obj = NULL;
4359 }
4360 }
841cf442
AD		 4361}
4362
963e81f9
AD		 4363static void cik_mec_fini(struct radeon_device *rdev)
4364{
4365 int r;
4366
4367 if (rdev->mec.hpd_eop_obj) {
4368 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4369 if (unlikely(r != 0))
4370 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4371 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4372 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4373
4374 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4375 rdev->mec.hpd_eop_obj = NULL;
4376 }
4377}
4378
4379#define MEC_HPD_SIZE 2048
4380
4381static int cik_mec_init(struct radeon_device *rdev)
4382{
4383 int r;
4384 u32 *hpd;
4385
4386 /*
4387 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4388 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4389 */
acaf6622
MD		 4390 if (rdev->family == CHIP_KAVERI)
4391 rdev->mec.num_mec = 2;
4392 else
4393 rdev->mec.num_mec = 1;
4394 rdev->mec.num_pipe = 4;
963e81f9
AD		 4395 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4396
4397 if (rdev->mec.hpd_eop_obj == NULL) {
4398 r = radeon_bo_create(rdev,
4399 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4400 PAGE_SIZE, true,
831b6966 4401 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
963e81f9
AD		 4402 &rdev->mec.hpd_eop_obj);
4403 if (r) {
4404 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4405 return r;
4406 }
4407 }
4408
4409 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4410 if (unlikely(r != 0)) {
4411 cik_mec_fini(rdev);
4412 return r;
4413 }
4414 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4415 &rdev->mec.hpd_eop_gpu_addr);
4416 if (r) {
4417 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4418 cik_mec_fini(rdev);
4419 return r;
4420 }
4421 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4422 if (r) {
4423 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4424 cik_mec_fini(rdev);
4425 return r;
4426 }
4427
4428 /* clear memory. Not sure if this is required or not */
4429 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4430
4431 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4432 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4433
4434 return 0;
4435}
4436
4437struct hqd_registers
4438{
4439 u32 cp_mqd_base_addr;
4440 u32 cp_mqd_base_addr_hi;
4441 u32 cp_hqd_active;
4442 u32 cp_hqd_vmid;
4443 u32 cp_hqd_persistent_state;
4444 u32 cp_hqd_pipe_priority;
4445 u32 cp_hqd_queue_priority;
4446 u32 cp_hqd_quantum;
4447 u32 cp_hqd_pq_base;
4448 u32 cp_hqd_pq_base_hi;
4449 u32 cp_hqd_pq_rptr;
4450 u32 cp_hqd_pq_rptr_report_addr;
4451 u32 cp_hqd_pq_rptr_report_addr_hi;
4452 u32 cp_hqd_pq_wptr_poll_addr;
4453 u32 cp_hqd_pq_wptr_poll_addr_hi;
4454 u32 cp_hqd_pq_doorbell_control;
4455 u32 cp_hqd_pq_wptr;
4456 u32 cp_hqd_pq_control;
4457 u32 cp_hqd_ib_base_addr;
4458 u32 cp_hqd_ib_base_addr_hi;
4459 u32 cp_hqd_ib_rptr;
4460 u32 cp_hqd_ib_control;
4461 u32 cp_hqd_iq_timer;
4462 u32 cp_hqd_iq_rptr;
4463 u32 cp_hqd_dequeue_request;
4464 u32 cp_hqd_dma_offload;
4465 u32 cp_hqd_sema_cmd;
4466 u32 cp_hqd_msg_type;
4467 u32 cp_hqd_atomic0_preop_lo;
4468 u32 cp_hqd_atomic0_preop_hi;
4469 u32 cp_hqd_atomic1_preop_lo;
4470 u32 cp_hqd_atomic1_preop_hi;
4471 u32 cp_hqd_hq_scheduler0;
4472 u32 cp_hqd_hq_scheduler1;
4473 u32 cp_mqd_control;
4474};
4475
4476struct bonaire_mqd
4477{
4478 u32 header;
4479 u32 dispatch_initiator;
4480 u32 dimensions[3];
4481 u32 start_idx[3];
4482 u32 num_threads[3];
4483 u32 pipeline_stat_enable;
4484 u32 perf_counter_enable;
4485 u32 pgm[2];
4486 u32 tba[2];
4487 u32 tma[2];
4488 u32 pgm_rsrc[2];
4489 u32 vmid;
4490 u32 resource_limits;
4491 u32 static_thread_mgmt01[2];
4492 u32 tmp_ring_size;
4493 u32 static_thread_mgmt23[2];
4494 u32 restart[3];
4495 u32 thread_trace_enable;
4496 u32 reserved1;
4497 u32 user_data[16];
4498 u32 vgtcs_invoke_count[2];
4499 struct hqd_registers queue_state;
4500 u32 dequeue_cntr;
4501 u32 interrupt_queue[64];
4502};
4503
841cf442
AD		 4504/**
4505 * cik_cp_compute_resume - setup the compute queue registers
4506 *
4507 * @rdev: radeon_device pointer
4508 *
4509 * Program the compute queues and test them to make sure they
4510 * are working.
4511 * Returns 0 for success, error for failure.
4512 */
4513static int cik_cp_compute_resume(struct radeon_device *rdev)
4514{
370ce45b 4515 int r, i, j, idx;
963e81f9
AD		 4516 u32 tmp;
4517 bool use_doorbell = true;
4518 u64 hqd_gpu_addr;
4519 u64 mqd_gpu_addr;
4520 u64 eop_gpu_addr;
4521 u64 wb_gpu_addr;
4522 u32 *buf;
4523 struct bonaire_mqd *mqd;
841cf442 4524
841cf442
AD		 4525 r = cik_cp_compute_start(rdev);
4526 if (r)
4527 return r;
963e81f9
AD		 4528
4529 /* fix up chicken bits */
4530 tmp = RREG32(CP_CPF_DEBUG);
4531 tmp |= (1 << 23);
4532 WREG32(CP_CPF_DEBUG, tmp);
4533
4534 /* init the pipes */
f61d5b46 4535 mutex_lock(&rdev->srbm_mutex);
963e81f9 4536
acaf6622
MD		 4537 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); ++i) {
4538 int me = (i < 4) ? 1 : 2;
4539 int pipe = (i < 4) ? i : (i - 4);
4540
4541 cik_srbm_select(rdev, me, pipe, 0, 0);
963e81f9 4542
d59095f7
AR		 4543 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2) ;
4544 /* write the EOP addr */
4545 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4546 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
963e81f9 4547
d59095f7
AR		 4548 /* set the VMID assigned */
4549 WREG32(CP_HPD_EOP_VMID, 0);
62a7b7fb 4550
d59095f7
AR		 4551 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4552 tmp = RREG32(CP_HPD_EOP_CONTROL);
4553 tmp &= ~EOP_SIZE_MASK;
4554 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4555 WREG32(CP_HPD_EOP_CONTROL, tmp);
963e81f9 4556
d59095f7 4557 }
acaf6622 4558 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4559 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4560
4561 /* init the queues. Just two for now. */
4562 for (i = 0; i < 2; i++) {
4563 if (i == 0)
4564 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4565 else
4566 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4567
4568 if (rdev->ring[idx].mqd_obj == NULL) {
4569 r = radeon_bo_create(rdev,
4570 sizeof(struct bonaire_mqd),
4571 PAGE_SIZE, true,
02376d82 4572 RADEON_GEM_DOMAIN_GTT, 0, NULL,
831b6966 4573 NULL, &rdev->ring[idx].mqd_obj);
963e81f9
AD		 4574 if (r) {
4575 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4576 return r;
4577 }
4578 }
4579
4580 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4581 if (unlikely(r != 0)) {
4582 cik_cp_compute_fini(rdev);
4583 return r;
4584 }
4585 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4586 &mqd_gpu_addr);
4587 if (r) {
4588 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4589 cik_cp_compute_fini(rdev);
4590 return r;
4591 }
4592 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4593 if (r) {
4594 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4595 cik_cp_compute_fini(rdev);
4596 return r;
4597 }
4598
963e81f9
AD		 4599 /* init the mqd struct */
4600 memset(buf, 0, sizeof(struct bonaire_mqd));
4601
4602 mqd = (struct bonaire_mqd *)buf;
4603 mqd->header = 0xC0310800;
4604 mqd->static_thread_mgmt01[0] = 0xffffffff;
4605 mqd->static_thread_mgmt01[1] = 0xffffffff;
4606 mqd->static_thread_mgmt23[0] = 0xffffffff;
4607 mqd->static_thread_mgmt23[1] = 0xffffffff;
4608
f61d5b46 4609 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4610 cik_srbm_select(rdev, rdev->ring[idx].me,
4611 rdev->ring[idx].pipe,
4612 rdev->ring[idx].queue, 0);
4613
4614 /* disable wptr polling */
4615 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4616 tmp &= ~WPTR_POLL_EN;
4617 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4618
4619 /* enable doorbell? */
4620 mqd->queue_state.cp_hqd_pq_doorbell_control =
4621 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4622 if (use_doorbell)
4623 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4624 else
4625 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4626 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4627 mqd->queue_state.cp_hqd_pq_doorbell_control);
4628
4629 /* disable the queue if it's active */
4630 mqd->queue_state.cp_hqd_dequeue_request = 0;
4631 mqd->queue_state.cp_hqd_pq_rptr = 0;
4632 mqd->queue_state.cp_hqd_pq_wptr= 0;
4633 if (RREG32(CP_HQD_ACTIVE) & 1) {
4634 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
370ce45b 4635 for (j = 0; j < rdev->usec_timeout; j++) {
963e81f9
AD		 4636 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4637 break;
4638 udelay(1);
4639 }
4640 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4641 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4642 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4643 }
4644
4645 /* set the pointer to the MQD */
4646 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4647 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4648 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4649 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4650 /* set MQD vmid to 0 */
4651 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4652 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4653 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4654
4655 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4656 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4657 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4658 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4659 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4660 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4661
4662 /* set up the HQD, this is similar to CP_RB0_CNTL */
4663 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4664 mqd->queue_state.cp_hqd_pq_control &=
4665 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4666
4667 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4668 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 4669 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4670 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 4671#ifdef __BIG_ENDIAN
4672 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4673#endif
4674 mqd->queue_state.cp_hqd_pq_control &=
4675 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4676 mqd->queue_state.cp_hqd_pq_control |=
4677 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4678 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4679
4680 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4681 if (i == 0)
4682 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4683 else
4684 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4685 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4686 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4687 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4688 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4689 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4690
4691 /* set the wb address wether it's enabled or not */
4692 if (i == 0)
4693 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4694 else
4695 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4696 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4697 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4698 upper_32_bits(wb_gpu_addr) & 0xffff;
4699 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4700 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4701 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4702 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4703
4704 /* enable the doorbell if requested */
4705 if (use_doorbell) {
4706 mqd->queue_state.cp_hqd_pq_doorbell_control =
4707 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4708 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4709 mqd->queue_state.cp_hqd_pq_doorbell_control |=
d5754ab8 4710 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
963e81f9
AD		 4711 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4712 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4713 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4714
4715 } else {
4716 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4717 }
4718 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4719 mqd->queue_state.cp_hqd_pq_doorbell_control);
4720
4721 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4722 rdev->ring[idx].wptr = 0;
4723 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4724 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
ff212f25 4725 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
963e81f9
AD		 4726
4727 /* set the vmid for the queue */
4728 mqd->queue_state.cp_hqd_vmid = 0;
4729 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4730
4731 /* activate the queue */
4732 mqd->queue_state.cp_hqd_active = 1;
4733 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4734
4735 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4736 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4737
4738 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4739 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4740
4741 rdev->ring[idx].ready = true;
4742 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4743 if (r)
4744 rdev->ring[idx].ready = false;
4745 }
4746
841cf442
AD		 4747 return 0;
4748}
4749
841cf442
AD		 4750static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4751{
4752 cik_cp_gfx_enable(rdev, enable);
4753 cik_cp_compute_enable(rdev, enable);
4754}
4755
841cf442
AD		 4756static int cik_cp_load_microcode(struct radeon_device *rdev)
4757{
4758 int r;
4759
4760 r = cik_cp_gfx_load_microcode(rdev);
4761 if (r)
4762 return r;
4763 r = cik_cp_compute_load_microcode(rdev);
4764 if (r)
4765 return r;
4766
4767 return 0;
4768}
4769
841cf442
AD		 4770static void cik_cp_fini(struct radeon_device *rdev)
4771{
4772 cik_cp_gfx_fini(rdev);
4773 cik_cp_compute_fini(rdev);
4774}
4775
841cf442
AD		 4776static int cik_cp_resume(struct radeon_device *rdev)
4777{
4778 int r;
4779
4214faf6
AD		 4780 cik_enable_gui_idle_interrupt(rdev, false);
4781
841cf442
AD		 4782 r = cik_cp_load_microcode(rdev);
4783 if (r)
4784 return r;
4785
4786 r = cik_cp_gfx_resume(rdev);
4787 if (r)
4788 return r;
4789 r = cik_cp_compute_resume(rdev);
4790 if (r)
4791 return r;
4792
4214faf6
AD		 4793 cik_enable_gui_idle_interrupt(rdev, true);
4794
841cf442
AD		 4795 return 0;
4796}
4797
cc066715 4798static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 4799{
6f2043ce
AD		 4800 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4801 RREG32(GRBM_STATUS));
4802 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4803 RREG32(GRBM_STATUS2));
4804 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4805 RREG32(GRBM_STATUS_SE0));
4806 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4807 RREG32(GRBM_STATUS_SE1));
4808 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4809 RREG32(GRBM_STATUS_SE2));
4810 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4811 RREG32(GRBM_STATUS_SE3));
4812 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4813 RREG32(SRBM_STATUS));
4814 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4815 RREG32(SRBM_STATUS2));
cc066715
AD		 4816 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4817 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4818 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4819 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 4820 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4821 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4822 RREG32(CP_STALLED_STAT1));
4823 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4824 RREG32(CP_STALLED_STAT2));
4825 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4826 RREG32(CP_STALLED_STAT3));
4827 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4828 RREG32(CP_CPF_BUSY_STAT));
4829 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4830 RREG32(CP_CPF_STALLED_STAT1));
4831 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4832 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4833 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4834 RREG32(CP_CPC_STALLED_STAT1));
4835 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 4836}
6f2043ce 4837
21a93e13 4838/**
cc066715 4839 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 4840 *
4841 * @rdev: radeon_device pointer
21a93e13 4842 *
cc066715
AD		 4843 * Check which blocks are busy and return the relevant reset
4844 * mask to be used by cik_gpu_soft_reset().
4845 * Returns a mask of the blocks to be reset.
21a93e13 4846 */
2483b4ea 4847u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 4848{
cc066715
AD		 4849 u32 reset_mask = 0;
4850 u32 tmp;
21a93e13 4851
cc066715
AD		 4852 /* GRBM_STATUS */
4853 tmp = RREG32(GRBM_STATUS);
4854 if (tmp & (PA_BUSY | SC_BUSY |
4855 BCI_BUSY | SX_BUSY |
4856 TA_BUSY | VGT_BUSY |
4857 DB_BUSY | CB_BUSY |
4858 GDS_BUSY | SPI_BUSY |
4859 IA_BUSY | IA_BUSY_NO_DMA))
4860 reset_mask |= RADEON_RESET_GFX;
21a93e13 4861
cc066715
AD		 4862 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4863 reset_mask |= RADEON_RESET_CP;
21a93e13 4864
cc066715
AD		 4865 /* GRBM_STATUS2 */
4866 tmp = RREG32(GRBM_STATUS2);
4867 if (tmp & RLC_BUSY)
4868 reset_mask |= RADEON_RESET_RLC;
21a93e13 4869
cc066715
AD		 4870 /* SDMA0_STATUS_REG */
4871 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4872 if (!(tmp & SDMA_IDLE))
4873 reset_mask |= RADEON_RESET_DMA;
21a93e13 4874
cc066715
AD		 4875 /* SDMA1_STATUS_REG */
4876 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4877 if (!(tmp & SDMA_IDLE))
4878 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4879
cc066715
AD		 4880 /* SRBM_STATUS2 */
4881 tmp = RREG32(SRBM_STATUS2);
4882 if (tmp & SDMA_BUSY)
4883 reset_mask |= RADEON_RESET_DMA;
21a93e13 4884
cc066715
AD		 4885 if (tmp & SDMA1_BUSY)
4886 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4887
cc066715
AD		 4888 /* SRBM_STATUS */
4889 tmp = RREG32(SRBM_STATUS);
21a93e13 4890
cc066715
AD		 4891 if (tmp & IH_BUSY)
4892 reset_mask |= RADEON_RESET_IH;
21a93e13 4893
cc066715
AD		 4894 if (tmp & SEM_BUSY)
4895 reset_mask |= RADEON_RESET_SEM;
21a93e13 4896
cc066715
AD		 4897 if (tmp & GRBM_RQ_PENDING)
4898 reset_mask |= RADEON_RESET_GRBM;
21a93e13 4899
cc066715
AD		 4900 if (tmp & VMC_BUSY)
4901 reset_mask |= RADEON_RESET_VMC;
21a93e13 4902
cc066715
AD		 4903 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4904 MCC_BUSY | MCD_BUSY))
4905 reset_mask |= RADEON_RESET_MC;
21a93e13 4906
cc066715
AD		 4907 if (evergreen_is_display_hung(rdev))
4908 reset_mask |= RADEON_RESET_DISPLAY;
4909
4910 /* Skip MC reset as it's mostly likely not hung, just busy */
4911 if (reset_mask & RADEON_RESET_MC) {
4912 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4913 reset_mask &= ~RADEON_RESET_MC;
21a93e13 4914 }
cc066715
AD		 4915
4916 return reset_mask;
21a93e13
AD		 4917}
4918
4919/**
cc066715 4920 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 4921 *
4922 * @rdev: radeon_device pointer
cc066715 4923 * @reset_mask: mask of which blocks to reset
21a93e13 4924 *
cc066715 4925 * Soft reset the blocks specified in @reset_mask.
21a93e13 4926 */
cc066715 4927static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 4928{
6f2043ce 4929 struct evergreen_mc_save save;
cc066715
AD		 4930 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4931 u32 tmp;
21a93e13 4932
cc066715
AD		 4933 if (reset_mask == 0)
4934 return;
21a93e13 4935
cc066715 4936 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 4937
cc066715
AD		 4938 cik_print_gpu_status_regs(rdev);
4939 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4940 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4941 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4942 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 4943
fb2c7f4d
AD		 4944 /* disable CG/PG */
4945 cik_fini_pg(rdev);
4946 cik_fini_cg(rdev);
4947
cc066715
AD		 4948 /* stop the rlc */
4949 cik_rlc_stop(rdev);
21a93e13 4950
cc066715
AD		 4951 /* Disable GFX parsing/prefetching */
4952 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 4953
cc066715
AD		 4954 /* Disable MEC parsing/prefetching */
4955 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 4956
cc066715
AD		 4957 if (reset_mask & RADEON_RESET_DMA) {
4958 /* sdma0 */
4959 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4960 tmp |= SDMA_HALT;
4961 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4962 }
4963 if (reset_mask & RADEON_RESET_DMA1) {
4964 /* sdma1 */
4965 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4966 tmp |= SDMA_HALT;
4967 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4968 }
21a93e13 4969
6f2043ce 4970 evergreen_mc_stop(rdev, &save);
cc066715 4971 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 4972 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4973 }
21a93e13 4974
cc066715
AD		 4975 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4976 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 4977
cc066715
AD		 4978 if (reset_mask & RADEON_RESET_CP) {
4979 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 4980
cc066715
AD		 4981 srbm_soft_reset |= SOFT_RESET_GRBM;
4982 }
21a93e13 4983
cc066715
AD		 4984 if (reset_mask & RADEON_RESET_DMA)
4985 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 4986
cc066715
AD		 4987 if (reset_mask & RADEON_RESET_DMA1)
4988 srbm_soft_reset |= SOFT_RESET_SDMA1;
4989
4990 if (reset_mask & RADEON_RESET_DISPLAY)
4991 srbm_soft_reset |= SOFT_RESET_DC;
4992
4993 if (reset_mask & RADEON_RESET_RLC)
4994 grbm_soft_reset |= SOFT_RESET_RLC;
4995
4996 if (reset_mask & RADEON_RESET_SEM)
4997 srbm_soft_reset |= SOFT_RESET_SEM;
4998
4999 if (reset_mask & RADEON_RESET_IH)
5000 srbm_soft_reset |= SOFT_RESET_IH;
5001
5002 if (reset_mask & RADEON_RESET_GRBM)
5003 srbm_soft_reset |= SOFT_RESET_GRBM;
5004
5005 if (reset_mask & RADEON_RESET_VMC)
5006 srbm_soft_reset |= SOFT_RESET_VMC;
5007
5008 if (!(rdev->flags & RADEON_IS_IGP)) {
5009 if (reset_mask & RADEON_RESET_MC)
5010 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 5011 }
5012
cc066715
AD		 5013 if (grbm_soft_reset) {
5014 tmp = RREG32(GRBM_SOFT_RESET);
5015 tmp |= grbm_soft_reset;
5016 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5017 WREG32(GRBM_SOFT_RESET, tmp);
5018 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 5019
cc066715 5020 udelay(50);
21a93e13 5021
cc066715
AD		 5022 tmp &= ~grbm_soft_reset;
5023 WREG32(GRBM_SOFT_RESET, tmp);
5024 tmp = RREG32(GRBM_SOFT_RESET);
5025 }
21a93e13 5026
cc066715
AD		 5027 if (srbm_soft_reset) {
5028 tmp = RREG32(SRBM_SOFT_RESET);
5029 tmp |= srbm_soft_reset;
5030 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5031 WREG32(SRBM_SOFT_RESET, tmp);
5032 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 5033
cc066715 5034 udelay(50);
21a93e13 5035
cc066715
AD		 5036 tmp &= ~srbm_soft_reset;
5037 WREG32(SRBM_SOFT_RESET, tmp);
5038 tmp = RREG32(SRBM_SOFT_RESET);
5039 }
21a93e13 5040
6f2043ce
AD		 5041 /* Wait a little for things to settle down */
5042 udelay(50);
21a93e13 5043
6f2043ce 5044 evergreen_mc_resume(rdev, &save);
cc066715
AD		 5045 udelay(50);
5046
5047 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 5048}
5049
0279ed19
AD		 5050struct kv_reset_save_regs {
5051 u32 gmcon_reng_execute;
5052 u32 gmcon_misc;
5053 u32 gmcon_misc3;
5054};
5055
5056static void kv_save_regs_for_reset(struct radeon_device *rdev,
5057 struct kv_reset_save_regs *save)
5058{
5059 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5060 save->gmcon_misc = RREG32(GMCON_MISC);
5061 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5062
5063 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5064 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5065 STCTRL_STUTTER_EN));
5066}
5067
5068static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5069 struct kv_reset_save_regs *save)
5070{
5071 int i;
5072
5073 WREG32(GMCON_PGFSM_WRITE, 0);
5074 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5075
5076 for (i = 0; i < 5; i++)
5077 WREG32(GMCON_PGFSM_WRITE, 0);
5078
5079 WREG32(GMCON_PGFSM_WRITE, 0);
5080 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5081
5082 for (i = 0; i < 5; i++)
5083 WREG32(GMCON_PGFSM_WRITE, 0);
5084
5085 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5086 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5087
5088 for (i = 0; i < 5; i++)
5089 WREG32(GMCON_PGFSM_WRITE, 0);
5090
5091 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5092 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5093
5094 for (i = 0; i < 5; i++)
5095 WREG32(GMCON_PGFSM_WRITE, 0);
5096
5097 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5098 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5099
5100 for (i = 0; i < 5; i++)
5101 WREG32(GMCON_PGFSM_WRITE, 0);
5102
5103 WREG32(GMCON_PGFSM_WRITE, 0);
5104 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5105
5106 for (i = 0; i < 5; i++)
5107 WREG32(GMCON_PGFSM_WRITE, 0);
5108
5109 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5110 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5111
5112 for (i = 0; i < 5; i++)
5113 WREG32(GMCON_PGFSM_WRITE, 0);
5114
5115 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5116 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5117
5118 for (i = 0; i < 5; i++)
5119 WREG32(GMCON_PGFSM_WRITE, 0);
5120
5121 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5122 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5123
5124 for (i = 0; i < 5; i++)
5125 WREG32(GMCON_PGFSM_WRITE, 0);
5126
5127 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5128 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5129
5130 for (i = 0; i < 5; i++)
5131 WREG32(GMCON_PGFSM_WRITE, 0);
5132
5133 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5134 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5135
5136 WREG32(GMCON_MISC3, save->gmcon_misc3);
5137 WREG32(GMCON_MISC, save->gmcon_misc);
5138 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5139}
5140
5141static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5142{
5143 struct evergreen_mc_save save;
5144 struct kv_reset_save_regs kv_save = { 0 };
5145 u32 tmp, i;
5146
5147 dev_info(rdev->dev, "GPU pci config reset\n");
5148
5149 /* disable dpm? */
5150
5151 /* disable cg/pg */
5152 cik_fini_pg(rdev);
5153 cik_fini_cg(rdev);
5154
5155 /* Disable GFX parsing/prefetching */
5156 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5157
5158 /* Disable MEC parsing/prefetching */
5159 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5160
5161 /* sdma0 */
5162 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5163 tmp |= SDMA_HALT;
5164 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5165 /* sdma1 */
5166 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5167 tmp |= SDMA_HALT;
5168 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5169 /* XXX other engines? */
5170
5171 /* halt the rlc, disable cp internal ints */
5172 cik_rlc_stop(rdev);
5173
5174 udelay(50);
5175
5176 /* disable mem access */
5177 evergreen_mc_stop(rdev, &save);
5178 if (evergreen_mc_wait_for_idle(rdev)) {
5179 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5180 }
5181
5182 if (rdev->flags & RADEON_IS_IGP)
5183 kv_save_regs_for_reset(rdev, &kv_save);
5184
5185 /* disable BM */
5186 pci_clear_master(rdev->pdev);
5187 /* reset */
5188 radeon_pci_config_reset(rdev);
5189
5190 udelay(100);
5191
5192 /* wait for asic to come out of reset */
5193 for (i = 0; i < rdev->usec_timeout; i++) {
5194 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5195 break;
5196 udelay(1);
5197 }
5198
5199 /* does asic init need to be run first??? */
5200 if (rdev->flags & RADEON_IS_IGP)
5201 kv_restore_regs_for_reset(rdev, &kv_save);
5202}
5203
21a93e13 5204/**
cc066715 5205 * cik_asic_reset - soft reset GPU
21a93e13
AD		 5206 *
5207 * @rdev: radeon_device pointer
71fe2899 5208 * @hard: force hard reset
21a93e13 5209 *
cc066715
AD		 5210 * Look up which blocks are hung and attempt
5211 * to reset them.
6f2043ce 5212 * Returns 0 for success.
21a93e13 5213 */
71fe2899 5214int cik_asic_reset(struct radeon_device *rdev, bool hard)
21a93e13 5215{
cc066715 5216 u32 reset_mask;
21a93e13 5217
71fe2899
JG		 5218 if (hard) {
5219 cik_gpu_pci_config_reset(rdev);
5220 return 0;
5221 }
5222
cc066715 5223 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5224
cc066715
AD		 5225 if (reset_mask)
5226 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 5227
0279ed19 5228 /* try soft reset */
cc066715 5229 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 5230
cc066715
AD		 5231 reset_mask = cik_gpu_check_soft_reset(rdev);
5232
0279ed19
AD		 5233 /* try pci config reset */
5234 if (reset_mask && radeon_hard_reset)
5235 cik_gpu_pci_config_reset(rdev);
5236
5237 reset_mask = cik_gpu_check_soft_reset(rdev);
5238
cc066715
AD		 5239 if (!reset_mask)
5240 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 5241
5242 return 0;
5243}
5244
5245/**
cc066715 5246 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 5247 *
5248 * @rdev: radeon_device pointer
cc066715 5249 * @ring: radeon_ring structure holding ring information
21a93e13 5250 *
cc066715
AD		 5251 * Check if the 3D engine is locked up (CIK).
5252 * Returns true if the engine is locked, false if not.
21a93e13 5253 */
cc066715 5254bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 5255{
cc066715 5256 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5257
cc066715
AD		 5258 if (!(reset_mask & (RADEON_RESET_GFX |
5259 RADEON_RESET_COMPUTE |
5260 RADEON_RESET_CP))) {
ff212f25 5261 radeon_ring_lockup_update(rdev, ring);
cc066715 5262 return false;
21a93e13 5263 }
cc066715 5264 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 5265}
5266
1c49165d 5267/* MC */
21a93e13 5268/**
1c49165d 5269 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 5270 *
5271 * @rdev: radeon_device pointer
21a93e13 5272 *
1c49165d
AD		 5273 * Set the location of vram, gart, and AGP in the GPU's
5274 * physical address space (CIK).
21a93e13 5275 */
1c49165d 5276static void cik_mc_program(struct radeon_device *rdev)
21a93e13 5277{
1c49165d 5278 struct evergreen_mc_save save;
21a93e13 5279 u32 tmp;
1c49165d 5280 int i, j;
21a93e13 5281
1c49165d
AD		 5282 /* Initialize HDP */
5283 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5284 WREG32((0x2c14 + j), 0x00000000);
5285 WREG32((0x2c18 + j), 0x00000000);
5286 WREG32((0x2c1c + j), 0x00000000);
5287 WREG32((0x2c20 + j), 0x00000000);
5288 WREG32((0x2c24 + j), 0x00000000);
21a93e13 5289 }
1c49165d 5290 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 5291
1c49165d
AD		 5292 evergreen_mc_stop(rdev, &save);
5293 if (radeon_mc_wait_for_idle(rdev)) {
5294 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5295 }
1c49165d
AD		 5296 /* Lockout access through VGA aperture*/
5297 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5298 /* Update configuration */
5299 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5300 rdev->mc.vram_start >> 12);
5301 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5302 rdev->mc.vram_end >> 12);
5303 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5304 rdev->vram_scratch.gpu_addr >> 12);
5305 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5306 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5307 WREG32(MC_VM_FB_LOCATION, tmp);
5308 /* XXX double check these! */
5309 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5310 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5311 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5312 WREG32(MC_VM_AGP_BASE, 0);
5313 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5314 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5315 if (radeon_mc_wait_for_idle(rdev)) {
5316 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5317 }
1c49165d
AD		 5318 evergreen_mc_resume(rdev, &save);
5319 /* we need to own VRAM, so turn off the VGA renderer here
5320 * to stop it overwriting our objects */
5321 rv515_vga_render_disable(rdev);
21a93e13
AD		 5322}
5323
5324/**
1c49165d 5325 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 5326 *
5327 * @rdev: radeon_device pointer
21a93e13 5328 *
1c49165d
AD		 5329 * Look up the amount of vram, vram width, and decide how to place
5330 * vram and gart within the GPU's physical address space (CIK).
5331 * Returns 0 for success.
21a93e13 5332 */
1c49165d 5333static int cik_mc_init(struct radeon_device *rdev)
21a93e13 5334{
1c49165d
AD		 5335 u32 tmp;
5336 int chansize, numchan;
21a93e13 5337
1c49165d
AD		 5338 /* Get VRAM informations */
5339 rdev->mc.vram_is_ddr = true;
5340 tmp = RREG32(MC_ARB_RAMCFG);
5341 if (tmp & CHANSIZE_MASK) {
5342 chansize = 64;
21a93e13 5343 } else {
1c49165d 5344 chansize = 32;
21a93e13 5345 }
1c49165d
AD		 5346 tmp = RREG32(MC_SHARED_CHMAP);
5347 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5348 case 0:
5349 default:
5350 numchan = 1;
5351 break;
5352 case 1:
5353 numchan = 2;
5354 break;
5355 case 2:
5356 numchan = 4;
5357 break;
5358 case 3:
5359 numchan = 8;
5360 break;
5361 case 4:
5362 numchan = 3;
5363 break;
5364 case 5:
5365 numchan = 6;
5366 break;
5367 case 6:
5368 numchan = 10;
5369 break;
5370 case 7:
5371 numchan = 12;
5372 break;
5373 case 8:
5374 numchan = 16;
5375 break;
5376 }
5377 rdev->mc.vram_width = numchan * chansize;
5378 /* Could aper size report 0 ? */
5379 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5380 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5381 /* size in MB on si */
13c5bfda
AD		 5382 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5383 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 5384 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5385 si_vram_gtt_location(rdev, &rdev->mc);
5386 radeon_update_bandwidth_info(rdev);
5387
5388 return 0;
5389}
5390
5391/*
5392 * GART
5393 * VMID 0 is the physical GPU addresses as used by the kernel.
5394 * VMIDs 1-15 are used for userspace clients and are handled
5395 * by the radeon vm/hsa code.
5396 */
5397/**
5398 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5399 *
5400 * @rdev: radeon_device pointer
5401 *
5402 * Flush the TLB for the VMID 0 page table (CIK).
5403 */
5404void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5405{
5406 /* flush hdp cache */
5407 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5408
5409 /* bits 0-15 are the VM contexts0-15 */
5410 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5411}
5412
5413/**
5414 * cik_pcie_gart_enable - gart enable
5415 *
5416 * @rdev: radeon_device pointer
5417 *
5418 * This sets up the TLBs, programs the page tables for VMID0,
5419 * sets up the hw for VMIDs 1-15 which are allocated on
5420 * demand, and sets up the global locations for the LDS, GDS,
5421 * and GPUVM for FSA64 clients (CIK).
5422 * Returns 0 for success, errors for failure.
5423 */
5424static int cik_pcie_gart_enable(struct radeon_device *rdev)
5425{
5426 int r, i;
5427
5428 if (rdev->gart.robj == NULL) {
5429 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5430 return -EINVAL;
5431 }
5432 r = radeon_gart_table_vram_pin(rdev);
5433 if (r)
5434 return r;
1c49165d
AD		 5435 /* Setup TLB control */
5436 WREG32(MC_VM_MX_L1_TLB_CNTL,
5437 (0xA << 7) |
5438 ENABLE_L1_TLB |
ec3dbbcb 5439 ENABLE_L1_FRAGMENT_PROCESSING |
1c49165d
AD		 5440 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5441 ENABLE_ADVANCED_DRIVER_MODEL |
5442 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5443 /* Setup L2 cache */
5444 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5445 ENABLE_L2_FRAGMENT_PROCESSING |
5446 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5447 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5448 EFFECTIVE_L2_QUEUE_SIZE(7) |
5449 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5450 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5451 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
ec3dbbcb
CK		 5452 BANK_SELECT(4) |
5453 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
1c49165d
AD		 5454 /* setup context0 */
5455 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
7c0411d2 5456 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1c49165d
AD		 5457 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5458 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5459 (u32)(rdev->dummy_page.addr >> 12));
5460 WREG32(VM_CONTEXT0_CNTL2, 0);
5461 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5462 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5463
5464 WREG32(0x15D4, 0);
5465 WREG32(0x15D8, 0);
5466 WREG32(0x15DC, 0);
5467
054e01d6 5468 /* restore context1-15 */
1c49165d
AD		 5469 /* set vm size, must be a multiple of 4 */
5470 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
607d4806 5471 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
1c49165d
AD		 5472 for (i = 1; i < 16; i++) {
5473 if (i < 8)
5474 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
054e01d6 5475 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5476 else
5477 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
054e01d6 5478 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5479 }
5480
5481 /* enable context1-15 */
5482 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5483 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 5484 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 5485 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4510fb98 5486 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
a00024b0
AD		 5487 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5488 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5489 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5490 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5491 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5492 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5493 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5494 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5495 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5496 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5497 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5498 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d 5499
1c49165d
AD		 5500 if (rdev->family == CHIP_KAVERI) {
5501 u32 tmp = RREG32(CHUB_CONTROL);
5502 tmp &= ~BYPASS_VM;
5503 WREG32(CHUB_CONTROL, tmp);
5504 }
5505
5506 /* XXX SH_MEM regs */
5507 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 5508 mutex_lock(&rdev->srbm_mutex);
1c49165d 5509 for (i = 0; i < 16; i++) {
b556b12e 5510 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 5511 /* CP and shaders */
75cb00dc 5512 WREG32(SH_MEM_CONFIG, SH_MEM_CONFIG_GFX_DEFAULT);
1c49165d
AD		 5513 WREG32(SH_MEM_APE1_BASE, 1);
5514 WREG32(SH_MEM_APE1_LIMIT, 0);
5515 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 5516 /* SDMA GFX */
5517 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5518 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5519 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5520 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5521 /* XXX SDMA RLC - todo */
1c49165d 5522 }
b556b12e 5523 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5524 mutex_unlock(&rdev->srbm_mutex);
1c49165d
AD		 5525
5526 cik_pcie_gart_tlb_flush(rdev);
5527 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5528 (unsigned)(rdev->mc.gtt_size >> 20),
5529 (unsigned long long)rdev->gart.table_addr);
5530 rdev->gart.ready = true;
5531 return 0;
5532}
5533
5534/**
5535 * cik_pcie_gart_disable - gart disable
5536 *
5537 * @rdev: radeon_device pointer
5538 *
5539 * This disables all VM page table (CIK).
5540 */
5541static void cik_pcie_gart_disable(struct radeon_device *rdev)
5542{
054e01d6
CK		 5543 unsigned i;
5544
5545 for (i = 1; i < 16; ++i) {
5546 uint32_t reg;
5547 if (i < 8)
5548 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5549 else
5550 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5551 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5552 }
5553
1c49165d
AD		 5554 /* Disable all tables */
5555 WREG32(VM_CONTEXT0_CNTL, 0);
5556 WREG32(VM_CONTEXT1_CNTL, 0);
5557 /* Setup TLB control */
5558 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5559 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5560 /* Setup L2 cache */
5561 WREG32(VM_L2_CNTL,
5562 ENABLE_L2_FRAGMENT_PROCESSING |
5563 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5564 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5565 EFFECTIVE_L2_QUEUE_SIZE(7) |
5566 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5567 WREG32(VM_L2_CNTL2, 0);
5568 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5569 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5570 radeon_gart_table_vram_unpin(rdev);
5571}
5572
5573/**
5574 * cik_pcie_gart_fini - vm fini callback
5575 *
5576 * @rdev: radeon_device pointer
5577 *
5578 * Tears down the driver GART/VM setup (CIK).
5579 */
5580static void cik_pcie_gart_fini(struct radeon_device *rdev)
5581{
5582 cik_pcie_gart_disable(rdev);
5583 radeon_gart_table_vram_free(rdev);
5584 radeon_gart_fini(rdev);
5585}
5586
5587/* vm parser */
5588/**
5589 * cik_ib_parse - vm ib_parse callback
5590 *
5591 * @rdev: radeon_device pointer
5592 * @ib: indirect buffer pointer
5593 *
5594 * CIK uses hw IB checking so this is a nop (CIK).
5595 */
5596int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5597{
5598 return 0;
5599}
5600
5601/*
5602 * vm
5603 * VMID 0 is the physical GPU addresses as used by the kernel.
5604 * VMIDs 1-15 are used for userspace clients and are handled
5605 * by the radeon vm/hsa code.
5606 */
5607/**
5608 * cik_vm_init - cik vm init callback
5609 *
5610 * @rdev: radeon_device pointer
5611 *
5612 * Inits cik specific vm parameters (number of VMs, base of vram for
5613 * VMIDs 1-15) (CIK).
5614 * Returns 0 for success.
5615 */
5616int cik_vm_init(struct radeon_device *rdev)
5617{
62a7b7fb
OG		 5618 /*
5619 * number of VMs
5620 * VMID 0 is reserved for System
f4fa88ab 5621 * radeon graphics/compute will use VMIDs 1-15
62a7b7fb 5622 */
f4fa88ab 5623 rdev->vm_manager.nvm = 16;
1c49165d
AD		 5624 /* base offset of vram pages */
5625 if (rdev->flags & RADEON_IS_IGP) {
5626 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5627 tmp <<= 22;
5628 rdev->vm_manager.vram_base_offset = tmp;
5629 } else
5630 rdev->vm_manager.vram_base_offset = 0;
5631
5632 return 0;
5633}
5634
5635/**
5636 * cik_vm_fini - cik vm fini callback
5637 *
5638 * @rdev: radeon_device pointer
5639 *
5640 * Tear down any asic specific VM setup (CIK).
5641 */
5642void cik_vm_fini(struct radeon_device *rdev)
5643{
5644}
5645
3ec7d11b
AD		 5646/**
5647 * cik_vm_decode_fault - print human readable fault info
5648 *
5649 * @rdev: radeon_device pointer
5650 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5651 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
28ae8ea4 5652 * @mc_client: VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT register value
3ec7d11b
AD		 5653 *
5654 * Print human readable fault information (CIK).
5655 */
5656static void cik_vm_decode_fault(struct radeon_device *rdev,
5657 u32 status, u32 addr, u32 mc_client)
5658{
939c0d3c 5659 u32 mc_id;
3ec7d11b
AD		 5660 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5661 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 5662 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5663 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 5664
939c0d3c
AD		 5665 if (rdev->family == CHIP_HAWAII)
5666 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5667 else
5668 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5669
328a50c7 5670 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 5671 protections, vmid, addr,
5672 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 5673 block, mc_client, mc_id);
3ec7d11b
AD		 5674}
5675
28ae8ea4 5676/*
f96ab484
AD		 5677 * cik_vm_flush - cik vm flush using the CP
5678 *
f96ab484
AD		 5679 * Update the page table base and flush the VM TLB
5680 * using the CP (CIK).
5681 */
faffaf62
CK		 5682void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5683 unsigned vm_id, uint64_t pd_addr)
f96ab484 5684{
faffaf62 5685 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
f96ab484
AD		 5686
5687 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
f1d2a26b 5688 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484 5689 WRITE_DATA_DST_SEL(0)));
faffaf62 5690 if (vm_id < 8) {
f96ab484 5691 radeon_ring_write(ring,
faffaf62 5692 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
f96ab484
AD		 5693 } else {
5694 radeon_ring_write(ring,
faffaf62 5695 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
f96ab484
AD		 5696 }
5697 radeon_ring_write(ring, 0);
faffaf62 5698 radeon_ring_write(ring, pd_addr >> 12);
f96ab484
AD		 5699
5700 /* update SH_MEM_* regs */
5701 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5702 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5703 WRITE_DATA_DST_SEL(0)));
5704 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5705 radeon_ring_write(ring, 0);
faffaf62 5706 radeon_ring_write(ring, VMID(vm_id));
f96ab484
AD		 5707
5708 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4fb0bbd5 5709 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5710 WRITE_DATA_DST_SEL(0)));
5711 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5712 radeon_ring_write(ring, 0);
5713
5714 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
75cb00dc 5715 radeon_ring_write(ring, SH_MEM_CONFIG_GFX_DEFAULT); /* SH_MEM_CONFIG */
f96ab484
AD		 5716 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5717 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 5718
f96ab484 5719 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5720 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5721 WRITE_DATA_DST_SEL(0)));
5722 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5723 radeon_ring_write(ring, 0);
5724 radeon_ring_write(ring, VMID(0));
6f2043ce 5725
f96ab484 5726 /* HDP flush */
faffaf62 5727 cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
f96ab484
AD		 5728
5729 /* bits 0-15 are the VM contexts0-15 */
5730 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5731 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5732 WRITE_DATA_DST_SEL(0)));
5733 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5734 radeon_ring_write(ring, 0);
faffaf62 5735 radeon_ring_write(ring, 1 << vm_id);
f96ab484 5736
3a01fd36
AD		 5737 /* wait for the invalidate to complete */
5738 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5739 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
5740 WAIT_REG_MEM_FUNCTION(0) | /* always */
5741 WAIT_REG_MEM_ENGINE(0))); /* me */
5742 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5743 radeon_ring_write(ring, 0);
5744 radeon_ring_write(ring, 0); /* ref */
5745 radeon_ring_write(ring, 0); /* mask */
5746 radeon_ring_write(ring, 0x20); /* poll interval */
5747
b07fdd38 5748 /* compute doesn't have PFP */
f1d2a26b 5749 if (usepfp) {
b07fdd38
AD		 5750 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5751 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5752 radeon_ring_write(ring, 0x0);
5753 }
cc066715 5754}
6f2043ce 5755
f6796cae
AD		 5756/*
5757 * RLC
5758 * The RLC is a multi-purpose microengine that handles a
5759 * variety of functions, the most important of which is
5760 * the interrupt controller.
5761 */
866d83de
AD		 5762static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5763 bool enable)
f6796cae 5764{
866d83de 5765 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 5766
866d83de
AD		 5767 if (enable)
5768 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5769 else
5770 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 5771 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 5772}
f6796cae 5773
866d83de 5774static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 5775{
cc066715 5776 u32 tmp;
6f2043ce 5777
866d83de
AD		 5778 tmp = RREG32(RLC_LB_CNTL);
5779 if (enable)
5780 tmp |= LOAD_BALANCE_ENABLE;
5781 else
5782 tmp &= ~LOAD_BALANCE_ENABLE;
5783 WREG32(RLC_LB_CNTL, tmp);
5784}
cc066715 5785
866d83de
AD		 5786static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5787{
5788 u32 i, j, k;
5789 u32 mask;
cc066715 5790
f6796cae
AD		 5791 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5792 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5793 cik_select_se_sh(rdev, i, j);
5794 for (k = 0; k < rdev->usec_timeout; k++) {
5795 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5796 break;
5797 udelay(1);
5798 }
5799 }
5800 }
5801 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
cc066715 5802
f6796cae
AD		 5803 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5804 for (k = 0; k < rdev->usec_timeout; k++) {
5805 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5806 break;
5807 udelay(1);
5808 }
5809}
cc066715 5810
22c775ce
AD		 5811static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5812{
5813 u32 tmp;
cc066715 5814
22c775ce
AD		 5815 tmp = RREG32(RLC_CNTL);
5816 if (tmp != rlc)
5817 WREG32(RLC_CNTL, rlc);
5818}
cc066715 5819
22c775ce
AD		 5820static u32 cik_halt_rlc(struct radeon_device *rdev)
5821{
5822 u32 data, orig;
cc066715 5823
22c775ce 5824 orig = data = RREG32(RLC_CNTL);
cc066715 5825
22c775ce
AD		 5826 if (data & RLC_ENABLE) {
5827 u32 i;
cc066715 5828
22c775ce
AD		 5829 data &= ~RLC_ENABLE;
5830 WREG32(RLC_CNTL, data);
cc066715 5831
22c775ce
AD		 5832 for (i = 0; i < rdev->usec_timeout; i++) {
5833 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5834 break;
5835 udelay(1);
5836 }
cc066715 5837
22c775ce
AD		 5838 cik_wait_for_rlc_serdes(rdev);
5839 }
cc066715 5840
22c775ce
AD		 5841 return orig;
5842}
cc066715 5843
a412fce0
AD		 5844void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5845{
5846 u32 tmp, i, mask;
5847
5848 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5849 WREG32(RLC_GPR_REG2, tmp);
5850
5851 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5852 for (i = 0; i < rdev->usec_timeout; i++) {
5853 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5854 break;
5855 udelay(1);
5856 }
5857
5858 for (i = 0; i < rdev->usec_timeout; i++) {
5859 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5860 break;
5861 udelay(1);
5862 }
5863}
5864
5865void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5866{
5867 u32 tmp;
5868
5869 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5870 WREG32(RLC_GPR_REG2, tmp);
5871}
5872
866d83de
AD		 5873/**
5874 * cik_rlc_stop - stop the RLC ME
5875 *
5876 * @rdev: radeon_device pointer
5877 *
5878 * Halt the RLC ME (MicroEngine) (CIK).
5879 */
5880static void cik_rlc_stop(struct radeon_device *rdev)
5881{
22c775ce 5882 WREG32(RLC_CNTL, 0);
866d83de
AD		 5883
5884 cik_enable_gui_idle_interrupt(rdev, false);
5885
866d83de
AD		 5886 cik_wait_for_rlc_serdes(rdev);
5887}
5888
f6796cae
AD		 5889/**
5890 * cik_rlc_start - start the RLC ME
5891 *
5892 * @rdev: radeon_device pointer
5893 *
5894 * Unhalt the RLC ME (MicroEngine) (CIK).
5895 */
5896static void cik_rlc_start(struct radeon_device *rdev)
5897{
f6796cae 5898 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 5899
866d83de 5900 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 5901
f6796cae 5902 udelay(50);
6f2043ce
AD		 5903}
5904
5905/**
f6796cae 5906 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 5907 *
5908 * @rdev: radeon_device pointer
5909 *
f6796cae
AD		 5910 * Initialize the RLC registers, load the ucode,
5911 * and start the RLC (CIK).
5912 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 5913 */
f6796cae 5914static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 5915{
22c775ce 5916 u32 i, size, tmp;
cc066715 5917
f6796cae
AD		 5918 if (!rdev->rlc_fw)
5919 return -EINVAL;
cc066715 5920
cc066715
AD		 5921 cik_rlc_stop(rdev);
5922
22c775ce
AD		 5923 /* disable CG */
5924 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5925 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 5926
866d83de 5927 si_rlc_reset(rdev);
6f2043ce 5928
22c775ce 5929 cik_init_pg(rdev);
6f2043ce 5930
22c775ce 5931 cik_init_cg(rdev);
cc066715 5932
f6796cae
AD		 5933 WREG32(RLC_LB_CNTR_INIT, 0);
5934 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 5935
f6796cae
AD		 5936 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5937 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5938 WREG32(RLC_LB_PARAMS, 0x00600408);
5939 WREG32(RLC_LB_CNTL, 0x80000004);
cc066715 5940
f6796cae
AD		 5941 WREG32(RLC_MC_CNTL, 0);
5942 WREG32(RLC_UCODE_CNTL, 0);
cc066715 5943
f2c6b0f4
AD		 5944 if (rdev->new_fw) {
5945 const struct rlc_firmware_header_v1_0 *hdr =
5946 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5947 const __le32 *fw_data = (const __le32 *)
5948 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5949
5950 radeon_ucode_print_rlc_hdr(&hdr->header);
5951
5952 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5953 WREG32(RLC_GPM_UCODE_ADDR, 0);
5954 for (i = 0; i < size; i++)
5955 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 5956 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
f2c6b0f4
AD		 5957 } else {
5958 const __be32 *fw_data;
5959
5960 switch (rdev->family) {
5961 case CHIP_BONAIRE:
5962 case CHIP_HAWAII:
5963 default:
5964 size = BONAIRE_RLC_UCODE_SIZE;
5965 break;
5966 case CHIP_KAVERI:
5967 size = KV_RLC_UCODE_SIZE;
5968 break;
5969 case CHIP_KABINI:
5970 size = KB_RLC_UCODE_SIZE;
5971 break;
5972 case CHIP_MULLINS:
5973 size = ML_RLC_UCODE_SIZE;
5974 break;
5975 }
5976
5977 fw_data = (const __be32 *)rdev->rlc_fw->data;
5978 WREG32(RLC_GPM_UCODE_ADDR, 0);
5979 for (i = 0; i < size; i++)
5980 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5981 WREG32(RLC_GPM_UCODE_ADDR, 0);
5982 }
cc066715 5983
866d83de
AD		 5984 /* XXX - find out what chips support lbpw */
5985 cik_enable_lbpw(rdev, false);
cc066715 5986
22c775ce
AD		 5987 if (rdev->family == CHIP_BONAIRE)
5988 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 5989
f6796cae 5990 cik_rlc_start(rdev);
cc066715 5991
f6796cae
AD		 5992 return 0;
5993}
cc066715 5994
22c775ce
AD		 5995static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5996{
5997 u32 data, orig, tmp, tmp2;
cc066715 5998
22c775ce 5999 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 6000
473359bc 6001 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 6002 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6003
22c775ce 6004 tmp = cik_halt_rlc(rdev);
cc066715 6005
22c775ce
AD		 6006 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6007 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6008 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6009 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6010 WREG32(RLC_SERDES_WR_CTRL, tmp2);
cc066715 6011
22c775ce 6012 cik_update_rlc(rdev, tmp);
cc066715 6013
22c775ce
AD		 6014 data |= CGCG_EN | CGLS_EN;
6015 } else {
ddc76ff6 6016 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 6017
22c775ce
AD		 6018 RREG32(CB_CGTT_SCLK_CTRL);
6019 RREG32(CB_CGTT_SCLK_CTRL);
6020 RREG32(CB_CGTT_SCLK_CTRL);
6021 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 6022
22c775ce 6023 data &= ~(CGCG_EN | CGLS_EN);
cc066715 6024 }
6f2043ce 6025
22c775ce
AD		 6026 if (orig != data)
6027 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 6028
6f2043ce
AD		 6029}
6030
22c775ce 6031static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 6032{
22c775ce
AD		 6033 u32 data, orig, tmp = 0;
6034
473359bc
AD		 6035 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6036 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6037 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6038 orig = data = RREG32(CP_MEM_SLP_CNTL);
6039 data |= CP_MEM_LS_EN;
6040 if (orig != data)
6041 WREG32(CP_MEM_SLP_CNTL, data);
6042 }
6043 }
cc066715 6044
22c775ce 6045 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4bb62c95 6046 data |= 0x00000001;
22c775ce
AD		 6047 data &= 0xfffffffd;
6048 if (orig != data)
6049 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6050
6051 tmp = cik_halt_rlc(rdev);
6052
6053 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6054 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6055 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6056 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6057 WREG32(RLC_SERDES_WR_CTRL, data);
6058
6059 cik_update_rlc(rdev, tmp);
6060
473359bc
AD		 6061 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6062 orig = data = RREG32(CGTS_SM_CTRL_REG);
6063 data &= ~SM_MODE_MASK;
6064 data |= SM_MODE(0x2);
6065 data |= SM_MODE_ENABLE;
6066 data &= ~CGTS_OVERRIDE;
6067 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6068 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6069 data &= ~CGTS_LS_OVERRIDE;
6070 data &= ~ON_MONITOR_ADD_MASK;
6071 data |= ON_MONITOR_ADD_EN;
6072 data |= ON_MONITOR_ADD(0x96);
6073 if (orig != data)
6074 WREG32(CGTS_SM_CTRL_REG, data);
6075 }
22c775ce
AD		 6076 } else {
6077 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4bb62c95 6078 data |= 0x00000003;
22c775ce
AD		 6079 if (orig != data)
6080 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6081
6082 data = RREG32(RLC_MEM_SLP_CNTL);
6083 if (data & RLC_MEM_LS_EN) {
6084 data &= ~RLC_MEM_LS_EN;
6085 WREG32(RLC_MEM_SLP_CNTL, data);
6086 }
6f2043ce 6087
22c775ce
AD		 6088 data = RREG32(CP_MEM_SLP_CNTL);
6089 if (data & CP_MEM_LS_EN) {
6090 data &= ~CP_MEM_LS_EN;
6091 WREG32(CP_MEM_SLP_CNTL, data);
6092 }
cc066715 6093
22c775ce
AD		 6094 orig = data = RREG32(CGTS_SM_CTRL_REG);
6095 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6096 if (orig != data)
6097 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 6098
22c775ce 6099 tmp = cik_halt_rlc(rdev);
cc066715 6100
22c775ce
AD		 6101 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6102 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6103 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6104 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6105 WREG32(RLC_SERDES_WR_CTRL, data);
cc066715 6106
22c775ce 6107 cik_update_rlc(rdev, tmp);
cc066715 6108 }
6f2043ce 6109}
1c49165d 6110
22c775ce 6111static const u32 mc_cg_registers[] =
21a93e13 6112{
22c775ce
AD		 6113 MC_HUB_MISC_HUB_CG,
6114 MC_HUB_MISC_SIP_CG,
6115 MC_HUB_MISC_VM_CG,
6116 MC_XPB_CLK_GAT,
6117 ATC_MISC_CG,
6118 MC_CITF_MISC_WR_CG,
6119 MC_CITF_MISC_RD_CG,
6120 MC_CITF_MISC_VM_CG,
6121 VM_L2_CG,
6122};
21a93e13 6123
22c775ce
AD		 6124static void cik_enable_mc_ls(struct radeon_device *rdev,
6125 bool enable)
1c49165d 6126{
22c775ce
AD		 6127 int i;
6128 u32 orig, data;
1c49165d 6129
22c775ce
AD		 6130 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6131 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6132 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 6133 data |= MC_LS_ENABLE;
6134 else
6135 data &= ~MC_LS_ENABLE;
6136 if (data != orig)
6137 WREG32(mc_cg_registers[i], data);
1c49165d 6138 }
22c775ce 6139}
1c49165d 6140
22c775ce
AD		 6141static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6142 bool enable)
6143{
6144 int i;
6145 u32 orig, data;
6146
6147 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6148 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6149 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 6150 data |= MC_CG_ENABLE;
6151 else
6152 data &= ~MC_CG_ENABLE;
6153 if (data != orig)
6154 WREG32(mc_cg_registers[i], data);
1c49165d 6155 }
1c49165d
AD		 6156}
6157
22c775ce
AD		 6158static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6159 bool enable)
1c49165d 6160{
22c775ce 6161 u32 orig, data;
1c49165d 6162
473359bc 6163 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 6164 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6165 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 6166 } else {
22c775ce
AD		 6167 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6168 data |= 0xff000000;
6169 if (data != orig)
6170 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6171
22c775ce
AD		 6172 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6173 data |= 0xff000000;
6174 if (data != orig)
6175 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6176 }
1c49165d
AD		 6177}
6178
22c775ce
AD		 6179static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6180 bool enable)
1c49165d 6181{
22c775ce
AD		 6182 u32 orig, data;
6183
473359bc 6184 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 6185 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6186 data |= 0x100;
6187 if (orig != data)
6188 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6189
6190 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6191 data |= 0x100;
6192 if (orig != data)
6193 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6194 } else {
6195 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6196 data &= ~0x100;
6197 if (orig != data)
6198 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6199
22c775ce
AD		 6200 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6201 data &= ~0x100;
6202 if (orig != data)
6203 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6204 }
1c49165d
AD		 6205}
6206
22c775ce
AD		 6207static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6208 bool enable)
1c49165d 6209{
22c775ce 6210 u32 orig, data;
1c49165d 6211
473359bc 6212 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 6213 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6214 data = 0xfff;
6215 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6216
22c775ce
AD		 6217 orig = data = RREG32(UVD_CGC_CTRL);
6218 data |= DCM;
6219 if (orig != data)
6220 WREG32(UVD_CGC_CTRL, data);
6221 } else {
6222 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6223 data &= ~0xfff;
6224 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6225
22c775ce
AD		 6226 orig = data = RREG32(UVD_CGC_CTRL);
6227 data &= ~DCM;
6228 if (orig != data)
6229 WREG32(UVD_CGC_CTRL, data);
1c49165d 6230 }
22c775ce 6231}
1c49165d 6232
473359bc
AD		 6233static void cik_enable_bif_mgls(struct radeon_device *rdev,
6234 bool enable)
6235{
6236 u32 orig, data;
1c49165d 6237
473359bc 6238 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 6239
473359bc
AD		 6240 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6241 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6242 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6243 else
6244 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6245 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 6246
473359bc
AD		 6247 if (orig != data)
6248 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6249}
1c49165d 6250
22c775ce
AD		 6251static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6252 bool enable)
6253{
6254 u32 orig, data;
1c49165d 6255
22c775ce 6256 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 6257
473359bc 6258 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 6259 data &= ~CLOCK_GATING_DIS;
6260 else
6261 data |= CLOCK_GATING_DIS;
6262
6263 if (orig != data)
6264 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 6265}
6266
22c775ce
AD		 6267static void cik_enable_hdp_ls(struct radeon_device *rdev,
6268 bool enable)
1c49165d 6269{
22c775ce
AD		 6270 u32 orig, data;
6271
6272 orig = data = RREG32(HDP_MEM_POWER_LS);
6273
473359bc 6274 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 6275 data |= HDP_LS_ENABLE;
6276 else
6277 data &= ~HDP_LS_ENABLE;
6278
6279 if (orig != data)
6280 WREG32(HDP_MEM_POWER_LS, data);
6281}
6282
6283void cik_update_cg(struct radeon_device *rdev,
6284 u32 block, bool enable)
6285{
4214faf6 6286
22c775ce 6287 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 6288 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 6289 /* order matters! */
6290 if (enable) {
6291 cik_enable_mgcg(rdev, true);
6292 cik_enable_cgcg(rdev, true);
6293 } else {
6294 cik_enable_cgcg(rdev, false);
6295 cik_enable_mgcg(rdev, false);
6296 }
4214faf6 6297 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 6298 }
6299
6300 if (block & RADEON_CG_BLOCK_MC) {
6301 if (!(rdev->flags & RADEON_IS_IGP)) {
6302 cik_enable_mc_mgcg(rdev, enable);
6303 cik_enable_mc_ls(rdev, enable);
6304 }
6305 }
6306
6307 if (block & RADEON_CG_BLOCK_SDMA) {
6308 cik_enable_sdma_mgcg(rdev, enable);
6309 cik_enable_sdma_mgls(rdev, enable);
6310 }
6311
473359bc
AD		 6312 if (block & RADEON_CG_BLOCK_BIF) {
6313 cik_enable_bif_mgls(rdev, enable);
6314 }
6315
22c775ce
AD		 6316 if (block & RADEON_CG_BLOCK_UVD) {
6317 if (rdev->has_uvd)
6318 cik_enable_uvd_mgcg(rdev, enable);
6319 }
6320
6321 if (block & RADEON_CG_BLOCK_HDP) {
6322 cik_enable_hdp_mgcg(rdev, enable);
6323 cik_enable_hdp_ls(rdev, enable);
6324 }
a1d6f97c
AD		 6325
6326 if (block & RADEON_CG_BLOCK_VCE) {
6327 vce_v2_0_enable_mgcg(rdev, enable);
6328 }
1c49165d
AD		 6329}
6330
22c775ce 6331static void cik_init_cg(struct radeon_device *rdev)
1c49165d 6332{
22c775ce 6333
ddc76ff6 6334 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 6335
6336 if (rdev->has_uvd)
6337 si_init_uvd_internal_cg(rdev);
6338
6339 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6340 RADEON_CG_BLOCK_SDMA |
473359bc 6341 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 6342 RADEON_CG_BLOCK_UVD |
6343 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 6344}
6345
473359bc 6346static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 6347{
473359bc
AD		 6348 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6349 RADEON_CG_BLOCK_SDMA |
6350 RADEON_CG_BLOCK_BIF |
6351 RADEON_CG_BLOCK_UVD |
6352 RADEON_CG_BLOCK_HDP), false);
6353
6354 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 6355}
6356
22c775ce
AD		 6357static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6358 bool enable)
1c49165d 6359{
22c775ce 6360 u32 data, orig;
1c49165d 6361
22c775ce 6362 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6363 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6364 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6365 else
6366 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6367 if (orig != data)
6368 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6369}
6370
22c775ce
AD		 6371static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6372 bool enable)
1c49165d 6373{
22c775ce
AD		 6374 u32 data, orig;
6375
6376 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6377 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6378 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6379 else
6380 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6381 if (orig != data)
6382 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6383}
6384
22c775ce 6385static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 6386{
22c775ce 6387 u32 data, orig;
3ec7d11b 6388
22c775ce 6389 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6390 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 6391 data &= ~DISABLE_CP_PG;
6392 else
6393 data |= DISABLE_CP_PG;
6394 if (orig != data)
6395 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 6396}
6397
22c775ce 6398static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 6399{
22c775ce 6400 u32 data, orig;
f96ab484 6401
22c775ce 6402 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6403 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 6404 data &= ~DISABLE_GDS_PG;
6405 else
6406 data |= DISABLE_GDS_PG;
6407 if (orig != data)
6408 WREG32(RLC_PG_CNTL, data);
6409}
6410
6411#define CP_ME_TABLE_SIZE 96
6412#define CP_ME_TABLE_OFFSET 2048
6413#define CP_MEC_TABLE_OFFSET 4096
6414
6415void cik_init_cp_pg_table(struct radeon_device *rdev)
6416{
22c775ce
AD		 6417 volatile u32 *dst_ptr;
6418 int me, i, max_me = 4;
6419 u32 bo_offset = 0;
f2c6b0f4 6420 u32 table_offset, table_size;
22c775ce
AD		 6421
6422 if (rdev->family == CHIP_KAVERI)
6423 max_me = 5;
6424
6425 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 6426 return;
6427
22c775ce
AD		 6428 /* write the cp table buffer */
6429 dst_ptr = rdev->rlc.cp_table_ptr;
6430 for (me = 0; me < max_me; me++) {
f2c6b0f4
AD		 6431 if (rdev->new_fw) {
6432 const __le32 *fw_data;
6433 const struct gfx_firmware_header_v1_0 *hdr;
6434
6435 if (me == 0) {
6436 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6437 fw_data = (const __le32 *)
6438 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6439 table_offset = le32_to_cpu(hdr->jt_offset);
6440 table_size = le32_to_cpu(hdr->jt_size);
6441 } else if (me == 1) {
6442 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6443 fw_data = (const __le32 *)
6444 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6445 table_offset = le32_to_cpu(hdr->jt_offset);
6446 table_size = le32_to_cpu(hdr->jt_size);
6447 } else if (me == 2) {
6448 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6449 fw_data = (const __le32 *)
6450 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6451 table_offset = le32_to_cpu(hdr->jt_offset);
6452 table_size = le32_to_cpu(hdr->jt_size);
6453 } else if (me == 3) {
6454 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6455 fw_data = (const __le32 *)
6456 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6457 table_offset = le32_to_cpu(hdr->jt_offset);
6458 table_size = le32_to_cpu(hdr->jt_size);
6459 } else {
6460 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6461 fw_data = (const __le32 *)
6462 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6463 table_offset = le32_to_cpu(hdr->jt_offset);
6464 table_size = le32_to_cpu(hdr->jt_size);
6465 }
6466
6467 for (i = 0; i < table_size; i ++) {
6468 dst_ptr[bo_offset + i] =
6469 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6470 }
6471 bo_offset += table_size;
22c775ce 6472 } else {
f2c6b0f4
AD		 6473 const __be32 *fw_data;
6474 table_size = CP_ME_TABLE_SIZE;
6475
6476 if (me == 0) {
6477 fw_data = (const __be32 *)rdev->ce_fw->data;
6478 table_offset = CP_ME_TABLE_OFFSET;
6479 } else if (me == 1) {
6480 fw_data = (const __be32 *)rdev->pfp_fw->data;
6481 table_offset = CP_ME_TABLE_OFFSET;
6482 } else if (me == 2) {
6483 fw_data = (const __be32 *)rdev->me_fw->data;
6484 table_offset = CP_ME_TABLE_OFFSET;
6485 } else {
6486 fw_data = (const __be32 *)rdev->mec_fw->data;
6487 table_offset = CP_MEC_TABLE_OFFSET;
6488 }
22c775ce 6489
f2c6b0f4
AD		 6490 for (i = 0; i < table_size; i ++) {
6491 dst_ptr[bo_offset + i] =
6492 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6493 }
6494 bo_offset += table_size;
22c775ce 6495 }
f96ab484 6496 }
22c775ce 6497}
f96ab484 6498
22c775ce
AD		 6499static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6500 bool enable)
6501{
6502 u32 data, orig;
6503
2b19d17f 6504 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 6505 orig = data = RREG32(RLC_PG_CNTL);
6506 data |= GFX_PG_ENABLE;
6507 if (orig != data)
6508 WREG32(RLC_PG_CNTL, data);
6509
6510 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6511 data |= AUTO_PG_EN;
6512 if (orig != data)
6513 WREG32(RLC_AUTO_PG_CTRL, data);
6514 } else {
6515 orig = data = RREG32(RLC_PG_CNTL);
6516 data &= ~GFX_PG_ENABLE;
6517 if (orig != data)
6518 WREG32(RLC_PG_CNTL, data);
f96ab484 6519
22c775ce
AD		 6520 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6521 data &= ~AUTO_PG_EN;
6522 if (orig != data)
6523 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 6524
22c775ce
AD		 6525 data = RREG32(DB_RENDER_CONTROL);
6526 }
6527}
f96ab484 6528
22c775ce
AD		 6529static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6530{
6531 u32 mask = 0, tmp, tmp1;
6532 int i;
f96ab484 6533
22c775ce
AD		 6534 cik_select_se_sh(rdev, se, sh);
6535 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6536 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6537 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
f96ab484 6538
22c775ce 6539 tmp &= 0xffff0000;
f96ab484 6540
22c775ce
AD		 6541 tmp |= tmp1;
6542 tmp >>= 16;
6543
6544 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6545 mask <<= 1;
6546 mask |= 1;
b07fdd38 6547 }
22c775ce
AD		 6548
6549 return (~tmp) & mask;
f96ab484
AD		 6550}
6551
22c775ce 6552static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 6553{
22c775ce
AD		 6554 u32 i, j, k, active_cu_number = 0;
6555 u32 mask, counter, cu_bitmap;
6556 u32 tmp = 0;
d0e092d9 6557
22c775ce
AD		 6558 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6559 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6560 mask = 1;
6561 cu_bitmap = 0;
6562 counter = 0;
6563 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6564 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6565 if (counter < 2)
6566 cu_bitmap |= mask;
6567 counter ++;
d0e092d9 6568 }
22c775ce 6569 mask <<= 1;
d0e092d9 6570 }
d0e092d9 6571
22c775ce
AD		 6572 active_cu_number += counter;
6573 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 6574 }
d0e092d9 6575 }
22c775ce
AD		 6576
6577 WREG32(RLC_PG_AO_CU_MASK, tmp);
6578
6579 tmp = RREG32(RLC_MAX_PG_CU);
6580 tmp &= ~MAX_PU_CU_MASK;
6581 tmp |= MAX_PU_CU(active_cu_number);
6582 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 6583}
6584
22c775ce
AD		 6585static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6586 bool enable)
605de6b9 6587{
22c775ce 6588 u32 data, orig;
605de6b9 6589
22c775ce 6590 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6591 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 6592 data |= STATIC_PER_CU_PG_ENABLE;
6593 else
6594 data &= ~STATIC_PER_CU_PG_ENABLE;
6595 if (orig != data)
6596 WREG32(RLC_PG_CNTL, data);
6597}
6598
6599static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6600 bool enable)
6601{
6602 u32 data, orig;
605de6b9 6603
22c775ce 6604 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6605 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 6606 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 6607 else
22c775ce
AD		 6608 data &= ~DYN_PER_CU_PG_ENABLE;
6609 if (orig != data)
6610 WREG32(RLC_PG_CNTL, data);
6611}
605de6b9 6612
22c775ce
AD		 6613#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6614#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6615
6616static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6617{
6618 u32 data, orig;
6619 u32 i;
6620
6621 if (rdev->rlc.cs_data) {
6622 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6623 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 6624 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 6625 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 6626 } else {
22c775ce
AD		 6627 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6628 for (i = 0; i < 3; i++)
6629 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6630 }
6631 if (rdev->rlc.reg_list) {
6632 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6633 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6634 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 6635 }
605de6b9 6636
22c775ce
AD		 6637 orig = data = RREG32(RLC_PG_CNTL);
6638 data |= GFX_PG_SRC;
6639 if (orig != data)
6640 WREG32(RLC_PG_CNTL, data);
605de6b9 6641
22c775ce
AD		 6642 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6643 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 6644
22c775ce
AD		 6645 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6646 data &= ~IDLE_POLL_COUNT_MASK;
6647 data |= IDLE_POLL_COUNT(0x60);
6648 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 6649
22c775ce
AD		 6650 data = 0x10101010;
6651 WREG32(RLC_PG_DELAY, data);
605de6b9 6652
22c775ce
AD		 6653 data = RREG32(RLC_PG_DELAY_2);
6654 data &= ~0xff;
6655 data |= 0x3;
6656 WREG32(RLC_PG_DELAY_2, data);
605de6b9 6657
22c775ce
AD		 6658 data = RREG32(RLC_AUTO_PG_CTRL);
6659 data &= ~GRBM_REG_SGIT_MASK;
6660 data |= GRBM_REG_SGIT(0x700);
6661 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 6662
605de6b9
AD		 6663}
6664
22c775ce 6665static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 6666{
473359bc
AD		 6667 cik_enable_gfx_cgpg(rdev, enable);
6668 cik_enable_gfx_static_mgpg(rdev, enable);
6669 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 6670}
f6796cae 6671
a0f38609
AD		 6672u32 cik_get_csb_size(struct radeon_device *rdev)
6673{
6674 u32 count = 0;
6675 const struct cs_section_def *sect = NULL;
6676 const struct cs_extent_def *ext = NULL;
f6796cae 6677
a0f38609
AD		 6678 if (rdev->rlc.cs_data == NULL)
6679 return 0;
f6796cae 6680
a0f38609
AD		 6681 /* begin clear state */
6682 count += 2;
6683 /* context control state */
6684 count += 3;
6685
6686 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6687 for (ext = sect->section; ext->extent != NULL; ++ext) {
6688 if (sect->id == SECT_CONTEXT)
6689 count += 2 + ext->reg_count;
6690 else
6691 return 0;
f6796cae
AD		 6692 }
6693 }
a0f38609
AD		 6694 /* pa_sc_raster_config/pa_sc_raster_config1 */
6695 count += 4;
6696 /* end clear state */
6697 count += 2;
6698 /* clear state */
6699 count += 2;
f6796cae 6700
a0f38609 6701 return count;
f6796cae
AD		 6702}
6703
a0f38609 6704void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 6705{
a0f38609
AD		 6706 u32 count = 0, i;
6707 const struct cs_section_def *sect = NULL;
6708 const struct cs_extent_def *ext = NULL;
f6796cae 6709
a0f38609
AD		 6710 if (rdev->rlc.cs_data == NULL)
6711 return;
6712 if (buffer == NULL)
6713 return;
f6796cae 6714
6ba81e53
AD		 6715 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6716 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
a0f38609 6717
6ba81e53
AD		 6718 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6719 buffer[count++] = cpu_to_le32(0x80000000);
6720 buffer[count++] = cpu_to_le32(0x80000000);
a0f38609
AD		 6721
6722 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6723 for (ext = sect->section; ext->extent != NULL; ++ext) {
6724 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 6725 buffer[count++] =
6726 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6727 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
a0f38609 6728 for (i = 0; i < ext->reg_count; i++)
6ba81e53 6729 buffer[count++] = cpu_to_le32(ext->extent[i]);
a0f38609
AD		 6730 } else {
6731 return;
6732 }
6733 }
6734 }
f6796cae 6735
6ba81e53
AD		 6736 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6737 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
f6796cae
AD		 6738 switch (rdev->family) {
6739 case CHIP_BONAIRE:
6ba81e53
AD		 6740 buffer[count++] = cpu_to_le32(0x16000012);
6741 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6742 break;
6743 case CHIP_KAVERI:
6ba81e53
AD		 6744 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6745 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6746 break;
6747 case CHIP_KABINI:
f73a9e83 6748 case CHIP_MULLINS:
6ba81e53
AD		 6749 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6750 buffer[count++] = cpu_to_le32(0x00000000);
a0f38609 6751 break;
bbfe90bd 6752 case CHIP_HAWAII:
a8947f57
AD		 6753 buffer[count++] = cpu_to_le32(0x3a00161a);
6754 buffer[count++] = cpu_to_le32(0x0000002e);
bbfe90bd 6755 break;
a0f38609 6756 default:
6ba81e53
AD		 6757 buffer[count++] = cpu_to_le32(0x00000000);
6758 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6759 break;
6760 }
6761
6ba81e53
AD		 6762 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6763 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
f6796cae 6764
6ba81e53
AD		 6765 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6766 buffer[count++] = cpu_to_le32(0);
a0f38609 6767}
f6796cae 6768
473359bc 6769static void cik_init_pg(struct radeon_device *rdev)
22c775ce 6770{
473359bc 6771 if (rdev->pg_flags) {
22c775ce
AD		 6772 cik_enable_sck_slowdown_on_pu(rdev, true);
6773 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 6774 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 6775 cik_init_gfx_cgpg(rdev);
6776 cik_enable_cp_pg(rdev, true);
6777 cik_enable_gds_pg(rdev, true);
6778 }
22c775ce
AD		 6779 cik_init_ao_cu_mask(rdev);
6780 cik_update_gfx_pg(rdev, true);
6781 }
6782}
f6796cae 6783
473359bc
AD		 6784static void cik_fini_pg(struct radeon_device *rdev)
6785{
6786 if (rdev->pg_flags) {
6787 cik_update_gfx_pg(rdev, false);
2b19d17f 6788 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 6789 cik_enable_cp_pg(rdev, false);
6790 cik_enable_gds_pg(rdev, false);
6791 }
6792 }
f6796cae 6793}
a59781bb
AD		 6794
6795/*
6796 * Interrupts
6797 * Starting with r6xx, interrupts are handled via a ring buffer.
6798 * Ring buffers are areas of GPU accessible memory that the GPU
6799 * writes interrupt vectors into and the host reads vectors out of.
6800 * There is a rptr (read pointer) that determines where the
6801 * host is currently reading, and a wptr (write pointer)
6802 * which determines where the GPU has written. When the
6803 * pointers are equal, the ring is idle. When the GPU
6804 * writes vectors to the ring buffer, it increments the
6805 * wptr. When there is an interrupt, the host then starts
6806 * fetching commands and processing them until the pointers are
6807 * equal again at which point it updates the rptr.
6808 */
6809
6810/**
6811 * cik_enable_interrupts - Enable the interrupt ring buffer
6812 *
6813 * @rdev: radeon_device pointer
6814 *
6815 * Enable the interrupt ring buffer (CIK).
6816 */
6817static void cik_enable_interrupts(struct radeon_device *rdev)
6818{
6819 u32 ih_cntl = RREG32(IH_CNTL);
6820 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6821
6822 ih_cntl |= ENABLE_INTR;
6823 ih_rb_cntl |= IH_RB_ENABLE;
6824 WREG32(IH_CNTL, ih_cntl);
6825 WREG32(IH_RB_CNTL, ih_rb_cntl);
6826 rdev->ih.enabled = true;
6827}
6828
6829/**
6830 * cik_disable_interrupts - Disable the interrupt ring buffer
6831 *
6832 * @rdev: radeon_device pointer
6833 *
6834 * Disable the interrupt ring buffer (CIK).
6835 */
6836static void cik_disable_interrupts(struct radeon_device *rdev)
6837{
6838 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6839 u32 ih_cntl = RREG32(IH_CNTL);
6840
6841 ih_rb_cntl &= ~IH_RB_ENABLE;
6842 ih_cntl &= ~ENABLE_INTR;
6843 WREG32(IH_RB_CNTL, ih_rb_cntl);
6844 WREG32(IH_CNTL, ih_cntl);
6845 /* set rptr, wptr to 0 */
6846 WREG32(IH_RB_RPTR, 0);
6847 WREG32(IH_RB_WPTR, 0);
6848 rdev->ih.enabled = false;
6849 rdev->ih.rptr = 0;
6850}
6851
6852/**
6853 * cik_disable_interrupt_state - Disable all interrupt sources
6854 *
6855 * @rdev: radeon_device pointer
6856 *
6857 * Clear all interrupt enable bits used by the driver (CIK).
6858 */
6859static void cik_disable_interrupt_state(struct radeon_device *rdev)
6860{
6861 u32 tmp;
6862
6863 /* gfx ring */
4214faf6
AD		 6864 tmp = RREG32(CP_INT_CNTL_RING0) &
6865 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6866 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 6867 /* sdma */
6868 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6869 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6870 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6871 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 6872 /* compute queues */
6873 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6874 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6875 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6876 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6877 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6878 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6879 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6880 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6881 /* grbm */
6882 WREG32(GRBM_INT_CNTL, 0);
dc12a3ec
LL		 6883 /* SRBM */
6884 WREG32(SRBM_INT_CNTL, 0);
a59781bb
AD		 6885 /* vline/vblank, etc. */
6886 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6887 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6888 if (rdev->num_crtc >= 4) {
6889 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6890 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6891 }
6892 if (rdev->num_crtc >= 6) {
6893 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6894 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6895 }
f5d636d2
CK		 6896 /* pflip */
6897 if (rdev->num_crtc >= 2) {
6898 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6899 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6900 }
6901 if (rdev->num_crtc >= 4) {
6902 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6903 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6904 }
6905 if (rdev->num_crtc >= 6) {
6906 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6907 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6908 }
a59781bb
AD		 6909
6910 /* dac hotplug */
6911 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6912
6913 /* digital hotplug */
6914 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6915 WREG32(DC_HPD1_INT_CONTROL, tmp);
6916 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6917 WREG32(DC_HPD2_INT_CONTROL, tmp);
6918 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6919 WREG32(DC_HPD3_INT_CONTROL, tmp);
6920 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6921 WREG32(DC_HPD4_INT_CONTROL, tmp);
6922 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6923 WREG32(DC_HPD5_INT_CONTROL, tmp);
6924 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6925 WREG32(DC_HPD6_INT_CONTROL, tmp);
6926
6927}
6928
6929/**
6930 * cik_irq_init - init and enable the interrupt ring
6931 *
6932 * @rdev: radeon_device pointer
6933 *
6934 * Allocate a ring buffer for the interrupt controller,
6935 * enable the RLC, disable interrupts, enable the IH
6936 * ring buffer and enable it (CIK).
6937 * Called at device load and reume.
6938 * Returns 0 for success, errors for failure.
6939 */
6940static int cik_irq_init(struct radeon_device *rdev)
6941{
6942 int ret = 0;
6943 int rb_bufsz;
6944 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6945
6946 /* allocate ring */
6947 ret = r600_ih_ring_alloc(rdev);
6948 if (ret)
6949 return ret;
6950
6951 /* disable irqs */
6952 cik_disable_interrupts(rdev);
6953
6954 /* init rlc */
6955 ret = cik_rlc_resume(rdev);
6956 if (ret) {
6957 r600_ih_ring_fini(rdev);
6958 return ret;
6959 }
6960
6961 /* setup interrupt control */
62d91dd2
SB		 6962 /* set dummy read address to dummy page address */
6963 WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);
a59781bb
AD		 6964 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6965 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6966 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6967 */
6968 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6969 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6970 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6971 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6972
6973 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 6974 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 6975
6976 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6977 IH_WPTR_OVERFLOW_CLEAR |
6978 (rb_bufsz << 1));
6979
6980 if (rdev->wb.enabled)
6981 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6982
6983 /* set the writeback address whether it's enabled or not */
6984 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6985 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6986
6987 WREG32(IH_RB_CNTL, ih_rb_cntl);
6988
6989 /* set rptr, wptr to 0 */
6990 WREG32(IH_RB_RPTR, 0);
6991 WREG32(IH_RB_WPTR, 0);
6992
6993 /* Default settings for IH_CNTL (disabled at first) */
6994 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6995 /* RPTR_REARM only works if msi's are enabled */
6996 if (rdev->msi_enabled)
6997 ih_cntl |= RPTR_REARM;
6998 WREG32(IH_CNTL, ih_cntl);
6999
7000 /* force the active interrupt state to all disabled */
7001 cik_disable_interrupt_state(rdev);
7002
7003 pci_set_master(rdev->pdev);
7004
7005 /* enable irqs */
7006 cik_enable_interrupts(rdev);
7007
7008 return ret;
7009}
7010
7011/**
7012 * cik_irq_set - enable/disable interrupt sources
7013 *
7014 * @rdev: radeon_device pointer
7015 *
7016 * Enable interrupt sources on the GPU (vblanks, hpd,
7017 * etc.) (CIK).
7018 * Returns 0 for success, errors for failure.
7019 */
7020int cik_irq_set(struct radeon_device *rdev)
7021{
4214faf6 7022 u32 cp_int_cntl;
7d752ea2
MD		 7023 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
7024 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
a59781bb
AD		 7025 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7026 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7027 u32 grbm_int_cntl = 0;
21a93e13 7028 u32 dma_cntl, dma_cntl1;
a59781bb
AD		 7029
7030 if (!rdev->irq.installed) {
7031 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7032 return -EINVAL;
7033 }
7034 /* don't enable anything if the ih is disabled */
7035 if (!rdev->ih.enabled) {
7036 cik_disable_interrupts(rdev);
7037 /* force the active interrupt state to all disabled */
7038 cik_disable_interrupt_state(rdev);
7039 return 0;
7040 }
7041
4214faf6
AD		 7042 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7043 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7044 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7045
f6b355dd
AD		 7046 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7047 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7048 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7049 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7050 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7051 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
a59781bb 7052
21a93e13
AD		 7053 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7054 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7055
2b0781a6 7056 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7d752ea2
MD		 7057 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7058 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7059 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7060 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7061 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7062 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7063 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
2b0781a6 7064
a59781bb
AD		 7065 /* enable CP interrupts on all rings */
7066 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7067 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7068 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7069 }
2b0781a6
AD		 7070 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7071 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7072 DRM_DEBUG("si_irq_set: sw int cp1\n");
7073 if (ring->me == 1) {
7074 switch (ring->pipe) {
7075 case 0:
7076 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7077 break;
7d752ea2
MD		 7078 case 1:
7079 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7080 break;
7081 case 2:
7082 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7083 break;
7084 case 3:
7085 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7086 break;
7087 default:
7088 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7089 break;
7090 }
7091 } else if (ring->me == 2) {
7092 switch (ring->pipe) {
7093 case 0:
7094 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7095 break;
7096 case 1:
7097 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7098 break;
7099 case 2:
7100 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7101 break;
7102 case 3:
7103 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7104 break;
2b0781a6
AD		 7105 default:
7106 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7107 break;
7108 }
7109 } else {
7110 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7111 }
7112 }
7113 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7114 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7115 DRM_DEBUG("si_irq_set: sw int cp2\n");
7116 if (ring->me == 1) {
7117 switch (ring->pipe) {
7118 case 0:
7119 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7120 break;
7d752ea2
MD		 7121 case 1:
7122 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7123 break;
7124 case 2:
7125 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7126 break;
7127 case 3:
7128 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7129 break;
7130 default:
7131 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7132 break;
7133 }
7134 } else if (ring->me == 2) {
7135 switch (ring->pipe) {
7136 case 0:
7137 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7138 break;
7139 case 1:
7140 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7141 break;
7142 case 2:
7143 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7144 break;
7145 case 3:
7146 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7147 break;
2b0781a6
AD		 7148 default:
7149 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7150 break;
7151 }
7152 } else {
7153 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7154 }
7155 }
a59781bb 7156
21a93e13
AD		 7157 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7158 DRM_DEBUG("cik_irq_set: sw int dma\n");
7159 dma_cntl |= TRAP_ENABLE;
7160 }
7161
7162 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7163 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7164 dma_cntl1 |= TRAP_ENABLE;
7165 }
7166
a59781bb
AD		 7167 if (rdev->irq.crtc_vblank_int[0] ||
7168 atomic_read(&rdev->irq.pflip[0])) {
7169 DRM_DEBUG("cik_irq_set: vblank 0\n");
7170 crtc1 |= VBLANK_INTERRUPT_MASK;
7171 }
7172 if (rdev->irq.crtc_vblank_int[1] ||
7173 atomic_read(&rdev->irq.pflip[1])) {
7174 DRM_DEBUG("cik_irq_set: vblank 1\n");
7175 crtc2 |= VBLANK_INTERRUPT_MASK;
7176 }
7177 if (rdev->irq.crtc_vblank_int[2] ||
7178 atomic_read(&rdev->irq.pflip[2])) {
7179 DRM_DEBUG("cik_irq_set: vblank 2\n");
7180 crtc3 |= VBLANK_INTERRUPT_MASK;
7181 }
7182 if (rdev->irq.crtc_vblank_int[3] ||
7183 atomic_read(&rdev->irq.pflip[3])) {
7184 DRM_DEBUG("cik_irq_set: vblank 3\n");
7185 crtc4 |= VBLANK_INTERRUPT_MASK;
7186 }
7187 if (rdev->irq.crtc_vblank_int[4] ||
7188 atomic_read(&rdev->irq.pflip[4])) {
7189 DRM_DEBUG("cik_irq_set: vblank 4\n");
7190 crtc5 |= VBLANK_INTERRUPT_MASK;
7191 }
7192 if (rdev->irq.crtc_vblank_int[5] ||
7193 atomic_read(&rdev->irq.pflip[5])) {
7194 DRM_DEBUG("cik_irq_set: vblank 5\n");
7195 crtc6 |= VBLANK_INTERRUPT_MASK;
7196 }
7197 if (rdev->irq.hpd[0]) {
7198 DRM_DEBUG("cik_irq_set: hpd 1\n");
f6b355dd 7199 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7200 }
7201 if (rdev->irq.hpd[1]) {
7202 DRM_DEBUG("cik_irq_set: hpd 2\n");
f6b355dd 7203 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7204 }
7205 if (rdev->irq.hpd[2]) {
7206 DRM_DEBUG("cik_irq_set: hpd 3\n");
f6b355dd 7207 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7208 }
7209 if (rdev->irq.hpd[3]) {
7210 DRM_DEBUG("cik_irq_set: hpd 4\n");
f6b355dd 7211 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7212 }
7213 if (rdev->irq.hpd[4]) {
7214 DRM_DEBUG("cik_irq_set: hpd 5\n");
f6b355dd 7215 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7216 }
7217 if (rdev->irq.hpd[5]) {
7218 DRM_DEBUG("cik_irq_set: hpd 6\n");
f6b355dd 7219 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7220 }
7221
7222 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7223
21a93e13
AD		 7224 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7225 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7226
2b0781a6 7227 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7d752ea2
MD		 7228 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7229 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7230 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7231 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7232 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7233 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7234 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
2b0781a6 7235
a59781bb
AD		 7236 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7237
7238 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7239 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7240 if (rdev->num_crtc >= 4) {
7241 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7242 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7243 }
7244 if (rdev->num_crtc >= 6) {
7245 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7246 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7247 }
7248
f5d636d2
CK		 7249 if (rdev->num_crtc >= 2) {
7250 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7251 GRPH_PFLIP_INT_MASK);
7252 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7253 GRPH_PFLIP_INT_MASK);
7254 }
7255 if (rdev->num_crtc >= 4) {
7256 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7257 GRPH_PFLIP_INT_MASK);
7258 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7259 GRPH_PFLIP_INT_MASK);
7260 }
7261 if (rdev->num_crtc >= 6) {
7262 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7263 GRPH_PFLIP_INT_MASK);
7264 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7265 GRPH_PFLIP_INT_MASK);
7266 }
7267
a59781bb
AD		 7268 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7269 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7270 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7271 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7272 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7273 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7274
cffefd9b
AD		 7275 /* posting read */
7276 RREG32(SRBM_STATUS);
7277
a59781bb
AD		 7278 return 0;
7279}
7280
7281/**
7282 * cik_irq_ack - ack interrupt sources
7283 *
7284 * @rdev: radeon_device pointer
7285 *
7286 * Ack interrupt sources on the GPU (vblanks, hpd,
7287 * etc.) (CIK). Certain interrupts sources are sw
7288 * generated and do not require an explicit ack.
7289 */
7290static inline void cik_irq_ack(struct radeon_device *rdev)
7291{
7292 u32 tmp;
7293
7294 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7295 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7296 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7297 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7298 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7299 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7300 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7301
f5d636d2
CK		 7302 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7303 EVERGREEN_CRTC0_REGISTER_OFFSET);
7304 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7305 EVERGREEN_CRTC1_REGISTER_OFFSET);
7306 if (rdev->num_crtc >= 4) {
7307 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7308 EVERGREEN_CRTC2_REGISTER_OFFSET);
7309 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7310 EVERGREEN_CRTC3_REGISTER_OFFSET);
7311 }
7312 if (rdev->num_crtc >= 6) {
7313 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7314 EVERGREEN_CRTC4_REGISTER_OFFSET);
7315 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7316 EVERGREEN_CRTC5_REGISTER_OFFSET);
7317 }
7318
7319 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7320 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7321 GRPH_PFLIP_INT_CLEAR);
7322 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7323 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7324 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7325 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7326 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7327 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7328 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7329 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7330 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7331 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7332 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7333
7334 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 7335 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7336 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7337 GRPH_PFLIP_INT_CLEAR);
7338 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7339 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7340 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7341 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7342 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7343 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7344 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7345 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7346 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7347 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7348 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7349 }
7350
7351 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 7352 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7353 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7354 GRPH_PFLIP_INT_CLEAR);
7355 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7356 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7357 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7358 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7359 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7360 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7361 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7362 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7363 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7364 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7365 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7366 }
7367
7368 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7369 tmp = RREG32(DC_HPD1_INT_CONTROL);
7370 tmp |= DC_HPDx_INT_ACK;
7371 WREG32(DC_HPD1_INT_CONTROL, tmp);
7372 }
7373 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7374 tmp = RREG32(DC_HPD2_INT_CONTROL);
7375 tmp |= DC_HPDx_INT_ACK;
7376 WREG32(DC_HPD2_INT_CONTROL, tmp);
7377 }
7378 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7379 tmp = RREG32(DC_HPD3_INT_CONTROL);
7380 tmp |= DC_HPDx_INT_ACK;
7381 WREG32(DC_HPD3_INT_CONTROL, tmp);
7382 }
7383 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7384 tmp = RREG32(DC_HPD4_INT_CONTROL);
7385 tmp |= DC_HPDx_INT_ACK;
7386 WREG32(DC_HPD4_INT_CONTROL, tmp);
7387 }
7388 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7389 tmp = RREG32(DC_HPD5_INT_CONTROL);
7390 tmp |= DC_HPDx_INT_ACK;
7391 WREG32(DC_HPD5_INT_CONTROL, tmp);
7392 }
7393 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
3d18e337 7394 tmp = RREG32(DC_HPD6_INT_CONTROL);
a59781bb
AD		 7395 tmp |= DC_HPDx_INT_ACK;
7396 WREG32(DC_HPD6_INT_CONTROL, tmp);
7397 }
f6b355dd
AD		 7398 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {
7399 tmp = RREG32(DC_HPD1_INT_CONTROL);
7400 tmp |= DC_HPDx_RX_INT_ACK;
7401 WREG32(DC_HPD1_INT_CONTROL, tmp);
7402 }
7403 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
7404 tmp = RREG32(DC_HPD2_INT_CONTROL);
7405 tmp |= DC_HPDx_RX_INT_ACK;
7406 WREG32(DC_HPD2_INT_CONTROL, tmp);
7407 }
7408 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
7409 tmp = RREG32(DC_HPD3_INT_CONTROL);
7410 tmp |= DC_HPDx_RX_INT_ACK;
7411 WREG32(DC_HPD3_INT_CONTROL, tmp);
7412 }
7413 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
7414 tmp = RREG32(DC_HPD4_INT_CONTROL);
7415 tmp |= DC_HPDx_RX_INT_ACK;
7416 WREG32(DC_HPD4_INT_CONTROL, tmp);
7417 }
7418 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
7419 tmp = RREG32(DC_HPD5_INT_CONTROL);
7420 tmp |= DC_HPDx_RX_INT_ACK;
7421 WREG32(DC_HPD5_INT_CONTROL, tmp);
7422 }
7423 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
3d18e337 7424 tmp = RREG32(DC_HPD6_INT_CONTROL);
f6b355dd
AD		 7425 tmp |= DC_HPDx_RX_INT_ACK;
7426 WREG32(DC_HPD6_INT_CONTROL, tmp);
7427 }
a59781bb
AD		 7428}
7429
7430/**
7431 * cik_irq_disable - disable interrupts
7432 *
7433 * @rdev: radeon_device pointer
7434 *
7435 * Disable interrupts on the hw (CIK).
7436 */
7437static void cik_irq_disable(struct radeon_device *rdev)
7438{
7439 cik_disable_interrupts(rdev);
7440 /* Wait and acknowledge irq */
7441 mdelay(1);
7442 cik_irq_ack(rdev);
7443 cik_disable_interrupt_state(rdev);
7444}
7445
7446/**
7447 * cik_irq_disable - disable interrupts for suspend
7448 *
7449 * @rdev: radeon_device pointer
7450 *
7451 * Disable interrupts and stop the RLC (CIK).
7452 * Used for suspend.
7453 */
7454static void cik_irq_suspend(struct radeon_device *rdev)
7455{
7456 cik_irq_disable(rdev);
7457 cik_rlc_stop(rdev);
7458}
7459
7460/**
7461 * cik_irq_fini - tear down interrupt support
7462 *
7463 * @rdev: radeon_device pointer
7464 *
7465 * Disable interrupts on the hw and free the IH ring
7466 * buffer (CIK).
7467 * Used for driver unload.
7468 */
7469static void cik_irq_fini(struct radeon_device *rdev)
7470{
7471 cik_irq_suspend(rdev);
7472 r600_ih_ring_fini(rdev);
7473}
7474
7475/**
7476 * cik_get_ih_wptr - get the IH ring buffer wptr
7477 *
7478 * @rdev: radeon_device pointer
7479 *
7480 * Get the IH ring buffer wptr from either the register
7481 * or the writeback memory buffer (CIK). Also check for
7482 * ring buffer overflow and deal with it.
7483 * Used by cik_irq_process().
7484 * Returns the value of the wptr.
7485 */
7486static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7487{
7488 u32 wptr, tmp;
7489
7490 if (rdev->wb.enabled)
7491 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7492 else
7493 wptr = RREG32(IH_RB_WPTR);
7494
7495 if (wptr & RB_OVERFLOW) {
11bab0ae 7496 wptr &= ~RB_OVERFLOW;
a59781bb
AD		 7497 /* When a ring buffer overflow happen start parsing interrupt
7498 * from the last not overwritten vector (wptr + 16). Hopefully
7499 * this should allow us to catchup.
7500 */
6cc2fda2
MD		 7501 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7502 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
a59781bb
AD		 7503 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7504 tmp = RREG32(IH_RB_CNTL);
7505 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7506 WREG32(IH_RB_CNTL, tmp);
7507 }
7508 return (wptr & rdev->ih.ptr_mask);
7509}
7510
7511/* CIK IV Ring
7512 * Each IV ring entry is 128 bits:
7513 * [7:0] - interrupt source id
7514 * [31:8] - reserved
7515 * [59:32] - interrupt source data
7516 * [63:60] - reserved
21a93e13
AD		 7517 * [71:64] - RINGID
7518 * CP:
7519 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 7520 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7521 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7522 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7523 * PIPE_ID - ME0 0=3D
7524 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 7525 * SDMA:
7526 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7527 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7528 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 7529 * [79:72] - VMID
7530 * [95:80] - PASID
7531 * [127:96] - reserved
7532 */
7533/**
7534 * cik_irq_process - interrupt handler
7535 *
7536 * @rdev: radeon_device pointer
7537 *
7538 * Interrupt hander (CIK). Walk the IH ring,
7539 * ack interrupts and schedule work to handle
7540 * interrupt events.
7541 * Returns irq process return code.
7542 */
7543int cik_irq_process(struct radeon_device *rdev)
7544{
2b0781a6
AD		 7545 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7546 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 7547 u32 wptr;
7548 u32 rptr;
7549 u32 src_id, src_data, ring_id;
7550 u8 me_id, pipe_id, queue_id;
7551 u32 ring_index;
7552 bool queue_hotplug = false;
f6b355dd 7553 bool queue_dp = false;
a59781bb 7554 bool queue_reset = false;
3ec7d11b 7555 u32 addr, status, mc_client;
41a524ab 7556 bool queue_thermal = false;
a59781bb
AD		 7557
7558 if (!rdev->ih.enabled || rdev->shutdown)
7559 return IRQ_NONE;
7560
7561 wptr = cik_get_ih_wptr(rdev);
7562
7563restart_ih:
7564 /* is somebody else already processing irqs? */
7565 if (atomic_xchg(&rdev->ih.lock, 1))
7566 return IRQ_NONE;
7567
7568 rptr = rdev->ih.rptr;
7569 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7570
7571 /* Order reading of wptr vs. reading of IH ring data */
7572 rmb();
7573
7574 /* display interrupts */
7575 cik_irq_ack(rdev);
7576
7577 while (rptr != wptr) {
7578 /* wptr/rptr are in bytes! */
7579 ring_index = rptr / 4;
e28740ec 7580
a59781bb
AD		 7581 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7582 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7583 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 7584
7585 switch (src_id) {
7586 case 1: /* D1 vblank/vline */
7587 switch (src_data) {
7588 case 0: /* D1 vblank */
07f18f0b
MK		 7589 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT))
7590 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7591
7592 if (rdev->irq.crtc_vblank_int[0]) {
7593 drm_handle_vblank(rdev->ddev, 0);
7594 rdev->pm.vblank_sync = true;
7595 wake_up(&rdev->irq.vblank_queue);
a59781bb 7596 }
07f18f0b
MK		 7597 if (atomic_read(&rdev->irq.pflip[0]))
7598 radeon_crtc_handle_vblank(rdev, 0);
7599 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7600 DRM_DEBUG("IH: D1 vblank\n");
7601
a59781bb
AD		 7602 break;
7603 case 1: /* D1 vline */
07f18f0b
MK		 7604 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT))
7605 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7606
7607 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7608 DRM_DEBUG("IH: D1 vline\n");
7609
a59781bb
AD		 7610 break;
7611 default:
7612 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7613 break;
7614 }
7615 break;
7616 case 2: /* D2 vblank/vline */
7617 switch (src_data) {
7618 case 0: /* D2 vblank */
07f18f0b
MK		 7619 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
7620 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7621
7622 if (rdev->irq.crtc_vblank_int[1]) {
7623 drm_handle_vblank(rdev->ddev, 1);
7624 rdev->pm.vblank_sync = true;
7625 wake_up(&rdev->irq.vblank_queue);
a59781bb 7626 }
07f18f0b
MK		 7627 if (atomic_read(&rdev->irq.pflip[1]))
7628 radeon_crtc_handle_vblank(rdev, 1);
7629 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7630 DRM_DEBUG("IH: D2 vblank\n");
7631
a59781bb
AD		 7632 break;
7633 case 1: /* D2 vline */
07f18f0b
MK		 7634 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT))
7635 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7636
7637 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7638 DRM_DEBUG("IH: D2 vline\n");
7639
a59781bb
AD		 7640 break;
7641 default:
7642 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7643 break;
7644 }
7645 break;
7646 case 3: /* D3 vblank/vline */
7647 switch (src_data) {
7648 case 0: /* D3 vblank */
07f18f0b
MK		 7649 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
7650 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7651
7652 if (rdev->irq.crtc_vblank_int[2]) {
7653 drm_handle_vblank(rdev->ddev, 2);
7654 rdev->pm.vblank_sync = true;
7655 wake_up(&rdev->irq.vblank_queue);
a59781bb 7656 }
07f18f0b
MK		 7657 if (atomic_read(&rdev->irq.pflip[2]))
7658 radeon_crtc_handle_vblank(rdev, 2);
7659 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7660 DRM_DEBUG("IH: D3 vblank\n");
7661
a59781bb
AD		 7662 break;
7663 case 1: /* D3 vline */
07f18f0b
MK		 7664 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
7665 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7666
7667 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7668 DRM_DEBUG("IH: D3 vline\n");
7669
a59781bb
AD		 7670 break;
7671 default:
7672 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7673 break;
7674 }
7675 break;
7676 case 4: /* D4 vblank/vline */
7677 switch (src_data) {
7678 case 0: /* D4 vblank */
07f18f0b
MK		 7679 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
7680 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7681
7682 if (rdev->irq.crtc_vblank_int[3]) {
7683 drm_handle_vblank(rdev->ddev, 3);
7684 rdev->pm.vblank_sync = true;
7685 wake_up(&rdev->irq.vblank_queue);
a59781bb 7686 }
07f18f0b
MK		 7687 if (atomic_read(&rdev->irq.pflip[3]))
7688 radeon_crtc_handle_vblank(rdev, 3);
7689 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7690 DRM_DEBUG("IH: D4 vblank\n");
7691
a59781bb
AD		 7692 break;
7693 case 1: /* D4 vline */
07f18f0b
MK		 7694 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
7695 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7696
7697 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7698 DRM_DEBUG("IH: D4 vline\n");
7699
a59781bb
AD		 7700 break;
7701 default:
7702 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7703 break;
7704 }
7705 break;
7706 case 5: /* D5 vblank/vline */
7707 switch (src_data) {
7708 case 0: /* D5 vblank */
07f18f0b
MK		 7709 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
7710 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7711
7712 if (rdev->irq.crtc_vblank_int[4]) {
7713 drm_handle_vblank(rdev->ddev, 4);
7714 rdev->pm.vblank_sync = true;
7715 wake_up(&rdev->irq.vblank_queue);
a59781bb 7716 }
07f18f0b
MK		 7717 if (atomic_read(&rdev->irq.pflip[4]))
7718 radeon_crtc_handle_vblank(rdev, 4);
7719 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7720 DRM_DEBUG("IH: D5 vblank\n");
7721
a59781bb
AD		 7722 break;
7723 case 1: /* D5 vline */
07f18f0b
MK		 7724 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
7725 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7726
7727 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7728 DRM_DEBUG("IH: D5 vline\n");
7729
a59781bb
AD		 7730 break;
7731 default:
7732 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7733 break;
7734 }
7735 break;
7736 case 6: /* D6 vblank/vline */
7737 switch (src_data) {
7738 case 0: /* D6 vblank */
07f18f0b
MK		 7739 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
7740 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7741
7742 if (rdev->irq.crtc_vblank_int[5]) {
7743 drm_handle_vblank(rdev->ddev, 5);
7744 rdev->pm.vblank_sync = true;
7745 wake_up(&rdev->irq.vblank_queue);
a59781bb 7746 }
07f18f0b
MK		 7747 if (atomic_read(&rdev->irq.pflip[5]))
7748 radeon_crtc_handle_vblank(rdev, 5);
7749 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7750 DRM_DEBUG("IH: D6 vblank\n");
7751
a59781bb
AD		 7752 break;
7753 case 1: /* D6 vline */
07f18f0b
MK		 7754 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
7755 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7756
7757 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7758 DRM_DEBUG("IH: D6 vline\n");
7759
a59781bb
AD		 7760 break;
7761 default:
7762 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7763 break;
7764 }
7765 break;
f5d636d2
CK		 7766 case 8: /* D1 page flip */
7767 case 10: /* D2 page flip */
7768 case 12: /* D3 page flip */
7769 case 14: /* D4 page flip */
7770 case 16: /* D5 page flip */
7771 case 18: /* D6 page flip */
7772 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 7773 if (radeon_use_pflipirq > 0)
7774 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 7775 break;
a59781bb
AD		 7776 case 42: /* HPD hotplug */
7777 switch (src_data) {
7778 case 0:
07f18f0b
MK		 7779 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT))
7780 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7781
7782 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7783 queue_hotplug = true;
7784 DRM_DEBUG("IH: HPD1\n");
7785
a59781bb
AD		 7786 break;
7787 case 1:
07f18f0b
MK		 7788 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT))
7789 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7790
7791 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7792 queue_hotplug = true;
7793 DRM_DEBUG("IH: HPD2\n");
7794
a59781bb
AD		 7795 break;
7796 case 2:
07f18f0b
MK		 7797 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT))
7798 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7799
7800 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7801 queue_hotplug = true;
7802 DRM_DEBUG("IH: HPD3\n");
7803
a59781bb
AD		 7804 break;
7805 case 3:
07f18f0b
MK		 7806 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT))
7807 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7808
7809 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7810 queue_hotplug = true;
7811 DRM_DEBUG("IH: HPD4\n");
7812
a59781bb
AD		 7813 break;
7814 case 4:
07f18f0b
MK		 7815 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT))
7816 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7817
7818 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7819 queue_hotplug = true;
7820 DRM_DEBUG("IH: HPD5\n");
7821
a59781bb
AD		 7822 break;
7823 case 5:
07f18f0b
MK		 7824 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT))
7825 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7826
7827 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7828 queue_hotplug = true;
7829 DRM_DEBUG("IH: HPD6\n");
7830
a59781bb 7831 break;
f6b355dd 7832 case 6:
07f18f0b
MK		 7833 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT))
7834 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7835
7836 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;
7837 queue_dp = true;
7838 DRM_DEBUG("IH: HPD_RX 1\n");
7839
f6b355dd
AD		 7840 break;
7841 case 7:
07f18f0b
MK		 7842 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT))
7843 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7844
7845 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
7846 queue_dp = true;
7847 DRM_DEBUG("IH: HPD_RX 2\n");
7848
f6b355dd
AD		 7849 break;
7850 case 8:
07f18f0b
MK		 7851 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
7852 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7853
7854 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
7855 queue_dp = true;
7856 DRM_DEBUG("IH: HPD_RX 3\n");
7857
f6b355dd
AD		 7858 break;
7859 case 9:
07f18f0b
MK		 7860 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
7861 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7862
7863 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
7864 queue_dp = true;
7865 DRM_DEBUG("IH: HPD_RX 4\n");
7866
f6b355dd
AD		 7867 break;
7868 case 10:
07f18f0b
MK		 7869 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
7870 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7871
7872 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
7873 queue_dp = true;
7874 DRM_DEBUG("IH: HPD_RX 5\n");
7875
f6b355dd
AD		 7876 break;
7877 case 11:
07f18f0b
MK		 7878 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
7879 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7880
7881 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
7882 queue_dp = true;
7883 DRM_DEBUG("IH: HPD_RX 6\n");
7884
f6b355dd 7885 break;
a59781bb
AD		 7886 default:
7887 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7888 break;
7889 }
7890 break;
dc12a3ec
LL		 7891 case 96:
7892 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
7893 WREG32(SRBM_INT_ACK, 0x1);
7894 break;
6a3808b8
CK		 7895 case 124: /* UVD */
7896 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7897 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7898 break;
9d97c99b
AD		 7899 case 146:
7900 case 147:
3ec7d11b
AD		 7901 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7902 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7903 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9b7d786b
CK		 7904 /* reset addr and status */
7905 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7906 if (addr == 0x0 && status == 0x0)
7907 break;
9d97c99b
AD		 7908 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7909 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 7910 addr);
9d97c99b 7911 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 7912 status);
7913 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b 7914 break;
d93f7937
CK		 7915 case 167: /* VCE */
7916 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7917 switch (src_data) {
7918 case 0:
7919 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7920 break;
7921 case 1:
7922 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7923 break;
7924 default:
7925 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7926 break;
7927 }
7928 break;
a59781bb
AD		 7929 case 176: /* GFX RB CP_INT */
7930 case 177: /* GFX IB CP_INT */
7931 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7932 break;
7933 case 181: /* CP EOP event */
7934 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 7935 /* XXX check the bitfield order! */
7936 me_id = (ring_id & 0x60) >> 5;
7937 pipe_id = (ring_id & 0x18) >> 3;
7938 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 7939 switch (me_id) {
7940 case 0:
7941 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7942 break;
7943 case 1:
a59781bb 7944 case 2:
2b0781a6
AD		 7945 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7946 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7947 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7948 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 7949 break;
7950 }
7951 break;
7952 case 184: /* CP Privileged reg access */
7953 DRM_ERROR("Illegal register access in command stream\n");
7954 /* XXX check the bitfield order! */
7955 me_id = (ring_id & 0x60) >> 5;
7956 pipe_id = (ring_id & 0x18) >> 3;
7957 queue_id = (ring_id & 0x7) >> 0;
7958 switch (me_id) {
7959 case 0:
7960 /* This results in a full GPU reset, but all we need to do is soft
7961 * reset the CP for gfx
7962 */
7963 queue_reset = true;
7964 break;
7965 case 1:
7966 /* XXX compute */
2b0781a6 7967 queue_reset = true;
a59781bb
AD		 7968 break;
7969 case 2:
7970 /* XXX compute */
2b0781a6 7971 queue_reset = true;
a59781bb
AD		 7972 break;
7973 }
7974 break;
7975 case 185: /* CP Privileged inst */
7976 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 7977 /* XXX check the bitfield order! */
7978 me_id = (ring_id & 0x60) >> 5;
7979 pipe_id = (ring_id & 0x18) >> 3;
7980 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 7981 switch (me_id) {
7982 case 0:
7983 /* This results in a full GPU reset, but all we need to do is soft
7984 * reset the CP for gfx
7985 */
7986 queue_reset = true;
7987 break;
7988 case 1:
7989 /* XXX compute */
2b0781a6 7990 queue_reset = true;
a59781bb
AD		 7991 break;
7992 case 2:
7993 /* XXX compute */
2b0781a6 7994 queue_reset = true;
a59781bb
AD		 7995 break;
7996 }
7997 break;
21a93e13
AD		 7998 case 224: /* SDMA trap event */
7999 /* XXX check the bitfield order! */
8000 me_id = (ring_id & 0x3) >> 0;
8001 queue_id = (ring_id & 0xc) >> 2;
8002 DRM_DEBUG("IH: SDMA trap\n");
8003 switch (me_id) {
8004 case 0:
8005 switch (queue_id) {
8006 case 0:
8007 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8008 break;
8009 case 1:
8010 /* XXX compute */
8011 break;
8012 case 2:
8013 /* XXX compute */
8014 break;
8015 }
8016 break;
8017 case 1:
8018 switch (queue_id) {
8019 case 0:
8020 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8021 break;
8022 case 1:
8023 /* XXX compute */
8024 break;
8025 case 2:
8026 /* XXX compute */
8027 break;
8028 }
8029 break;
8030 }
8031 break;
41a524ab
AD		 8032 case 230: /* thermal low to high */
8033 DRM_DEBUG("IH: thermal low to high\n");
8034 rdev->pm.dpm.thermal.high_to_low = false;
8035 queue_thermal = true;
8036 break;
8037 case 231: /* thermal high to low */
8038 DRM_DEBUG("IH: thermal high to low\n");
8039 rdev->pm.dpm.thermal.high_to_low = true;
8040 queue_thermal = true;
8041 break;
8042 case 233: /* GUI IDLE */
8043 DRM_DEBUG("IH: GUI idle\n");
8044 break;
21a93e13
AD		 8045 case 241: /* SDMA Privileged inst */
8046 case 247: /* SDMA Privileged inst */
8047 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8048 /* XXX check the bitfield order! */
8049 me_id = (ring_id & 0x3) >> 0;
8050 queue_id = (ring_id & 0xc) >> 2;
8051 switch (me_id) {
8052 case 0:
8053 switch (queue_id) {
8054 case 0:
8055 queue_reset = true;
8056 break;
8057 case 1:
8058 /* XXX compute */
8059 queue_reset = true;
8060 break;
8061 case 2:
8062 /* XXX compute */
8063 queue_reset = true;
8064 break;
8065 }
8066 break;
8067 case 1:
8068 switch (queue_id) {
8069 case 0:
8070 queue_reset = true;
8071 break;
8072 case 1:
8073 /* XXX compute */
8074 queue_reset = true;
8075 break;
8076 case 2:
8077 /* XXX compute */
8078 queue_reset = true;
8079 break;
8080 }
8081 break;
8082 }
8083 break;
a59781bb
AD		 8084 default:
8085 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8086 break;
8087 }
8088
8089 /* wptr/rptr are in bytes! */
8090 rptr += 16;
8091 rptr &= rdev->ih.ptr_mask;
f55e03b9 8092 WREG32(IH_RB_RPTR, rptr);
a59781bb 8093 }
f6b355dd
AD		 8094 if (queue_dp)
8095 schedule_work(&rdev->dp_work);
a59781bb 8096 if (queue_hotplug)
cb5d4166 8097 schedule_delayed_work(&rdev->hotplug_work, 0);
3c036389
CK		 8098 if (queue_reset) {
8099 rdev->needs_reset = true;
8100 wake_up_all(&rdev->fence_queue);
8101 }
41a524ab
AD		 8102 if (queue_thermal)
8103 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb 8104 rdev->ih.rptr = rptr;
a59781bb
AD		 8105 atomic_set(&rdev->ih.lock, 0);
8106
8107 /* make sure wptr hasn't changed while processing */
8108 wptr = cik_get_ih_wptr(rdev);
8109 if (wptr != rptr)
8110 goto restart_ih;
8111
8112 return IRQ_HANDLED;
8113}
7bf94a2c
AD		 8114
8115/*
8116 * startup/shutdown callbacks
8117 */
bc48a15a
JG		 8118static void cik_uvd_init(struct radeon_device *rdev)
8119{
8120 int r;
8121
8122 if (!rdev->has_uvd)
8123 return;
8124
8125 r = radeon_uvd_init(rdev);
8126 if (r) {
8127 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
8128 /*
8129 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
8130 * to early fails cik_uvd_start() and thus nothing happens
8131 * there. So it is pointless to try to go through that code
8132 * hence why we disable uvd here.
8133 */
45b2de28 8134 rdev->has_uvd = false;
bc48a15a
JG		 8135 return;
8136 }
8137 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
8138 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
8139}
8140
8141static void cik_uvd_start(struct radeon_device *rdev)
8142{
8143 int r;
8144
8145 if (!rdev->has_uvd)
8146 return;
8147
8148 r = radeon_uvd_resume(rdev);
8149 if (r) {
8150 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
8151 goto error;
8152 }
8153 r = uvd_v4_2_resume(rdev);
8154 if (r) {
8155 dev_err(rdev->dev, "failed UVD 4.2 resume (%d).\n", r);
8156 goto error;
8157 }
8158 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
8159 if (r) {
8160 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
8161 goto error;
8162 }
8163 return;
8164
8165error:
8166 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8167}
8168
8169static void cik_uvd_resume(struct radeon_device *rdev)
8170{
8171 struct radeon_ring *ring;
8172 int r;
8173
8174 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
8175 return;
8176
8177 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
70a033d2 8178 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
bc48a15a
JG		 8179 if (r) {
8180 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
8181 return;
8182 }
8183 r = uvd_v1_0_init(rdev);
8184 if (r) {
8185 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
8186 return;
8187 }
8188}
8189
cb25f7e0
JG		 8190static void cik_vce_init(struct radeon_device *rdev)
8191{
8192 int r;
8193
8194 if (!rdev->has_vce)
8195 return;
8196
8197 r = radeon_vce_init(rdev);
8198 if (r) {
8199 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
8200 /*
8201 * At this point rdev->vce.vcpu_bo is NULL which trickles down
8202 * to early fails cik_vce_start() and thus nothing happens
8203 * there. So it is pointless to try to go through that code
8204 * hence why we disable vce here.
8205 */
45b2de28 8206 rdev->has_vce = false;
cb25f7e0
JG		 8207 return;
8208 }
8209 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
8210 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
8211 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
8212 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
8213}
8214
8215static void cik_vce_start(struct radeon_device *rdev)
8216{
8217 int r;
8218
8219 if (!rdev->has_vce)
8220 return;
8221
8222 r = radeon_vce_resume(rdev);
8223 if (r) {
8224 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8225 goto error;
8226 }
8227 r = vce_v2_0_resume(rdev);
8228 if (r) {
8229 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8230 goto error;
8231 }
8232 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
8233 if (r) {
8234 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
8235 goto error;
8236 }
8237 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
8238 if (r) {
8239 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
8240 goto error;
8241 }
8242 return;
8243
8244error:
8245 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8246 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8247}
8248
8249static void cik_vce_resume(struct radeon_device *rdev)
8250{
8251 struct radeon_ring *ring;
8252 int r;
8253
8254 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
8255 return;
8256
8257 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8258 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8259 if (r) {
8260 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8261 return;
8262 }
8263 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8264 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8265 if (r) {
8266 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8267 return;
8268 }
8269 r = vce_v1_0_init(rdev);
8270 if (r) {
8271 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
8272 return;
8273 }
8274}
8275
7bf94a2c
AD		 8276/**
8277 * cik_startup - program the asic to a functional state
8278 *
8279 * @rdev: radeon_device pointer
8280 *
8281 * Programs the asic to a functional state (CIK).
8282 * Called by cik_init() and cik_resume().
8283 * Returns 0 for success, error for failure.
8284 */
8285static int cik_startup(struct radeon_device *rdev)
8286{
8287 struct radeon_ring *ring;
0e16e4cf 8288 u32 nop;
7bf94a2c
AD		 8289 int r;
8290
8a7cd276
AD		 8291 /* enable pcie gen2/3 link */
8292 cik_pcie_gen3_enable(rdev);
7235711a
AD		 8293 /* enable aspm */
8294 cik_program_aspm(rdev);
8a7cd276 8295
e5903d39
AD		 8296 /* scratch needs to be initialized before MC */
8297 r = r600_vram_scratch_init(rdev);
8298 if (r)
8299 return r;
8300
6fab3feb
AD		 8301 cik_mc_program(rdev);
8302
6c7bccea 8303 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7bf94a2c
AD		 8304 r = ci_mc_load_microcode(rdev);
8305 if (r) {
8306 DRM_ERROR("Failed to load MC firmware!\n");
8307 return r;
8308 }
8309 }
8310
7bf94a2c
AD		 8311 r = cik_pcie_gart_enable(rdev);
8312 if (r)
8313 return r;
8314 cik_gpu_init(rdev);
8315
8316 /* allocate rlc buffers */
22c775ce
AD		 8317 if (rdev->flags & RADEON_IS_IGP) {
8318 if (rdev->family == CHIP_KAVERI) {
8319 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8320 rdev->rlc.reg_list_size =
8321 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8322 } else {
8323 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8324 rdev->rlc.reg_list_size =
8325 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8326 }
8327 }
8328 rdev->rlc.cs_data = ci_cs_data;
e70a15f5 8329 rdev->rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */
9406d216 8330 rdev->rlc.cp_table_size += 64 * 1024; /* GDS */
1fd11777 8331 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 8332 if (r) {
8333 DRM_ERROR("Failed to init rlc BOs!\n");
8334 return r;
8335 }
8336
8337 /* allocate wb buffer */
8338 r = radeon_wb_init(rdev);
8339 if (r)
8340 return r;
8341
963e81f9
AD		 8342 /* allocate mec buffers */
8343 r = cik_mec_init(rdev);
8344 if (r) {
8345 DRM_ERROR("Failed to init MEC BOs!\n");
8346 return r;
8347 }
8348
7bf94a2c
AD		 8349 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8350 if (r) {
8351 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8352 return r;
8353 }
8354
963e81f9
AD		 8355 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8356 if (r) {
8357 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8358 return r;
8359 }
8360
8361 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8362 if (r) {
8363 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8364 return r;
8365 }
8366
7bf94a2c
AD		 8367 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8368 if (r) {
8369 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8370 return r;
8371 }
8372
8373 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8374 if (r) {
8375 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8376 return r;
8377 }
8378
bc48a15a 8379 cik_uvd_start(rdev);
cb25f7e0 8380 cik_vce_start(rdev);
d93f7937 8381
7bf94a2c
AD		 8382 /* Enable IRQ */
8383 if (!rdev->irq.installed) {
8384 r = radeon_irq_kms_init(rdev);
8385 if (r)
8386 return r;
8387 }
8388
8389 r = cik_irq_init(rdev);
8390 if (r) {
8391 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8392 radeon_irq_kms_fini(rdev);
8393 return r;
8394 }
8395 cik_irq_set(rdev);
8396
0e16e4cf 8397 if (rdev->family == CHIP_HAWAII) {
78cd3661
AD		 8398 if (rdev->new_fw)
8399 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8400 else
8401 nop = RADEON_CP_PACKET2;
0e16e4cf
AD		 8402 } else {
8403 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8404 }
8405
7bf94a2c
AD		 8406 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8407 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
0e16e4cf 8408 nop);
7bf94a2c
AD		 8409 if (r)
8410 return r;
8411
963e81f9 8412 /* set up the compute queues */
2615b53a 8413 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8414 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8415 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
0e16e4cf 8416 nop);
963e81f9
AD		 8417 if (r)
8418 return r;
8419 ring->me = 1; /* first MEC */
8420 ring->pipe = 0; /* first pipe */
8421 ring->queue = 0; /* first queue */
8422 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8423
2615b53a 8424 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8425 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8426 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
0e16e4cf 8427 nop);
963e81f9
AD		 8428 if (r)
8429 return r;
8430 /* dGPU only have 1 MEC */
8431 ring->me = 1; /* first MEC */
8432 ring->pipe = 0; /* first pipe */
8433 ring->queue = 1; /* second queue */
8434 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8435
7bf94a2c
AD		 8436 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8437 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 8438 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8439 if (r)
8440 return r;
8441
8442 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8443 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 8444 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8445 if (r)
8446 return r;
8447
8448 r = cik_cp_resume(rdev);
8449 if (r)
8450 return r;
8451
8452 r = cik_sdma_resume(rdev);
8453 if (r)
8454 return r;
8455
bc48a15a 8456 cik_uvd_resume(rdev);
cb25f7e0 8457 cik_vce_resume(rdev);
d93f7937 8458
7bf94a2c
AD		 8459 r = radeon_ib_pool_init(rdev);
8460 if (r) {
8461 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8462 return r;
8463 }
8464
8465 r = radeon_vm_manager_init(rdev);
8466 if (r) {
8467 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8468 return r;
8469 }
8470
bfc1f97d 8471 r = radeon_audio_init(rdev);
b530602f
AD		 8472 if (r)
8473 return r;
8474
7bf94a2c
AD		 8475 return 0;
8476}
8477
8478/**
8479 * cik_resume - resume the asic to a functional state
8480 *
8481 * @rdev: radeon_device pointer
8482 *
8483 * Programs the asic to a functional state (CIK).
8484 * Called at resume.
8485 * Returns 0 for success, error for failure.
8486 */
8487int cik_resume(struct radeon_device *rdev)
8488{
8489 int r;
8490
8491 /* post card */
8492 atom_asic_init(rdev->mode_info.atom_context);
8493
0aafd313
AD		 8494 /* init golden registers */
8495 cik_init_golden_registers(rdev);
8496
bc6a6295
AD		 8497 if (rdev->pm.pm_method == PM_METHOD_DPM)
8498 radeon_pm_resume(rdev);
6c7bccea 8499
7bf94a2c
AD		 8500 rdev->accel_working = true;
8501 r = cik_startup(rdev);
8502 if (r) {
8503 DRM_ERROR("cik startup failed on resume\n");
8504 rdev->accel_working = false;
8505 return r;
8506 }
8507
8508 return r;
8509
8510}
8511
8512/**
8513 * cik_suspend - suspend the asic
8514 *
8515 * @rdev: radeon_device pointer
8516 *
8517 * Bring the chip into a state suitable for suspend (CIK).
8518 * Called at suspend.
8519 * Returns 0 for success.
8520 */
8521int cik_suspend(struct radeon_device *rdev)
8522{
6c7bccea 8523 radeon_pm_suspend(rdev);
7991d665 8524 radeon_audio_fini(rdev);
7bf94a2c
AD		 8525 radeon_vm_manager_fini(rdev);
8526 cik_cp_enable(rdev, false);
8527 cik_sdma_enable(rdev, false);
bc48a15a
JG		 8528 if (rdev->has_uvd) {
8529 uvd_v1_0_fini(rdev);
8530 radeon_uvd_suspend(rdev);
8531 }
cb25f7e0
JG		 8532 if (rdev->has_vce)
8533 radeon_vce_suspend(rdev);
473359bc
AD		 8534 cik_fini_pg(rdev);
8535 cik_fini_cg(rdev);
7bf94a2c
AD		 8536 cik_irq_suspend(rdev);
8537 radeon_wb_disable(rdev);
8538 cik_pcie_gart_disable(rdev);
8539 return 0;
8540}
8541
8542/* Plan is to move initialization in that function and use
8543 * helper function so that radeon_device_init pretty much
8544 * do nothing more than calling asic specific function. This
8545 * should also allow to remove a bunch of callback function
8546 * like vram_info.
8547 */
8548/**
8549 * cik_init - asic specific driver and hw init
8550 *
8551 * @rdev: radeon_device pointer
8552 *
8553 * Setup asic specific driver variables and program the hw
8554 * to a functional state (CIK).
8555 * Called at driver startup.
8556 * Returns 0 for success, errors for failure.
8557 */
8558int cik_init(struct radeon_device *rdev)
8559{
8560 struct radeon_ring *ring;
8561 int r;
8562
8563 /* Read BIOS */
8564 if (!radeon_get_bios(rdev)) {
8565 if (ASIC_IS_AVIVO(rdev))
8566 return -EINVAL;
8567 }
8568 /* Must be an ATOMBIOS */
8569 if (!rdev->is_atom_bios) {
8570 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8571 return -EINVAL;
8572 }
8573 r = radeon_atombios_init(rdev);
8574 if (r)
8575 return r;
8576
8577 /* Post card if necessary */
8578 if (!radeon_card_posted(rdev)) {
8579 if (!rdev->bios) {
8580 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8581 return -EINVAL;
8582 }
8583 DRM_INFO("GPU not posted. posting now...\n");
8584 atom_asic_init(rdev->mode_info.atom_context);
8585 }
0aafd313
AD		 8586 /* init golden registers */
8587 cik_init_golden_registers(rdev);
7bf94a2c
AD		 8588 /* Initialize scratch registers */
8589 cik_scratch_init(rdev);
8590 /* Initialize surface registers */
8591 radeon_surface_init(rdev);
8592 /* Initialize clocks */
8593 radeon_get_clock_info(rdev->ddev);
8594
8595 /* Fence driver */
8596 r = radeon_fence_driver_init(rdev);
8597 if (r)
8598 return r;
8599
8600 /* initialize memory controller */
8601 r = cik_mc_init(rdev);
8602 if (r)
8603 return r;
8604 /* Memory manager */
8605 r = radeon_bo_init(rdev);
8606 if (r)
8607 return r;
8608
01ac8794
AD		 8609 if (rdev->flags & RADEON_IS_IGP) {
8610 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8611 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8612 r = cik_init_microcode(rdev);
8613 if (r) {
8614 DRM_ERROR("Failed to load firmware!\n");
8615 return r;
8616 }
8617 }
8618 } else {
8619 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8620 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8621 !rdev->mc_fw) {
8622 r = cik_init_microcode(rdev);
8623 if (r) {
8624 DRM_ERROR("Failed to load firmware!\n");
8625 return r;
8626 }
8627 }
8628 }
8629
6c7bccea
AD		 8630 /* Initialize power management */
8631 radeon_pm_init(rdev);
8632
7bf94a2c
AD		 8633 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8634 ring->ring_obj = NULL;
8635 r600_ring_init(rdev, ring, 1024 * 1024);
8636
963e81f9
AD		 8637 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8638 ring->ring_obj = NULL;
8639 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8640 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8641 if (r)
8642 return r;
8643
8644 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8645 ring->ring_obj = NULL;
8646 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8647 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8648 if (r)
8649 return r;
8650
7bf94a2c
AD		 8651 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8652 ring->ring_obj = NULL;
8653 r600_ring_init(rdev, ring, 256 * 1024);
8654
8655 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8656 ring->ring_obj = NULL;
8657 r600_ring_init(rdev, ring, 256 * 1024);
8658
bc48a15a 8659 cik_uvd_init(rdev);
cb25f7e0 8660 cik_vce_init(rdev);
d93f7937 8661
7bf94a2c
AD		 8662 rdev->ih.ring_obj = NULL;
8663 r600_ih_ring_init(rdev, 64 * 1024);
8664
8665 r = r600_pcie_gart_init(rdev);
8666 if (r)
8667 return r;
8668
8669 rdev->accel_working = true;
8670 r = cik_startup(rdev);
8671 if (r) {
8672 dev_err(rdev->dev, "disabling GPU acceleration\n");
8673 cik_cp_fini(rdev);
8674 cik_sdma_fini(rdev);
8675 cik_irq_fini(rdev);
1fd11777 8676 sumo_rlc_fini(rdev);
963e81f9 8677 cik_mec_fini(rdev);
7bf94a2c
AD		 8678 radeon_wb_fini(rdev);
8679 radeon_ib_pool_fini(rdev);
8680 radeon_vm_manager_fini(rdev);
8681 radeon_irq_kms_fini(rdev);
8682 cik_pcie_gart_fini(rdev);
8683 rdev->accel_working = false;
8684 }
8685
8686 /* Don't start up if the MC ucode is missing.
8687 * The default clocks and voltages before the MC ucode
8688 * is loaded are not suffient for advanced operations.
8689 */
8690 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8691 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8692 return -EINVAL;
8693 }
8694
8695 return 0;
8696}
8697
8698/**
8699 * cik_fini - asic specific driver and hw fini
8700 *
8701 * @rdev: radeon_device pointer
8702 *
8703 * Tear down the asic specific driver variables and program the hw
8704 * to an idle state (CIK).
8705 * Called at driver unload.
8706 */
8707void cik_fini(struct radeon_device *rdev)
8708{
6c7bccea 8709 radeon_pm_fini(rdev);
7bf94a2c
AD		 8710 cik_cp_fini(rdev);
8711 cik_sdma_fini(rdev);
473359bc
AD		 8712 cik_fini_pg(rdev);
8713 cik_fini_cg(rdev);
7bf94a2c 8714 cik_irq_fini(rdev);
1fd11777 8715 sumo_rlc_fini(rdev);
963e81f9 8716 cik_mec_fini(rdev);
7bf94a2c
AD		 8717 radeon_wb_fini(rdev);
8718 radeon_vm_manager_fini(rdev);
8719 radeon_ib_pool_fini(rdev);
8720 radeon_irq_kms_fini(rdev);
e409b128 8721 uvd_v1_0_fini(rdev);
87167bb1 8722 radeon_uvd_fini(rdev);
d93f7937 8723 radeon_vce_fini(rdev);
7bf94a2c
AD		 8724 cik_pcie_gart_fini(rdev);
8725 r600_vram_scratch_fini(rdev);
8726 radeon_gem_fini(rdev);
8727 radeon_fence_driver_fini(rdev);
8728 radeon_bo_fini(rdev);
8729 radeon_atombios_fini(rdev);
8730 kfree(rdev->bios);
8731 rdev->bios = NULL;
8732}
cd84a27d 8733
134b480f
AD		 8734void dce8_program_fmt(struct drm_encoder *encoder)
8735{
8736 struct drm_device *dev = encoder->dev;
8737 struct radeon_device *rdev = dev->dev_private;
8738 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8739 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8740 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8741 int bpc = 0;
8742 u32 tmp = 0;
6214bb74 8743 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 8744
6214bb74
AD		 8745 if (connector) {
8746 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 8747 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 8748 dither = radeon_connector->dither;
8749 }
134b480f
AD		 8750
8751 /* LVDS/eDP FMT is set up by atom */
8752 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8753 return;
8754
8755 /* not needed for analog */
8756 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8757 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8758 return;
8759
8760 if (bpc == 0)
8761 return;
8762
8763 switch (bpc) {
8764 case 6:
6214bb74 8765 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8766 /* XXX sort out optimal dither settings */
8767 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8768 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8769 else
8770 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8771 break;
8772 case 8:
6214bb74 8773 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8774 /* XXX sort out optimal dither settings */
8775 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8776 FMT_RGB_RANDOM_ENABLE |
8777 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8778 else
8779 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8780 break;
8781 case 10:
6214bb74 8782 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8783 /* XXX sort out optimal dither settings */
8784 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8785 FMT_RGB_RANDOM_ENABLE |
8786 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8787 else
8788 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8789 break;
8790 default:
8791 /* not needed */
8792 break;
8793 }
8794
8795 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8796}
8797
cd84a27d
AD		 8798/* display watermark setup */
8799/**
8800 * dce8_line_buffer_adjust - Set up the line buffer
8801 *
8802 * @rdev: radeon_device pointer
8803 * @radeon_crtc: the selected display controller
8804 * @mode: the current display mode on the selected display
8805 * controller
8806 *
8807 * Setup up the line buffer allocation for
8808 * the selected display controller (CIK).
8809 * Returns the line buffer size in pixels.
8810 */
8811static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8812 struct radeon_crtc *radeon_crtc,
8813 struct drm_display_mode *mode)
8814{
bc01a8c7
AD		 8815 u32 tmp, buffer_alloc, i;
8816 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 8817 /*
8818 * Line Buffer Setup
8819 * There are 6 line buffers, one for each display controllers.
8820 * There are 3 partitions per LB. Select the number of partitions
8821 * to enable based on the display width. For display widths larger
8822 * than 4096, you need use to use 2 display controllers and combine
8823 * them using the stereo blender.
8824 */
8825 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 8826 if (mode->crtc_hdisplay < 1920) {
cd84a27d 8827 tmp = 1;
bc01a8c7
AD		 8828 buffer_alloc = 2;
8829 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 8830 tmp = 2;
bc01a8c7
AD		 8831 buffer_alloc = 2;
8832 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 8833 tmp = 0;
bc01a8c7
AD		 8834 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8835 } else {
cd84a27d
AD		 8836 DRM_DEBUG_KMS("Mode too big for LB!\n");
8837 tmp = 0;
bc01a8c7 8838 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 8839 }
bc01a8c7 8840 } else {
cd84a27d 8841 tmp = 1;
bc01a8c7
AD		 8842 buffer_alloc = 0;
8843 }
cd84a27d
AD		 8844
8845 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8846 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8847
bc01a8c7
AD		 8848 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8849 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8850 for (i = 0; i < rdev->usec_timeout; i++) {
8851 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8852 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8853 break;
8854 udelay(1);
8855 }
8856
cd84a27d
AD		 8857 if (radeon_crtc->base.enabled && mode) {
8858 switch (tmp) {
8859 case 0:
8860 default:
8861 return 4096 * 2;
8862 case 1:
8863 return 1920 * 2;
8864 case 2:
8865 return 2560 * 2;
8866 }
8867 }
8868
8869 /* controller not enabled, so no lb used */
8870 return 0;
8871}
8872
8873/**
8874 * cik_get_number_of_dram_channels - get the number of dram channels
8875 *
8876 * @rdev: radeon_device pointer
8877 *
8878 * Look up the number of video ram channels (CIK).
8879 * Used for display watermark bandwidth calculations
8880 * Returns the number of dram channels
8881 */
8882static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8883{
8884 u32 tmp = RREG32(MC_SHARED_CHMAP);
8885
8886 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8887 case 0:
8888 default:
8889 return 1;
8890 case 1:
8891 return 2;
8892 case 2:
8893 return 4;
8894 case 3:
8895 return 8;
8896 case 4:
8897 return 3;
8898 case 5:
8899 return 6;
8900 case 6:
8901 return 10;
8902 case 7:
8903 return 12;
8904 case 8:
8905 return 16;
8906 }
8907}
8908
8909struct dce8_wm_params {
8910 u32 dram_channels; /* number of dram channels */
8911 u32 yclk; /* bandwidth per dram data pin in kHz */
8912 u32 sclk; /* engine clock in kHz */
8913 u32 disp_clk; /* display clock in kHz */
8914 u32 src_width; /* viewport width */
8915 u32 active_time; /* active display time in ns */
8916 u32 blank_time; /* blank time in ns */
8917 bool interlaced; /* mode is interlaced */
8918 fixed20_12 vsc; /* vertical scale ratio */
8919 u32 num_heads; /* number of active crtcs */
8920 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8921 u32 lb_size; /* line buffer allocated to pipe */
8922 u32 vtaps; /* vertical scaler taps */
8923};
8924
8925/**
8926 * dce8_dram_bandwidth - get the dram bandwidth
8927 *
8928 * @wm: watermark calculation data
8929 *
8930 * Calculate the raw dram bandwidth (CIK).
8931 * Used for display watermark bandwidth calculations
8932 * Returns the dram bandwidth in MBytes/s
8933 */
8934static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8935{
8936 /* Calculate raw DRAM Bandwidth */
8937 fixed20_12 dram_efficiency; /* 0.7 */
8938 fixed20_12 yclk, dram_channels, bandwidth;
8939 fixed20_12 a;
8940
8941 a.full = dfixed_const(1000);
8942 yclk.full = dfixed_const(wm->yclk);
8943 yclk.full = dfixed_div(yclk, a);
8944 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8945 a.full = dfixed_const(10);
8946 dram_efficiency.full = dfixed_const(7);
8947 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8948 bandwidth.full = dfixed_mul(dram_channels, yclk);
8949 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8950
8951 return dfixed_trunc(bandwidth);
8952}
8953
8954/**
8955 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8956 *
8957 * @wm: watermark calculation data
8958 *
8959 * Calculate the dram bandwidth used for display (CIK).
8960 * Used for display watermark bandwidth calculations
8961 * Returns the dram bandwidth for display in MBytes/s
8962 */
8963static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8964{
8965 /* Calculate DRAM Bandwidth and the part allocated to display. */
8966 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8967 fixed20_12 yclk, dram_channels, bandwidth;
8968 fixed20_12 a;
8969
8970 a.full = dfixed_const(1000);
8971 yclk.full = dfixed_const(wm->yclk);
8972 yclk.full = dfixed_div(yclk, a);
8973 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8974 a.full = dfixed_const(10);
8975 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8976 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8977 bandwidth.full = dfixed_mul(dram_channels, yclk);
8978 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8979
8980 return dfixed_trunc(bandwidth);
8981}
8982
8983/**
8984 * dce8_data_return_bandwidth - get the data return bandwidth
8985 *
8986 * @wm: watermark calculation data
8987 *
8988 * Calculate the data return bandwidth used for display (CIK).
8989 * Used for display watermark bandwidth calculations
8990 * Returns the data return bandwidth in MBytes/s
8991 */
8992static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8993{
8994 /* Calculate the display Data return Bandwidth */
8995 fixed20_12 return_efficiency; /* 0.8 */
8996 fixed20_12 sclk, bandwidth;
8997 fixed20_12 a;
8998
8999 a.full = dfixed_const(1000);
9000 sclk.full = dfixed_const(wm->sclk);
9001 sclk.full = dfixed_div(sclk, a);
9002 a.full = dfixed_const(10);
9003 return_efficiency.full = dfixed_const(8);
9004 return_efficiency.full = dfixed_div(return_efficiency, a);
9005 a.full = dfixed_const(32);
9006 bandwidth.full = dfixed_mul(a, sclk);
9007 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9008
9009 return dfixed_trunc(bandwidth);
9010}
9011
9012/**
9013 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9014 *
9015 * @wm: watermark calculation data
9016 *
9017 * Calculate the dmif bandwidth used for display (CIK).
9018 * Used for display watermark bandwidth calculations
9019 * Returns the dmif bandwidth in MBytes/s
9020 */
9021static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9022{
9023 /* Calculate the DMIF Request Bandwidth */
9024 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9025 fixed20_12 disp_clk, bandwidth;
9026 fixed20_12 a, b;
9027
9028 a.full = dfixed_const(1000);
9029 disp_clk.full = dfixed_const(wm->disp_clk);
9030 disp_clk.full = dfixed_div(disp_clk, a);
9031 a.full = dfixed_const(32);
9032 b.full = dfixed_mul(a, disp_clk);
9033
9034 a.full = dfixed_const(10);
9035 disp_clk_request_efficiency.full = dfixed_const(8);
9036 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9037
9038 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9039
9040 return dfixed_trunc(bandwidth);
9041}
9042
9043/**
9044 * dce8_available_bandwidth - get the min available bandwidth
9045 *
9046 * @wm: watermark calculation data
9047 *
9048 * Calculate the min available bandwidth used for display (CIK).
9049 * Used for display watermark bandwidth calculations
9050 * Returns the min available bandwidth in MBytes/s
9051 */
9052static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9053{
9054 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9055 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9056 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9057 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9058
9059 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9060}
9061
9062/**
9063 * dce8_average_bandwidth - get the average available bandwidth
9064 *
9065 * @wm: watermark calculation data
9066 *
9067 * Calculate the average available bandwidth used for display (CIK).
9068 * Used for display watermark bandwidth calculations
9069 * Returns the average available bandwidth in MBytes/s
9070 */
9071static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9072{
9073 /* Calculate the display mode Average Bandwidth
9074 * DisplayMode should contain the source and destination dimensions,
9075 * timing, etc.
9076 */
9077 fixed20_12 bpp;
9078 fixed20_12 line_time;
9079 fixed20_12 src_width;
9080 fixed20_12 bandwidth;
9081 fixed20_12 a;
9082
9083 a.full = dfixed_const(1000);
9084 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9085 line_time.full = dfixed_div(line_time, a);
9086 bpp.full = dfixed_const(wm->bytes_per_pixel);
9087 src_width.full = dfixed_const(wm->src_width);
9088 bandwidth.full = dfixed_mul(src_width, bpp);
9089 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9090 bandwidth.full = dfixed_div(bandwidth, line_time);
9091
9092 return dfixed_trunc(bandwidth);
9093}
9094
9095/**
9096 * dce8_latency_watermark - get the latency watermark
9097 *
9098 * @wm: watermark calculation data
9099 *
9100 * Calculate the latency watermark (CIK).
9101 * Used for display watermark bandwidth calculations
9102 * Returns the latency watermark in ns
9103 */
9104static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9105{
9106 /* First calculate the latency in ns */
9107 u32 mc_latency = 2000; /* 2000 ns. */
9108 u32 available_bandwidth = dce8_available_bandwidth(wm);
9109 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9110 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9111 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9112 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9113 (wm->num_heads * cursor_line_pair_return_time);
9114 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9115 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9116 u32 tmp, dmif_size = 12288;
9117 fixed20_12 a, b, c;
9118
9119 if (wm->num_heads == 0)
9120 return 0;
9121
9122 a.full = dfixed_const(2);
9123 b.full = dfixed_const(1);
9124 if ((wm->vsc.full > a.full) ||
9125 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9126 (wm->vtaps >= 5) ||
9127 ((wm->vsc.full >= a.full) && wm->interlaced))
9128 max_src_lines_per_dst_line = 4;
9129 else
9130 max_src_lines_per_dst_line = 2;
9131
9132 a.full = dfixed_const(available_bandwidth);
9133 b.full = dfixed_const(wm->num_heads);
9134 a.full = dfixed_div(a, b);
ae45bbc2
MK		 9135 tmp = div_u64((u64) dmif_size * (u64) wm->disp_clk, mc_latency + 512);
9136 tmp = min(dfixed_trunc(a), tmp);
cd84a27d 9137
ae45bbc2 9138 lb_fill_bw = min(tmp, wm->disp_clk * wm->bytes_per_pixel / 1000);
cd84a27d
AD		 9139
9140 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9141 b.full = dfixed_const(1000);
9142 c.full = dfixed_const(lb_fill_bw);
9143 b.full = dfixed_div(c, b);
9144 a.full = dfixed_div(a, b);
9145 line_fill_time = dfixed_trunc(a);
9146
9147 if (line_fill_time < wm->active_time)
9148 return latency;
9149 else
9150 return latency + (line_fill_time - wm->active_time);
9151
9152}
9153
9154/**
9155 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9156 * average and available dram bandwidth
9157 *
9158 * @wm: watermark calculation data
9159 *
9160 * Check if the display average bandwidth fits in the display
9161 * dram bandwidth (CIK).
9162 * Used for display watermark bandwidth calculations
9163 * Returns true if the display fits, false if not.
9164 */
9165static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9166{
9167 if (dce8_average_bandwidth(wm) <=
9168 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9169 return true;
9170 else
9171 return false;
9172}
9173
9174/**
9175 * dce8_average_bandwidth_vs_available_bandwidth - check
9176 * average and available bandwidth
9177 *
9178 * @wm: watermark calculation data
9179 *
9180 * Check if the display average bandwidth fits in the display
9181 * available bandwidth (CIK).
9182 * Used for display watermark bandwidth calculations
9183 * Returns true if the display fits, false if not.
9184 */
9185static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9186{
9187 if (dce8_average_bandwidth(wm) <=
9188 (dce8_available_bandwidth(wm) / wm->num_heads))
9189 return true;
9190 else
9191 return false;
9192}
9193
9194/**
9195 * dce8_check_latency_hiding - check latency hiding
9196 *
9197 * @wm: watermark calculation data
9198 *
9199 * Check latency hiding (CIK).
9200 * Used for display watermark bandwidth calculations
9201 * Returns true if the display fits, false if not.
9202 */
9203static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9204{
9205 u32 lb_partitions = wm->lb_size / wm->src_width;
9206 u32 line_time = wm->active_time + wm->blank_time;
9207 u32 latency_tolerant_lines;
9208 u32 latency_hiding;
9209 fixed20_12 a;
9210
9211 a.full = dfixed_const(1);
9212 if (wm->vsc.full > a.full)
9213 latency_tolerant_lines = 1;
9214 else {
9215 if (lb_partitions <= (wm->vtaps + 1))
9216 latency_tolerant_lines = 1;
9217 else
9218 latency_tolerant_lines = 2;
9219 }
9220
9221 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9222
9223 if (dce8_latency_watermark(wm) <= latency_hiding)
9224 return true;
9225 else
9226 return false;
9227}
9228
9229/**
9230 * dce8_program_watermarks - program display watermarks
9231 *
9232 * @rdev: radeon_device pointer
9233 * @radeon_crtc: the selected display controller
9234 * @lb_size: line buffer size
9235 * @num_heads: number of display controllers in use
9236 *
9237 * Calculate and program the display watermarks for the
9238 * selected display controller (CIK).
9239 */
9240static void dce8_program_watermarks(struct radeon_device *rdev,
9241 struct radeon_crtc *radeon_crtc,
9242 u32 lb_size, u32 num_heads)
9243{
9244 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 9245 struct dce8_wm_params wm_low, wm_high;
e6b9a6c8 9246 u32 active_time;
cd84a27d
AD		 9247 u32 line_time = 0;
9248 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9249 u32 tmp, wm_mask;
9250
9251 if (radeon_crtc->base.enabled && num_heads && mode) {
55f61a04
MK		 9252 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
9253 (u32)mode->clock);
9254 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
9255 (u32)mode->clock);
9256 line_time = min(line_time, (u32)65535);
cd84a27d 9257
58ea2dea
AD		 9258 /* watermark for high clocks */
9259 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9260 rdev->pm.dpm_enabled) {
9261 wm_high.yclk =
9262 radeon_dpm_get_mclk(rdev, false) * 10;
9263 wm_high.sclk =
9264 radeon_dpm_get_sclk(rdev, false) * 10;
9265 } else {
9266 wm_high.yclk = rdev->pm.current_mclk * 10;
9267 wm_high.sclk = rdev->pm.current_sclk * 10;
9268 }
9269
9270 wm_high.disp_clk = mode->clock;
9271 wm_high.src_width = mode->crtc_hdisplay;
e6b9a6c8 9272 wm_high.active_time = active_time;
58ea2dea
AD		 9273 wm_high.blank_time = line_time - wm_high.active_time;
9274 wm_high.interlaced = false;
cd84a27d 9275 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 9276 wm_high.interlaced = true;
9277 wm_high.vsc = radeon_crtc->vsc;
9278 wm_high.vtaps = 1;
cd84a27d 9279 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 9280 wm_high.vtaps = 2;
9281 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9282 wm_high.lb_size = lb_size;
9283 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9284 wm_high.num_heads = num_heads;
cd84a27d
AD		 9285
9286 /* set for high clocks */
58ea2dea
AD		 9287 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9288
9289 /* possibly force display priority to high */
9290 /* should really do this at mode validation time... */
9291 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9292 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9293 !dce8_check_latency_hiding(&wm_high) ||
9294 (rdev->disp_priority == 2)) {
9295 DRM_DEBUG_KMS("force priority to high\n");
9296 }
9297
9298 /* watermark for low clocks */
9299 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9300 rdev->pm.dpm_enabled) {
9301 wm_low.yclk =
9302 radeon_dpm_get_mclk(rdev, true) * 10;
9303 wm_low.sclk =
9304 radeon_dpm_get_sclk(rdev, true) * 10;
9305 } else {
9306 wm_low.yclk = rdev->pm.current_mclk * 10;
9307 wm_low.sclk = rdev->pm.current_sclk * 10;
9308 }
9309
9310 wm_low.disp_clk = mode->clock;
9311 wm_low.src_width = mode->crtc_hdisplay;
e6b9a6c8 9312 wm_low.active_time = active_time;
58ea2dea
AD		 9313 wm_low.blank_time = line_time - wm_low.active_time;
9314 wm_low.interlaced = false;
9315 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9316 wm_low.interlaced = true;
9317 wm_low.vsc = radeon_crtc->vsc;
9318 wm_low.vtaps = 1;
9319 if (radeon_crtc->rmx_type != RMX_OFF)
9320 wm_low.vtaps = 2;
9321 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9322 wm_low.lb_size = lb_size;
9323 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9324 wm_low.num_heads = num_heads;
9325
cd84a27d 9326 /* set for low clocks */
58ea2dea 9327 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 9328
9329 /* possibly force display priority to high */
9330 /* should really do this at mode validation time... */
58ea2dea
AD		 9331 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9332 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9333 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 9334 (rdev->disp_priority == 2)) {
9335 DRM_DEBUG_KMS("force priority to high\n");
9336 }
5b5561b3
MK		 9337
9338 /* Save number of lines the linebuffer leads before the scanout */
9339 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
cd84a27d
AD		 9340 }
9341
9342 /* select wm A */
9343 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9344 tmp = wm_mask;
9345 tmp &= ~LATENCY_WATERMARK_MASK(3);
9346 tmp |= LATENCY_WATERMARK_MASK(1);
9347 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9348 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9349 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9350 LATENCY_HIGH_WATERMARK(line_time)));
9351 /* select wm B */
9352 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9353 tmp &= ~LATENCY_WATERMARK_MASK(3);
9354 tmp |= LATENCY_WATERMARK_MASK(2);
9355 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9356 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9357 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9358 LATENCY_HIGH_WATERMARK(line_time)));
9359 /* restore original selection */
9360 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 9361
9362 /* save values for DPM */
9363 radeon_crtc->line_time = line_time;
9364 radeon_crtc->wm_high = latency_watermark_a;
9365 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 9366}
9367
9368/**
9369 * dce8_bandwidth_update - program display watermarks
9370 *
9371 * @rdev: radeon_device pointer
9372 *
9373 * Calculate and program the display watermarks and line
9374 * buffer allocation (CIK).
9375 */
9376void dce8_bandwidth_update(struct radeon_device *rdev)
9377{
9378 struct drm_display_mode *mode = NULL;
9379 u32 num_heads = 0, lb_size;
9380 int i;
9381
8efe82ca
AD		 9382 if (!rdev->mode_info.mode_config_initialized)
9383 return;
9384
cd84a27d
AD		 9385 radeon_update_display_priority(rdev);
9386
9387 for (i = 0; i < rdev->num_crtc; i++) {
9388 if (rdev->mode_info.crtcs[i]->base.enabled)
9389 num_heads++;
9390 }
9391 for (i = 0; i < rdev->num_crtc; i++) {
9392 mode = &rdev->mode_info.crtcs[i]->base.mode;
9393 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9394 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9395 }
9396}
44fa346f
AD		 9397
9398/**
9399 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9400 *
9401 * @rdev: radeon_device pointer
9402 *
9403 * Fetches a GPU clock counter snapshot (SI).
9404 * Returns the 64 bit clock counter snapshot.
9405 */
9406uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9407{
9408 uint64_t clock;
9409
9410 mutex_lock(&rdev->gpu_clock_mutex);
9411 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9412 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
3cf8bb1a 9413 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
44fa346f
AD		 9414 mutex_unlock(&rdev->gpu_clock_mutex);
9415 return clock;
9416}
9417
87167bb1 9418static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
3cf8bb1a 9419 u32 cntl_reg, u32 status_reg)
87167bb1
CK		 9420{
9421 int r, i;
9422 struct atom_clock_dividers dividers;
9423 uint32_t tmp;
9424
9425 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9426 clock, false, &dividers);
9427 if (r)
9428 return r;
9429
9430 tmp = RREG32_SMC(cntl_reg);
9431 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9432 tmp |= dividers.post_divider;
9433 WREG32_SMC(cntl_reg, tmp);
9434
9435 for (i = 0; i < 100; i++) {
9436 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9437 break;
9438 mdelay(10);
9439 }
9440 if (i == 100)
9441 return -ETIMEDOUT;
9442
9443 return 0;
9444}
9445
9446int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9447{
9448 int r = 0;
9449
9450 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9451 if (r)
9452 return r;
9453
9454 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9455 return r;
9456}
9457
5ad6bf91
AD		 9458int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9459{
9460 int r, i;
9461 struct atom_clock_dividers dividers;
9462 u32 tmp;
9463
9464 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9465 ecclk, false, &dividers);
9466 if (r)
9467 return r;
9468
9469 for (i = 0; i < 100; i++) {
9470 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9471 break;
9472 mdelay(10);
9473 }
9474 if (i == 100)
9475 return -ETIMEDOUT;
9476
9477 tmp = RREG32_SMC(CG_ECLK_CNTL);
9478 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9479 tmp |= dividers.post_divider;
9480 WREG32_SMC(CG_ECLK_CNTL, tmp);
9481
9482 for (i = 0; i < 100; i++) {
9483 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9484 break;
9485 mdelay(10);
9486 }
9487 if (i == 100)
9488 return -ETIMEDOUT;
9489
9490 return 0;
9491}
9492
8a7cd276 9493static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 9494{
8a7cd276 9495 struct pci_dev *root = rdev->pdev->bus->self;
5f152a57 9496 enum pci_bus_speed speed_cap;
5f152a57
AD		 9497 u32 speed_cntl, current_data_rate;
9498 int i;
8a7cd276 9499 u16 tmp16;
87167bb1 9500
0bd252de
AW		 9501 if (pci_is_root_bus(rdev->pdev->bus))
9502 return;
9503
8a7cd276
AD		 9504 if (radeon_pcie_gen2 == 0)
9505 return;
87167bb1 9506
8a7cd276
AD		 9507 if (rdev->flags & RADEON_IS_IGP)
9508 return;
87167bb1 9509
8a7cd276
AD		 9510 if (!(rdev->flags & RADEON_IS_PCIE))
9511 return;
87167bb1 9512
5f152a57
AD		 9513 speed_cap = pcie_get_speed_cap(root);
9514 if (speed_cap == PCI_SPEED_UNKNOWN)
8a7cd276 9515 return;
87167bb1 9516
5f152a57
AD		 9517 if ((speed_cap != PCIE_SPEED_8_0GT) &&
9518 (speed_cap != PCIE_SPEED_5_0GT))
8a7cd276 9519 return;
87167bb1 9520
8a7cd276
AD		 9521 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9522 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9523 LC_CURRENT_DATA_RATE_SHIFT;
5f152a57 9524 if (speed_cap == PCIE_SPEED_8_0GT) {
8a7cd276
AD		 9525 if (current_data_rate == 2) {
9526 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9527 return;
9528 }
9529 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
5f152a57 9530 } else if (speed_cap == PCIE_SPEED_5_0GT) {
8a7cd276
AD		 9531 if (current_data_rate == 1) {
9532 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9533 return;
9534 }
9535 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9536 }
87167bb1 9537
3d581b11 9538 if (!pci_is_pcie(root) || !pci_is_pcie(rdev->pdev))
8a7cd276
AD		 9539 return;
9540
5f152a57 9541 if (speed_cap == PCIE_SPEED_8_0GT) {
8a7cd276
AD		 9542 /* re-try equalization if gen3 is not already enabled */
9543 if (current_data_rate != 2) {
9544 u16 bridge_cfg, gpu_cfg;
9545 u16 bridge_cfg2, gpu_cfg2;
9546 u32 max_lw, current_lw, tmp;
9547
3d581b11
FL		 9548 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
9549 &bridge_cfg);
9550 pcie_capability_read_word(rdev->pdev, PCI_EXP_LNKCTL,
9551 &gpu_cfg);
8a7cd276
AD		 9552
9553 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
3d581b11 9554 pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
8a7cd276
AD		 9555
9556 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
3d581b11
FL		 9557 pcie_capability_write_word(rdev->pdev, PCI_EXP_LNKCTL,
9558 tmp16);
8a7cd276
AD		 9559
9560 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9561 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9562 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9563
9564 if (current_lw < max_lw) {
9565 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9566 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9567 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9568 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9569 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9570 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9571 }
9572 }
9573
9574 for (i = 0; i < 10; i++) {
9575 /* check status */
3d581b11
FL		 9576 pcie_capability_read_word(rdev->pdev,
9577 PCI_EXP_DEVSTA,
9578 &tmp16);
8a7cd276
AD		 9579 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9580 break;
9581
3d581b11
FL		 9582 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
9583 &bridge_cfg);
9584 pcie_capability_read_word(rdev->pdev,
9585 PCI_EXP_LNKCTL,
9586 &gpu_cfg);
8a7cd276 9587
3d581b11
FL		 9588 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
9589 &bridge_cfg2);
9590 pcie_capability_read_word(rdev->pdev,
9591 PCI_EXP_LNKCTL2,
9592 &gpu_cfg2);
8a7cd276
AD		 9593
9594 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9595 tmp |= LC_SET_QUIESCE;
9596 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9597
9598 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9599 tmp |= LC_REDO_EQ;
9600 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9601
c51cebb8 9602 msleep(100);
8a7cd276
AD		 9603
9604 /* linkctl */
3d581b11
FL		 9605 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
9606 &tmp16);
8a7cd276
AD		 9607 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9608 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
3d581b11
FL		 9609 pcie_capability_write_word(root, PCI_EXP_LNKCTL,
9610 tmp16);
8a7cd276 9611
3d581b11
FL		 9612 pcie_capability_read_word(rdev->pdev,
9613 PCI_EXP_LNKCTL,
9614 &tmp16);
8a7cd276
AD		 9615 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9616 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
3d581b11
FL		 9617 pcie_capability_write_word(rdev->pdev,
9618 PCI_EXP_LNKCTL,
9619 tmp16);
8a7cd276
AD		 9620
9621 /* linkctl2 */
3d581b11
FL		 9622 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
9623 &tmp16);
ca56f99c
BH		 9624 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
9625 PCI_EXP_LNKCTL2_TX_MARGIN);
9626 tmp16 |= (bridge_cfg2 &
9627 (PCI_EXP_LNKCTL2_ENTER_COMP |
9628 PCI_EXP_LNKCTL2_TX_MARGIN));
3d581b11
FL		 9629 pcie_capability_write_word(root,
9630 PCI_EXP_LNKCTL2,
9631 tmp16);
8a7cd276 9632
3d581b11
FL		 9633 pcie_capability_read_word(rdev->pdev,
9634 PCI_EXP_LNKCTL2,
9635 &tmp16);
ca56f99c
BH		 9636 tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
9637 PCI_EXP_LNKCTL2_TX_MARGIN);
9638 tmp16 |= (gpu_cfg2 &
9639 (PCI_EXP_LNKCTL2_ENTER_COMP |
9640 PCI_EXP_LNKCTL2_TX_MARGIN));
3d581b11
FL		 9641 pcie_capability_write_word(rdev->pdev,
9642 PCI_EXP_LNKCTL2,
9643 tmp16);
8a7cd276
AD		 9644
9645 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9646 tmp &= ~LC_SET_QUIESCE;
9647 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9648 }
9649 }
9650 }
9651
9652 /* set the link speed */
9653 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9654 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9655 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9656
3d581b11 9657 pcie_capability_read_word(rdev->pdev, PCI_EXP_LNKCTL2, &tmp16);
ca56f99c 9658 tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
5f152a57 9659 if (speed_cap == PCIE_SPEED_8_0GT)
ca56f99c 9660 tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
5f152a57 9661 else if (speed_cap == PCIE_SPEED_5_0GT)
ca56f99c 9662 tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
8a7cd276 9663 else
ca56f99c 9664 tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
3d581b11 9665 pcie_capability_write_word(rdev->pdev, PCI_EXP_LNKCTL2, tmp16);
8a7cd276
AD		 9666
9667 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9668 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9669 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9670
9671 for (i = 0; i < rdev->usec_timeout; i++) {
9672 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9673 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9674 break;
9675 udelay(1);
9676 }
9677}
7235711a
AD		 9678
9679static void cik_program_aspm(struct radeon_device *rdev)
9680{
9681 u32 data, orig;
9682 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9683 bool disable_clkreq = false;
9684
9685 if (radeon_aspm == 0)
9686 return;
9687
9688 /* XXX double check IGPs */
9689 if (rdev->flags & RADEON_IS_IGP)
9690 return;
9691
9692 if (!(rdev->flags & RADEON_IS_PCIE))
9693 return;
9694
9695 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9696 data &= ~LC_XMIT_N_FTS_MASK;
9697 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9698 if (orig != data)
9699 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9700
9701 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9702 data |= LC_GO_TO_RECOVERY;
9703 if (orig != data)
9704 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9705
9706 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9707 data |= P_IGNORE_EDB_ERR;
9708 if (orig != data)
9709 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9710
9711 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9712 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9713 data |= LC_PMI_TO_L1_DIS;
9714 if (!disable_l0s)
9715 data |= LC_L0S_INACTIVITY(7);
9716
9717 if (!disable_l1) {
9718 data |= LC_L1_INACTIVITY(7);
9719 data &= ~LC_PMI_TO_L1_DIS;
9720 if (orig != data)
9721 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9722
9723 if (!disable_plloff_in_l1) {
9724 bool clk_req_support;
9725
9726 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9727 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9728 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9729 if (orig != data)
9730 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9731
9732 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9733 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9734 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9735 if (orig != data)
9736 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9737
9738 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9739 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9740 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9741 if (orig != data)
9742 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9743
9744 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9745 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9746 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9747 if (orig != data)
9748 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9749
9750 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9751 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9752 data |= LC_DYN_LANES_PWR_STATE(3);
9753 if (orig != data)
9754 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9755
0bd252de
AW		 9756 if (!disable_clkreq &&
9757 !pci_is_root_bus(rdev->pdev->bus)) {
7235711a
AD		 9758 struct pci_dev *root = rdev->pdev->bus->self;
9759 u32 lnkcap;
9760
9761 clk_req_support = false;
9762 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9763 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9764 clk_req_support = true;
9765 } else {
9766 clk_req_support = false;
9767 }
9768
9769 if (clk_req_support) {
9770 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9771 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9772 if (orig != data)
9773 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9774
9775 orig = data = RREG32_SMC(THM_CLK_CNTL);
9776 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9777 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9778 if (orig != data)
9779 WREG32_SMC(THM_CLK_CNTL, data);
9780
9781 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9782 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9783 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9784 if (orig != data)
9785 WREG32_SMC(MISC_CLK_CTRL, data);
9786
9787 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9788 data &= ~BCLK_AS_XCLK;
9789 if (orig != data)
9790 WREG32_SMC(CG_CLKPIN_CNTL, data);
9791
9792 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9793 data &= ~FORCE_BIF_REFCLK_EN;
9794 if (orig != data)
9795 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9796
9797 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9798 data &= ~MPLL_CLKOUT_SEL_MASK;
9799 data |= MPLL_CLKOUT_SEL(4);
9800 if (orig != data)
9801 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9802 }
9803 }
9804 } else {
9805 if (orig != data)
9806 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9807 }
9808
9809 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9810 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9811 if (orig != data)
9812 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9813
9814 if (!disable_l0s) {
9815 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9816 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9817 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9818 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9819 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9820 data &= ~LC_L0S_INACTIVITY_MASK;
9821 if (orig != data)
9822 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9823 }
9824 }
9825 }
87167bb1 9826}
Linux 6.x block layer and io_uring tree(s)