projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm: Use pr_cont where appropriate
[linux-2.6-block.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
bfc1f97d 30#include "radeon_audio.h"
8cc1a532
AD		 31#include "cikd.h"
32#include "atom.h"
841cf442 33#include "cik_blit_shaders.h"
8c68e393 34#include "radeon_ucode.h"
22c775ce 35#include "clearstate_ci.h"
e28740ec 36#include "radeon_kfd.h"
02c81327 37
75cb00dc
MO		 38#define SH_MEM_CONFIG_GFX_DEFAULT \
39 ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED)
40
02c81327
AD		 41MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
42MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
43MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
44MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
45MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
277babc3 46MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
02c81327 47MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 48MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 49MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
f2c6b0f4
AD		 50
51MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
52MODULE_FIRMWARE("radeon/bonaire_me.bin");
53MODULE_FIRMWARE("radeon/bonaire_ce.bin");
54MODULE_FIRMWARE("radeon/bonaire_mec.bin");
55MODULE_FIRMWARE("radeon/bonaire_mc.bin");
56MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
57MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
58MODULE_FIRMWARE("radeon/bonaire_smc.bin");
b2ea0dcd 59MODULE_FIRMWARE("radeon/bonaire_k_smc.bin");
f2c6b0f4 60
d4775655
AD		 61MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
62MODULE_FIRMWARE("radeon/HAWAII_me.bin");
63MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
64MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
65MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
277babc3 66MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
d4775655
AD		 67MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
68MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
69MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
f2c6b0f4
AD		 70
71MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
72MODULE_FIRMWARE("radeon/hawaii_me.bin");
73MODULE_FIRMWARE("radeon/hawaii_ce.bin");
74MODULE_FIRMWARE("radeon/hawaii_mec.bin");
75MODULE_FIRMWARE("radeon/hawaii_mc.bin");
76MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
77MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
78MODULE_FIRMWARE("radeon/hawaii_smc.bin");
b2ea0dcd 79MODULE_FIRMWARE("radeon/hawaii_k_smc.bin");
f2c6b0f4 80
02c81327
AD		 81MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
82MODULE_FIRMWARE("radeon/KAVERI_me.bin");
83MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
84MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
85MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 86MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
f2c6b0f4
AD		 87
88MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
89MODULE_FIRMWARE("radeon/kaveri_me.bin");
90MODULE_FIRMWARE("radeon/kaveri_ce.bin");
91MODULE_FIRMWARE("radeon/kaveri_mec.bin");
92MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
93MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
94MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
95
02c81327
AD		 96MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
97MODULE_FIRMWARE("radeon/KABINI_me.bin");
98MODULE_FIRMWARE("radeon/KABINI_ce.bin");
99MODULE_FIRMWARE("radeon/KABINI_mec.bin");
100MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 101MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
f2c6b0f4
AD		 102
103MODULE_FIRMWARE("radeon/kabini_pfp.bin");
104MODULE_FIRMWARE("radeon/kabini_me.bin");
105MODULE_FIRMWARE("radeon/kabini_ce.bin");
106MODULE_FIRMWARE("radeon/kabini_mec.bin");
107MODULE_FIRMWARE("radeon/kabini_rlc.bin");
108MODULE_FIRMWARE("radeon/kabini_sdma.bin");
109
f73a9e83
SL		 110MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
111MODULE_FIRMWARE("radeon/MULLINS_me.bin");
112MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
113MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
114MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
115MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
02c81327 116
f2c6b0f4
AD		 117MODULE_FIRMWARE("radeon/mullins_pfp.bin");
118MODULE_FIRMWARE("radeon/mullins_me.bin");
119MODULE_FIRMWARE("radeon/mullins_ce.bin");
120MODULE_FIRMWARE("radeon/mullins_mec.bin");
121MODULE_FIRMWARE("radeon/mullins_rlc.bin");
122MODULE_FIRMWARE("radeon/mullins_sdma.bin");
123
a59781bb
AD		 124extern int r600_ih_ring_alloc(struct radeon_device *rdev);
125extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 126extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
127extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 128extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 129extern void sumo_rlc_fini(struct radeon_device *rdev);
130extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 131extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 132extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 133extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
65fcf668 134static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
2483b4ea
CK		 135extern int cik_sdma_resume(struct radeon_device *rdev);
136extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
137extern void cik_sdma_fini(struct radeon_device *rdev);
a1d6f97c 138extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
cc066715 139static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 140static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 141static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 142static void cik_init_pg(struct radeon_device *rdev);
143static void cik_init_cg(struct radeon_device *rdev);
fb2c7f4d
AD		 144static void cik_fini_pg(struct radeon_device *rdev);
145static void cik_fini_cg(struct radeon_device *rdev);
4214faf6
AD		 146static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
147 bool enable);
6f2043ce 148
353eec2a
AD		 149/**
150 * cik_get_allowed_info_register - fetch the register for the info ioctl
151 *
152 * @rdev: radeon_device pointer
153 * @reg: register offset in bytes
154 * @val: register value
155 *
156 * Returns 0 for success or -EINVAL for an invalid register
157 *
158 */
159int cik_get_allowed_info_register(struct radeon_device *rdev,
160 u32 reg, u32 *val)
161{
162 switch (reg) {
163 case GRBM_STATUS:
164 case GRBM_STATUS2:
165 case GRBM_STATUS_SE0:
166 case GRBM_STATUS_SE1:
167 case GRBM_STATUS_SE2:
168 case GRBM_STATUS_SE3:
169 case SRBM_STATUS:
170 case SRBM_STATUS2:
171 case (SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET):
172 case (SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET):
173 case UVD_STATUS:
174 /* TODO VCE */
175 *val = RREG32(reg);
176 return 0;
177 default:
178 return -EINVAL;
179 }
180}
181
9e5acbc2
DV		 182/*
183 * Indirect registers accessor
184 */
185u32 cik_didt_rreg(struct radeon_device *rdev, u32 reg)
186{
187 unsigned long flags;
188 u32 r;
189
190 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
191 WREG32(CIK_DIDT_IND_INDEX, (reg));
192 r = RREG32(CIK_DIDT_IND_DATA);
193 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
194 return r;
195}
196
197void cik_didt_wreg(struct radeon_device *rdev, u32 reg, u32 v)
198{
199 unsigned long flags;
200
201 spin_lock_irqsave(&rdev->didt_idx_lock, flags);
202 WREG32(CIK_DIDT_IND_INDEX, (reg));
203 WREG32(CIK_DIDT_IND_DATA, (v));
204 spin_unlock_irqrestore(&rdev->didt_idx_lock, flags);
205}
206
286d9cc6
AD		 207/* get temperature in millidegrees */
208int ci_get_temp(struct radeon_device *rdev)
209{
210 u32 temp;
211 int actual_temp = 0;
212
213 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
214 CTF_TEMP_SHIFT;
215
216 if (temp & 0x200)
217 actual_temp = 255;
218 else
219 actual_temp = temp & 0x1ff;
220
221 actual_temp = actual_temp * 1000;
222
223 return actual_temp;
224}
225
226/* get temperature in millidegrees */
227int kv_get_temp(struct radeon_device *rdev)
228{
229 u32 temp;
230 int actual_temp = 0;
231
232 temp = RREG32_SMC(0xC0300E0C);
233
234 if (temp)
235 actual_temp = (temp / 8) - 49;
236 else
237 actual_temp = 0;
238
239 actual_temp = actual_temp * 1000;
240
241 return actual_temp;
242}
6f2043ce 243
6e2c3c0a
AD		 244/*
245 * Indirect registers accessor
246 */
247u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
248{
0a5b7b0b 249 unsigned long flags;
6e2c3c0a
AD		 250 u32 r;
251
0a5b7b0b 252 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 253 WREG32(PCIE_INDEX, reg);
254 (void)RREG32(PCIE_INDEX);
255 r = RREG32(PCIE_DATA);
0a5b7b0b 256 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 257 return r;
258}
259
260void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
261{
0a5b7b0b
AD		 262 unsigned long flags;
263
264 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 265 WREG32(PCIE_INDEX, reg);
266 (void)RREG32(PCIE_INDEX);
267 WREG32(PCIE_DATA, v);
268 (void)RREG32(PCIE_DATA);
0a5b7b0b 269 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 270}
271
22c775ce
AD		 272static const u32 spectre_rlc_save_restore_register_list[] =
273{
274 (0x0e00 << 16) | (0xc12c >> 2),
275 0x00000000,
276 (0x0e00 << 16) | (0xc140 >> 2),
277 0x00000000,
278 (0x0e00 << 16) | (0xc150 >> 2),
279 0x00000000,
280 (0x0e00 << 16) | (0xc15c >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0xc168 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0xc170 >> 2),
285 0x00000000,
286 (0x0e00 << 16) | (0xc178 >> 2),
287 0x00000000,
288 (0x0e00 << 16) | (0xc204 >> 2),
289 0x00000000,
290 (0x0e00 << 16) | (0xc2b4 >> 2),
291 0x00000000,
292 (0x0e00 << 16) | (0xc2b8 >> 2),
293 0x00000000,
294 (0x0e00 << 16) | (0xc2bc >> 2),
295 0x00000000,
296 (0x0e00 << 16) | (0xc2c0 >> 2),
297 0x00000000,
298 (0x0e00 << 16) | (0x8228 >> 2),
299 0x00000000,
300 (0x0e00 << 16) | (0x829c >> 2),
301 0x00000000,
302 (0x0e00 << 16) | (0x869c >> 2),
303 0x00000000,
304 (0x0600 << 16) | (0x98f4 >> 2),
305 0x00000000,
306 (0x0e00 << 16) | (0x98f8 >> 2),
307 0x00000000,
308 (0x0e00 << 16) | (0x9900 >> 2),
309 0x00000000,
310 (0x0e00 << 16) | (0xc260 >> 2),
311 0x00000000,
312 (0x0e00 << 16) | (0x90e8 >> 2),
313 0x00000000,
314 (0x0e00 << 16) | (0x3c000 >> 2),
315 0x00000000,
316 (0x0e00 << 16) | (0x3c00c >> 2),
317 0x00000000,
318 (0x0e00 << 16) | (0x8c1c >> 2),
319 0x00000000,
320 (0x0e00 << 16) | (0x9700 >> 2),
321 0x00000000,
322 (0x0e00 << 16) | (0xcd20 >> 2),
323 0x00000000,
324 (0x4e00 << 16) | (0xcd20 >> 2),
325 0x00000000,
326 (0x5e00 << 16) | (0xcd20 >> 2),
327 0x00000000,
328 (0x6e00 << 16) | (0xcd20 >> 2),
329 0x00000000,
330 (0x7e00 << 16) | (0xcd20 >> 2),
331 0x00000000,
332 (0x8e00 << 16) | (0xcd20 >> 2),
333 0x00000000,
334 (0x9e00 << 16) | (0xcd20 >> 2),
335 0x00000000,
336 (0xae00 << 16) | (0xcd20 >> 2),
337 0x00000000,
338 (0xbe00 << 16) | (0xcd20 >> 2),
339 0x00000000,
340 (0x0e00 << 16) | (0x89bc >> 2),
341 0x00000000,
342 (0x0e00 << 16) | (0x8900 >> 2),
343 0x00000000,
344 0x3,
345 (0x0e00 << 16) | (0xc130 >> 2),
346 0x00000000,
347 (0x0e00 << 16) | (0xc134 >> 2),
348 0x00000000,
349 (0x0e00 << 16) | (0xc1fc >> 2),
350 0x00000000,
351 (0x0e00 << 16) | (0xc208 >> 2),
352 0x00000000,
353 (0x0e00 << 16) | (0xc264 >> 2),
354 0x00000000,
355 (0x0e00 << 16) | (0xc268 >> 2),
356 0x00000000,
357 (0x0e00 << 16) | (0xc26c >> 2),
358 0x00000000,
359 (0x0e00 << 16) | (0xc270 >> 2),
360 0x00000000,
361 (0x0e00 << 16) | (0xc274 >> 2),
362 0x00000000,
363 (0x0e00 << 16) | (0xc278 >> 2),
364 0x00000000,
365 (0x0e00 << 16) | (0xc27c >> 2),
366 0x00000000,
367 (0x0e00 << 16) | (0xc280 >> 2),
368 0x00000000,
369 (0x0e00 << 16) | (0xc284 >> 2),
370 0x00000000,
371 (0x0e00 << 16) | (0xc288 >> 2),
372 0x00000000,
373 (0x0e00 << 16) | (0xc28c >> 2),
374 0x00000000,
375 (0x0e00 << 16) | (0xc290 >> 2),
376 0x00000000,
377 (0x0e00 << 16) | (0xc294 >> 2),
378 0x00000000,
379 (0x0e00 << 16) | (0xc298 >> 2),
380 0x00000000,
381 (0x0e00 << 16) | (0xc29c >> 2),
382 0x00000000,
383 (0x0e00 << 16) | (0xc2a0 >> 2),
384 0x00000000,
385 (0x0e00 << 16) | (0xc2a4 >> 2),
386 0x00000000,
387 (0x0e00 << 16) | (0xc2a8 >> 2),
388 0x00000000,
389 (0x0e00 << 16) | (0xc2ac >> 2),
390 0x00000000,
391 (0x0e00 << 16) | (0xc2b0 >> 2),
392 0x00000000,
393 (0x0e00 << 16) | (0x301d0 >> 2),
394 0x00000000,
395 (0x0e00 << 16) | (0x30238 >> 2),
396 0x00000000,
397 (0x0e00 << 16) | (0x30250 >> 2),
398 0x00000000,
399 (0x0e00 << 16) | (0x30254 >> 2),
400 0x00000000,
401 (0x0e00 << 16) | (0x30258 >> 2),
402 0x00000000,
403 (0x0e00 << 16) | (0x3025c >> 2),
404 0x00000000,
405 (0x4e00 << 16) | (0xc900 >> 2),
406 0x00000000,
407 (0x5e00 << 16) | (0xc900 >> 2),
408 0x00000000,
409 (0x6e00 << 16) | (0xc900 >> 2),
410 0x00000000,
411 (0x7e00 << 16) | (0xc900 >> 2),
412 0x00000000,
413 (0x8e00 << 16) | (0xc900 >> 2),
414 0x00000000,
415 (0x9e00 << 16) | (0xc900 >> 2),
416 0x00000000,
417 (0xae00 << 16) | (0xc900 >> 2),
418 0x00000000,
419 (0xbe00 << 16) | (0xc900 >> 2),
420 0x00000000,
421 (0x4e00 << 16) | (0xc904 >> 2),
422 0x00000000,
423 (0x5e00 << 16) | (0xc904 >> 2),
424 0x00000000,
425 (0x6e00 << 16) | (0xc904 >> 2),
426 0x00000000,
427 (0x7e00 << 16) | (0xc904 >> 2),
428 0x00000000,
429 (0x8e00 << 16) | (0xc904 >> 2),
430 0x00000000,
431 (0x9e00 << 16) | (0xc904 >> 2),
432 0x00000000,
433 (0xae00 << 16) | (0xc904 >> 2),
434 0x00000000,
435 (0xbe00 << 16) | (0xc904 >> 2),
436 0x00000000,
437 (0x4e00 << 16) | (0xc908 >> 2),
438 0x00000000,
439 (0x5e00 << 16) | (0xc908 >> 2),
440 0x00000000,
441 (0x6e00 << 16) | (0xc908 >> 2),
442 0x00000000,
443 (0x7e00 << 16) | (0xc908 >> 2),
444 0x00000000,
445 (0x8e00 << 16) | (0xc908 >> 2),
446 0x00000000,
447 (0x9e00 << 16) | (0xc908 >> 2),
448 0x00000000,
449 (0xae00 << 16) | (0xc908 >> 2),
450 0x00000000,
451 (0xbe00 << 16) | (0xc908 >> 2),
452 0x00000000,
453 (0x4e00 << 16) | (0xc90c >> 2),
454 0x00000000,
455 (0x5e00 << 16) | (0xc90c >> 2),
456 0x00000000,
457 (0x6e00 << 16) | (0xc90c >> 2),
458 0x00000000,
459 (0x7e00 << 16) | (0xc90c >> 2),
460 0x00000000,
461 (0x8e00 << 16) | (0xc90c >> 2),
462 0x00000000,
463 (0x9e00 << 16) | (0xc90c >> 2),
464 0x00000000,
465 (0xae00 << 16) | (0xc90c >> 2),
466 0x00000000,
467 (0xbe00 << 16) | (0xc90c >> 2),
468 0x00000000,
469 (0x4e00 << 16) | (0xc910 >> 2),
470 0x00000000,
471 (0x5e00 << 16) | (0xc910 >> 2),
472 0x00000000,
473 (0x6e00 << 16) | (0xc910 >> 2),
474 0x00000000,
475 (0x7e00 << 16) | (0xc910 >> 2),
476 0x00000000,
477 (0x8e00 << 16) | (0xc910 >> 2),
478 0x00000000,
479 (0x9e00 << 16) | (0xc910 >> 2),
480 0x00000000,
481 (0xae00 << 16) | (0xc910 >> 2),
482 0x00000000,
483 (0xbe00 << 16) | (0xc910 >> 2),
484 0x00000000,
485 (0x0e00 << 16) | (0xc99c >> 2),
486 0x00000000,
487 (0x0e00 << 16) | (0x9834 >> 2),
488 0x00000000,
489 (0x0000 << 16) | (0x30f00 >> 2),
490 0x00000000,
491 (0x0001 << 16) | (0x30f00 >> 2),
492 0x00000000,
493 (0x0000 << 16) | (0x30f04 >> 2),
494 0x00000000,
495 (0x0001 << 16) | (0x30f04 >> 2),
496 0x00000000,
497 (0x0000 << 16) | (0x30f08 >> 2),
498 0x00000000,
499 (0x0001 << 16) | (0x30f08 >> 2),
500 0x00000000,
501 (0x0000 << 16) | (0x30f0c >> 2),
502 0x00000000,
503 (0x0001 << 16) | (0x30f0c >> 2),
504 0x00000000,
505 (0x0600 << 16) | (0x9b7c >> 2),
506 0x00000000,
507 (0x0e00 << 16) | (0x8a14 >> 2),
508 0x00000000,
509 (0x0e00 << 16) | (0x8a18 >> 2),
510 0x00000000,
511 (0x0600 << 16) | (0x30a00 >> 2),
512 0x00000000,
513 (0x0e00 << 16) | (0x8bf0 >> 2),
514 0x00000000,
515 (0x0e00 << 16) | (0x8bcc >> 2),
516 0x00000000,
517 (0x0e00 << 16) | (0x8b24 >> 2),
518 0x00000000,
519 (0x0e00 << 16) | (0x30a04 >> 2),
520 0x00000000,
521 (0x0600 << 16) | (0x30a10 >> 2),
522 0x00000000,
523 (0x0600 << 16) | (0x30a14 >> 2),
524 0x00000000,
525 (0x0600 << 16) | (0x30a18 >> 2),
526 0x00000000,
527 (0x0600 << 16) | (0x30a2c >> 2),
528 0x00000000,
529 (0x0e00 << 16) | (0xc700 >> 2),
530 0x00000000,
531 (0x0e00 << 16) | (0xc704 >> 2),
532 0x00000000,
533 (0x0e00 << 16) | (0xc708 >> 2),
534 0x00000000,
535 (0x0e00 << 16) | (0xc768 >> 2),
536 0x00000000,
537 (0x0400 << 16) | (0xc770 >> 2),
538 0x00000000,
539 (0x0400 << 16) | (0xc774 >> 2),
540 0x00000000,
541 (0x0400 << 16) | (0xc778 >> 2),
542 0x00000000,
543 (0x0400 << 16) | (0xc77c >> 2),
544 0x00000000,
545 (0x0400 << 16) | (0xc780 >> 2),
546 0x00000000,
547 (0x0400 << 16) | (0xc784 >> 2),
548 0x00000000,
549 (0x0400 << 16) | (0xc788 >> 2),
550 0x00000000,
551 (0x0400 << 16) | (0xc78c >> 2),
552 0x00000000,
553 (0x0400 << 16) | (0xc798 >> 2),
554 0x00000000,
555 (0x0400 << 16) | (0xc79c >> 2),
556 0x00000000,
557 (0x0400 << 16) | (0xc7a0 >> 2),
558 0x00000000,
559 (0x0400 << 16) | (0xc7a4 >> 2),
560 0x00000000,
561 (0x0400 << 16) | (0xc7a8 >> 2),
562 0x00000000,
563 (0x0400 << 16) | (0xc7ac >> 2),
564 0x00000000,
565 (0x0400 << 16) | (0xc7b0 >> 2),
566 0x00000000,
567 (0x0400 << 16) | (0xc7b4 >> 2),
568 0x00000000,
569 (0x0e00 << 16) | (0x9100 >> 2),
570 0x00000000,
571 (0x0e00 << 16) | (0x3c010 >> 2),
572 0x00000000,
573 (0x0e00 << 16) | (0x92a8 >> 2),
574 0x00000000,
575 (0x0e00 << 16) | (0x92ac >> 2),
576 0x00000000,
577 (0x0e00 << 16) | (0x92b4 >> 2),
578 0x00000000,
579 (0x0e00 << 16) | (0x92b8 >> 2),
580 0x00000000,
581 (0x0e00 << 16) | (0x92bc >> 2),
582 0x00000000,
583 (0x0e00 << 16) | (0x92c0 >> 2),
584 0x00000000,
585 (0x0e00 << 16) | (0x92c4 >> 2),
586 0x00000000,
587 (0x0e00 << 16) | (0x92c8 >> 2),
588 0x00000000,
589 (0x0e00 << 16) | (0x92cc >> 2),
590 0x00000000,
591 (0x0e00 << 16) | (0x92d0 >> 2),
592 0x00000000,
593 (0x0e00 << 16) | (0x8c00 >> 2),
594 0x00000000,
595 (0x0e00 << 16) | (0x8c04 >> 2),
596 0x00000000,
597 (0x0e00 << 16) | (0x8c20 >> 2),
598 0x00000000,
599 (0x0e00 << 16) | (0x8c38 >> 2),
600 0x00000000,
601 (0x0e00 << 16) | (0x8c3c >> 2),
602 0x00000000,
603 (0x0e00 << 16) | (0xae00 >> 2),
604 0x00000000,
605 (0x0e00 << 16) | (0x9604 >> 2),
606 0x00000000,
607 (0x0e00 << 16) | (0xac08 >> 2),
608 0x00000000,
609 (0x0e00 << 16) | (0xac0c >> 2),
610 0x00000000,
611 (0x0e00 << 16) | (0xac10 >> 2),
612 0x00000000,
613 (0x0e00 << 16) | (0xac14 >> 2),
614 0x00000000,
615 (0x0e00 << 16) | (0xac58 >> 2),
616 0x00000000,
617 (0x0e00 << 16) | (0xac68 >> 2),
618 0x00000000,
619 (0x0e00 << 16) | (0xac6c >> 2),
620 0x00000000,
621 (0x0e00 << 16) | (0xac70 >> 2),
622 0x00000000,
623 (0x0e00 << 16) | (0xac74 >> 2),
624 0x00000000,
625 (0x0e00 << 16) | (0xac78 >> 2),
626 0x00000000,
627 (0x0e00 << 16) | (0xac7c >> 2),
628 0x00000000,
629 (0x0e00 << 16) | (0xac80 >> 2),
630 0x00000000,
631 (0x0e00 << 16) | (0xac84 >> 2),
632 0x00000000,
633 (0x0e00 << 16) | (0xac88 >> 2),
634 0x00000000,
635 (0x0e00 << 16) | (0xac8c >> 2),
636 0x00000000,
637 (0x0e00 << 16) | (0x970c >> 2),
638 0x00000000,
639 (0x0e00 << 16) | (0x9714 >> 2),
640 0x00000000,
641 (0x0e00 << 16) | (0x9718 >> 2),
642 0x00000000,
643 (0x0e00 << 16) | (0x971c >> 2),
644 0x00000000,
645 (0x0e00 << 16) | (0x31068 >> 2),
646 0x00000000,
647 (0x4e00 << 16) | (0x31068 >> 2),
648 0x00000000,
649 (0x5e00 << 16) | (0x31068 >> 2),
650 0x00000000,
651 (0x6e00 << 16) | (0x31068 >> 2),
652 0x00000000,
653 (0x7e00 << 16) | (0x31068 >> 2),
654 0x00000000,
655 (0x8e00 << 16) | (0x31068 >> 2),
656 0x00000000,
657 (0x9e00 << 16) | (0x31068 >> 2),
658 0x00000000,
659 (0xae00 << 16) | (0x31068 >> 2),
660 0x00000000,
661 (0xbe00 << 16) | (0x31068 >> 2),
662 0x00000000,
663 (0x0e00 << 16) | (0xcd10 >> 2),
664 0x00000000,
665 (0x0e00 << 16) | (0xcd14 >> 2),
666 0x00000000,
667 (0x0e00 << 16) | (0x88b0 >> 2),
668 0x00000000,
669 (0x0e00 << 16) | (0x88b4 >> 2),
670 0x00000000,
671 (0x0e00 << 16) | (0x88b8 >> 2),
672 0x00000000,
673 (0x0e00 << 16) | (0x88bc >> 2),
674 0x00000000,
675 (0x0400 << 16) | (0x89c0 >> 2),
676 0x00000000,
677 (0x0e00 << 16) | (0x88c4 >> 2),
678 0x00000000,
679 (0x0e00 << 16) | (0x88c8 >> 2),
680 0x00000000,
681 (0x0e00 << 16) | (0x88d0 >> 2),
682 0x00000000,
683 (0x0e00 << 16) | (0x88d4 >> 2),
684 0x00000000,
685 (0x0e00 << 16) | (0x88d8 >> 2),
686 0x00000000,
687 (0x0e00 << 16) | (0x8980 >> 2),
688 0x00000000,
689 (0x0e00 << 16) | (0x30938 >> 2),
690 0x00000000,
691 (0x0e00 << 16) | (0x3093c >> 2),
692 0x00000000,
693 (0x0e00 << 16) | (0x30940 >> 2),
694 0x00000000,
695 (0x0e00 << 16) | (0x89a0 >> 2),
696 0x00000000,
697 (0x0e00 << 16) | (0x30900 >> 2),
698 0x00000000,
699 (0x0e00 << 16) | (0x30904 >> 2),
700 0x00000000,
701 (0x0e00 << 16) | (0x89b4 >> 2),
702 0x00000000,
703 (0x0e00 << 16) | (0x3c210 >> 2),
704 0x00000000,
705 (0x0e00 << 16) | (0x3c214 >> 2),
706 0x00000000,
707 (0x0e00 << 16) | (0x3c218 >> 2),
708 0x00000000,
709 (0x0e00 << 16) | (0x8904 >> 2),
710 0x00000000,
711 0x5,
712 (0x0e00 << 16) | (0x8c28 >> 2),
713 (0x0e00 << 16) | (0x8c2c >> 2),
714 (0x0e00 << 16) | (0x8c30 >> 2),
715 (0x0e00 << 16) | (0x8c34 >> 2),
716 (0x0e00 << 16) | (0x9600 >> 2),
717};
718
719static const u32 kalindi_rlc_save_restore_register_list[] =
720{
721 (0x0e00 << 16) | (0xc12c >> 2),
722 0x00000000,
723 (0x0e00 << 16) | (0xc140 >> 2),
724 0x00000000,
725 (0x0e00 << 16) | (0xc150 >> 2),
726 0x00000000,
727 (0x0e00 << 16) | (0xc15c >> 2),
728 0x00000000,
729 (0x0e00 << 16) | (0xc168 >> 2),
730 0x00000000,
731 (0x0e00 << 16) | (0xc170 >> 2),
732 0x00000000,
733 (0x0e00 << 16) | (0xc204 >> 2),
734 0x00000000,
735 (0x0e00 << 16) | (0xc2b4 >> 2),
736 0x00000000,
737 (0x0e00 << 16) | (0xc2b8 >> 2),
738 0x00000000,
739 (0x0e00 << 16) | (0xc2bc >> 2),
740 0x00000000,
741 (0x0e00 << 16) | (0xc2c0 >> 2),
742 0x00000000,
743 (0x0e00 << 16) | (0x8228 >> 2),
744 0x00000000,
745 (0x0e00 << 16) | (0x829c >> 2),
746 0x00000000,
747 (0x0e00 << 16) | (0x869c >> 2),
748 0x00000000,
749 (0x0600 << 16) | (0x98f4 >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0x98f8 >> 2),
752 0x00000000,
753 (0x0e00 << 16) | (0x9900 >> 2),
754 0x00000000,
755 (0x0e00 << 16) | (0xc260 >> 2),
756 0x00000000,
757 (0x0e00 << 16) | (0x90e8 >> 2),
758 0x00000000,
759 (0x0e00 << 16) | (0x3c000 >> 2),
760 0x00000000,
761 (0x0e00 << 16) | (0x3c00c >> 2),
762 0x00000000,
763 (0x0e00 << 16) | (0x8c1c >> 2),
764 0x00000000,
765 (0x0e00 << 16) | (0x9700 >> 2),
766 0x00000000,
767 (0x0e00 << 16) | (0xcd20 >> 2),
768 0x00000000,
769 (0x4e00 << 16) | (0xcd20 >> 2),
770 0x00000000,
771 (0x5e00 << 16) | (0xcd20 >> 2),
772 0x00000000,
773 (0x6e00 << 16) | (0xcd20 >> 2),
774 0x00000000,
775 (0x7e00 << 16) | (0xcd20 >> 2),
776 0x00000000,
777 (0x0e00 << 16) | (0x89bc >> 2),
778 0x00000000,
779 (0x0e00 << 16) | (0x8900 >> 2),
780 0x00000000,
781 0x3,
782 (0x0e00 << 16) | (0xc130 >> 2),
783 0x00000000,
784 (0x0e00 << 16) | (0xc134 >> 2),
785 0x00000000,
786 (0x0e00 << 16) | (0xc1fc >> 2),
787 0x00000000,
788 (0x0e00 << 16) | (0xc208 >> 2),
789 0x00000000,
790 (0x0e00 << 16) | (0xc264 >> 2),
791 0x00000000,
792 (0x0e00 << 16) | (0xc268 >> 2),
793 0x00000000,
794 (0x0e00 << 16) | (0xc26c >> 2),
795 0x00000000,
796 (0x0e00 << 16) | (0xc270 >> 2),
797 0x00000000,
798 (0x0e00 << 16) | (0xc274 >> 2),
799 0x00000000,
800 (0x0e00 << 16) | (0xc28c >> 2),
801 0x00000000,
802 (0x0e00 << 16) | (0xc290 >> 2),
803 0x00000000,
804 (0x0e00 << 16) | (0xc294 >> 2),
805 0x00000000,
806 (0x0e00 << 16) | (0xc298 >> 2),
807 0x00000000,
808 (0x0e00 << 16) | (0xc2a0 >> 2),
809 0x00000000,
810 (0x0e00 << 16) | (0xc2a4 >> 2),
811 0x00000000,
812 (0x0e00 << 16) | (0xc2a8 >> 2),
813 0x00000000,
814 (0x0e00 << 16) | (0xc2ac >> 2),
815 0x00000000,
816 (0x0e00 << 16) | (0x301d0 >> 2),
817 0x00000000,
818 (0x0e00 << 16) | (0x30238 >> 2),
819 0x00000000,
820 (0x0e00 << 16) | (0x30250 >> 2),
821 0x00000000,
822 (0x0e00 << 16) | (0x30254 >> 2),
823 0x00000000,
824 (0x0e00 << 16) | (0x30258 >> 2),
825 0x00000000,
826 (0x0e00 << 16) | (0x3025c >> 2),
827 0x00000000,
828 (0x4e00 << 16) | (0xc900 >> 2),
829 0x00000000,
830 (0x5e00 << 16) | (0xc900 >> 2),
831 0x00000000,
832 (0x6e00 << 16) | (0xc900 >> 2),
833 0x00000000,
834 (0x7e00 << 16) | (0xc900 >> 2),
835 0x00000000,
836 (0x4e00 << 16) | (0xc904 >> 2),
837 0x00000000,
838 (0x5e00 << 16) | (0xc904 >> 2),
839 0x00000000,
840 (0x6e00 << 16) | (0xc904 >> 2),
841 0x00000000,
842 (0x7e00 << 16) | (0xc904 >> 2),
843 0x00000000,
844 (0x4e00 << 16) | (0xc908 >> 2),
845 0x00000000,
846 (0x5e00 << 16) | (0xc908 >> 2),
847 0x00000000,
848 (0x6e00 << 16) | (0xc908 >> 2),
849 0x00000000,
850 (0x7e00 << 16) | (0xc908 >> 2),
851 0x00000000,
852 (0x4e00 << 16) | (0xc90c >> 2),
853 0x00000000,
854 (0x5e00 << 16) | (0xc90c >> 2),
855 0x00000000,
856 (0x6e00 << 16) | (0xc90c >> 2),
857 0x00000000,
858 (0x7e00 << 16) | (0xc90c >> 2),
859 0x00000000,
860 (0x4e00 << 16) | (0xc910 >> 2),
861 0x00000000,
862 (0x5e00 << 16) | (0xc910 >> 2),
863 0x00000000,
864 (0x6e00 << 16) | (0xc910 >> 2),
865 0x00000000,
866 (0x7e00 << 16) | (0xc910 >> 2),
867 0x00000000,
868 (0x0e00 << 16) | (0xc99c >> 2),
869 0x00000000,
870 (0x0e00 << 16) | (0x9834 >> 2),
871 0x00000000,
872 (0x0000 << 16) | (0x30f00 >> 2),
873 0x00000000,
874 (0x0000 << 16) | (0x30f04 >> 2),
875 0x00000000,
876 (0x0000 << 16) | (0x30f08 >> 2),
877 0x00000000,
878 (0x0000 << 16) | (0x30f0c >> 2),
879 0x00000000,
880 (0x0600 << 16) | (0x9b7c >> 2),
881 0x00000000,
882 (0x0e00 << 16) | (0x8a14 >> 2),
883 0x00000000,
884 (0x0e00 << 16) | (0x8a18 >> 2),
885 0x00000000,
886 (0x0600 << 16) | (0x30a00 >> 2),
887 0x00000000,
888 (0x0e00 << 16) | (0x8bf0 >> 2),
889 0x00000000,
890 (0x0e00 << 16) | (0x8bcc >> 2),
891 0x00000000,
892 (0x0e00 << 16) | (0x8b24 >> 2),
893 0x00000000,
894 (0x0e00 << 16) | (0x30a04 >> 2),
895 0x00000000,
896 (0x0600 << 16) | (0x30a10 >> 2),
897 0x00000000,
898 (0x0600 << 16) | (0x30a14 >> 2),
899 0x00000000,
900 (0x0600 << 16) | (0x30a18 >> 2),
901 0x00000000,
902 (0x0600 << 16) | (0x30a2c >> 2),
903 0x00000000,
904 (0x0e00 << 16) | (0xc700 >> 2),
905 0x00000000,
906 (0x0e00 << 16) | (0xc704 >> 2),
907 0x00000000,
908 (0x0e00 << 16) | (0xc708 >> 2),
909 0x00000000,
910 (0x0e00 << 16) | (0xc768 >> 2),
911 0x00000000,
912 (0x0400 << 16) | (0xc770 >> 2),
913 0x00000000,
914 (0x0400 << 16) | (0xc774 >> 2),
915 0x00000000,
916 (0x0400 << 16) | (0xc798 >> 2),
917 0x00000000,
918 (0x0400 << 16) | (0xc79c >> 2),
919 0x00000000,
920 (0x0e00 << 16) | (0x9100 >> 2),
921 0x00000000,
922 (0x0e00 << 16) | (0x3c010 >> 2),
923 0x00000000,
924 (0x0e00 << 16) | (0x8c00 >> 2),
925 0x00000000,
926 (0x0e00 << 16) | (0x8c04 >> 2),
927 0x00000000,
928 (0x0e00 << 16) | (0x8c20 >> 2),
929 0x00000000,
930 (0x0e00 << 16) | (0x8c38 >> 2),
931 0x00000000,
932 (0x0e00 << 16) | (0x8c3c >> 2),
933 0x00000000,
934 (0x0e00 << 16) | (0xae00 >> 2),
935 0x00000000,
936 (0x0e00 << 16) | (0x9604 >> 2),
937 0x00000000,
938 (0x0e00 << 16) | (0xac08 >> 2),
939 0x00000000,
940 (0x0e00 << 16) | (0xac0c >> 2),
941 0x00000000,
942 (0x0e00 << 16) | (0xac10 >> 2),
943 0x00000000,
944 (0x0e00 << 16) | (0xac14 >> 2),
945 0x00000000,
946 (0x0e00 << 16) | (0xac58 >> 2),
947 0x00000000,
948 (0x0e00 << 16) | (0xac68 >> 2),
949 0x00000000,
950 (0x0e00 << 16) | (0xac6c >> 2),
951 0x00000000,
952 (0x0e00 << 16) | (0xac70 >> 2),
953 0x00000000,
954 (0x0e00 << 16) | (0xac74 >> 2),
955 0x00000000,
956 (0x0e00 << 16) | (0xac78 >> 2),
957 0x00000000,
958 (0x0e00 << 16) | (0xac7c >> 2),
959 0x00000000,
960 (0x0e00 << 16) | (0xac80 >> 2),
961 0x00000000,
962 (0x0e00 << 16) | (0xac84 >> 2),
963 0x00000000,
964 (0x0e00 << 16) | (0xac88 >> 2),
965 0x00000000,
966 (0x0e00 << 16) | (0xac8c >> 2),
967 0x00000000,
968 (0x0e00 << 16) | (0x970c >> 2),
969 0x00000000,
970 (0x0e00 << 16) | (0x9714 >> 2),
971 0x00000000,
972 (0x0e00 << 16) | (0x9718 >> 2),
973 0x00000000,
974 (0x0e00 << 16) | (0x971c >> 2),
975 0x00000000,
976 (0x0e00 << 16) | (0x31068 >> 2),
977 0x00000000,
978 (0x4e00 << 16) | (0x31068 >> 2),
979 0x00000000,
980 (0x5e00 << 16) | (0x31068 >> 2),
981 0x00000000,
982 (0x6e00 << 16) | (0x31068 >> 2),
983 0x00000000,
984 (0x7e00 << 16) | (0x31068 >> 2),
985 0x00000000,
986 (0x0e00 << 16) | (0xcd10 >> 2),
987 0x00000000,
988 (0x0e00 << 16) | (0xcd14 >> 2),
989 0x00000000,
990 (0x0e00 << 16) | (0x88b0 >> 2),
991 0x00000000,
992 (0x0e00 << 16) | (0x88b4 >> 2),
993 0x00000000,
994 (0x0e00 << 16) | (0x88b8 >> 2),
995 0x00000000,
996 (0x0e00 << 16) | (0x88bc >> 2),
997 0x00000000,
998 (0x0400 << 16) | (0x89c0 >> 2),
999 0x00000000,
1000 (0x0e00 << 16) | (0x88c4 >> 2),
1001 0x00000000,
1002 (0x0e00 << 16) | (0x88c8 >> 2),
1003 0x00000000,
1004 (0x0e00 << 16) | (0x88d0 >> 2),
1005 0x00000000,
1006 (0x0e00 << 16) | (0x88d4 >> 2),
1007 0x00000000,
1008 (0x0e00 << 16) | (0x88d8 >> 2),
1009 0x00000000,
1010 (0x0e00 << 16) | (0x8980 >> 2),
1011 0x00000000,
1012 (0x0e00 << 16) | (0x30938 >> 2),
1013 0x00000000,
1014 (0x0e00 << 16) | (0x3093c >> 2),
1015 0x00000000,
1016 (0x0e00 << 16) | (0x30940 >> 2),
1017 0x00000000,
1018 (0x0e00 << 16) | (0x89a0 >> 2),
1019 0x00000000,
1020 (0x0e00 << 16) | (0x30900 >> 2),
1021 0x00000000,
1022 (0x0e00 << 16) | (0x30904 >> 2),
1023 0x00000000,
1024 (0x0e00 << 16) | (0x89b4 >> 2),
1025 0x00000000,
1026 (0x0e00 << 16) | (0x3e1fc >> 2),
1027 0x00000000,
1028 (0x0e00 << 16) | (0x3c210 >> 2),
1029 0x00000000,
1030 (0x0e00 << 16) | (0x3c214 >> 2),
1031 0x00000000,
1032 (0x0e00 << 16) | (0x3c218 >> 2),
1033 0x00000000,
1034 (0x0e00 << 16) | (0x8904 >> 2),
1035 0x00000000,
1036 0x5,
1037 (0x0e00 << 16) | (0x8c28 >> 2),
1038 (0x0e00 << 16) | (0x8c2c >> 2),
1039 (0x0e00 << 16) | (0x8c30 >> 2),
1040 (0x0e00 << 16) | (0x8c34 >> 2),
1041 (0x0e00 << 16) | (0x9600 >> 2),
1042};
1043
0aafd313
AD		 1044static const u32 bonaire_golden_spm_registers[] =
1045{
1046 0x30800, 0xe0ffffff, 0xe0000000
1047};
1048
1049static const u32 bonaire_golden_common_registers[] =
1050{
1051 0xc770, 0xffffffff, 0x00000800,
1052 0xc774, 0xffffffff, 0x00000800,
1053 0xc798, 0xffffffff, 0x00007fbf,
1054 0xc79c, 0xffffffff, 0x00007faf
1055};
1056
1057static const u32 bonaire_golden_registers[] =
1058{
1059 0x3354, 0x00000333, 0x00000333,
1060 0x3350, 0x000c0fc0, 0x00040200,
1061 0x9a10, 0x00010000, 0x00058208,
1062 0x3c000, 0xffff1fff, 0x00140000,
1063 0x3c200, 0xfdfc0fff, 0x00000100,
1064 0x3c234, 0x40000000, 0x40000200,
1065 0x9830, 0xffffffff, 0x00000000,
1066 0x9834, 0xf00fffff, 0x00000400,
1067 0x9838, 0x0002021c, 0x00020200,
1068 0xc78, 0x00000080, 0x00000000,
1069 0x5bb0, 0x000000f0, 0x00000070,
1070 0x5bc0, 0xf0311fff, 0x80300000,
1071 0x98f8, 0x73773777, 0x12010001,
1072 0x350c, 0x00810000, 0x408af000,
1073 0x7030, 0x31000111, 0x00000011,
1074 0x2f48, 0x73773777, 0x12010001,
1075 0x220c, 0x00007fb6, 0x0021a1b1,
1076 0x2210, 0x00007fb6, 0x002021b1,
1077 0x2180, 0x00007fb6, 0x00002191,
1078 0x2218, 0x00007fb6, 0x002121b1,
1079 0x221c, 0x00007fb6, 0x002021b1,
1080 0x21dc, 0x00007fb6, 0x00002191,
1081 0x21e0, 0x00007fb6, 0x00002191,
1082 0x3628, 0x0000003f, 0x0000000a,
1083 0x362c, 0x0000003f, 0x0000000a,
1084 0x2ae4, 0x00073ffe, 0x000022a2,
1085 0x240c, 0x000007ff, 0x00000000,
1086 0x8a14, 0xf000003f, 0x00000007,
1087 0x8bf0, 0x00002001, 0x00000001,
1088 0x8b24, 0xffffffff, 0x00ffffff,
1089 0x30a04, 0x0000ff0f, 0x00000000,
1090 0x28a4c, 0x07ffffff, 0x06000000,
1091 0x4d8, 0x00000fff, 0x00000100,
1092 0x3e78, 0x00000001, 0x00000002,
1093 0x9100, 0x03000000, 0x0362c688,
1094 0x8c00, 0x000000ff, 0x00000001,
1095 0xe40, 0x00001fff, 0x00001fff,
1096 0x9060, 0x0000007f, 0x00000020,
1097 0x9508, 0x00010000, 0x00010000,
1098 0xac14, 0x000003ff, 0x000000f3,
1099 0xac0c, 0xffffffff, 0x00001032
1100};
1101
1102static const u32 bonaire_mgcg_cgcg_init[] =
1103{
1104 0xc420, 0xffffffff, 0xfffffffc,
1105 0x30800, 0xffffffff, 0xe0000000,
1106 0x3c2a0, 0xffffffff, 0x00000100,
1107 0x3c208, 0xffffffff, 0x00000100,
1108 0x3c2c0, 0xffffffff, 0xc0000100,
1109 0x3c2c8, 0xffffffff, 0xc0000100,
1110 0x3c2c4, 0xffffffff, 0xc0000100,
1111 0x55e4, 0xffffffff, 0x00600100,
1112 0x3c280, 0xffffffff, 0x00000100,
1113 0x3c214, 0xffffffff, 0x06000100,
1114 0x3c220, 0xffffffff, 0x00000100,
1115 0x3c218, 0xffffffff, 0x06000100,
1116 0x3c204, 0xffffffff, 0x00000100,
1117 0x3c2e0, 0xffffffff, 0x00000100,
1118 0x3c224, 0xffffffff, 0x00000100,
1119 0x3c200, 0xffffffff, 0x00000100,
1120 0x3c230, 0xffffffff, 0x00000100,
1121 0x3c234, 0xffffffff, 0x00000100,
1122 0x3c250, 0xffffffff, 0x00000100,
1123 0x3c254, 0xffffffff, 0x00000100,
1124 0x3c258, 0xffffffff, 0x00000100,
1125 0x3c25c, 0xffffffff, 0x00000100,
1126 0x3c260, 0xffffffff, 0x00000100,
1127 0x3c27c, 0xffffffff, 0x00000100,
1128 0x3c278, 0xffffffff, 0x00000100,
1129 0x3c210, 0xffffffff, 0x06000100,
1130 0x3c290, 0xffffffff, 0x00000100,
1131 0x3c274, 0xffffffff, 0x00000100,
1132 0x3c2b4, 0xffffffff, 0x00000100,
1133 0x3c2b0, 0xffffffff, 0x00000100,
1134 0x3c270, 0xffffffff, 0x00000100,
1135 0x30800, 0xffffffff, 0xe0000000,
1136 0x3c020, 0xffffffff, 0x00010000,
1137 0x3c024, 0xffffffff, 0x00030002,
1138 0x3c028, 0xffffffff, 0x00040007,
1139 0x3c02c, 0xffffffff, 0x00060005,
1140 0x3c030, 0xffffffff, 0x00090008,
1141 0x3c034, 0xffffffff, 0x00010000,
1142 0x3c038, 0xffffffff, 0x00030002,
1143 0x3c03c, 0xffffffff, 0x00040007,
1144 0x3c040, 0xffffffff, 0x00060005,
1145 0x3c044, 0xffffffff, 0x00090008,
1146 0x3c048, 0xffffffff, 0x00010000,
1147 0x3c04c, 0xffffffff, 0x00030002,
1148 0x3c050, 0xffffffff, 0x00040007,
1149 0x3c054, 0xffffffff, 0x00060005,
1150 0x3c058, 0xffffffff, 0x00090008,
1151 0x3c05c, 0xffffffff, 0x00010000,
1152 0x3c060, 0xffffffff, 0x00030002,
1153 0x3c064, 0xffffffff, 0x00040007,
1154 0x3c068, 0xffffffff, 0x00060005,
1155 0x3c06c, 0xffffffff, 0x00090008,
1156 0x3c070, 0xffffffff, 0x00010000,
1157 0x3c074, 0xffffffff, 0x00030002,
1158 0x3c078, 0xffffffff, 0x00040007,
1159 0x3c07c, 0xffffffff, 0x00060005,
1160 0x3c080, 0xffffffff, 0x00090008,
1161 0x3c084, 0xffffffff, 0x00010000,
1162 0x3c088, 0xffffffff, 0x00030002,
1163 0x3c08c, 0xffffffff, 0x00040007,
1164 0x3c090, 0xffffffff, 0x00060005,
1165 0x3c094, 0xffffffff, 0x00090008,
1166 0x3c098, 0xffffffff, 0x00010000,
1167 0x3c09c, 0xffffffff, 0x00030002,
1168 0x3c0a0, 0xffffffff, 0x00040007,
1169 0x3c0a4, 0xffffffff, 0x00060005,
1170 0x3c0a8, 0xffffffff, 0x00090008,
1171 0x3c000, 0xffffffff, 0x96e00200,
1172 0x8708, 0xffffffff, 0x00900100,
1173 0xc424, 0xffffffff, 0x0020003f,
1174 0x38, 0xffffffff, 0x0140001c,
1175 0x3c, 0x000f0000, 0x000f0000,
1176 0x220, 0xffffffff, 0xC060000C,
1177 0x224, 0xc0000fff, 0x00000100,
1178 0xf90, 0xffffffff, 0x00000100,
1179 0xf98, 0x00000101, 0x00000000,
1180 0x20a8, 0xffffffff, 0x00000104,
1181 0x55e4, 0xff000fff, 0x00000100,
1182 0x30cc, 0xc0000fff, 0x00000104,
1183 0xc1e4, 0x00000001, 0x00000001,
1184 0xd00c, 0xff000ff0, 0x00000100,
1185 0xd80c, 0xff000ff0, 0x00000100
1186};
1187
1188static const u32 spectre_golden_spm_registers[] =
1189{
1190 0x30800, 0xe0ffffff, 0xe0000000
1191};
1192
1193static const u32 spectre_golden_common_registers[] =
1194{
1195 0xc770, 0xffffffff, 0x00000800,
1196 0xc774, 0xffffffff, 0x00000800,
1197 0xc798, 0xffffffff, 0x00007fbf,
1198 0xc79c, 0xffffffff, 0x00007faf
1199};
1200
1201static const u32 spectre_golden_registers[] =
1202{
1203 0x3c000, 0xffff1fff, 0x96940200,
1204 0x3c00c, 0xffff0001, 0xff000000,
1205 0x3c200, 0xfffc0fff, 0x00000100,
1206 0x6ed8, 0x00010101, 0x00010000,
1207 0x9834, 0xf00fffff, 0x00000400,
1208 0x9838, 0xfffffffc, 0x00020200,
1209 0x5bb0, 0x000000f0, 0x00000070,
1210 0x5bc0, 0xf0311fff, 0x80300000,
1211 0x98f8, 0x73773777, 0x12010001,
1212 0x9b7c, 0x00ff0000, 0x00fc0000,
1213 0x2f48, 0x73773777, 0x12010001,
1214 0x8a14, 0xf000003f, 0x00000007,
1215 0x8b24, 0xffffffff, 0x00ffffff,
1216 0x28350, 0x3f3f3fff, 0x00000082,
f1553174 1217 0x28354, 0x0000003f, 0x00000000,
0aafd313
AD		 1218 0x3e78, 0x00000001, 0x00000002,
1219 0x913c, 0xffff03df, 0x00000004,
1220 0xc768, 0x00000008, 0x00000008,
1221 0x8c00, 0x000008ff, 0x00000800,
1222 0x9508, 0x00010000, 0x00010000,
1223 0xac0c, 0xffffffff, 0x54763210,
1224 0x214f8, 0x01ff01ff, 0x00000002,
1225 0x21498, 0x007ff800, 0x00200000,
1226 0x2015c, 0xffffffff, 0x00000f40,
1227 0x30934, 0xffffffff, 0x00000001
1228};
1229
1230static const u32 spectre_mgcg_cgcg_init[] =
1231{
1232 0xc420, 0xffffffff, 0xfffffffc,
1233 0x30800, 0xffffffff, 0xe0000000,
1234 0x3c2a0, 0xffffffff, 0x00000100,
1235 0x3c208, 0xffffffff, 0x00000100,
1236 0x3c2c0, 0xffffffff, 0x00000100,
1237 0x3c2c8, 0xffffffff, 0x00000100,
1238 0x3c2c4, 0xffffffff, 0x00000100,
1239 0x55e4, 0xffffffff, 0x00600100,
1240 0x3c280, 0xffffffff, 0x00000100,
1241 0x3c214, 0xffffffff, 0x06000100,
1242 0x3c220, 0xffffffff, 0x00000100,
1243 0x3c218, 0xffffffff, 0x06000100,
1244 0x3c204, 0xffffffff, 0x00000100,
1245 0x3c2e0, 0xffffffff, 0x00000100,
1246 0x3c224, 0xffffffff, 0x00000100,
1247 0x3c200, 0xffffffff, 0x00000100,
1248 0x3c230, 0xffffffff, 0x00000100,
1249 0x3c234, 0xffffffff, 0x00000100,
1250 0x3c250, 0xffffffff, 0x00000100,
1251 0x3c254, 0xffffffff, 0x00000100,
1252 0x3c258, 0xffffffff, 0x00000100,
1253 0x3c25c, 0xffffffff, 0x00000100,
1254 0x3c260, 0xffffffff, 0x00000100,
1255 0x3c27c, 0xffffffff, 0x00000100,
1256 0x3c278, 0xffffffff, 0x00000100,
1257 0x3c210, 0xffffffff, 0x06000100,
1258 0x3c290, 0xffffffff, 0x00000100,
1259 0x3c274, 0xffffffff, 0x00000100,
1260 0x3c2b4, 0xffffffff, 0x00000100,
1261 0x3c2b0, 0xffffffff, 0x00000100,
1262 0x3c270, 0xffffffff, 0x00000100,
1263 0x30800, 0xffffffff, 0xe0000000,
1264 0x3c020, 0xffffffff, 0x00010000,
1265 0x3c024, 0xffffffff, 0x00030002,
1266 0x3c028, 0xffffffff, 0x00040007,
1267 0x3c02c, 0xffffffff, 0x00060005,
1268 0x3c030, 0xffffffff, 0x00090008,
1269 0x3c034, 0xffffffff, 0x00010000,
1270 0x3c038, 0xffffffff, 0x00030002,
1271 0x3c03c, 0xffffffff, 0x00040007,
1272 0x3c040, 0xffffffff, 0x00060005,
1273 0x3c044, 0xffffffff, 0x00090008,
1274 0x3c048, 0xffffffff, 0x00010000,
1275 0x3c04c, 0xffffffff, 0x00030002,
1276 0x3c050, 0xffffffff, 0x00040007,
1277 0x3c054, 0xffffffff, 0x00060005,
1278 0x3c058, 0xffffffff, 0x00090008,
1279 0x3c05c, 0xffffffff, 0x00010000,
1280 0x3c060, 0xffffffff, 0x00030002,
1281 0x3c064, 0xffffffff, 0x00040007,
1282 0x3c068, 0xffffffff, 0x00060005,
1283 0x3c06c, 0xffffffff, 0x00090008,
1284 0x3c070, 0xffffffff, 0x00010000,
1285 0x3c074, 0xffffffff, 0x00030002,
1286 0x3c078, 0xffffffff, 0x00040007,
1287 0x3c07c, 0xffffffff, 0x00060005,
1288 0x3c080, 0xffffffff, 0x00090008,
1289 0x3c084, 0xffffffff, 0x00010000,
1290 0x3c088, 0xffffffff, 0x00030002,
1291 0x3c08c, 0xffffffff, 0x00040007,
1292 0x3c090, 0xffffffff, 0x00060005,
1293 0x3c094, 0xffffffff, 0x00090008,
1294 0x3c098, 0xffffffff, 0x00010000,
1295 0x3c09c, 0xffffffff, 0x00030002,
1296 0x3c0a0, 0xffffffff, 0x00040007,
1297 0x3c0a4, 0xffffffff, 0x00060005,
1298 0x3c0a8, 0xffffffff, 0x00090008,
1299 0x3c0ac, 0xffffffff, 0x00010000,
1300 0x3c0b0, 0xffffffff, 0x00030002,
1301 0x3c0b4, 0xffffffff, 0x00040007,
1302 0x3c0b8, 0xffffffff, 0x00060005,
1303 0x3c0bc, 0xffffffff, 0x00090008,
1304 0x3c000, 0xffffffff, 0x96e00200,
1305 0x8708, 0xffffffff, 0x00900100,
1306 0xc424, 0xffffffff, 0x0020003f,
1307 0x38, 0xffffffff, 0x0140001c,
1308 0x3c, 0x000f0000, 0x000f0000,
1309 0x220, 0xffffffff, 0xC060000C,
1310 0x224, 0xc0000fff, 0x00000100,
1311 0xf90, 0xffffffff, 0x00000100,
1312 0xf98, 0x00000101, 0x00000000,
1313 0x20a8, 0xffffffff, 0x00000104,
1314 0x55e4, 0xff000fff, 0x00000100,
1315 0x30cc, 0xc0000fff, 0x00000104,
1316 0xc1e4, 0x00000001, 0x00000001,
1317 0xd00c, 0xff000ff0, 0x00000100,
1318 0xd80c, 0xff000ff0, 0x00000100
1319};
1320
1321static const u32 kalindi_golden_spm_registers[] =
1322{
1323 0x30800, 0xe0ffffff, 0xe0000000
1324};
1325
1326static const u32 kalindi_golden_common_registers[] =
1327{
1328 0xc770, 0xffffffff, 0x00000800,
1329 0xc774, 0xffffffff, 0x00000800,
1330 0xc798, 0xffffffff, 0x00007fbf,
1331 0xc79c, 0xffffffff, 0x00007faf
1332};
1333
1334static const u32 kalindi_golden_registers[] =
1335{
1336 0x3c000, 0xffffdfff, 0x6e944040,
1337 0x55e4, 0xff607fff, 0xfc000100,
1338 0x3c220, 0xff000fff, 0x00000100,
1339 0x3c224, 0xff000fff, 0x00000100,
1340 0x3c200, 0xfffc0fff, 0x00000100,
1341 0x6ed8, 0x00010101, 0x00010000,
1342 0x9830, 0xffffffff, 0x00000000,
1343 0x9834, 0xf00fffff, 0x00000400,
1344 0x5bb0, 0x000000f0, 0x00000070,
1345 0x5bc0, 0xf0311fff, 0x80300000,
1346 0x98f8, 0x73773777, 0x12010001,
1347 0x98fc, 0xffffffff, 0x00000010,
1348 0x9b7c, 0x00ff0000, 0x00fc0000,
1349 0x8030, 0x00001f0f, 0x0000100a,
1350 0x2f48, 0x73773777, 0x12010001,
1351 0x2408, 0x000fffff, 0x000c007f,
1352 0x8a14, 0xf000003f, 0x00000007,
1353 0x8b24, 0x3fff3fff, 0x00ffcfff,
1354 0x30a04, 0x0000ff0f, 0x00000000,
1355 0x28a4c, 0x07ffffff, 0x06000000,
1356 0x4d8, 0x00000fff, 0x00000100,
1357 0x3e78, 0x00000001, 0x00000002,
1358 0xc768, 0x00000008, 0x00000008,
1359 0x8c00, 0x000000ff, 0x00000003,
1360 0x214f8, 0x01ff01ff, 0x00000002,
1361 0x21498, 0x007ff800, 0x00200000,
1362 0x2015c, 0xffffffff, 0x00000f40,
1363 0x88c4, 0x001f3ae3, 0x00000082,
1364 0x88d4, 0x0000001f, 0x00000010,
1365 0x30934, 0xffffffff, 0x00000000
1366};
1367
1368static const u32 kalindi_mgcg_cgcg_init[] =
1369{
1370 0xc420, 0xffffffff, 0xfffffffc,
1371 0x30800, 0xffffffff, 0xe0000000,
1372 0x3c2a0, 0xffffffff, 0x00000100,
1373 0x3c208, 0xffffffff, 0x00000100,
1374 0x3c2c0, 0xffffffff, 0x00000100,
1375 0x3c2c8, 0xffffffff, 0x00000100,
1376 0x3c2c4, 0xffffffff, 0x00000100,
1377 0x55e4, 0xffffffff, 0x00600100,
1378 0x3c280, 0xffffffff, 0x00000100,
1379 0x3c214, 0xffffffff, 0x06000100,
1380 0x3c220, 0xffffffff, 0x00000100,
1381 0x3c218, 0xffffffff, 0x06000100,
1382 0x3c204, 0xffffffff, 0x00000100,
1383 0x3c2e0, 0xffffffff, 0x00000100,
1384 0x3c224, 0xffffffff, 0x00000100,
1385 0x3c200, 0xffffffff, 0x00000100,
1386 0x3c230, 0xffffffff, 0x00000100,
1387 0x3c234, 0xffffffff, 0x00000100,
1388 0x3c250, 0xffffffff, 0x00000100,
1389 0x3c254, 0xffffffff, 0x00000100,
1390 0x3c258, 0xffffffff, 0x00000100,
1391 0x3c25c, 0xffffffff, 0x00000100,
1392 0x3c260, 0xffffffff, 0x00000100,
1393 0x3c27c, 0xffffffff, 0x00000100,
1394 0x3c278, 0xffffffff, 0x00000100,
1395 0x3c210, 0xffffffff, 0x06000100,
1396 0x3c290, 0xffffffff, 0x00000100,
1397 0x3c274, 0xffffffff, 0x00000100,
1398 0x3c2b4, 0xffffffff, 0x00000100,
1399 0x3c2b0, 0xffffffff, 0x00000100,
1400 0x3c270, 0xffffffff, 0x00000100,
1401 0x30800, 0xffffffff, 0xe0000000,
1402 0x3c020, 0xffffffff, 0x00010000,
1403 0x3c024, 0xffffffff, 0x00030002,
1404 0x3c028, 0xffffffff, 0x00040007,
1405 0x3c02c, 0xffffffff, 0x00060005,
1406 0x3c030, 0xffffffff, 0x00090008,
1407 0x3c034, 0xffffffff, 0x00010000,
1408 0x3c038, 0xffffffff, 0x00030002,
1409 0x3c03c, 0xffffffff, 0x00040007,
1410 0x3c040, 0xffffffff, 0x00060005,
1411 0x3c044, 0xffffffff, 0x00090008,
1412 0x3c000, 0xffffffff, 0x96e00200,
1413 0x8708, 0xffffffff, 0x00900100,
1414 0xc424, 0xffffffff, 0x0020003f,
1415 0x38, 0xffffffff, 0x0140001c,
1416 0x3c, 0x000f0000, 0x000f0000,
1417 0x220, 0xffffffff, 0xC060000C,
1418 0x224, 0xc0000fff, 0x00000100,
1419 0x20a8, 0xffffffff, 0x00000104,
1420 0x55e4, 0xff000fff, 0x00000100,
1421 0x30cc, 0xc0000fff, 0x00000104,
1422 0xc1e4, 0x00000001, 0x00000001,
1423 0xd00c, 0xff000ff0, 0x00000100,
1424 0xd80c, 0xff000ff0, 0x00000100
1425};
1426
8efff337
AD		 1427static const u32 hawaii_golden_spm_registers[] =
1428{
1429 0x30800, 0xe0ffffff, 0xe0000000
1430};
1431
1432static const u32 hawaii_golden_common_registers[] =
1433{
1434 0x30800, 0xffffffff, 0xe0000000,
1435 0x28350, 0xffffffff, 0x3a00161a,
1436 0x28354, 0xffffffff, 0x0000002e,
1437 0x9a10, 0xffffffff, 0x00018208,
1438 0x98f8, 0xffffffff, 0x12011003
1439};
1440
1441static const u32 hawaii_golden_registers[] =
1442{
1443 0x3354, 0x00000333, 0x00000333,
1444 0x9a10, 0x00010000, 0x00058208,
1445 0x9830, 0xffffffff, 0x00000000,
1446 0x9834, 0xf00fffff, 0x00000400,
1447 0x9838, 0x0002021c, 0x00020200,
1448 0xc78, 0x00000080, 0x00000000,
1449 0x5bb0, 0x000000f0, 0x00000070,
1450 0x5bc0, 0xf0311fff, 0x80300000,
1451 0x350c, 0x00810000, 0x408af000,
1452 0x7030, 0x31000111, 0x00000011,
1453 0x2f48, 0x73773777, 0x12010001,
1454 0x2120, 0x0000007f, 0x0000001b,
1455 0x21dc, 0x00007fb6, 0x00002191,
1456 0x3628, 0x0000003f, 0x0000000a,
1457 0x362c, 0x0000003f, 0x0000000a,
1458 0x2ae4, 0x00073ffe, 0x000022a2,
1459 0x240c, 0x000007ff, 0x00000000,
1460 0x8bf0, 0x00002001, 0x00000001,
1461 0x8b24, 0xffffffff, 0x00ffffff,
1462 0x30a04, 0x0000ff0f, 0x00000000,
1463 0x28a4c, 0x07ffffff, 0x06000000,
1464 0x3e78, 0x00000001, 0x00000002,
1465 0xc768, 0x00000008, 0x00000008,
1466 0xc770, 0x00000f00, 0x00000800,
1467 0xc774, 0x00000f00, 0x00000800,
1468 0xc798, 0x00ffffff, 0x00ff7fbf,
1469 0xc79c, 0x00ffffff, 0x00ff7faf,
1470 0x8c00, 0x000000ff, 0x00000800,
1471 0xe40, 0x00001fff, 0x00001fff,
1472 0x9060, 0x0000007f, 0x00000020,
1473 0x9508, 0x00010000, 0x00010000,
1474 0xae00, 0x00100000, 0x000ff07c,
1475 0xac14, 0x000003ff, 0x0000000f,
1476 0xac10, 0xffffffff, 0x7564fdec,
1477 0xac0c, 0xffffffff, 0x3120b9a8,
1478 0xac08, 0x20000000, 0x0f9c0000
1479};
1480
1481static const u32 hawaii_mgcg_cgcg_init[] =
1482{
1483 0xc420, 0xffffffff, 0xfffffffd,
1484 0x30800, 0xffffffff, 0xe0000000,
1485 0x3c2a0, 0xffffffff, 0x00000100,
1486 0x3c208, 0xffffffff, 0x00000100,
1487 0x3c2c0, 0xffffffff, 0x00000100,
1488 0x3c2c8, 0xffffffff, 0x00000100,
1489 0x3c2c4, 0xffffffff, 0x00000100,
1490 0x55e4, 0xffffffff, 0x00200100,
1491 0x3c280, 0xffffffff, 0x00000100,
1492 0x3c214, 0xffffffff, 0x06000100,
1493 0x3c220, 0xffffffff, 0x00000100,
1494 0x3c218, 0xffffffff, 0x06000100,
1495 0x3c204, 0xffffffff, 0x00000100,
1496 0x3c2e0, 0xffffffff, 0x00000100,
1497 0x3c224, 0xffffffff, 0x00000100,
1498 0x3c200, 0xffffffff, 0x00000100,
1499 0x3c230, 0xffffffff, 0x00000100,
1500 0x3c234, 0xffffffff, 0x00000100,
1501 0x3c250, 0xffffffff, 0x00000100,
1502 0x3c254, 0xffffffff, 0x00000100,
1503 0x3c258, 0xffffffff, 0x00000100,
1504 0x3c25c, 0xffffffff, 0x00000100,
1505 0x3c260, 0xffffffff, 0x00000100,
1506 0x3c27c, 0xffffffff, 0x00000100,
1507 0x3c278, 0xffffffff, 0x00000100,
1508 0x3c210, 0xffffffff, 0x06000100,
1509 0x3c290, 0xffffffff, 0x00000100,
1510 0x3c274, 0xffffffff, 0x00000100,
1511 0x3c2b4, 0xffffffff, 0x00000100,
1512 0x3c2b0, 0xffffffff, 0x00000100,
1513 0x3c270, 0xffffffff, 0x00000100,
1514 0x30800, 0xffffffff, 0xe0000000,
1515 0x3c020, 0xffffffff, 0x00010000,
1516 0x3c024, 0xffffffff, 0x00030002,
1517 0x3c028, 0xffffffff, 0x00040007,
1518 0x3c02c, 0xffffffff, 0x00060005,
1519 0x3c030, 0xffffffff, 0x00090008,
1520 0x3c034, 0xffffffff, 0x00010000,
1521 0x3c038, 0xffffffff, 0x00030002,
1522 0x3c03c, 0xffffffff, 0x00040007,
1523 0x3c040, 0xffffffff, 0x00060005,
1524 0x3c044, 0xffffffff, 0x00090008,
1525 0x3c048, 0xffffffff, 0x00010000,
1526 0x3c04c, 0xffffffff, 0x00030002,
1527 0x3c050, 0xffffffff, 0x00040007,
1528 0x3c054, 0xffffffff, 0x00060005,
1529 0x3c058, 0xffffffff, 0x00090008,
1530 0x3c05c, 0xffffffff, 0x00010000,
1531 0x3c060, 0xffffffff, 0x00030002,
1532 0x3c064, 0xffffffff, 0x00040007,
1533 0x3c068, 0xffffffff, 0x00060005,
1534 0x3c06c, 0xffffffff, 0x00090008,
1535 0x3c070, 0xffffffff, 0x00010000,
1536 0x3c074, 0xffffffff, 0x00030002,
1537 0x3c078, 0xffffffff, 0x00040007,
1538 0x3c07c, 0xffffffff, 0x00060005,
1539 0x3c080, 0xffffffff, 0x00090008,
1540 0x3c084, 0xffffffff, 0x00010000,
1541 0x3c088, 0xffffffff, 0x00030002,
1542 0x3c08c, 0xffffffff, 0x00040007,
1543 0x3c090, 0xffffffff, 0x00060005,
1544 0x3c094, 0xffffffff, 0x00090008,
1545 0x3c098, 0xffffffff, 0x00010000,
1546 0x3c09c, 0xffffffff, 0x00030002,
1547 0x3c0a0, 0xffffffff, 0x00040007,
1548 0x3c0a4, 0xffffffff, 0x00060005,
1549 0x3c0a8, 0xffffffff, 0x00090008,
1550 0x3c0ac, 0xffffffff, 0x00010000,
1551 0x3c0b0, 0xffffffff, 0x00030002,
1552 0x3c0b4, 0xffffffff, 0x00040007,
1553 0x3c0b8, 0xffffffff, 0x00060005,
1554 0x3c0bc, 0xffffffff, 0x00090008,
1555 0x3c0c0, 0xffffffff, 0x00010000,
1556 0x3c0c4, 0xffffffff, 0x00030002,
1557 0x3c0c8, 0xffffffff, 0x00040007,
1558 0x3c0cc, 0xffffffff, 0x00060005,
1559 0x3c0d0, 0xffffffff, 0x00090008,
1560 0x3c0d4, 0xffffffff, 0x00010000,
1561 0x3c0d8, 0xffffffff, 0x00030002,
1562 0x3c0dc, 0xffffffff, 0x00040007,
1563 0x3c0e0, 0xffffffff, 0x00060005,
1564 0x3c0e4, 0xffffffff, 0x00090008,
1565 0x3c0e8, 0xffffffff, 0x00010000,
1566 0x3c0ec, 0xffffffff, 0x00030002,
1567 0x3c0f0, 0xffffffff, 0x00040007,
1568 0x3c0f4, 0xffffffff, 0x00060005,
1569 0x3c0f8, 0xffffffff, 0x00090008,
1570 0xc318, 0xffffffff, 0x00020200,
1571 0x3350, 0xffffffff, 0x00000200,
1572 0x15c0, 0xffffffff, 0x00000400,
1573 0x55e8, 0xffffffff, 0x00000000,
1574 0x2f50, 0xffffffff, 0x00000902,
1575 0x3c000, 0xffffffff, 0x96940200,
1576 0x8708, 0xffffffff, 0x00900100,
1577 0xc424, 0xffffffff, 0x0020003f,
1578 0x38, 0xffffffff, 0x0140001c,
1579 0x3c, 0x000f0000, 0x000f0000,
1580 0x220, 0xffffffff, 0xc060000c,
1581 0x224, 0xc0000fff, 0x00000100,
1582 0xf90, 0xffffffff, 0x00000100,
1583 0xf98, 0x00000101, 0x00000000,
1584 0x20a8, 0xffffffff, 0x00000104,
1585 0x55e4, 0xff000fff, 0x00000100,
1586 0x30cc, 0xc0000fff, 0x00000104,
1587 0xc1e4, 0x00000001, 0x00000001,
1588 0xd00c, 0xff000ff0, 0x00000100,
1589 0xd80c, 0xff000ff0, 0x00000100
1590};
1591
f73a9e83
SL		 1592static const u32 godavari_golden_registers[] =
1593{
1594 0x55e4, 0xff607fff, 0xfc000100,
1595 0x6ed8, 0x00010101, 0x00010000,
1596 0x9830, 0xffffffff, 0x00000000,
1597 0x98302, 0xf00fffff, 0x00000400,
1598 0x6130, 0xffffffff, 0x00010000,
1599 0x5bb0, 0x000000f0, 0x00000070,
1600 0x5bc0, 0xf0311fff, 0x80300000,
1601 0x98f8, 0x73773777, 0x12010001,
1602 0x98fc, 0xffffffff, 0x00000010,
1603 0x8030, 0x00001f0f, 0x0000100a,
1604 0x2f48, 0x73773777, 0x12010001,
1605 0x2408, 0x000fffff, 0x000c007f,
1606 0x8a14, 0xf000003f, 0x00000007,
1607 0x8b24, 0xffffffff, 0x00ff0fff,
1608 0x30a04, 0x0000ff0f, 0x00000000,
1609 0x28a4c, 0x07ffffff, 0x06000000,
1610 0x4d8, 0x00000fff, 0x00000100,
1611 0xd014, 0x00010000, 0x00810001,
1612 0xd814, 0x00010000, 0x00810001,
1613 0x3e78, 0x00000001, 0x00000002,
1614 0xc768, 0x00000008, 0x00000008,
1615 0xc770, 0x00000f00, 0x00000800,
1616 0xc774, 0x00000f00, 0x00000800,
1617 0xc798, 0x00ffffff, 0x00ff7fbf,
1618 0xc79c, 0x00ffffff, 0x00ff7faf,
1619 0x8c00, 0x000000ff, 0x00000001,
1620 0x214f8, 0x01ff01ff, 0x00000002,
1621 0x21498, 0x007ff800, 0x00200000,
1622 0x2015c, 0xffffffff, 0x00000f40,
1623 0x88c4, 0x001f3ae3, 0x00000082,
1624 0x88d4, 0x0000001f, 0x00000010,
1625 0x30934, 0xffffffff, 0x00000000
1626};
1627
1628
0aafd313
AD		 1629static void cik_init_golden_registers(struct radeon_device *rdev)
1630{
1c0a4625
OG		 1631 /* Some of the registers might be dependent on GRBM_GFX_INDEX */
1632 mutex_lock(&rdev->grbm_idx_mutex);
0aafd313
AD		 1633 switch (rdev->family) {
1634 case CHIP_BONAIRE:
1635 radeon_program_register_sequence(rdev,
1636 bonaire_mgcg_cgcg_init,
1637 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1638 radeon_program_register_sequence(rdev,
1639 bonaire_golden_registers,
1640 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1641 radeon_program_register_sequence(rdev,
1642 bonaire_golden_common_registers,
1643 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1644 radeon_program_register_sequence(rdev,
1645 bonaire_golden_spm_registers,
1646 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1647 break;
1648 case CHIP_KABINI:
1649 radeon_program_register_sequence(rdev,
1650 kalindi_mgcg_cgcg_init,
1651 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1652 radeon_program_register_sequence(rdev,
1653 kalindi_golden_registers,
1654 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1655 radeon_program_register_sequence(rdev,
1656 kalindi_golden_common_registers,
1657 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1658 radeon_program_register_sequence(rdev,
1659 kalindi_golden_spm_registers,
1660 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1661 break;
f73a9e83
SL		 1662 case CHIP_MULLINS:
1663 radeon_program_register_sequence(rdev,
1664 kalindi_mgcg_cgcg_init,
1665 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1666 radeon_program_register_sequence(rdev,
1667 godavari_golden_registers,
1668 (const u32)ARRAY_SIZE(godavari_golden_registers));
1669 radeon_program_register_sequence(rdev,
1670 kalindi_golden_common_registers,
1671 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1672 radeon_program_register_sequence(rdev,
1673 kalindi_golden_spm_registers,
1674 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1675 break;
0aafd313
AD		 1676 case CHIP_KAVERI:
1677 radeon_program_register_sequence(rdev,
1678 spectre_mgcg_cgcg_init,
1679 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1680 radeon_program_register_sequence(rdev,
1681 spectre_golden_registers,
1682 (const u32)ARRAY_SIZE(spectre_golden_registers));
1683 radeon_program_register_sequence(rdev,
1684 spectre_golden_common_registers,
1685 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1686 radeon_program_register_sequence(rdev,
1687 spectre_golden_spm_registers,
1688 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1689 break;
8efff337
AD		 1690 case CHIP_HAWAII:
1691 radeon_program_register_sequence(rdev,
1692 hawaii_mgcg_cgcg_init,
1693 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1694 radeon_program_register_sequence(rdev,
1695 hawaii_golden_registers,
1696 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1697 radeon_program_register_sequence(rdev,
1698 hawaii_golden_common_registers,
1699 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1700 radeon_program_register_sequence(rdev,
1701 hawaii_golden_spm_registers,
1702 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1703 break;
0aafd313
AD		 1704 default:
1705 break;
1706 }
1c0a4625 1707 mutex_unlock(&rdev->grbm_idx_mutex);
0aafd313
AD		 1708}
1709
2c67912c
AD		 1710/**
1711 * cik_get_xclk - get the xclk
1712 *
1713 * @rdev: radeon_device pointer
1714 *
1715 * Returns the reference clock used by the gfx engine
1716 * (CIK).
1717 */
1718u32 cik_get_xclk(struct radeon_device *rdev)
1719{
3cf8bb1a 1720 u32 reference_clock = rdev->clock.spll.reference_freq;
2c67912c
AD		 1721
1722 if (rdev->flags & RADEON_IS_IGP) {
1723 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1724 return reference_clock / 2;
1725 } else {
1726 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1727 return reference_clock / 4;
1728 }
1729 return reference_clock;
1730}
1731
75efdee1
AD		 1732/**
1733 * cik_mm_rdoorbell - read a doorbell dword
1734 *
1735 * @rdev: radeon_device pointer
d5754ab8 1736 * @index: doorbell index
75efdee1
AD		 1737 *
1738 * Returns the value in the doorbell aperture at the
d5754ab8 1739 * requested doorbell index (CIK).
75efdee1 1740 */
d5754ab8 1741u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
75efdee1 1742{
d5754ab8
AL		 1743 if (index < rdev->doorbell.num_doorbells) {
1744 return readl(rdev->doorbell.ptr + index);
75efdee1 1745 } else {
d5754ab8 1746 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1747 return 0;
1748 }
1749}
1750
1751/**
1752 * cik_mm_wdoorbell - write a doorbell dword
1753 *
1754 * @rdev: radeon_device pointer
d5754ab8 1755 * @index: doorbell index
75efdee1
AD		 1756 * @v: value to write
1757 *
1758 * Writes @v to the doorbell aperture at the
d5754ab8 1759 * requested doorbell index (CIK).
75efdee1 1760 */
d5754ab8 1761void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
75efdee1 1762{
d5754ab8
AL		 1763 if (index < rdev->doorbell.num_doorbells) {
1764 writel(v, rdev->doorbell.ptr + index);
75efdee1 1765 } else {
d5754ab8 1766 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1767 }
1768}
1769
bc8273fe
AD		 1770#define BONAIRE_IO_MC_REGS_SIZE 36
1771
1772static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1773{
1774 {0x00000070, 0x04400000},
1775 {0x00000071, 0x80c01803},
1776 {0x00000072, 0x00004004},
1777 {0x00000073, 0x00000100},
1778 {0x00000074, 0x00ff0000},
1779 {0x00000075, 0x34000000},
1780 {0x00000076, 0x08000014},
1781 {0x00000077, 0x00cc08ec},
1782 {0x00000078, 0x00000400},
1783 {0x00000079, 0x00000000},
1784 {0x0000007a, 0x04090000},
1785 {0x0000007c, 0x00000000},
1786 {0x0000007e, 0x4408a8e8},
1787 {0x0000007f, 0x00000304},
1788 {0x00000080, 0x00000000},
1789 {0x00000082, 0x00000001},
1790 {0x00000083, 0x00000002},
1791 {0x00000084, 0xf3e4f400},
1792 {0x00000085, 0x052024e3},
1793 {0x00000087, 0x00000000},
1794 {0x00000088, 0x01000000},
1795 {0x0000008a, 0x1c0a0000},
1796 {0x0000008b, 0xff010000},
1797 {0x0000008d, 0xffffefff},
1798 {0x0000008e, 0xfff3efff},
1799 {0x0000008f, 0xfff3efbf},
1800 {0x00000092, 0xf7ffffff},
1801 {0x00000093, 0xffffff7f},
1802 {0x00000095, 0x00101101},
1803 {0x00000096, 0x00000fff},
1804 {0x00000097, 0x00116fff},
1805 {0x00000098, 0x60010000},
1806 {0x00000099, 0x10010000},
1807 {0x0000009a, 0x00006000},
1808 {0x0000009b, 0x00001000},
1809 {0x0000009f, 0x00b48000}
1810};
1811
d4775655
AD		 1812#define HAWAII_IO_MC_REGS_SIZE 22
1813
1814static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1815{
1816 {0x0000007d, 0x40000000},
1817 {0x0000007e, 0x40180304},
1818 {0x0000007f, 0x0000ff00},
1819 {0x00000081, 0x00000000},
1820 {0x00000083, 0x00000800},
1821 {0x00000086, 0x00000000},
1822 {0x00000087, 0x00000100},
1823 {0x00000088, 0x00020100},
1824 {0x00000089, 0x00000000},
1825 {0x0000008b, 0x00040000},
1826 {0x0000008c, 0x00000100},
1827 {0x0000008e, 0xff010000},
1828 {0x00000090, 0xffffefff},
1829 {0x00000091, 0xfff3efff},
1830 {0x00000092, 0xfff3efbf},
1831 {0x00000093, 0xf7ffffff},
1832 {0x00000094, 0xffffff7f},
1833 {0x00000095, 0x00000fff},
1834 {0x00000096, 0x00116fff},
1835 {0x00000097, 0x60010000},
1836 {0x00000098, 0x10010000},
1837 {0x0000009f, 0x00c79000}
1838};
1839
1840
b556b12e
AD		 1841/**
1842 * cik_srbm_select - select specific register instances
1843 *
1844 * @rdev: radeon_device pointer
1845 * @me: selected ME (micro engine)
1846 * @pipe: pipe
1847 * @queue: queue
1848 * @vmid: VMID
1849 *
1850 * Switches the currently active registers instances. Some
1851 * registers are instanced per VMID, others are instanced per
1852 * me/pipe/queue combination.
1853 */
1854static void cik_srbm_select(struct radeon_device *rdev,
1855 u32 me, u32 pipe, u32 queue, u32 vmid)
1856{
1857 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1858 MEID(me & 0x3) |
1859 VMID(vmid & 0xf) |
1860 QUEUEID(queue & 0x7));
1861 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1862}
1863
bc8273fe
AD		 1864/* ucode loading */
1865/**
1866 * ci_mc_load_microcode - load MC ucode into the hw
1867 *
1868 * @rdev: radeon_device pointer
1869 *
1870 * Load the GDDR MC ucode into the hw (CIK).
1871 * Returns 0 on success, error on failure.
1872 */
6c7bccea 1873int ci_mc_load_microcode(struct radeon_device *rdev)
bc8273fe 1874{
f2c6b0f4
AD		 1875 const __be32 *fw_data = NULL;
1876 const __le32 *new_fw_data = NULL;
6e4b070e 1877 u32 running, tmp;
f2c6b0f4
AD		 1878 u32 *io_mc_regs = NULL;
1879 const __le32 *new_io_mc_regs = NULL;
bcddee29 1880 int i, regs_size, ucode_size;
bc8273fe
AD		 1881
1882 if (!rdev->mc_fw)
1883 return -EINVAL;
1884
f2c6b0f4
AD		 1885 if (rdev->new_fw) {
1886 const struct mc_firmware_header_v1_0 *hdr =
1887 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
bcddee29 1888
f2c6b0f4
AD		 1889 radeon_ucode_print_mc_hdr(&hdr->header);
1890
1891 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1892 new_io_mc_regs = (const __le32 *)
1893 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1894 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1895 new_fw_data = (const __le32 *)
1896 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1897 } else {
1898 ucode_size = rdev->mc_fw->size / 4;
1899
1900 switch (rdev->family) {
1901 case CHIP_BONAIRE:
1902 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1903 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1904 break;
1905 case CHIP_HAWAII:
1906 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1907 regs_size = HAWAII_IO_MC_REGS_SIZE;
1908 break;
1909 default:
1910 return -EINVAL;
1911 }
1912 fw_data = (const __be32 *)rdev->mc_fw->data;
bc8273fe
AD		 1913 }
1914
1915 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1916
1917 if (running == 0) {
bc8273fe
AD		 1918 /* reset the engine and set to writable */
1919 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1920 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1921
1922 /* load mc io regs */
1923 for (i = 0; i < regs_size; i++) {
f2c6b0f4
AD		 1924 if (rdev->new_fw) {
1925 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1926 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1927 } else {
1928 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1929 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1930 }
bc8273fe 1931 }
9feb3dda
AD		 1932
1933 tmp = RREG32(MC_SEQ_MISC0);
1934 if ((rdev->pdev->device == 0x6649) && ((tmp & 0xff00) == 0x5600)) {
1935 WREG32(MC_SEQ_IO_DEBUG_INDEX, 5);
1936 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x00000023);
1937 WREG32(MC_SEQ_IO_DEBUG_INDEX, 9);
1938 WREG32(MC_SEQ_IO_DEBUG_DATA, 0x000001f0);
1939 }
1940
bc8273fe 1941 /* load the MC ucode */
f2c6b0f4
AD		 1942 for (i = 0; i < ucode_size; i++) {
1943 if (rdev->new_fw)
1944 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1945 else
1946 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1947 }
bc8273fe
AD		 1948
1949 /* put the engine back into the active state */
1950 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1951 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1952 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1953
1954 /* wait for training to complete */
1955 for (i = 0; i < rdev->usec_timeout; i++) {
1956 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1957 break;
1958 udelay(1);
1959 }
1960 for (i = 0; i < rdev->usec_timeout; i++) {
1961 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1962 break;
1963 udelay(1);
1964 }
bc8273fe
AD		 1965 }
1966
1967 return 0;
1968}
1969
02c81327
AD		 1970/**
1971 * cik_init_microcode - load ucode images from disk
1972 *
1973 * @rdev: radeon_device pointer
1974 *
1975 * Use the firmware interface to load the ucode images into
1976 * the driver (not loaded into hw).
1977 * Returns 0 on success, error on failure.
1978 */
1979static int cik_init_microcode(struct radeon_device *rdev)
1980{
02c81327 1981 const char *chip_name;
f2c6b0f4 1982 const char *new_chip_name;
02c81327 1983 size_t pfp_req_size, me_req_size, ce_req_size,
d4775655 1984 mec_req_size, rlc_req_size, mc_req_size = 0,
277babc3 1985 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
02c81327 1986 char fw_name[30];
f2c6b0f4 1987 int new_fw = 0;
02c81327 1988 int err;
f2c6b0f4 1989 int num_fw;
b2ea0dcd 1990 bool new_smc = false;
02c81327
AD		 1991
1992 DRM_DEBUG("\n");
1993
02c81327
AD		 1994 switch (rdev->family) {
1995 case CHIP_BONAIRE:
1996 chip_name = "BONAIRE";
b2ea0dcd
AD		 1997 if ((rdev->pdev->revision == 0x80) ||
1998 (rdev->pdev->revision == 0x81) ||
1999 (rdev->pdev->device == 0x665f))
2000 new_smc = true;
f2c6b0f4 2001 new_chip_name = "bonaire";
02c81327
AD		 2002 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2003 me_req_size = CIK_ME_UCODE_SIZE * 4;
2004 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2005 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2006 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
277babc3
AD		 2007 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
2008 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
21a93e13 2009 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 2010 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
f2c6b0f4 2011 num_fw = 8;
02c81327 2012 break;
d4775655
AD		 2013 case CHIP_HAWAII:
2014 chip_name = "HAWAII";
b2ea0dcd
AD		 2015 if (rdev->pdev->revision == 0x80)
2016 new_smc = true;
f2c6b0f4 2017 new_chip_name = "hawaii";
d4775655
AD		 2018 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2019 me_req_size = CIK_ME_UCODE_SIZE * 4;
2020 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2021 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2022 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
2023 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
277babc3 2024 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
d4775655
AD		 2025 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
2026 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
f2c6b0f4 2027 num_fw = 8;
d4775655 2028 break;
02c81327
AD		 2029 case CHIP_KAVERI:
2030 chip_name = "KAVERI";
f2c6b0f4 2031 new_chip_name = "kaveri";
02c81327
AD		 2032 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2033 me_req_size = CIK_ME_UCODE_SIZE * 4;
2034 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2035 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2036 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 2037 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2038 num_fw = 7;
02c81327
AD		 2039 break;
2040 case CHIP_KABINI:
2041 chip_name = "KABINI";
f2c6b0f4 2042 new_chip_name = "kabini";
02c81327
AD		 2043 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2044 me_req_size = CIK_ME_UCODE_SIZE * 4;
2045 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2046 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2047 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 2048 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2049 num_fw = 6;
02c81327 2050 break;
f73a9e83
SL		 2051 case CHIP_MULLINS:
2052 chip_name = "MULLINS";
f2c6b0f4 2053 new_chip_name = "mullins";
f73a9e83
SL		 2054 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
2055 me_req_size = CIK_ME_UCODE_SIZE * 4;
2056 ce_req_size = CIK_CE_UCODE_SIZE * 4;
2057 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
2058 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
2059 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 2060 num_fw = 6;
f73a9e83 2061 break;
02c81327
AD		 2062 default: BUG();
2063 }
2064
f2c6b0f4 2065 DRM_INFO("Loading %s Microcode\n", new_chip_name);
02c81327 2066
f2c6b0f4 2067 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
0a168933 2068 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2069 if (err) {
2070 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
2071 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
2072 if (err)
2073 goto out;
2074 if (rdev->pfp_fw->size != pfp_req_size) {
2075 printk(KERN_ERR
2076 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2077 rdev->pfp_fw->size, fw_name);
2078 err = -EINVAL;
2079 goto out;
2080 }
2081 } else {
2082 err = radeon_ucode_validate(rdev->pfp_fw);
2083 if (err) {
2084 printk(KERN_ERR
2085 "cik_fw: validation failed for firmware \"%s\"\n",
2086 fw_name);
2087 goto out;
2088 } else {
2089 new_fw++;
2090 }
02c81327
AD		 2091 }
2092
f2c6b0f4 2093 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
0a168933 2094 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2095 if (err) {
2096 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2097 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2098 if (err)
2099 goto out;
2100 if (rdev->me_fw->size != me_req_size) {
2101 printk(KERN_ERR
2102 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2103 rdev->me_fw->size, fw_name);
2104 err = -EINVAL;
2105 }
2106 } else {
2107 err = radeon_ucode_validate(rdev->me_fw);
2108 if (err) {
2109 printk(KERN_ERR
2110 "cik_fw: validation failed for firmware \"%s\"\n",
2111 fw_name);
2112 goto out;
2113 } else {
2114 new_fw++;
2115 }
02c81327
AD		 2116 }
2117
f2c6b0f4 2118 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
0a168933 2119 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2120 if (err) {
2121 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2122 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2123 if (err)
2124 goto out;
2125 if (rdev->ce_fw->size != ce_req_size) {
2126 printk(KERN_ERR
2127 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2128 rdev->ce_fw->size, fw_name);
2129 err = -EINVAL;
2130 }
2131 } else {
2132 err = radeon_ucode_validate(rdev->ce_fw);
2133 if (err) {
2134 printk(KERN_ERR
2135 "cik_fw: validation failed for firmware \"%s\"\n",
2136 fw_name);
2137 goto out;
2138 } else {
2139 new_fw++;
2140 }
02c81327
AD		 2141 }
2142
f2c6b0f4 2143 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
0a168933 2144 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2145 if (err) {
2146 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2147 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2148 if (err)
2149 goto out;
2150 if (rdev->mec_fw->size != mec_req_size) {
2151 printk(KERN_ERR
2152 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2153 rdev->mec_fw->size, fw_name);
2154 err = -EINVAL;
2155 }
2156 } else {
2157 err = radeon_ucode_validate(rdev->mec_fw);
2158 if (err) {
2159 printk(KERN_ERR
2160 "cik_fw: validation failed for firmware \"%s\"\n",
2161 fw_name);
2162 goto out;
2163 } else {
2164 new_fw++;
2165 }
2166 }
2167
2168 if (rdev->family == CHIP_KAVERI) {
2169 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2170 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2171 if (err) {
2172 goto out;
2173 } else {
2174 err = radeon_ucode_validate(rdev->mec2_fw);
2175 if (err) {
2176 goto out;
2177 } else {
2178 new_fw++;
2179 }
2180 }
02c81327
AD		 2181 }
2182
f2c6b0f4 2183 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
0a168933 2184 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2185 if (err) {
2186 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2187 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2188 if (err)
2189 goto out;
2190 if (rdev->rlc_fw->size != rlc_req_size) {
2191 printk(KERN_ERR
2192 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2193 rdev->rlc_fw->size, fw_name);
2194 err = -EINVAL;
2195 }
2196 } else {
2197 err = radeon_ucode_validate(rdev->rlc_fw);
2198 if (err) {
2199 printk(KERN_ERR
2200 "cik_fw: validation failed for firmware \"%s\"\n",
2201 fw_name);
2202 goto out;
2203 } else {
2204 new_fw++;
2205 }
02c81327
AD		 2206 }
2207
f2c6b0f4 2208 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
0a168933 2209 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2210 if (err) {
2211 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2212 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2213 if (err)
2214 goto out;
2215 if (rdev->sdma_fw->size != sdma_req_size) {
2216 printk(KERN_ERR
2217 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2218 rdev->sdma_fw->size, fw_name);
2219 err = -EINVAL;
2220 }
2221 } else {
2222 err = radeon_ucode_validate(rdev->sdma_fw);
2223 if (err) {
2224 printk(KERN_ERR
2225 "cik_fw: validation failed for firmware \"%s\"\n",
2226 fw_name);
2227 goto out;
2228 } else {
2229 new_fw++;
2230 }
21a93e13
AD		 2231 }
2232
cc8dbbb4 2233 /* No SMC, MC ucode on APUs */
02c81327 2234 if (!(rdev->flags & RADEON_IS_IGP)) {
f2c6b0f4 2235 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
0a168933 2236 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
277babc3 2237 if (err) {
f2c6b0f4 2238 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
277babc3 2239 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2240 if (err) {
2241 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2242 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2243 if (err)
2244 goto out;
2245 }
2246 if ((rdev->mc_fw->size != mc_req_size) &&
2247 (rdev->mc_fw->size != mc2_req_size)){
2248 printk(KERN_ERR
2249 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2250 rdev->mc_fw->size, fw_name);
2251 err = -EINVAL;
2252 }
2253 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2254 } else {
2255 err = radeon_ucode_validate(rdev->mc_fw);
2256 if (err) {
2257 printk(KERN_ERR
2258 "cik_fw: validation failed for firmware \"%s\"\n",
2259 fw_name);
277babc3 2260 goto out;
f2c6b0f4
AD		 2261 } else {
2262 new_fw++;
2263 }
277babc3 2264 }
cc8dbbb4 2265
b2ea0dcd
AD		 2266 if (new_smc)
2267 snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
2268 else
2269 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
cc8dbbb4
AD		 2270 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2271 if (err) {
f2c6b0f4
AD		 2272 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2273 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2274 if (err) {
2275 printk(KERN_ERR
2276 "smc: error loading firmware \"%s\"\n",
2277 fw_name);
2278 release_firmware(rdev->smc_fw);
2279 rdev->smc_fw = NULL;
2280 err = 0;
2281 } else if (rdev->smc_fw->size != smc_req_size) {
2282 printk(KERN_ERR
2283 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2284 rdev->smc_fw->size, fw_name);
2285 err = -EINVAL;
2286 }
2287 } else {
2288 err = radeon_ucode_validate(rdev->smc_fw);
2289 if (err) {
2290 printk(KERN_ERR
2291 "cik_fw: validation failed for firmware \"%s\"\n",
2292 fw_name);
2293 goto out;
2294 } else {
2295 new_fw++;
2296 }
cc8dbbb4 2297 }
02c81327
AD		 2298 }
2299
f2c6b0f4
AD		 2300 if (new_fw == 0) {
2301 rdev->new_fw = false;
2302 } else if (new_fw < num_fw) {
2303 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2304 err = -EINVAL;
2305 } else {
2306 rdev->new_fw = true;
2307 }
2308
02c81327 2309out:
02c81327
AD		 2310 if (err) {
2311 if (err != -EINVAL)
2312 printk(KERN_ERR
2313 "cik_cp: Failed to load firmware \"%s\"\n",
2314 fw_name);
2315 release_firmware(rdev->pfp_fw);
2316 rdev->pfp_fw = NULL;
2317 release_firmware(rdev->me_fw);
2318 rdev->me_fw = NULL;
2319 release_firmware(rdev->ce_fw);
2320 rdev->ce_fw = NULL;
f2c6b0f4
AD		 2321 release_firmware(rdev->mec_fw);
2322 rdev->mec_fw = NULL;
2323 release_firmware(rdev->mec2_fw);
2324 rdev->mec2_fw = NULL;
02c81327
AD		 2325 release_firmware(rdev->rlc_fw);
2326 rdev->rlc_fw = NULL;
f2c6b0f4
AD		 2327 release_firmware(rdev->sdma_fw);
2328 rdev->sdma_fw = NULL;
02c81327
AD		 2329 release_firmware(rdev->mc_fw);
2330 rdev->mc_fw = NULL;
cc8dbbb4
AD		 2331 release_firmware(rdev->smc_fw);
2332 rdev->smc_fw = NULL;
02c81327
AD		 2333 }
2334 return err;
2335}
2336
8cc1a532
AD		 2337/*
2338 * Core functions
2339 */
2340/**
2341 * cik_tiling_mode_table_init - init the hw tiling table
2342 *
2343 * @rdev: radeon_device pointer
2344 *
2345 * Starting with SI, the tiling setup is done globally in a
2346 * set of 32 tiling modes. Rather than selecting each set of
2347 * parameters per surface as on older asics, we just select
2348 * which index in the tiling table we want to use, and the
2349 * surface uses those parameters (CIK).
2350 */
2351static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2352{
f0e201f2
JP		 2353 u32 *tile = rdev->config.cik.tile_mode_array;
2354 u32 *macrotile = rdev->config.cik.macrotile_mode_array;
2355 const u32 num_tile_mode_states =
2356 ARRAY_SIZE(rdev->config.cik.tile_mode_array);
2357 const u32 num_secondary_tile_mode_states =
2358 ARRAY_SIZE(rdev->config.cik.macrotile_mode_array);
2359 u32 reg_offset, split_equal_to_row_size;
8cc1a532
AD		 2360 u32 num_pipe_configs;
2361 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2362 rdev->config.cik.max_shader_engines;
2363
2364 switch (rdev->config.cik.mem_row_size_in_kb) {
2365 case 1:
2366 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2367 break;
2368 case 2:
2369 default:
2370 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2371 break;
2372 case 4:
2373 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2374 break;
2375 }
2376
2377 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2378 if (num_pipe_configs > 8)
21e438af 2379 num_pipe_configs = 16;
8cc1a532 2380
f0e201f2
JP		 2381 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2382 tile[reg_offset] = 0;
2383 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2384 macrotile[reg_offset] = 0;
2385
2386 switch(num_pipe_configs) {
2387 case 16:
2388 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2389 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2390 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2391 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2392 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2393 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2394 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2395 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2396 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2397 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2398 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2399 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2400 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2401 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2402 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2403 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2404 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2405 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2406 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2407 TILE_SPLIT(split_equal_to_row_size));
2408 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2409 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2410 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2411 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2412 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2413 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2414 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2415 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2416 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2417 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2418 TILE_SPLIT(split_equal_to_row_size));
2419 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2420 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2421 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2422 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2423 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2424 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2425 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2426 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2427 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2428 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2429 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2430 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2431 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2432 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2433 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2434 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2435 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2436 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2437 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2438 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2439 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2440 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2441 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2442 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2443 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2444 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2445 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2446 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2447 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2448 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2449 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2450 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2451 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2452 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2453 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2454 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2455 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2456 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2457 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2458 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2459 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2460 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2461 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2462 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2463 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2464 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2465 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2466
2467 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2468 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2469 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2470 NUM_BANKS(ADDR_SURF_16_BANK));
2471 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2472 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2473 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2474 NUM_BANKS(ADDR_SURF_16_BANK));
2475 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2476 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2477 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2478 NUM_BANKS(ADDR_SURF_16_BANK));
2479 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2480 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2481 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2482 NUM_BANKS(ADDR_SURF_16_BANK));
2483 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2484 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2485 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2486 NUM_BANKS(ADDR_SURF_8_BANK));
2487 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2488 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2489 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2490 NUM_BANKS(ADDR_SURF_4_BANK));
2491 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2492 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2493 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2494 NUM_BANKS(ADDR_SURF_2_BANK));
2495 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2496 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2497 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2498 NUM_BANKS(ADDR_SURF_16_BANK));
2499 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2500 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2501 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2502 NUM_BANKS(ADDR_SURF_16_BANK));
2503 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2504 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2505 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2506 NUM_BANKS(ADDR_SURF_16_BANK));
2507 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2508 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2509 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2510 NUM_BANKS(ADDR_SURF_8_BANK));
2511 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2512 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2513 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2514 NUM_BANKS(ADDR_SURF_4_BANK));
2515 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2516 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2517 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2518 NUM_BANKS(ADDR_SURF_2_BANK));
2519 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2520 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2521 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2522 NUM_BANKS(ADDR_SURF_2_BANK));
2523
2524 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2525 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2526 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2527 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2528 break;
2529
2530 case 8:
2531 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2532 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2533 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2534 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2535 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2536 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2537 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2538 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2539 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2540 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2542 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2543 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2544 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2545 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2546 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2547 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2548 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2549 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2550 TILE_SPLIT(split_equal_to_row_size));
2551 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2552 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2553 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2554 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2555 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2556 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2557 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2558 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2559 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2560 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2561 TILE_SPLIT(split_equal_to_row_size));
2562 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2563 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2564 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2566 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2567 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2568 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2569 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2570 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2571 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2572 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2573 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2574 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2575 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2576 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2577 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2578 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2579 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2580 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2581 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2582 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2583 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2584 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2585 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2586 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2587 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2588 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2589 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2590 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2591 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2592 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2593 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2594 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2595 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2596 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2597 tile[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2598 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2599 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2600 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2601 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2602 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2603 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2604 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2605 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2606 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2607 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2608 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2609
2610 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2611 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2612 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2613 NUM_BANKS(ADDR_SURF_16_BANK));
2614 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2615 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2616 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2617 NUM_BANKS(ADDR_SURF_16_BANK));
2618 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2619 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2620 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2621 NUM_BANKS(ADDR_SURF_16_BANK));
2622 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2623 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2624 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2625 NUM_BANKS(ADDR_SURF_16_BANK));
2626 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2627 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2628 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2629 NUM_BANKS(ADDR_SURF_8_BANK));
2630 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2631 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2632 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2633 NUM_BANKS(ADDR_SURF_4_BANK));
2634 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2635 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2636 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2637 NUM_BANKS(ADDR_SURF_2_BANK));
2638 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2639 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2640 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2641 NUM_BANKS(ADDR_SURF_16_BANK));
2642 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2643 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2644 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2645 NUM_BANKS(ADDR_SURF_16_BANK));
2646 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2647 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2648 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2649 NUM_BANKS(ADDR_SURF_16_BANK));
2650 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2651 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2652 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2653 NUM_BANKS(ADDR_SURF_16_BANK));
2654 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2656 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2657 NUM_BANKS(ADDR_SURF_8_BANK));
2658 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2659 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2660 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2661 NUM_BANKS(ADDR_SURF_4_BANK));
2662 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2663 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2664 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2665 NUM_BANKS(ADDR_SURF_2_BANK));
2666
2667 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2668 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2669 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2670 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2671 break;
2672
2673 case 4:
8cc1a532 2674 if (num_rbs == 4) {
f0e201f2
JP		 2675 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2676 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2677 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2678 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2679 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2680 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2681 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2682 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2683 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2684 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2685 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2686 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2687 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2688 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2689 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2690 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2691 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2692 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2693 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2694 TILE_SPLIT(split_equal_to_row_size));
2695 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2696 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2697 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2698 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2699 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2700 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2701 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2702 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2703 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2704 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2705 TILE_SPLIT(split_equal_to_row_size));
2706 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2707 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2708 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2709 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2710 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2711 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2712 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2713 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2714 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2715 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2716 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2717 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2718 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2719 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2720 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2721 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2722 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2723 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2724 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2725 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2726 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2727 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2728 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2729 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2730 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2731 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2732 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2733 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2734 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2735 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2736 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2737 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2738 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2739 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2740 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2741 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2742 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2743 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2744 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2745 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2746 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2747 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2748 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2749 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2750 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2751 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2752 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2753
8cc1a532 2754 } else if (num_rbs < 4) {
f0e201f2
JP		 2755 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2756 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2757 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2758 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2759 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2760 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2761 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2762 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2763 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2764 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2765 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2766 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2767 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2768 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2769 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2770 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2771 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2772 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2773 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2774 TILE_SPLIT(split_equal_to_row_size));
2775 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2776 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2777 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2778 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2779 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2780 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2781 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2782 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2783 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2784 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2785 TILE_SPLIT(split_equal_to_row_size));
2786 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2787 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2788 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2789 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2790 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2791 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2792 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2793 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2794 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2795 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2796 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2797 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2798 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2799 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2800 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2801 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2802 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2803 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2804 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2805 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2806 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2807 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2808 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2809 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2810 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2811 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2812 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2813 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2814 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2815 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2816 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2817 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2818 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2819 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2820 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2821 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2822 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2823 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2824 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2825 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2826 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2827 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2828 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2829 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2830 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2831 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2832 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
8cc1a532 2833 }
f0e201f2
JP		 2834
2835 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2836 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2837 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2838 NUM_BANKS(ADDR_SURF_16_BANK));
2839 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2840 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2841 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2842 NUM_BANKS(ADDR_SURF_16_BANK));
2843 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2844 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2845 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2846 NUM_BANKS(ADDR_SURF_16_BANK));
2847 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2848 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2849 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2850 NUM_BANKS(ADDR_SURF_16_BANK));
2851 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2852 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2853 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2854 NUM_BANKS(ADDR_SURF_16_BANK));
2855 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2856 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2857 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2858 NUM_BANKS(ADDR_SURF_8_BANK));
2859 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2860 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2861 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2862 NUM_BANKS(ADDR_SURF_4_BANK));
2863 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2864 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2865 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2866 NUM_BANKS(ADDR_SURF_16_BANK));
2867 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2868 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2869 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2870 NUM_BANKS(ADDR_SURF_16_BANK));
2871 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2872 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2873 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2874 NUM_BANKS(ADDR_SURF_16_BANK));
2875 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2876 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2877 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2878 NUM_BANKS(ADDR_SURF_16_BANK));
2879 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2880 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2881 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2882 NUM_BANKS(ADDR_SURF_16_BANK));
2883 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2884 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2885 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2886 NUM_BANKS(ADDR_SURF_8_BANK));
2887 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2888 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2889 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2890 NUM_BANKS(ADDR_SURF_4_BANK));
2891
2892 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2893 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2894 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2895 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
2896 break;
2897
2898 case 2:
2899 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2900 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2901 PIPE_CONFIG(ADDR_SURF_P2) |
2902 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2903 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2904 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2905 PIPE_CONFIG(ADDR_SURF_P2) |
2906 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2907 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2908 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2909 PIPE_CONFIG(ADDR_SURF_P2) |
2910 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2911 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2912 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2913 PIPE_CONFIG(ADDR_SURF_P2) |
2914 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2915 tile[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2916 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2917 PIPE_CONFIG(ADDR_SURF_P2) |
2918 TILE_SPLIT(split_equal_to_row_size));
2919 tile[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2920 PIPE_CONFIG(ADDR_SURF_P2) |
2921 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2922 tile[6] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2923 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2924 PIPE_CONFIG(ADDR_SURF_P2) |
2925 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2926 tile[7] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2927 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2928 PIPE_CONFIG(ADDR_SURF_P2) |
2929 TILE_SPLIT(split_equal_to_row_size));
2930 tile[8] = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2931 PIPE_CONFIG(ADDR_SURF_P2);
2932 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2933 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2934 PIPE_CONFIG(ADDR_SURF_P2));
2935 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2936 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2937 PIPE_CONFIG(ADDR_SURF_P2) |
2938 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2939 tile[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2940 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2941 PIPE_CONFIG(ADDR_SURF_P2) |
2942 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2943 tile[12] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2944 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2945 PIPE_CONFIG(ADDR_SURF_P2) |
2946 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2947 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2948 PIPE_CONFIG(ADDR_SURF_P2) |
2949 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2950 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2951 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2952 PIPE_CONFIG(ADDR_SURF_P2) |
2953 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2954 tile[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2955 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2956 PIPE_CONFIG(ADDR_SURF_P2) |
2957 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2958 tile[17] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2959 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2960 PIPE_CONFIG(ADDR_SURF_P2) |
2961 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2962 tile[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2963 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2964 PIPE_CONFIG(ADDR_SURF_P2));
2965 tile[28] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2966 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2967 PIPE_CONFIG(ADDR_SURF_P2) |
2968 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2969 tile[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2970 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2971 PIPE_CONFIG(ADDR_SURF_P2) |
2972 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2973 tile[30] = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2974 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2975 PIPE_CONFIG(ADDR_SURF_P2) |
2976 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2977
2978 macrotile[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2979 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2980 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2981 NUM_BANKS(ADDR_SURF_16_BANK));
2982 macrotile[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2983 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2984 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2985 NUM_BANKS(ADDR_SURF_16_BANK));
2986 macrotile[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2987 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2988 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2989 NUM_BANKS(ADDR_SURF_16_BANK));
2990 macrotile[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2991 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2992 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2993 NUM_BANKS(ADDR_SURF_16_BANK));
2994 macrotile[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2995 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2996 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2997 NUM_BANKS(ADDR_SURF_16_BANK));
2998 macrotile[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2999 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3000 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3001 NUM_BANKS(ADDR_SURF_16_BANK));
3002 macrotile[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3003 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3004 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3005 NUM_BANKS(ADDR_SURF_8_BANK));
3006 macrotile[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3007 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3008 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3009 NUM_BANKS(ADDR_SURF_16_BANK));
3010 macrotile[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3011 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3012 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3013 NUM_BANKS(ADDR_SURF_16_BANK));
3014 macrotile[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3015 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3016 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3017 NUM_BANKS(ADDR_SURF_16_BANK));
3018 macrotile[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3019 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3020 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3021 NUM_BANKS(ADDR_SURF_16_BANK));
3022 macrotile[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3023 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3024 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3025 NUM_BANKS(ADDR_SURF_16_BANK));
3026 macrotile[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3027 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3028 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3029 NUM_BANKS(ADDR_SURF_16_BANK));
3030 macrotile[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3031 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3032 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3033 NUM_BANKS(ADDR_SURF_8_BANK));
3034
3035 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3036 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
3037 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3038 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), macrotile[reg_offset]);
3039 break;
3040
3041 default:
8cc1a532 3042 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
f0e201f2 3043 }
8cc1a532
AD		 3044}
3045
3046/**
3047 * cik_select_se_sh - select which SE, SH to address
3048 *
3049 * @rdev: radeon_device pointer
3050 * @se_num: shader engine to address
3051 * @sh_num: sh block to address
3052 *
3053 * Select which SE, SH combinations to address. Certain
3054 * registers are instanced per SE or SH. 0xffffffff means
3055 * broadcast to all SEs or SHs (CIK).
3056 */
3057static void cik_select_se_sh(struct radeon_device *rdev,
3058 u32 se_num, u32 sh_num)
3059{
3060 u32 data = INSTANCE_BROADCAST_WRITES;
3061
3062 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 3063 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 3064 else if (se_num == 0xffffffff)
3065 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3066 else if (sh_num == 0xffffffff)
3067 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3068 else
3069 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3070 WREG32(GRBM_GFX_INDEX, data);
3071}
3072
3073/**
3074 * cik_create_bitmask - create a bitmask
3075 *
3076 * @bit_width: length of the mask
3077 *
3078 * create a variable length bit mask (CIK).
3079 * Returns the bitmask.
3080 */
3081static u32 cik_create_bitmask(u32 bit_width)
3082{
3083 u32 i, mask = 0;
3084
3085 for (i = 0; i < bit_width; i++) {
3086 mask <<= 1;
3087 mask |= 1;
3088 }
3089 return mask;
3090}
3091
3092/**
972c5ddb 3093 * cik_get_rb_disabled - computes the mask of disabled RBs
8cc1a532
AD		 3094 *
3095 * @rdev: radeon_device pointer
3096 * @max_rb_num: max RBs (render backends) for the asic
3097 * @se_num: number of SEs (shader engines) for the asic
3098 * @sh_per_se: number of SH blocks per SE for the asic
3099 *
3100 * Calculates the bitmask of disabled RBs (CIK).
3101 * Returns the disabled RB bitmask.
3102 */
3103static u32 cik_get_rb_disabled(struct radeon_device *rdev,
9fadb352 3104 u32 max_rb_num_per_se,
8cc1a532
AD		 3105 u32 sh_per_se)
3106{
3107 u32 data, mask;
3108
3109 data = RREG32(CC_RB_BACKEND_DISABLE);
3110 if (data & 1)
3111 data &= BACKEND_DISABLE_MASK;
3112 else
3113 data = 0;
3114 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3115
3116 data >>= BACKEND_DISABLE_SHIFT;
3117
9fadb352 3118 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
8cc1a532
AD		 3119
3120 return data & mask;
3121}
3122
3123/**
3124 * cik_setup_rb - setup the RBs on the asic
3125 *
3126 * @rdev: radeon_device pointer
3127 * @se_num: number of SEs (shader engines) for the asic
3128 * @sh_per_se: number of SH blocks per SE for the asic
3129 * @max_rb_num: max RBs (render backends) for the asic
3130 *
3131 * Configures per-SE/SH RB registers (CIK).
3132 */
3133static void cik_setup_rb(struct radeon_device *rdev,
3134 u32 se_num, u32 sh_per_se,
9fadb352 3135 u32 max_rb_num_per_se)
8cc1a532
AD		 3136{
3137 int i, j;
3138 u32 data, mask;
3139 u32 disabled_rbs = 0;
3140 u32 enabled_rbs = 0;
3141
1c0a4625 3142 mutex_lock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3143 for (i = 0; i < se_num; i++) {
3144 for (j = 0; j < sh_per_se; j++) {
3145 cik_select_se_sh(rdev, i, j);
9fadb352 3146 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
fc821b70
AD		 3147 if (rdev->family == CHIP_HAWAII)
3148 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3149 else
3150 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
8cc1a532
AD		 3151 }
3152 }
3153 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 3154 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3155
3156 mask = 1;
9fadb352 3157 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
8cc1a532
AD		 3158 if (!(disabled_rbs & mask))
3159 enabled_rbs |= mask;
3160 mask <<= 1;
3161 }
3162
439a1cff
MO		 3163 rdev->config.cik.backend_enable_mask = enabled_rbs;
3164
1c0a4625 3165 mutex_lock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3166 for (i = 0; i < se_num; i++) {
3167 cik_select_se_sh(rdev, i, 0xffffffff);
3168 data = 0;
3169 for (j = 0; j < sh_per_se; j++) {
3170 switch (enabled_rbs & 3) {
fc821b70
AD		 3171 case 0:
3172 if (j == 0)
3173 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3174 else
3175 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3176 break;
8cc1a532
AD		 3177 case 1:
3178 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3179 break;
3180 case 2:
3181 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3182 break;
3183 case 3:
3184 default:
3185 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3186 break;
3187 }
3188 enabled_rbs >>= 2;
3189 }
3190 WREG32(PA_SC_RASTER_CONFIG, data);
3191 }
3192 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 3193 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3194}
3195
3196/**
3197 * cik_gpu_init - setup the 3D engine
3198 *
3199 * @rdev: radeon_device pointer
3200 *
3201 * Configures the 3D engine and tiling configuration
3202 * registers so that the 3D engine is usable.
3203 */
3204static void cik_gpu_init(struct radeon_device *rdev)
3205{
3206 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3207 u32 mc_shared_chmap, mc_arb_ramcfg;
3208 u32 hdp_host_path_cntl;
3209 u32 tmp;
6101b3ae 3210 int i, j;
8cc1a532
AD		 3211
3212 switch (rdev->family) {
3213 case CHIP_BONAIRE:
3214 rdev->config.cik.max_shader_engines = 2;
3215 rdev->config.cik.max_tile_pipes = 4;
3216 rdev->config.cik.max_cu_per_sh = 7;
3217 rdev->config.cik.max_sh_per_se = 1;
3218 rdev->config.cik.max_backends_per_se = 2;
3219 rdev->config.cik.max_texture_channel_caches = 4;
3220 rdev->config.cik.max_gprs = 256;
3221 rdev->config.cik.max_gs_threads = 32;
3222 rdev->config.cik.max_hw_contexts = 8;
3223
3224 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3225 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3226 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3227 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3228 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3229 break;
b496038b
AD		 3230 case CHIP_HAWAII:
3231 rdev->config.cik.max_shader_engines = 4;
3232 rdev->config.cik.max_tile_pipes = 16;
3233 rdev->config.cik.max_cu_per_sh = 11;
3234 rdev->config.cik.max_sh_per_se = 1;
3235 rdev->config.cik.max_backends_per_se = 4;
3236 rdev->config.cik.max_texture_channel_caches = 16;
3237 rdev->config.cik.max_gprs = 256;
3238 rdev->config.cik.max_gs_threads = 32;
3239 rdev->config.cik.max_hw_contexts = 8;
3240
3241 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3242 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3243 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3244 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3245 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3246 break;
8cc1a532 3247 case CHIP_KAVERI:
b2e4c70a
AD		 3248 rdev->config.cik.max_shader_engines = 1;
3249 rdev->config.cik.max_tile_pipes = 4;
3250 if ((rdev->pdev->device == 0x1304) ||
3251 (rdev->pdev->device == 0x1305) ||
3252 (rdev->pdev->device == 0x130C) ||
3253 (rdev->pdev->device == 0x130F) ||
3254 (rdev->pdev->device == 0x1310) ||
3255 (rdev->pdev->device == 0x1311) ||
3256 (rdev->pdev->device == 0x131C)) {
3257 rdev->config.cik.max_cu_per_sh = 8;
3258 rdev->config.cik.max_backends_per_se = 2;
3259 } else if ((rdev->pdev->device == 0x1309) ||
3260 (rdev->pdev->device == 0x130A) ||
3261 (rdev->pdev->device == 0x130D) ||
7c4622d5
AD		 3262 (rdev->pdev->device == 0x1313) ||
3263 (rdev->pdev->device == 0x131D)) {
b2e4c70a
AD		 3264 rdev->config.cik.max_cu_per_sh = 6;
3265 rdev->config.cik.max_backends_per_se = 2;
3266 } else if ((rdev->pdev->device == 0x1306) ||
3267 (rdev->pdev->device == 0x1307) ||
3268 (rdev->pdev->device == 0x130B) ||
3269 (rdev->pdev->device == 0x130E) ||
3270 (rdev->pdev->device == 0x1315) ||
6dc14baf 3271 (rdev->pdev->device == 0x1318) ||
b2e4c70a
AD		 3272 (rdev->pdev->device == 0x131B)) {
3273 rdev->config.cik.max_cu_per_sh = 4;
3274 rdev->config.cik.max_backends_per_se = 1;
3275 } else {
3276 rdev->config.cik.max_cu_per_sh = 3;
3277 rdev->config.cik.max_backends_per_se = 1;
3278 }
3279 rdev->config.cik.max_sh_per_se = 1;
3280 rdev->config.cik.max_texture_channel_caches = 4;
3281 rdev->config.cik.max_gprs = 256;
3282 rdev->config.cik.max_gs_threads = 16;
3283 rdev->config.cik.max_hw_contexts = 8;
3284
3285 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3286 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3287 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3288 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3289 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 3290 break;
3291 case CHIP_KABINI:
f73a9e83 3292 case CHIP_MULLINS:
8cc1a532
AD		 3293 default:
3294 rdev->config.cik.max_shader_engines = 1;
3295 rdev->config.cik.max_tile_pipes = 2;
3296 rdev->config.cik.max_cu_per_sh = 2;
3297 rdev->config.cik.max_sh_per_se = 1;
3298 rdev->config.cik.max_backends_per_se = 1;
3299 rdev->config.cik.max_texture_channel_caches = 2;
3300 rdev->config.cik.max_gprs = 256;
3301 rdev->config.cik.max_gs_threads = 16;
3302 rdev->config.cik.max_hw_contexts = 8;
3303
3304 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3305 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3306 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3307 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3308 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3309 break;
3310 }
3311
3312 /* Initialize HDP */
3313 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3314 WREG32((0x2c14 + j), 0x00000000);
3315 WREG32((0x2c18 + j), 0x00000000);
3316 WREG32((0x2c1c + j), 0x00000000);
3317 WREG32((0x2c20 + j), 0x00000000);
3318 WREG32((0x2c24 + j), 0x00000000);
3319 }
3320
3321 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
dc12a3ec
LL		 3322 WREG32(SRBM_INT_CNTL, 0x1);
3323 WREG32(SRBM_INT_ACK, 0x1);
8cc1a532
AD		 3324
3325 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3326
3327 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3328 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3329
3330 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3331 rdev->config.cik.mem_max_burst_length_bytes = 256;
3332 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3333 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3334 if (rdev->config.cik.mem_row_size_in_kb > 4)
3335 rdev->config.cik.mem_row_size_in_kb = 4;
3336 /* XXX use MC settings? */
3337 rdev->config.cik.shader_engine_tile_size = 32;
3338 rdev->config.cik.num_gpus = 1;
3339 rdev->config.cik.multi_gpu_tile_size = 64;
3340
3341 /* fix up row size */
3342 gb_addr_config &= ~ROW_SIZE_MASK;
3343 switch (rdev->config.cik.mem_row_size_in_kb) {
3344 case 1:
3345 default:
3346 gb_addr_config |= ROW_SIZE(0);
3347 break;
3348 case 2:
3349 gb_addr_config |= ROW_SIZE(1);
3350 break;
3351 case 4:
3352 gb_addr_config |= ROW_SIZE(2);
3353 break;
3354 }
3355
3356 /* setup tiling info dword. gb_addr_config is not adequate since it does
3357 * not have bank info, so create a custom tiling dword.
3358 * bits 3:0 num_pipes
3359 * bits 7:4 num_banks
3360 * bits 11:8 group_size
3361 * bits 15:12 row_size
3362 */
3363 rdev->config.cik.tile_config = 0;
3364 switch (rdev->config.cik.num_tile_pipes) {
3365 case 1:
3366 rdev->config.cik.tile_config |= (0 << 0);
3367 break;
3368 case 2:
3369 rdev->config.cik.tile_config |= (1 << 0);
3370 break;
3371 case 4:
3372 rdev->config.cik.tile_config |= (2 << 0);
3373 break;
3374 case 8:
3375 default:
3376 /* XXX what about 12? */
3377 rdev->config.cik.tile_config |= (3 << 0);
3378 break;
3379 }
a537314e
MD		 3380 rdev->config.cik.tile_config |=
3381 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 3382 rdev->config.cik.tile_config |=
3383 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3384 rdev->config.cik.tile_config |=
3385 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3386
3387 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3388 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3389 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 3390 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3391 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 3392 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3393 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3394 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 3395
3396 cik_tiling_mode_table_init(rdev);
3397
3398 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3399 rdev->config.cik.max_sh_per_se,
3400 rdev->config.cik.max_backends_per_se);
3401
52da51f0 3402 rdev->config.cik.active_cus = 0;
65fcf668
AD		 3403 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3404 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6101b3ae
AD		 3405 rdev->config.cik.active_cus +=
3406 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
65fcf668
AD		 3407 }
3408 }
3409
8cc1a532
AD		 3410 /* set HW defaults for 3D engine */
3411 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3412
1c0a4625
OG		 3413 mutex_lock(&rdev->grbm_idx_mutex);
3414 /*
3415 * making sure that the following register writes will be broadcasted
3416 * to all the shaders
3417 */
3418 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
8cc1a532
AD		 3419 WREG32(SX_DEBUG_1, 0x20);
3420
3421 WREG32(TA_CNTL_AUX, 0x00010000);
3422
3423 tmp = RREG32(SPI_CONFIG_CNTL);
3424 tmp |= 0x03000000;
3425 WREG32(SPI_CONFIG_CNTL, tmp);
3426
3427 WREG32(SQ_CONFIG, 1);
3428
3429 WREG32(DB_DEBUG, 0);
3430
3431 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3432 tmp |= 0x00000400;
3433 WREG32(DB_DEBUG2, tmp);
3434
3435 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3436 tmp |= 0x00020200;
3437 WREG32(DB_DEBUG3, tmp);
3438
3439 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3440 tmp |= 0x00018208;
3441 WREG32(CB_HW_CONTROL, tmp);
3442
3443 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3444
3445 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3446 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3447 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3448 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3449
3450 WREG32(VGT_NUM_INSTANCES, 1);
3451
3452 WREG32(CP_PERFMON_CNTL, 0);
3453
3454 WREG32(SQ_CONFIG, 0);
3455
3456 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3457 FORCE_EOV_MAX_REZ_CNT(255)));
3458
3459 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3460 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3461
3462 WREG32(VGT_GS_VERTEX_REUSE, 16);
3463 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3464
3465 tmp = RREG32(HDP_MISC_CNTL);
3466 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3467 WREG32(HDP_MISC_CNTL, tmp);
3468
3469 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3470 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3471
3472 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3473 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
1c0a4625 3474 mutex_unlock(&rdev->grbm_idx_mutex);
8cc1a532
AD		 3475
3476 udelay(50);
3477}
3478
2cae3bc3
AD		 3479/*
3480 * GPU scratch registers helpers function.
3481 */
3482/**
3483 * cik_scratch_init - setup driver info for CP scratch regs
3484 *
3485 * @rdev: radeon_device pointer
3486 *
3487 * Set up the number and offset of the CP scratch registers.
3488 * NOTE: use of CP scratch registers is a legacy inferface and
3489 * is not used by default on newer asics (r6xx+). On newer asics,
3490 * memory buffers are used for fences rather than scratch regs.
3491 */
3492static void cik_scratch_init(struct radeon_device *rdev)
3493{
3494 int i;
3495
3496 rdev->scratch.num_reg = 7;
3497 rdev->scratch.reg_base = SCRATCH_REG0;
3498 for (i = 0; i < rdev->scratch.num_reg; i++) {
3499 rdev->scratch.free[i] = true;
3500 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3501 }
3502}
3503
fbc832c7
AD		 3504/**
3505 * cik_ring_test - basic gfx ring test
3506 *
3507 * @rdev: radeon_device pointer
3508 * @ring: radeon_ring structure holding ring information
3509 *
3510 * Allocate a scratch register and write to it using the gfx ring (CIK).
3511 * Provides a basic gfx ring test to verify that the ring is working.
3512 * Used by cik_cp_gfx_resume();
3513 * Returns 0 on success, error on failure.
3514 */
3515int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3516{
3517 uint32_t scratch;
3518 uint32_t tmp = 0;
3519 unsigned i;
3520 int r;
3521
3522 r = radeon_scratch_get(rdev, &scratch);
3523 if (r) {
3524 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3525 return r;
3526 }
3527 WREG32(scratch, 0xCAFEDEAD);
3528 r = radeon_ring_lock(rdev, ring, 3);
3529 if (r) {
3530 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3531 radeon_scratch_free(rdev, scratch);
3532 return r;
3533 }
3534 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3535 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3536 radeon_ring_write(ring, 0xDEADBEEF);
1538a9e0 3537 radeon_ring_unlock_commit(rdev, ring, false);
963e81f9 3538
fbc832c7
AD		 3539 for (i = 0; i < rdev->usec_timeout; i++) {
3540 tmp = RREG32(scratch);
3541 if (tmp == 0xDEADBEEF)
3542 break;
3543 DRM_UDELAY(1);
3544 }
3545 if (i < rdev->usec_timeout) {
3546 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3547 } else {
3548 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3549 ring->idx, scratch, tmp);
3550 r = -EINVAL;
3551 }
3552 radeon_scratch_free(rdev, scratch);
3553 return r;
3554}
3555
780f5ddd
AD		 3556/**
3557 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3558 *
3559 * @rdev: radeon_device pointer
3560 * @ridx: radeon ring index
3561 *
3562 * Emits an hdp flush on the cp.
3563 */
3564static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3565 int ridx)
3566{
3567 struct radeon_ring *ring = &rdev->ring[ridx];
5d259067 3568 u32 ref_and_mask;
780f5ddd 3569
5d259067
AD		 3570 switch (ring->idx) {
3571 case CAYMAN_RING_TYPE_CP1_INDEX:
3572 case CAYMAN_RING_TYPE_CP2_INDEX:
3573 default:
3574 switch (ring->me) {
3575 case 0:
3576 ref_and_mask = CP2 << ring->pipe;
3577 break;
3578 case 1:
3579 ref_and_mask = CP6 << ring->pipe;
3580 break;
3581 default:
3582 return;
3583 }
3584 break;
3585 case RADEON_RING_TYPE_GFX_INDEX:
3586 ref_and_mask = CP0;
3587 break;
3588 }
3589
3590 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3591 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3592 WAIT_REG_MEM_FUNCTION(3) | /* == */
3593 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3594 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3595 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3596 radeon_ring_write(ring, ref_and_mask);
3597 radeon_ring_write(ring, ref_and_mask);
3598 radeon_ring_write(ring, 0x20); /* poll interval */
780f5ddd
AD		 3599}
3600
2cae3bc3 3601/**
b07fdd38 3602 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3603 *
3604 * @rdev: radeon_device pointer
3605 * @fence: radeon fence object
3606 *
3607 * Emits a fence sequnce number on the gfx ring and flushes
3608 * GPU caches.
3609 */
b07fdd38
AD		 3610void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3611 struct radeon_fence *fence)
2cae3bc3
AD		 3612{
3613 struct radeon_ring *ring = &rdev->ring[fence->ring];
3614 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3615
a9c73a0e
CK		 3616 /* Workaround for cache flush problems. First send a dummy EOP
3617 * event down the pipe with seq one below.
3618 */
3619 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3620 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3621 EOP_TC_ACTION_EN |
3622 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3623 EVENT_INDEX(5)));
3624 radeon_ring_write(ring, addr & 0xfffffffc);
3625 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
3626 DATA_SEL(1) | INT_SEL(0));
3627 radeon_ring_write(ring, fence->seq - 1);
3628 radeon_ring_write(ring, 0);
3629
3630 /* Then send the real EOP event down the pipe. */
2cae3bc3
AD		 3631 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3632 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3633 EOP_TC_ACTION_EN |
3634 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3635 EVENT_INDEX(5)));
3636 radeon_ring_write(ring, addr & 0xfffffffc);
3637 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3638 radeon_ring_write(ring, fence->seq);
3639 radeon_ring_write(ring, 0);
2cae3bc3
AD		 3640}
3641
b07fdd38
AD		 3642/**
3643 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3644 *
3645 * @rdev: radeon_device pointer
3646 * @fence: radeon fence object
3647 *
3648 * Emits a fence sequnce number on the compute ring and flushes
3649 * GPU caches.
3650 */
3651void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3652 struct radeon_fence *fence)
3653{
3654 struct radeon_ring *ring = &rdev->ring[fence->ring];
3655 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3656
3657 /* RELEASE_MEM - flush caches, send int */
3658 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3659 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3660 EOP_TC_ACTION_EN |
3661 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3662 EVENT_INDEX(5)));
3663 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3664 radeon_ring_write(ring, addr & 0xfffffffc);
3665 radeon_ring_write(ring, upper_32_bits(addr));
3666 radeon_ring_write(ring, fence->seq);
3667 radeon_ring_write(ring, 0);
b07fdd38
AD		 3668}
3669
86302eea
CK		 3670/**
3671 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3672 *
3673 * @rdev: radeon_device pointer
3674 * @ring: radeon ring buffer object
3675 * @semaphore: radeon semaphore object
3676 * @emit_wait: Is this a sempahore wait?
3677 *
3678 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3679 * from running ahead of semaphore waits.
3680 */
1654b817 3681bool cik_semaphore_ring_emit(struct radeon_device *rdev,
2cae3bc3
AD		 3682 struct radeon_ring *ring,
3683 struct radeon_semaphore *semaphore,
3684 bool emit_wait)
3685{
3686 uint64_t addr = semaphore->gpu_addr;
3687 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3688
3689 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
5e167cdb 3690 radeon_ring_write(ring, lower_32_bits(addr));
2cae3bc3 3691 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
1654b817 3692
86302eea
CK		 3693 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3694 /* Prevent the PFP from running ahead of the semaphore wait */
3695 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3696 radeon_ring_write(ring, 0x0);
3697 }
3698
1654b817 3699 return true;
2cae3bc3
AD		 3700}
3701
c9dbd705
AD		 3702/**
3703 * cik_copy_cpdma - copy pages using the CP DMA engine
3704 *
3705 * @rdev: radeon_device pointer
3706 * @src_offset: src GPU address
3707 * @dst_offset: dst GPU address
3708 * @num_gpu_pages: number of GPU pages to xfer
57d20a43 3709 * @resv: reservation object to sync to
c9dbd705
AD		 3710 *
3711 * Copy GPU paging using the CP DMA engine (CIK+).
3712 * Used by the radeon ttm implementation to move pages if
3713 * registered as the asic copy callback.
3714 */
57d20a43
CK		 3715struct radeon_fence *cik_copy_cpdma(struct radeon_device *rdev,
3716 uint64_t src_offset, uint64_t dst_offset,
3717 unsigned num_gpu_pages,
3718 struct reservation_object *resv)
c9dbd705 3719{
57d20a43 3720 struct radeon_fence *fence;
975700d2 3721 struct radeon_sync sync;
c9dbd705
AD		 3722 int ring_index = rdev->asic->copy.blit_ring_index;
3723 struct radeon_ring *ring = &rdev->ring[ring_index];
3724 u32 size_in_bytes, cur_size_in_bytes, control;
3725 int i, num_loops;
3726 int r = 0;
3727
975700d2 3728 radeon_sync_create(&sync);
c9dbd705
AD		 3729
3730 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3731 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3732 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3733 if (r) {
3734 DRM_ERROR("radeon: moving bo (%d).\n", r);
975700d2 3735 radeon_sync_free(rdev, &sync, NULL);
57d20a43 3736 return ERR_PTR(r);
c9dbd705
AD		 3737 }
3738
975700d2
CK		 3739 radeon_sync_resv(rdev, &sync, resv, false);
3740 radeon_sync_rings(rdev, &sync, ring->idx);
c9dbd705
AD		 3741
3742 for (i = 0; i < num_loops; i++) {
3743 cur_size_in_bytes = size_in_bytes;
3744 if (cur_size_in_bytes > 0x1fffff)
3745 cur_size_in_bytes = 0x1fffff;
3746 size_in_bytes -= cur_size_in_bytes;
3747 control = 0;
3748 if (size_in_bytes == 0)
3749 control |= PACKET3_DMA_DATA_CP_SYNC;
3750 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3751 radeon_ring_write(ring, control);
3752 radeon_ring_write(ring, lower_32_bits(src_offset));
3753 radeon_ring_write(ring, upper_32_bits(src_offset));
3754 radeon_ring_write(ring, lower_32_bits(dst_offset));
3755 radeon_ring_write(ring, upper_32_bits(dst_offset));
3756 radeon_ring_write(ring, cur_size_in_bytes);
3757 src_offset += cur_size_in_bytes;
3758 dst_offset += cur_size_in_bytes;
3759 }
3760
57d20a43 3761 r = radeon_fence_emit(rdev, &fence, ring->idx);
c9dbd705
AD		 3762 if (r) {
3763 radeon_ring_unlock_undo(rdev, ring);
975700d2 3764 radeon_sync_free(rdev, &sync, NULL);
57d20a43 3765 return ERR_PTR(r);
c9dbd705
AD		 3766 }
3767
1538a9e0 3768 radeon_ring_unlock_commit(rdev, ring, false);
975700d2 3769 radeon_sync_free(rdev, &sync, fence);
c9dbd705 3770
57d20a43 3771 return fence;
c9dbd705
AD		 3772}
3773
2cae3bc3
AD		 3774/*
3775 * IB stuff
3776 */
3777/**
3778 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3779 *
3780 * @rdev: radeon_device pointer
3781 * @ib: radeon indirect buffer object
3782 *
5fc45397 3783 * Emits a DE (drawing engine) or CE (constant engine) IB
2cae3bc3
AD		 3784 * on the gfx ring. IBs are usually generated by userspace
3785 * acceleration drivers and submitted to the kernel for
5fc45397 3786 * scheduling on the ring. This function schedules the IB
2cae3bc3
AD		 3787 * on the gfx ring for execution by the GPU.
3788 */
3789void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3790{
3791 struct radeon_ring *ring = &rdev->ring[ib->ring];
7c42bc1a 3792 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
2cae3bc3
AD		 3793 u32 header, control = INDIRECT_BUFFER_VALID;
3794
3795 if (ib->is_const_ib) {
3796 /* set switch buffer packet before const IB */
3797 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3798 radeon_ring_write(ring, 0);
3799
3800 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3801 } else {
3802 u32 next_rptr;
3803 if (ring->rptr_save_reg) {
3804 next_rptr = ring->wptr + 3 + 4;
3805 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3806 radeon_ring_write(ring, ((ring->rptr_save_reg -
3807 PACKET3_SET_UCONFIG_REG_START) >> 2));
3808 radeon_ring_write(ring, next_rptr);
3809 } else if (rdev->wb.enabled) {
3810 next_rptr = ring->wptr + 5 + 4;
3811 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3812 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3813 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5e167cdb 3814 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
2cae3bc3
AD		 3815 radeon_ring_write(ring, next_rptr);
3816 }
3817
3818 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3819 }
3820
7c42bc1a 3821 control |= ib->length_dw | (vm_id << 24);
2cae3bc3
AD		 3822
3823 radeon_ring_write(ring, header);
5f3e226f 3824 radeon_ring_write(ring, (ib->gpu_addr & 0xFFFFFFFC));
2cae3bc3
AD		 3825 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3826 radeon_ring_write(ring, control);
3827}
3828
fbc832c7
AD		 3829/**
3830 * cik_ib_test - basic gfx ring IB test
3831 *
3832 * @rdev: radeon_device pointer
3833 * @ring: radeon_ring structure holding ring information
3834 *
3835 * Allocate an IB and execute it on the gfx ring (CIK).
3836 * Provides a basic gfx ring test to verify that IBs are working.
3837 * Returns 0 on success, error on failure.
3838 */
3839int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3840{
3841 struct radeon_ib ib;
3842 uint32_t scratch;
3843 uint32_t tmp = 0;
3844 unsigned i;
3845 int r;
3846
3847 r = radeon_scratch_get(rdev, &scratch);
3848 if (r) {
3849 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3850 return r;
3851 }
3852 WREG32(scratch, 0xCAFEDEAD);
3853 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3854 if (r) {
3855 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
5510f124 3856 radeon_scratch_free(rdev, scratch);
fbc832c7
AD		 3857 return r;
3858 }
3859 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3860 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3861 ib.ptr[2] = 0xDEADBEEF;
3862 ib.length_dw = 3;
1538a9e0 3863 r = radeon_ib_schedule(rdev, &ib, NULL, false);
fbc832c7
AD		 3864 if (r) {
3865 radeon_scratch_free(rdev, scratch);
3866 radeon_ib_free(rdev, &ib);
3867 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3868 return r;
3869 }
04db4caf
MD		 3870 r = radeon_fence_wait_timeout(ib.fence, false, usecs_to_jiffies(
3871 RADEON_USEC_IB_TEST_TIMEOUT));
3872 if (r < 0) {
fbc832c7 3873 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
5510f124
CK		 3874 radeon_scratch_free(rdev, scratch);
3875 radeon_ib_free(rdev, &ib);
fbc832c7 3876 return r;
04db4caf
MD		 3877 } else if (r == 0) {
3878 DRM_ERROR("radeon: fence wait timed out.\n");
3879 radeon_scratch_free(rdev, scratch);
3880 radeon_ib_free(rdev, &ib);
3881 return -ETIMEDOUT;
fbc832c7 3882 }
04db4caf 3883 r = 0;
fbc832c7
AD		 3884 for (i = 0; i < rdev->usec_timeout; i++) {
3885 tmp = RREG32(scratch);
3886 if (tmp == 0xDEADBEEF)
3887 break;
3888 DRM_UDELAY(1);
3889 }
3890 if (i < rdev->usec_timeout) {
3891 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3892 } else {
3893 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3894 scratch, tmp);
3895 r = -EINVAL;
3896 }
3897 radeon_scratch_free(rdev, scratch);
3898 radeon_ib_free(rdev, &ib);
3899 return r;
3900}
3901
841cf442
AD		 3902/*
3903 * CP.
3904 * On CIK, gfx and compute now have independant command processors.
3905 *
3906 * GFX
3907 * Gfx consists of a single ring and can process both gfx jobs and
3908 * compute jobs. The gfx CP consists of three microengines (ME):
3909 * PFP - Pre-Fetch Parser
3910 * ME - Micro Engine
3911 * CE - Constant Engine
3912 * The PFP and ME make up what is considered the Drawing Engine (DE).
3913 * The CE is an asynchronous engine used for updating buffer desciptors
3914 * used by the DE so that they can be loaded into cache in parallel
3915 * while the DE is processing state update packets.
3916 *
3917 * Compute
3918 * The compute CP consists of two microengines (ME):
3919 * MEC1 - Compute MicroEngine 1
3920 * MEC2 - Compute MicroEngine 2
3921 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3922 * The queues are exposed to userspace and are programmed directly
3923 * by the compute runtime.
3924 */
3925/**
3926 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3927 *
3928 * @rdev: radeon_device pointer
3929 * @enable: enable or disable the MEs
3930 *
3931 * Halts or unhalts the gfx MEs.
3932 */
3933static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3934{
3935 if (enable)
3936 WREG32(CP_ME_CNTL, 0);
3937 else {
50efa51a
AD		 3938 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3939 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
841cf442
AD		 3940 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3941 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3942 }
3943 udelay(50);
3944}
3945
3946/**
3947 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3948 *
3949 * @rdev: radeon_device pointer
3950 *
3951 * Loads the gfx PFP, ME, and CE ucode.
3952 * Returns 0 for success, -EINVAL if the ucode is not available.
3953 */
3954static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3955{
841cf442
AD		 3956 int i;
3957
3958 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3959 return -EINVAL;
3960
3961 cik_cp_gfx_enable(rdev, false);
3962
f2c6b0f4
AD		 3963 if (rdev->new_fw) {
3964 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3965 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3966 const struct gfx_firmware_header_v1_0 *ce_hdr =
3967 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3968 const struct gfx_firmware_header_v1_0 *me_hdr =
3969 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3970 const __le32 *fw_data;
3971 u32 fw_size;
3972
3973 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3974 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3975 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3976
3977 /* PFP */
3978 fw_data = (const __le32 *)
3979 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3980 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3981 WREG32(CP_PFP_UCODE_ADDR, 0);
3982 for (i = 0; i < fw_size; i++)
3983 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 3984 WREG32(CP_PFP_UCODE_ADDR, le32_to_cpu(pfp_hdr->header.ucode_version));
f2c6b0f4
AD		 3985
3986 /* CE */
3987 fw_data = (const __le32 *)
3988 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3989 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3990 WREG32(CP_CE_UCODE_ADDR, 0);
3991 for (i = 0; i < fw_size; i++)
3992 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 3993 WREG32(CP_CE_UCODE_ADDR, le32_to_cpu(ce_hdr->header.ucode_version));
f2c6b0f4
AD		 3994
3995 /* ME */
3996 fw_data = (const __be32 *)
3997 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3998 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3999 WREG32(CP_ME_RAM_WADDR, 0);
4000 for (i = 0; i < fw_size; i++)
4001 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
38aea071
AD		 4002 WREG32(CP_ME_RAM_WADDR, le32_to_cpu(me_hdr->header.ucode_version));
4003 WREG32(CP_ME_RAM_RADDR, le32_to_cpu(me_hdr->header.ucode_version));
f2c6b0f4
AD		 4004 } else {
4005 const __be32 *fw_data;
4006
4007 /* PFP */
4008 fw_data = (const __be32 *)rdev->pfp_fw->data;
4009 WREG32(CP_PFP_UCODE_ADDR, 0);
4010 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4011 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4012 WREG32(CP_PFP_UCODE_ADDR, 0);
4013
4014 /* CE */
4015 fw_data = (const __be32 *)rdev->ce_fw->data;
4016 WREG32(CP_CE_UCODE_ADDR, 0);
4017 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4018 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4019 WREG32(CP_CE_UCODE_ADDR, 0);
4020
4021 /* ME */
4022 fw_data = (const __be32 *)rdev->me_fw->data;
4023 WREG32(CP_ME_RAM_WADDR, 0);
4024 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4025 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4026 WREG32(CP_ME_RAM_WADDR, 0);
4027 }
841cf442 4028
841cf442
AD		 4029 return 0;
4030}
4031
4032/**
4033 * cik_cp_gfx_start - start the gfx ring
4034 *
4035 * @rdev: radeon_device pointer
4036 *
4037 * Enables the ring and loads the clear state context and other
4038 * packets required to init the ring.
4039 * Returns 0 for success, error for failure.
4040 */
4041static int cik_cp_gfx_start(struct radeon_device *rdev)
4042{
4043 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4044 int r, i;
4045
4046 /* init the CP */
4047 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4048 WREG32(CP_ENDIAN_SWAP, 0);
4049 WREG32(CP_DEVICE_ID, 1);
4050
4051 cik_cp_gfx_enable(rdev, true);
4052
4053 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4054 if (r) {
4055 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4056 return r;
4057 }
4058
4059 /* init the CE partitions. CE only used for gfx on CIK */
4060 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4061 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
dc4edad6
JZ		 4062 radeon_ring_write(ring, 0x8000);
4063 radeon_ring_write(ring, 0x8000);
841cf442
AD		 4064
4065 /* setup clear context state */
4066 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4067 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4068
4069 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4070 radeon_ring_write(ring, 0x80000000);
4071 radeon_ring_write(ring, 0x80000000);
4072
4073 for (i = 0; i < cik_default_size; i++)
4074 radeon_ring_write(ring, cik_default_state[i]);
4075
4076 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4077 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4078
4079 /* set clear context state */
4080 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4081 radeon_ring_write(ring, 0);
4082
4083 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4084 radeon_ring_write(ring, 0x00000316);
4085 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4086 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4087
1538a9e0 4088 radeon_ring_unlock_commit(rdev, ring, false);
841cf442
AD		 4089
4090 return 0;
4091}
4092
4093/**
4094 * cik_cp_gfx_fini - stop the gfx ring
4095 *
4096 * @rdev: radeon_device pointer
4097 *
4098 * Stop the gfx ring and tear down the driver ring
4099 * info.
4100 */
4101static void cik_cp_gfx_fini(struct radeon_device *rdev)
4102{
4103 cik_cp_gfx_enable(rdev, false);
4104 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4105}
4106
4107/**
4108 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4109 *
4110 * @rdev: radeon_device pointer
4111 *
4112 * Program the location and size of the gfx ring buffer
4113 * and test it to make sure it's working.
4114 * Returns 0 for success, error for failure.
4115 */
4116static int cik_cp_gfx_resume(struct radeon_device *rdev)
4117{
4118 struct radeon_ring *ring;
4119 u32 tmp;
4120 u32 rb_bufsz;
4121 u64 rb_addr;
4122 int r;
4123
4124 WREG32(CP_SEM_WAIT_TIMER, 0x0);
939c0d3c
AD		 4125 if (rdev->family != CHIP_HAWAII)
4126 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
841cf442
AD		 4127
4128 /* Set the write pointer delay */
4129 WREG32(CP_RB_WPTR_DELAY, 0);
4130
4131 /* set the RB to use vmid 0 */
4132 WREG32(CP_RB_VMID, 0);
4133
4134 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4135
4136 /* ring 0 - compute and gfx */
4137 /* Set ring buffer size */
4138 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 4139 rb_bufsz = order_base_2(ring->ring_size / 8);
4140 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 4141#ifdef __BIG_ENDIAN
4142 tmp |= BUF_SWAP_32BIT;
4143#endif
4144 WREG32(CP_RB0_CNTL, tmp);
4145
4146 /* Initialize the ring buffer's read and write pointers */
4147 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4148 ring->wptr = 0;
4149 WREG32(CP_RB0_WPTR, ring->wptr);
4150
4151 /* set the wb address wether it's enabled or not */
4152 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4153 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4154
4155 /* scratch register shadowing is no longer supported */
4156 WREG32(SCRATCH_UMSK, 0);
4157
4158 if (!rdev->wb.enabled)
4159 tmp |= RB_NO_UPDATE;
4160
4161 mdelay(1);
4162 WREG32(CP_RB0_CNTL, tmp);
4163
4164 rb_addr = ring->gpu_addr >> 8;
4165 WREG32(CP_RB0_BASE, rb_addr);
4166 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4167
841cf442
AD		 4168 /* start the ring */
4169 cik_cp_gfx_start(rdev);
4170 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4171 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4172 if (r) {
4173 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4174 return r;
4175 }
50efa51a
AD		 4176
4177 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4178 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4179
841cf442
AD		 4180 return 0;
4181}
4182
ea31bf69
AD		 4183u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4184 struct radeon_ring *ring)
963e81f9
AD		 4185{
4186 u32 rptr;
4187
ea31bf69
AD		 4188 if (rdev->wb.enabled)
4189 rptr = rdev->wb.wb[ring->rptr_offs/4];
4190 else
4191 rptr = RREG32(CP_RB0_RPTR);
4192
4193 return rptr;
4194}
4195
4196u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4197 struct radeon_ring *ring)
4198{
0003b8d2 4199 return RREG32(CP_RB0_WPTR);
ea31bf69
AD		 4200}
4201
4202void cik_gfx_set_wptr(struct radeon_device *rdev,
4203 struct radeon_ring *ring)
4204{
4205 WREG32(CP_RB0_WPTR, ring->wptr);
4206 (void)RREG32(CP_RB0_WPTR);
4207}
4208
4209u32 cik_compute_get_rptr(struct radeon_device *rdev,
4210 struct radeon_ring *ring)
4211{
4212 u32 rptr;
963e81f9
AD		 4213
4214 if (rdev->wb.enabled) {
ea31bf69 4215 rptr = rdev->wb.wb[ring->rptr_offs/4];
963e81f9 4216 } else {
f61d5b46 4217 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4218 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4219 rptr = RREG32(CP_HQD_PQ_RPTR);
4220 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4221 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4222 }
963e81f9
AD		 4223
4224 return rptr;
4225}
4226
ea31bf69
AD		 4227u32 cik_compute_get_wptr(struct radeon_device *rdev,
4228 struct radeon_ring *ring)
963e81f9
AD		 4229{
4230 u32 wptr;
4231
4232 if (rdev->wb.enabled) {
ea31bf69
AD		 4233 /* XXX check if swapping is necessary on BE */
4234 wptr = rdev->wb.wb[ring->wptr_offs/4];
963e81f9 4235 } else {
f61d5b46 4236 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4237 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4238 wptr = RREG32(CP_HQD_PQ_WPTR);
4239 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4240 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4241 }
963e81f9
AD		 4242
4243 return wptr;
4244}
4245
ea31bf69
AD		 4246void cik_compute_set_wptr(struct radeon_device *rdev,
4247 struct radeon_ring *ring)
963e81f9 4248{
ea31bf69
AD		 4249 /* XXX check if swapping is necessary on BE */
4250 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
d5754ab8 4251 WDOORBELL32(ring->doorbell_index, ring->wptr);
963e81f9
AD		 4252}
4253
161569de
JG		 4254static void cik_compute_stop(struct radeon_device *rdev,
4255 struct radeon_ring *ring)
4256{
4257 u32 j, tmp;
4258
4259 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4260 /* Disable wptr polling. */
4261 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4262 tmp &= ~WPTR_POLL_EN;
4263 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4264 /* Disable HQD. */
4265 if (RREG32(CP_HQD_ACTIVE) & 1) {
4266 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4267 for (j = 0; j < rdev->usec_timeout; j++) {
4268 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4269 break;
4270 udelay(1);
4271 }
4272 WREG32(CP_HQD_DEQUEUE_REQUEST, 0);
4273 WREG32(CP_HQD_PQ_RPTR, 0);
4274 WREG32(CP_HQD_PQ_WPTR, 0);
4275 }
4276 cik_srbm_select(rdev, 0, 0, 0, 0);
4277}
4278
841cf442
AD		 4279/**
4280 * cik_cp_compute_enable - enable/disable the compute CP MEs
4281 *
4282 * @rdev: radeon_device pointer
4283 * @enable: enable or disable the MEs
4284 *
4285 * Halts or unhalts the compute MEs.
4286 */
4287static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4288{
4289 if (enable)
4290 WREG32(CP_MEC_CNTL, 0);
b2b3d8d9 4291 else {
161569de
JG		 4292 /*
4293 * To make hibernation reliable we need to clear compute ring
4294 * configuration before halting the compute ring.
4295 */
4296 mutex_lock(&rdev->srbm_mutex);
4297 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
4298 cik_compute_stop(rdev,&rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
4299 mutex_unlock(&rdev->srbm_mutex);
4300
841cf442 4301 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
b2b3d8d9
AD		 4302 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4303 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4304 }
841cf442
AD		 4305 udelay(50);
4306}
4307
4308/**
4309 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4310 *
4311 * @rdev: radeon_device pointer
4312 *
4313 * Loads the compute MEC1&2 ucode.
4314 * Returns 0 for success, -EINVAL if the ucode is not available.
4315 */
4316static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4317{
841cf442
AD		 4318 int i;
4319
4320 if (!rdev->mec_fw)
4321 return -EINVAL;
4322
4323 cik_cp_compute_enable(rdev, false);
4324
f2c6b0f4
AD		 4325 if (rdev->new_fw) {
4326 const struct gfx_firmware_header_v1_0 *mec_hdr =
4327 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4328 const __le32 *fw_data;
4329 u32 fw_size;
4330
4331 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4332
4333 /* MEC1 */
4334 fw_data = (const __le32 *)
4335 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4336 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4337 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4338 for (i = 0; i < fw_size; i++)
4339 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4340 WREG32(CP_MEC_ME1_UCODE_ADDR, le32_to_cpu(mec_hdr->header.ucode_version));
841cf442 4341
841cf442 4342 /* MEC2 */
f2c6b0f4
AD		 4343 if (rdev->family == CHIP_KAVERI) {
4344 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4345 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4346
4347 fw_data = (const __le32 *)
4348 (rdev->mec2_fw->data +
4349 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4350 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4351 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4352 for (i = 0; i < fw_size; i++)
4353 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 4354 WREG32(CP_MEC_ME2_UCODE_ADDR, le32_to_cpu(mec2_hdr->header.ucode_version));
f2c6b0f4
AD		 4355 }
4356 } else {
4357 const __be32 *fw_data;
4358
4359 /* MEC1 */
841cf442 4360 fw_data = (const __be32 *)rdev->mec_fw->data;
f2c6b0f4 4361 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
841cf442 4362 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
f2c6b0f4
AD		 4363 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4364 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4365
4366 if (rdev->family == CHIP_KAVERI) {
4367 /* MEC2 */
4368 fw_data = (const __be32 *)rdev->mec_fw->data;
4369 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4370 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4371 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4372 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4373 }
841cf442
AD		 4374 }
4375
4376 return 0;
4377}
4378
4379/**
4380 * cik_cp_compute_start - start the compute queues
4381 *
4382 * @rdev: radeon_device pointer
4383 *
4384 * Enable the compute queues.
4385 * Returns 0 for success, error for failure.
4386 */
4387static int cik_cp_compute_start(struct radeon_device *rdev)
4388{
963e81f9
AD		 4389 cik_cp_compute_enable(rdev, true);
4390
841cf442
AD		 4391 return 0;
4392}
4393
4394/**
4395 * cik_cp_compute_fini - stop the compute queues
4396 *
4397 * @rdev: radeon_device pointer
4398 *
4399 * Stop the compute queues and tear down the driver queue
4400 * info.
4401 */
4402static void cik_cp_compute_fini(struct radeon_device *rdev)
4403{
963e81f9
AD		 4404 int i, idx, r;
4405
841cf442 4406 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 4407
4408 for (i = 0; i < 2; i++) {
4409 if (i == 0)
4410 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4411 else
4412 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4413
4414 if (rdev->ring[idx].mqd_obj) {
4415 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4416 if (unlikely(r != 0))
4417 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4418
4419 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4420 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4421
4422 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4423 rdev->ring[idx].mqd_obj = NULL;
4424 }
4425 }
841cf442
AD		 4426}
4427
963e81f9
AD		 4428static void cik_mec_fini(struct radeon_device *rdev)
4429{
4430 int r;
4431
4432 if (rdev->mec.hpd_eop_obj) {
4433 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4434 if (unlikely(r != 0))
4435 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4436 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4437 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4438
4439 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4440 rdev->mec.hpd_eop_obj = NULL;
4441 }
4442}
4443
4444#define MEC_HPD_SIZE 2048
4445
4446static int cik_mec_init(struct radeon_device *rdev)
4447{
4448 int r;
4449 u32 *hpd;
4450
4451 /*
4452 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4453 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
62a7b7fb
OG		 4454 * Nonetheless, we assign only 1 pipe because all other pipes will
4455 * be handled by KFD
963e81f9 4456 */
62a7b7fb
OG		 4457 rdev->mec.num_mec = 1;
4458 rdev->mec.num_pipe = 1;
963e81f9
AD		 4459 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4460
4461 if (rdev->mec.hpd_eop_obj == NULL) {
4462 r = radeon_bo_create(rdev,
4463 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4464 PAGE_SIZE, true,
831b6966 4465 RADEON_GEM_DOMAIN_GTT, 0, NULL, NULL,
963e81f9
AD		 4466 &rdev->mec.hpd_eop_obj);
4467 if (r) {
4468 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4469 return r;
4470 }
4471 }
4472
4473 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4474 if (unlikely(r != 0)) {
4475 cik_mec_fini(rdev);
4476 return r;
4477 }
4478 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4479 &rdev->mec.hpd_eop_gpu_addr);
4480 if (r) {
4481 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4482 cik_mec_fini(rdev);
4483 return r;
4484 }
4485 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4486 if (r) {
4487 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4488 cik_mec_fini(rdev);
4489 return r;
4490 }
4491
4492 /* clear memory. Not sure if this is required or not */
4493 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4494
4495 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4496 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4497
4498 return 0;
4499}
4500
4501struct hqd_registers
4502{
4503 u32 cp_mqd_base_addr;
4504 u32 cp_mqd_base_addr_hi;
4505 u32 cp_hqd_active;
4506 u32 cp_hqd_vmid;
4507 u32 cp_hqd_persistent_state;
4508 u32 cp_hqd_pipe_priority;
4509 u32 cp_hqd_queue_priority;
4510 u32 cp_hqd_quantum;
4511 u32 cp_hqd_pq_base;
4512 u32 cp_hqd_pq_base_hi;
4513 u32 cp_hqd_pq_rptr;
4514 u32 cp_hqd_pq_rptr_report_addr;
4515 u32 cp_hqd_pq_rptr_report_addr_hi;
4516 u32 cp_hqd_pq_wptr_poll_addr;
4517 u32 cp_hqd_pq_wptr_poll_addr_hi;
4518 u32 cp_hqd_pq_doorbell_control;
4519 u32 cp_hqd_pq_wptr;
4520 u32 cp_hqd_pq_control;
4521 u32 cp_hqd_ib_base_addr;
4522 u32 cp_hqd_ib_base_addr_hi;
4523 u32 cp_hqd_ib_rptr;
4524 u32 cp_hqd_ib_control;
4525 u32 cp_hqd_iq_timer;
4526 u32 cp_hqd_iq_rptr;
4527 u32 cp_hqd_dequeue_request;
4528 u32 cp_hqd_dma_offload;
4529 u32 cp_hqd_sema_cmd;
4530 u32 cp_hqd_msg_type;
4531 u32 cp_hqd_atomic0_preop_lo;
4532 u32 cp_hqd_atomic0_preop_hi;
4533 u32 cp_hqd_atomic1_preop_lo;
4534 u32 cp_hqd_atomic1_preop_hi;
4535 u32 cp_hqd_hq_scheduler0;
4536 u32 cp_hqd_hq_scheduler1;
4537 u32 cp_mqd_control;
4538};
4539
4540struct bonaire_mqd
4541{
4542 u32 header;
4543 u32 dispatch_initiator;
4544 u32 dimensions[3];
4545 u32 start_idx[3];
4546 u32 num_threads[3];
4547 u32 pipeline_stat_enable;
4548 u32 perf_counter_enable;
4549 u32 pgm[2];
4550 u32 tba[2];
4551 u32 tma[2];
4552 u32 pgm_rsrc[2];
4553 u32 vmid;
4554 u32 resource_limits;
4555 u32 static_thread_mgmt01[2];
4556 u32 tmp_ring_size;
4557 u32 static_thread_mgmt23[2];
4558 u32 restart[3];
4559 u32 thread_trace_enable;
4560 u32 reserved1;
4561 u32 user_data[16];
4562 u32 vgtcs_invoke_count[2];
4563 struct hqd_registers queue_state;
4564 u32 dequeue_cntr;
4565 u32 interrupt_queue[64];
4566};
4567
841cf442
AD		 4568/**
4569 * cik_cp_compute_resume - setup the compute queue registers
4570 *
4571 * @rdev: radeon_device pointer
4572 *
4573 * Program the compute queues and test them to make sure they
4574 * are working.
4575 * Returns 0 for success, error for failure.
4576 */
4577static int cik_cp_compute_resume(struct radeon_device *rdev)
4578{
370ce45b 4579 int r, i, j, idx;
963e81f9
AD		 4580 u32 tmp;
4581 bool use_doorbell = true;
4582 u64 hqd_gpu_addr;
4583 u64 mqd_gpu_addr;
4584 u64 eop_gpu_addr;
4585 u64 wb_gpu_addr;
4586 u32 *buf;
4587 struct bonaire_mqd *mqd;
841cf442 4588
841cf442
AD		 4589 r = cik_cp_compute_start(rdev);
4590 if (r)
4591 return r;
963e81f9
AD		 4592
4593 /* fix up chicken bits */
4594 tmp = RREG32(CP_CPF_DEBUG);
4595 tmp |= (1 << 23);
4596 WREG32(CP_CPF_DEBUG, tmp);
4597
4598 /* init the pipes */
f61d5b46 4599 mutex_lock(&rdev->srbm_mutex);
963e81f9 4600
62a7b7fb 4601 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr;
963e81f9 4602
62a7b7fb 4603 cik_srbm_select(rdev, 0, 0, 0, 0);
963e81f9 4604
62a7b7fb
OG		 4605 /* write the EOP addr */
4606 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4607 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
963e81f9 4608
62a7b7fb
OG		 4609 /* set the VMID assigned */
4610 WREG32(CP_HPD_EOP_VMID, 0);
4611
4612 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4613 tmp = RREG32(CP_HPD_EOP_CONTROL);
4614 tmp &= ~EOP_SIZE_MASK;
4615 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4616 WREG32(CP_HPD_EOP_CONTROL, tmp);
963e81f9 4617
f61d5b46 4618 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4619
4620 /* init the queues. Just two for now. */
4621 for (i = 0; i < 2; i++) {
4622 if (i == 0)
4623 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4624 else
4625 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4626
4627 if (rdev->ring[idx].mqd_obj == NULL) {
4628 r = radeon_bo_create(rdev,
4629 sizeof(struct bonaire_mqd),
4630 PAGE_SIZE, true,
02376d82 4631 RADEON_GEM_DOMAIN_GTT, 0, NULL,
831b6966 4632 NULL, &rdev->ring[idx].mqd_obj);
963e81f9
AD		 4633 if (r) {
4634 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4635 return r;
4636 }
4637 }
4638
4639 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4640 if (unlikely(r != 0)) {
4641 cik_cp_compute_fini(rdev);
4642 return r;
4643 }
4644 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4645 &mqd_gpu_addr);
4646 if (r) {
4647 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4648 cik_cp_compute_fini(rdev);
4649 return r;
4650 }
4651 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4652 if (r) {
4653 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4654 cik_cp_compute_fini(rdev);
4655 return r;
4656 }
4657
963e81f9
AD		 4658 /* init the mqd struct */
4659 memset(buf, 0, sizeof(struct bonaire_mqd));
4660
4661 mqd = (struct bonaire_mqd *)buf;
4662 mqd->header = 0xC0310800;
4663 mqd->static_thread_mgmt01[0] = 0xffffffff;
4664 mqd->static_thread_mgmt01[1] = 0xffffffff;
4665 mqd->static_thread_mgmt23[0] = 0xffffffff;
4666 mqd->static_thread_mgmt23[1] = 0xffffffff;
4667
f61d5b46 4668 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4669 cik_srbm_select(rdev, rdev->ring[idx].me,
4670 rdev->ring[idx].pipe,
4671 rdev->ring[idx].queue, 0);
4672
4673 /* disable wptr polling */
4674 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4675 tmp &= ~WPTR_POLL_EN;
4676 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4677
4678 /* enable doorbell? */
4679 mqd->queue_state.cp_hqd_pq_doorbell_control =
4680 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4681 if (use_doorbell)
4682 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4683 else
4684 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4685 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4686 mqd->queue_state.cp_hqd_pq_doorbell_control);
4687
4688 /* disable the queue if it's active */
4689 mqd->queue_state.cp_hqd_dequeue_request = 0;
4690 mqd->queue_state.cp_hqd_pq_rptr = 0;
4691 mqd->queue_state.cp_hqd_pq_wptr= 0;
4692 if (RREG32(CP_HQD_ACTIVE) & 1) {
4693 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
370ce45b 4694 for (j = 0; j < rdev->usec_timeout; j++) {
963e81f9
AD		 4695 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4696 break;
4697 udelay(1);
4698 }
4699 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4700 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4701 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4702 }
4703
4704 /* set the pointer to the MQD */
4705 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4706 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4707 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4708 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4709 /* set MQD vmid to 0 */
4710 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4711 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4712 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4713
4714 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4715 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4716 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4717 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4718 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4719 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4720
4721 /* set up the HQD, this is similar to CP_RB0_CNTL */
4722 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4723 mqd->queue_state.cp_hqd_pq_control &=
4724 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4725
4726 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4727 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 4728 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4729 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 4730#ifdef __BIG_ENDIAN
4731 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4732#endif
4733 mqd->queue_state.cp_hqd_pq_control &=
4734 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4735 mqd->queue_state.cp_hqd_pq_control |=
4736 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4737 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4738
4739 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4740 if (i == 0)
4741 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4742 else
4743 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4744 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4745 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4746 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4747 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4748 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4749
4750 /* set the wb address wether it's enabled or not */
4751 if (i == 0)
4752 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4753 else
4754 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4755 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4756 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4757 upper_32_bits(wb_gpu_addr) & 0xffff;
4758 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4759 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4760 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4761 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4762
4763 /* enable the doorbell if requested */
4764 if (use_doorbell) {
4765 mqd->queue_state.cp_hqd_pq_doorbell_control =
4766 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4767 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4768 mqd->queue_state.cp_hqd_pq_doorbell_control |=
d5754ab8 4769 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
963e81f9
AD		 4770 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4771 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4772 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4773
4774 } else {
4775 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4776 }
4777 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4778 mqd->queue_state.cp_hqd_pq_doorbell_control);
4779
4780 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4781 rdev->ring[idx].wptr = 0;
4782 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4783 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
ff212f25 4784 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
963e81f9
AD		 4785
4786 /* set the vmid for the queue */
4787 mqd->queue_state.cp_hqd_vmid = 0;
4788 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4789
4790 /* activate the queue */
4791 mqd->queue_state.cp_hqd_active = 1;
4792 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4793
4794 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4795 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4796
4797 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4798 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4799
4800 rdev->ring[idx].ready = true;
4801 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4802 if (r)
4803 rdev->ring[idx].ready = false;
4804 }
4805
841cf442
AD		 4806 return 0;
4807}
4808
841cf442
AD		 4809static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4810{
4811 cik_cp_gfx_enable(rdev, enable);
4812 cik_cp_compute_enable(rdev, enable);
4813}
4814
841cf442
AD		 4815static int cik_cp_load_microcode(struct radeon_device *rdev)
4816{
4817 int r;
4818
4819 r = cik_cp_gfx_load_microcode(rdev);
4820 if (r)
4821 return r;
4822 r = cik_cp_compute_load_microcode(rdev);
4823 if (r)
4824 return r;
4825
4826 return 0;
4827}
4828
841cf442
AD		 4829static void cik_cp_fini(struct radeon_device *rdev)
4830{
4831 cik_cp_gfx_fini(rdev);
4832 cik_cp_compute_fini(rdev);
4833}
4834
841cf442
AD		 4835static int cik_cp_resume(struct radeon_device *rdev)
4836{
4837 int r;
4838
4214faf6
AD		 4839 cik_enable_gui_idle_interrupt(rdev, false);
4840
841cf442
AD		 4841 r = cik_cp_load_microcode(rdev);
4842 if (r)
4843 return r;
4844
4845 r = cik_cp_gfx_resume(rdev);
4846 if (r)
4847 return r;
4848 r = cik_cp_compute_resume(rdev);
4849 if (r)
4850 return r;
4851
4214faf6
AD		 4852 cik_enable_gui_idle_interrupt(rdev, true);
4853
841cf442
AD		 4854 return 0;
4855}
4856
cc066715 4857static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 4858{
6f2043ce
AD		 4859 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4860 RREG32(GRBM_STATUS));
4861 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4862 RREG32(GRBM_STATUS2));
4863 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4864 RREG32(GRBM_STATUS_SE0));
4865 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4866 RREG32(GRBM_STATUS_SE1));
4867 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4868 RREG32(GRBM_STATUS_SE2));
4869 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4870 RREG32(GRBM_STATUS_SE3));
4871 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4872 RREG32(SRBM_STATUS));
4873 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4874 RREG32(SRBM_STATUS2));
cc066715
AD		 4875 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4876 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4877 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4878 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 4879 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4880 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4881 RREG32(CP_STALLED_STAT1));
4882 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4883 RREG32(CP_STALLED_STAT2));
4884 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4885 RREG32(CP_STALLED_STAT3));
4886 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4887 RREG32(CP_CPF_BUSY_STAT));
4888 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4889 RREG32(CP_CPF_STALLED_STAT1));
4890 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4891 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4892 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4893 RREG32(CP_CPC_STALLED_STAT1));
4894 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 4895}
6f2043ce 4896
21a93e13 4897/**
cc066715 4898 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 4899 *
4900 * @rdev: radeon_device pointer
21a93e13 4901 *
cc066715
AD		 4902 * Check which blocks are busy and return the relevant reset
4903 * mask to be used by cik_gpu_soft_reset().
4904 * Returns a mask of the blocks to be reset.
21a93e13 4905 */
2483b4ea 4906u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 4907{
cc066715
AD		 4908 u32 reset_mask = 0;
4909 u32 tmp;
21a93e13 4910
cc066715
AD		 4911 /* GRBM_STATUS */
4912 tmp = RREG32(GRBM_STATUS);
4913 if (tmp & (PA_BUSY | SC_BUSY |
4914 BCI_BUSY | SX_BUSY |
4915 TA_BUSY | VGT_BUSY |
4916 DB_BUSY | CB_BUSY |
4917 GDS_BUSY | SPI_BUSY |
4918 IA_BUSY | IA_BUSY_NO_DMA))
4919 reset_mask |= RADEON_RESET_GFX;
21a93e13 4920
cc066715
AD		 4921 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4922 reset_mask |= RADEON_RESET_CP;
21a93e13 4923
cc066715
AD		 4924 /* GRBM_STATUS2 */
4925 tmp = RREG32(GRBM_STATUS2);
4926 if (tmp & RLC_BUSY)
4927 reset_mask |= RADEON_RESET_RLC;
21a93e13 4928
cc066715
AD		 4929 /* SDMA0_STATUS_REG */
4930 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4931 if (!(tmp & SDMA_IDLE))
4932 reset_mask |= RADEON_RESET_DMA;
21a93e13 4933
cc066715
AD		 4934 /* SDMA1_STATUS_REG */
4935 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4936 if (!(tmp & SDMA_IDLE))
4937 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4938
cc066715
AD		 4939 /* SRBM_STATUS2 */
4940 tmp = RREG32(SRBM_STATUS2);
4941 if (tmp & SDMA_BUSY)
4942 reset_mask |= RADEON_RESET_DMA;
21a93e13 4943
cc066715
AD		 4944 if (tmp & SDMA1_BUSY)
4945 reset_mask |= RADEON_RESET_DMA1;
21a93e13 4946
cc066715
AD		 4947 /* SRBM_STATUS */
4948 tmp = RREG32(SRBM_STATUS);
21a93e13 4949
cc066715
AD		 4950 if (tmp & IH_BUSY)
4951 reset_mask |= RADEON_RESET_IH;
21a93e13 4952
cc066715
AD		 4953 if (tmp & SEM_BUSY)
4954 reset_mask |= RADEON_RESET_SEM;
21a93e13 4955
cc066715
AD		 4956 if (tmp & GRBM_RQ_PENDING)
4957 reset_mask |= RADEON_RESET_GRBM;
21a93e13 4958
cc066715
AD		 4959 if (tmp & VMC_BUSY)
4960 reset_mask |= RADEON_RESET_VMC;
21a93e13 4961
cc066715
AD		 4962 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4963 MCC_BUSY | MCD_BUSY))
4964 reset_mask |= RADEON_RESET_MC;
21a93e13 4965
cc066715
AD		 4966 if (evergreen_is_display_hung(rdev))
4967 reset_mask |= RADEON_RESET_DISPLAY;
4968
4969 /* Skip MC reset as it's mostly likely not hung, just busy */
4970 if (reset_mask & RADEON_RESET_MC) {
4971 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4972 reset_mask &= ~RADEON_RESET_MC;
21a93e13 4973 }
cc066715
AD		 4974
4975 return reset_mask;
21a93e13
AD		 4976}
4977
4978/**
cc066715 4979 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 4980 *
4981 * @rdev: radeon_device pointer
cc066715 4982 * @reset_mask: mask of which blocks to reset
21a93e13 4983 *
cc066715 4984 * Soft reset the blocks specified in @reset_mask.
21a93e13 4985 */
cc066715 4986static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 4987{
6f2043ce 4988 struct evergreen_mc_save save;
cc066715
AD		 4989 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4990 u32 tmp;
21a93e13 4991
cc066715
AD		 4992 if (reset_mask == 0)
4993 return;
21a93e13 4994
cc066715 4995 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 4996
cc066715
AD		 4997 cik_print_gpu_status_regs(rdev);
4998 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4999 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5000 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5001 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 5002
fb2c7f4d
AD		 5003 /* disable CG/PG */
5004 cik_fini_pg(rdev);
5005 cik_fini_cg(rdev);
5006
cc066715
AD		 5007 /* stop the rlc */
5008 cik_rlc_stop(rdev);
21a93e13 5009
cc066715
AD		 5010 /* Disable GFX parsing/prefetching */
5011 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 5012
cc066715
AD		 5013 /* Disable MEC parsing/prefetching */
5014 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 5015
cc066715
AD		 5016 if (reset_mask & RADEON_RESET_DMA) {
5017 /* sdma0 */
5018 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5019 tmp |= SDMA_HALT;
5020 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5021 }
5022 if (reset_mask & RADEON_RESET_DMA1) {
5023 /* sdma1 */
5024 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5025 tmp |= SDMA_HALT;
5026 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5027 }
21a93e13 5028
6f2043ce 5029 evergreen_mc_stop(rdev, &save);
cc066715 5030 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 5031 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5032 }
21a93e13 5033
cc066715
AD		 5034 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5035 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 5036
cc066715
AD		 5037 if (reset_mask & RADEON_RESET_CP) {
5038 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 5039
cc066715
AD		 5040 srbm_soft_reset |= SOFT_RESET_GRBM;
5041 }
21a93e13 5042
cc066715
AD		 5043 if (reset_mask & RADEON_RESET_DMA)
5044 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 5045
cc066715
AD		 5046 if (reset_mask & RADEON_RESET_DMA1)
5047 srbm_soft_reset |= SOFT_RESET_SDMA1;
5048
5049 if (reset_mask & RADEON_RESET_DISPLAY)
5050 srbm_soft_reset |= SOFT_RESET_DC;
5051
5052 if (reset_mask & RADEON_RESET_RLC)
5053 grbm_soft_reset |= SOFT_RESET_RLC;
5054
5055 if (reset_mask & RADEON_RESET_SEM)
5056 srbm_soft_reset |= SOFT_RESET_SEM;
5057
5058 if (reset_mask & RADEON_RESET_IH)
5059 srbm_soft_reset |= SOFT_RESET_IH;
5060
5061 if (reset_mask & RADEON_RESET_GRBM)
5062 srbm_soft_reset |= SOFT_RESET_GRBM;
5063
5064 if (reset_mask & RADEON_RESET_VMC)
5065 srbm_soft_reset |= SOFT_RESET_VMC;
5066
5067 if (!(rdev->flags & RADEON_IS_IGP)) {
5068 if (reset_mask & RADEON_RESET_MC)
5069 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 5070 }
5071
cc066715
AD		 5072 if (grbm_soft_reset) {
5073 tmp = RREG32(GRBM_SOFT_RESET);
5074 tmp |= grbm_soft_reset;
5075 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5076 WREG32(GRBM_SOFT_RESET, tmp);
5077 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 5078
cc066715 5079 udelay(50);
21a93e13 5080
cc066715
AD		 5081 tmp &= ~grbm_soft_reset;
5082 WREG32(GRBM_SOFT_RESET, tmp);
5083 tmp = RREG32(GRBM_SOFT_RESET);
5084 }
21a93e13 5085
cc066715
AD		 5086 if (srbm_soft_reset) {
5087 tmp = RREG32(SRBM_SOFT_RESET);
5088 tmp |= srbm_soft_reset;
5089 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5090 WREG32(SRBM_SOFT_RESET, tmp);
5091 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 5092
cc066715 5093 udelay(50);
21a93e13 5094
cc066715
AD		 5095 tmp &= ~srbm_soft_reset;
5096 WREG32(SRBM_SOFT_RESET, tmp);
5097 tmp = RREG32(SRBM_SOFT_RESET);
5098 }
21a93e13 5099
6f2043ce
AD		 5100 /* Wait a little for things to settle down */
5101 udelay(50);
21a93e13 5102
6f2043ce 5103 evergreen_mc_resume(rdev, &save);
cc066715
AD		 5104 udelay(50);
5105
5106 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 5107}
5108
0279ed19
AD		 5109struct kv_reset_save_regs {
5110 u32 gmcon_reng_execute;
5111 u32 gmcon_misc;
5112 u32 gmcon_misc3;
5113};
5114
5115static void kv_save_regs_for_reset(struct radeon_device *rdev,
5116 struct kv_reset_save_regs *save)
5117{
5118 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5119 save->gmcon_misc = RREG32(GMCON_MISC);
5120 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5121
5122 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5123 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5124 STCTRL_STUTTER_EN));
5125}
5126
5127static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5128 struct kv_reset_save_regs *save)
5129{
5130 int i;
5131
5132 WREG32(GMCON_PGFSM_WRITE, 0);
5133 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5134
5135 for (i = 0; i < 5; i++)
5136 WREG32(GMCON_PGFSM_WRITE, 0);
5137
5138 WREG32(GMCON_PGFSM_WRITE, 0);
5139 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5140
5141 for (i = 0; i < 5; i++)
5142 WREG32(GMCON_PGFSM_WRITE, 0);
5143
5144 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5145 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5146
5147 for (i = 0; i < 5; i++)
5148 WREG32(GMCON_PGFSM_WRITE, 0);
5149
5150 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5151 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5152
5153 for (i = 0; i < 5; i++)
5154 WREG32(GMCON_PGFSM_WRITE, 0);
5155
5156 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5157 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5158
5159 for (i = 0; i < 5; i++)
5160 WREG32(GMCON_PGFSM_WRITE, 0);
5161
5162 WREG32(GMCON_PGFSM_WRITE, 0);
5163 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5164
5165 for (i = 0; i < 5; i++)
5166 WREG32(GMCON_PGFSM_WRITE, 0);
5167
5168 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5169 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5170
5171 for (i = 0; i < 5; i++)
5172 WREG32(GMCON_PGFSM_WRITE, 0);
5173
5174 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5175 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5176
5177 for (i = 0; i < 5; i++)
5178 WREG32(GMCON_PGFSM_WRITE, 0);
5179
5180 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5181 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5182
5183 for (i = 0; i < 5; i++)
5184 WREG32(GMCON_PGFSM_WRITE, 0);
5185
5186 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5187 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5188
5189 for (i = 0; i < 5; i++)
5190 WREG32(GMCON_PGFSM_WRITE, 0);
5191
5192 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5193 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5194
5195 WREG32(GMCON_MISC3, save->gmcon_misc3);
5196 WREG32(GMCON_MISC, save->gmcon_misc);
5197 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5198}
5199
5200static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5201{
5202 struct evergreen_mc_save save;
5203 struct kv_reset_save_regs kv_save = { 0 };
5204 u32 tmp, i;
5205
5206 dev_info(rdev->dev, "GPU pci config reset\n");
5207
5208 /* disable dpm? */
5209
5210 /* disable cg/pg */
5211 cik_fini_pg(rdev);
5212 cik_fini_cg(rdev);
5213
5214 /* Disable GFX parsing/prefetching */
5215 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5216
5217 /* Disable MEC parsing/prefetching */
5218 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5219
5220 /* sdma0 */
5221 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5222 tmp |= SDMA_HALT;
5223 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5224 /* sdma1 */
5225 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5226 tmp |= SDMA_HALT;
5227 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5228 /* XXX other engines? */
5229
5230 /* halt the rlc, disable cp internal ints */
5231 cik_rlc_stop(rdev);
5232
5233 udelay(50);
5234
5235 /* disable mem access */
5236 evergreen_mc_stop(rdev, &save);
5237 if (evergreen_mc_wait_for_idle(rdev)) {
5238 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5239 }
5240
5241 if (rdev->flags & RADEON_IS_IGP)
5242 kv_save_regs_for_reset(rdev, &kv_save);
5243
5244 /* disable BM */
5245 pci_clear_master(rdev->pdev);
5246 /* reset */
5247 radeon_pci_config_reset(rdev);
5248
5249 udelay(100);
5250
5251 /* wait for asic to come out of reset */
5252 for (i = 0; i < rdev->usec_timeout; i++) {
5253 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5254 break;
5255 udelay(1);
5256 }
5257
5258 /* does asic init need to be run first??? */
5259 if (rdev->flags & RADEON_IS_IGP)
5260 kv_restore_regs_for_reset(rdev, &kv_save);
5261}
5262
21a93e13 5263/**
cc066715 5264 * cik_asic_reset - soft reset GPU
21a93e13
AD		 5265 *
5266 * @rdev: radeon_device pointer
71fe2899 5267 * @hard: force hard reset
21a93e13 5268 *
cc066715
AD		 5269 * Look up which blocks are hung and attempt
5270 * to reset them.
6f2043ce 5271 * Returns 0 for success.
21a93e13 5272 */
71fe2899 5273int cik_asic_reset(struct radeon_device *rdev, bool hard)
21a93e13 5274{
cc066715 5275 u32 reset_mask;
21a93e13 5276
71fe2899
JG		 5277 if (hard) {
5278 cik_gpu_pci_config_reset(rdev);
5279 return 0;
5280 }
5281
cc066715 5282 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5283
cc066715
AD		 5284 if (reset_mask)
5285 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 5286
0279ed19 5287 /* try soft reset */
cc066715 5288 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 5289
cc066715
AD		 5290 reset_mask = cik_gpu_check_soft_reset(rdev);
5291
0279ed19
AD		 5292 /* try pci config reset */
5293 if (reset_mask && radeon_hard_reset)
5294 cik_gpu_pci_config_reset(rdev);
5295
5296 reset_mask = cik_gpu_check_soft_reset(rdev);
5297
cc066715
AD		 5298 if (!reset_mask)
5299 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 5300
5301 return 0;
5302}
5303
5304/**
cc066715 5305 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 5306 *
5307 * @rdev: radeon_device pointer
cc066715 5308 * @ring: radeon_ring structure holding ring information
21a93e13 5309 *
cc066715
AD		 5310 * Check if the 3D engine is locked up (CIK).
5311 * Returns true if the engine is locked, false if not.
21a93e13 5312 */
cc066715 5313bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 5314{
cc066715 5315 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5316
cc066715
AD		 5317 if (!(reset_mask & (RADEON_RESET_GFX |
5318 RADEON_RESET_COMPUTE |
5319 RADEON_RESET_CP))) {
ff212f25 5320 radeon_ring_lockup_update(rdev, ring);
cc066715 5321 return false;
21a93e13 5322 }
cc066715 5323 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 5324}
5325
1c49165d 5326/* MC */
21a93e13 5327/**
1c49165d 5328 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 5329 *
5330 * @rdev: radeon_device pointer
21a93e13 5331 *
1c49165d
AD		 5332 * Set the location of vram, gart, and AGP in the GPU's
5333 * physical address space (CIK).
21a93e13 5334 */
1c49165d 5335static void cik_mc_program(struct radeon_device *rdev)
21a93e13 5336{
1c49165d 5337 struct evergreen_mc_save save;
21a93e13 5338 u32 tmp;
1c49165d 5339 int i, j;
21a93e13 5340
1c49165d
AD		 5341 /* Initialize HDP */
5342 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5343 WREG32((0x2c14 + j), 0x00000000);
5344 WREG32((0x2c18 + j), 0x00000000);
5345 WREG32((0x2c1c + j), 0x00000000);
5346 WREG32((0x2c20 + j), 0x00000000);
5347 WREG32((0x2c24 + j), 0x00000000);
21a93e13 5348 }
1c49165d 5349 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 5350
1c49165d
AD		 5351 evergreen_mc_stop(rdev, &save);
5352 if (radeon_mc_wait_for_idle(rdev)) {
5353 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5354 }
1c49165d
AD		 5355 /* Lockout access through VGA aperture*/
5356 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5357 /* Update configuration */
5358 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5359 rdev->mc.vram_start >> 12);
5360 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5361 rdev->mc.vram_end >> 12);
5362 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5363 rdev->vram_scratch.gpu_addr >> 12);
5364 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5365 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5366 WREG32(MC_VM_FB_LOCATION, tmp);
5367 /* XXX double check these! */
5368 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5369 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5370 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5371 WREG32(MC_VM_AGP_BASE, 0);
5372 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5373 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5374 if (radeon_mc_wait_for_idle(rdev)) {
5375 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5376 }
1c49165d
AD		 5377 evergreen_mc_resume(rdev, &save);
5378 /* we need to own VRAM, so turn off the VGA renderer here
5379 * to stop it overwriting our objects */
5380 rv515_vga_render_disable(rdev);
21a93e13
AD		 5381}
5382
5383/**
1c49165d 5384 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 5385 *
5386 * @rdev: radeon_device pointer
21a93e13 5387 *
1c49165d
AD		 5388 * Look up the amount of vram, vram width, and decide how to place
5389 * vram and gart within the GPU's physical address space (CIK).
5390 * Returns 0 for success.
21a93e13 5391 */
1c49165d 5392static int cik_mc_init(struct radeon_device *rdev)
21a93e13 5393{
1c49165d
AD		 5394 u32 tmp;
5395 int chansize, numchan;
21a93e13 5396
1c49165d
AD		 5397 /* Get VRAM informations */
5398 rdev->mc.vram_is_ddr = true;
5399 tmp = RREG32(MC_ARB_RAMCFG);
5400 if (tmp & CHANSIZE_MASK) {
5401 chansize = 64;
21a93e13 5402 } else {
1c49165d 5403 chansize = 32;
21a93e13 5404 }
1c49165d
AD		 5405 tmp = RREG32(MC_SHARED_CHMAP);
5406 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5407 case 0:
5408 default:
5409 numchan = 1;
5410 break;
5411 case 1:
5412 numchan = 2;
5413 break;
5414 case 2:
5415 numchan = 4;
5416 break;
5417 case 3:
5418 numchan = 8;
5419 break;
5420 case 4:
5421 numchan = 3;
5422 break;
5423 case 5:
5424 numchan = 6;
5425 break;
5426 case 6:
5427 numchan = 10;
5428 break;
5429 case 7:
5430 numchan = 12;
5431 break;
5432 case 8:
5433 numchan = 16;
5434 break;
5435 }
5436 rdev->mc.vram_width = numchan * chansize;
5437 /* Could aper size report 0 ? */
5438 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5439 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5440 /* size in MB on si */
13c5bfda
AD		 5441 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5442 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 5443 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5444 si_vram_gtt_location(rdev, &rdev->mc);
5445 radeon_update_bandwidth_info(rdev);
5446
5447 return 0;
5448}
5449
5450/*
5451 * GART
5452 * VMID 0 is the physical GPU addresses as used by the kernel.
5453 * VMIDs 1-15 are used for userspace clients and are handled
5454 * by the radeon vm/hsa code.
5455 */
5456/**
5457 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5458 *
5459 * @rdev: radeon_device pointer
5460 *
5461 * Flush the TLB for the VMID 0 page table (CIK).
5462 */
5463void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5464{
5465 /* flush hdp cache */
5466 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5467
5468 /* bits 0-15 are the VM contexts0-15 */
5469 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5470}
5471
08dcc57f
BG		 5472static void cik_pcie_init_compute_vmid(struct radeon_device *rdev)
5473{
5474 int i;
5475 uint32_t sh_mem_bases, sh_mem_config;
5476
5477 sh_mem_bases = 0x6000 | 0x6000 << 16;
5478 sh_mem_config = ALIGNMENT_MODE(SH_MEM_ALIGNMENT_MODE_UNALIGNED);
5479 sh_mem_config |= DEFAULT_MTYPE(MTYPE_NONCACHED);
5480
5481 mutex_lock(&rdev->srbm_mutex);
5482 for (i = 8; i < 16; i++) {
5483 cik_srbm_select(rdev, 0, 0, 0, i);
5484 /* CP and shaders */
5485 WREG32(SH_MEM_CONFIG, sh_mem_config);
5486 WREG32(SH_MEM_APE1_BASE, 1);
5487 WREG32(SH_MEM_APE1_LIMIT, 0);
5488 WREG32(SH_MEM_BASES, sh_mem_bases);
5489 }
5490 cik_srbm_select(rdev, 0, 0, 0, 0);
5491 mutex_unlock(&rdev->srbm_mutex);
5492}
5493
1c49165d
AD		 5494/**
5495 * cik_pcie_gart_enable - gart enable
5496 *
5497 * @rdev: radeon_device pointer
5498 *
5499 * This sets up the TLBs, programs the page tables for VMID0,
5500 * sets up the hw for VMIDs 1-15 which are allocated on
5501 * demand, and sets up the global locations for the LDS, GDS,
5502 * and GPUVM for FSA64 clients (CIK).
5503 * Returns 0 for success, errors for failure.
5504 */
5505static int cik_pcie_gart_enable(struct radeon_device *rdev)
5506{
5507 int r, i;
5508
5509 if (rdev->gart.robj == NULL) {
5510 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5511 return -EINVAL;
5512 }
5513 r = radeon_gart_table_vram_pin(rdev);
5514 if (r)
5515 return r;
1c49165d
AD		 5516 /* Setup TLB control */
5517 WREG32(MC_VM_MX_L1_TLB_CNTL,
5518 (0xA << 7) |
5519 ENABLE_L1_TLB |
ec3dbbcb 5520 ENABLE_L1_FRAGMENT_PROCESSING |
1c49165d
AD		 5521 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5522 ENABLE_ADVANCED_DRIVER_MODEL |
5523 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5524 /* Setup L2 cache */
5525 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5526 ENABLE_L2_FRAGMENT_PROCESSING |
5527 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5528 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5529 EFFECTIVE_L2_QUEUE_SIZE(7) |
5530 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5531 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5532 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
ec3dbbcb
CK		 5533 BANK_SELECT(4) |
5534 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
1c49165d
AD		 5535 /* setup context0 */
5536 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
7c0411d2 5537 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1c49165d
AD		 5538 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5539 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5540 (u32)(rdev->dummy_page.addr >> 12));
5541 WREG32(VM_CONTEXT0_CNTL2, 0);
5542 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5543 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5544
5545 WREG32(0x15D4, 0);
5546 WREG32(0x15D8, 0);
5547 WREG32(0x15DC, 0);
5548
054e01d6 5549 /* restore context1-15 */
1c49165d
AD		 5550 /* set vm size, must be a multiple of 4 */
5551 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
607d4806 5552 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
1c49165d
AD		 5553 for (i = 1; i < 16; i++) {
5554 if (i < 8)
5555 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
054e01d6 5556 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5557 else
5558 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
054e01d6 5559 rdev->vm_manager.saved_table_addr[i]);
1c49165d
AD		 5560 }
5561
5562 /* enable context1-15 */
5563 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5564 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 5565 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 5566 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4510fb98 5567 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
a00024b0
AD		 5568 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5569 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5570 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5571 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5572 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5573 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5574 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5575 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5576 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5577 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5578 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5579 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d 5580
1c49165d
AD		 5581 if (rdev->family == CHIP_KAVERI) {
5582 u32 tmp = RREG32(CHUB_CONTROL);
5583 tmp &= ~BYPASS_VM;
5584 WREG32(CHUB_CONTROL, tmp);
5585 }
5586
5587 /* XXX SH_MEM regs */
5588 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 5589 mutex_lock(&rdev->srbm_mutex);
1c49165d 5590 for (i = 0; i < 16; i++) {
b556b12e 5591 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 5592 /* CP and shaders */
75cb00dc 5593 WREG32(SH_MEM_CONFIG, SH_MEM_CONFIG_GFX_DEFAULT);
1c49165d
AD		 5594 WREG32(SH_MEM_APE1_BASE, 1);
5595 WREG32(SH_MEM_APE1_LIMIT, 0);
5596 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 5597 /* SDMA GFX */
5598 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5599 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5600 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5601 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5602 /* XXX SDMA RLC - todo */
1c49165d 5603 }
b556b12e 5604 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5605 mutex_unlock(&rdev->srbm_mutex);
1c49165d 5606
08dcc57f
BG		 5607 cik_pcie_init_compute_vmid(rdev);
5608
1c49165d
AD		 5609 cik_pcie_gart_tlb_flush(rdev);
5610 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5611 (unsigned)(rdev->mc.gtt_size >> 20),
5612 (unsigned long long)rdev->gart.table_addr);
5613 rdev->gart.ready = true;
5614 return 0;
5615}
5616
5617/**
5618 * cik_pcie_gart_disable - gart disable
5619 *
5620 * @rdev: radeon_device pointer
5621 *
5622 * This disables all VM page table (CIK).
5623 */
5624static void cik_pcie_gart_disable(struct radeon_device *rdev)
5625{
054e01d6
CK		 5626 unsigned i;
5627
5628 for (i = 1; i < 16; ++i) {
5629 uint32_t reg;
5630 if (i < 8)
5631 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
5632 else
5633 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
5634 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
5635 }
5636
1c49165d
AD		 5637 /* Disable all tables */
5638 WREG32(VM_CONTEXT0_CNTL, 0);
5639 WREG32(VM_CONTEXT1_CNTL, 0);
5640 /* Setup TLB control */
5641 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5642 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5643 /* Setup L2 cache */
5644 WREG32(VM_L2_CNTL,
5645 ENABLE_L2_FRAGMENT_PROCESSING |
5646 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5647 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5648 EFFECTIVE_L2_QUEUE_SIZE(7) |
5649 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5650 WREG32(VM_L2_CNTL2, 0);
5651 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5652 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5653 radeon_gart_table_vram_unpin(rdev);
5654}
5655
5656/**
5657 * cik_pcie_gart_fini - vm fini callback
5658 *
5659 * @rdev: radeon_device pointer
5660 *
5661 * Tears down the driver GART/VM setup (CIK).
5662 */
5663static void cik_pcie_gart_fini(struct radeon_device *rdev)
5664{
5665 cik_pcie_gart_disable(rdev);
5666 radeon_gart_table_vram_free(rdev);
5667 radeon_gart_fini(rdev);
5668}
5669
5670/* vm parser */
5671/**
5672 * cik_ib_parse - vm ib_parse callback
5673 *
5674 * @rdev: radeon_device pointer
5675 * @ib: indirect buffer pointer
5676 *
5677 * CIK uses hw IB checking so this is a nop (CIK).
5678 */
5679int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5680{
5681 return 0;
5682}
5683
5684/*
5685 * vm
5686 * VMID 0 is the physical GPU addresses as used by the kernel.
5687 * VMIDs 1-15 are used for userspace clients and are handled
5688 * by the radeon vm/hsa code.
5689 */
5690/**
5691 * cik_vm_init - cik vm init callback
5692 *
5693 * @rdev: radeon_device pointer
5694 *
5695 * Inits cik specific vm parameters (number of VMs, base of vram for
5696 * VMIDs 1-15) (CIK).
5697 * Returns 0 for success.
5698 */
5699int cik_vm_init(struct radeon_device *rdev)
5700{
62a7b7fb
OG		 5701 /*
5702 * number of VMs
5703 * VMID 0 is reserved for System
5704 * radeon graphics/compute will use VMIDs 1-7
5705 * amdkfd will use VMIDs 8-15
5706 */
5707 rdev->vm_manager.nvm = RADEON_NUM_OF_VMIDS;
1c49165d
AD		 5708 /* base offset of vram pages */
5709 if (rdev->flags & RADEON_IS_IGP) {
5710 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5711 tmp <<= 22;
5712 rdev->vm_manager.vram_base_offset = tmp;
5713 } else
5714 rdev->vm_manager.vram_base_offset = 0;
5715
5716 return 0;
5717}
5718
5719/**
5720 * cik_vm_fini - cik vm fini callback
5721 *
5722 * @rdev: radeon_device pointer
5723 *
5724 * Tear down any asic specific VM setup (CIK).
5725 */
5726void cik_vm_fini(struct radeon_device *rdev)
5727{
5728}
5729
3ec7d11b
AD		 5730/**
5731 * cik_vm_decode_fault - print human readable fault info
5732 *
5733 * @rdev: radeon_device pointer
5734 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5735 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5736 *
5737 * Print human readable fault information (CIK).
5738 */
5739static void cik_vm_decode_fault(struct radeon_device *rdev,
5740 u32 status, u32 addr, u32 mc_client)
5741{
939c0d3c 5742 u32 mc_id;
3ec7d11b
AD		 5743 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5744 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 5745 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5746 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 5747
939c0d3c
AD		 5748 if (rdev->family == CHIP_HAWAII)
5749 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5750 else
5751 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5752
328a50c7 5753 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 5754 protections, vmid, addr,
5755 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 5756 block, mc_client, mc_id);
3ec7d11b
AD		 5757}
5758
f96ab484
AD		 5759/**
5760 * cik_vm_flush - cik vm flush using the CP
5761 *
5762 * @rdev: radeon_device pointer
5763 *
5764 * Update the page table base and flush the VM TLB
5765 * using the CP (CIK).
5766 */
faffaf62
CK		 5767void cik_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5768 unsigned vm_id, uint64_t pd_addr)
f96ab484 5769{
faffaf62 5770 int usepfp = (ring->idx == RADEON_RING_TYPE_GFX_INDEX);
f96ab484
AD		 5771
5772 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
f1d2a26b 5773 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484 5774 WRITE_DATA_DST_SEL(0)));
faffaf62 5775 if (vm_id < 8) {
f96ab484 5776 radeon_ring_write(ring,
faffaf62 5777 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
f96ab484
AD		 5778 } else {
5779 radeon_ring_write(ring,
faffaf62 5780 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
f96ab484
AD		 5781 }
5782 radeon_ring_write(ring, 0);
faffaf62 5783 radeon_ring_write(ring, pd_addr >> 12);
f96ab484
AD		 5784
5785 /* update SH_MEM_* regs */
5786 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5787 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5788 WRITE_DATA_DST_SEL(0)));
5789 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5790 radeon_ring_write(ring, 0);
faffaf62 5791 radeon_ring_write(ring, VMID(vm_id));
f96ab484
AD		 5792
5793 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4fb0bbd5 5794 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5795 WRITE_DATA_DST_SEL(0)));
5796 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5797 radeon_ring_write(ring, 0);
5798
5799 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
75cb00dc 5800 radeon_ring_write(ring, SH_MEM_CONFIG_GFX_DEFAULT); /* SH_MEM_CONFIG */
f96ab484
AD		 5801 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5802 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 5803
f96ab484 5804 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5805 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5806 WRITE_DATA_DST_SEL(0)));
5807 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5808 radeon_ring_write(ring, 0);
5809 radeon_ring_write(ring, VMID(0));
6f2043ce 5810
f96ab484 5811 /* HDP flush */
faffaf62 5812 cik_hdp_flush_cp_ring_emit(rdev, ring->idx);
f96ab484
AD		 5813
5814 /* bits 0-15 are the VM contexts0-15 */
5815 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5816 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5817 WRITE_DATA_DST_SEL(0)));
5818 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5819 radeon_ring_write(ring, 0);
faffaf62 5820 radeon_ring_write(ring, 1 << vm_id);
f96ab484 5821
3a01fd36
AD		 5822 /* wait for the invalidate to complete */
5823 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5824 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
5825 WAIT_REG_MEM_FUNCTION(0) | /* always */
5826 WAIT_REG_MEM_ENGINE(0))); /* me */
5827 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5828 radeon_ring_write(ring, 0);
5829 radeon_ring_write(ring, 0); /* ref */
5830 radeon_ring_write(ring, 0); /* mask */
5831 radeon_ring_write(ring, 0x20); /* poll interval */
5832
b07fdd38 5833 /* compute doesn't have PFP */
f1d2a26b 5834 if (usepfp) {
b07fdd38
AD		 5835 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5836 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5837 radeon_ring_write(ring, 0x0);
5838 }
cc066715 5839}
6f2043ce 5840
f6796cae
AD		 5841/*
5842 * RLC
5843 * The RLC is a multi-purpose microengine that handles a
5844 * variety of functions, the most important of which is
5845 * the interrupt controller.
5846 */
866d83de
AD		 5847static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5848 bool enable)
f6796cae 5849{
866d83de 5850 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 5851
866d83de
AD		 5852 if (enable)
5853 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5854 else
5855 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 5856 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 5857}
f6796cae 5858
866d83de 5859static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 5860{
cc066715 5861 u32 tmp;
6f2043ce 5862
866d83de
AD		 5863 tmp = RREG32(RLC_LB_CNTL);
5864 if (enable)
5865 tmp |= LOAD_BALANCE_ENABLE;
5866 else
5867 tmp &= ~LOAD_BALANCE_ENABLE;
5868 WREG32(RLC_LB_CNTL, tmp);
5869}
cc066715 5870
866d83de
AD		 5871static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5872{
5873 u32 i, j, k;
5874 u32 mask;
cc066715 5875
1c0a4625 5876 mutex_lock(&rdev->grbm_idx_mutex);
f6796cae
AD		 5877 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5878 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5879 cik_select_se_sh(rdev, i, j);
5880 for (k = 0; k < rdev->usec_timeout; k++) {
5881 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5882 break;
5883 udelay(1);
5884 }
5885 }
5886 }
5887 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 5888 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 5889
f6796cae
AD		 5890 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5891 for (k = 0; k < rdev->usec_timeout; k++) {
5892 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5893 break;
5894 udelay(1);
5895 }
5896}
cc066715 5897
22c775ce
AD		 5898static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5899{
5900 u32 tmp;
cc066715 5901
22c775ce
AD		 5902 tmp = RREG32(RLC_CNTL);
5903 if (tmp != rlc)
5904 WREG32(RLC_CNTL, rlc);
5905}
cc066715 5906
22c775ce
AD		 5907static u32 cik_halt_rlc(struct radeon_device *rdev)
5908{
5909 u32 data, orig;
cc066715 5910
22c775ce 5911 orig = data = RREG32(RLC_CNTL);
cc066715 5912
22c775ce
AD		 5913 if (data & RLC_ENABLE) {
5914 u32 i;
cc066715 5915
22c775ce
AD		 5916 data &= ~RLC_ENABLE;
5917 WREG32(RLC_CNTL, data);
cc066715 5918
22c775ce
AD		 5919 for (i = 0; i < rdev->usec_timeout; i++) {
5920 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5921 break;
5922 udelay(1);
5923 }
cc066715 5924
22c775ce
AD		 5925 cik_wait_for_rlc_serdes(rdev);
5926 }
cc066715 5927
22c775ce
AD		 5928 return orig;
5929}
cc066715 5930
a412fce0
AD		 5931void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5932{
5933 u32 tmp, i, mask;
5934
5935 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5936 WREG32(RLC_GPR_REG2, tmp);
5937
5938 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5939 for (i = 0; i < rdev->usec_timeout; i++) {
5940 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5941 break;
5942 udelay(1);
5943 }
5944
5945 for (i = 0; i < rdev->usec_timeout; i++) {
5946 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5947 break;
5948 udelay(1);
5949 }
5950}
5951
5952void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5953{
5954 u32 tmp;
5955
5956 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5957 WREG32(RLC_GPR_REG2, tmp);
5958}
5959
866d83de
AD		 5960/**
5961 * cik_rlc_stop - stop the RLC ME
5962 *
5963 * @rdev: radeon_device pointer
5964 *
5965 * Halt the RLC ME (MicroEngine) (CIK).
5966 */
5967static void cik_rlc_stop(struct radeon_device *rdev)
5968{
22c775ce 5969 WREG32(RLC_CNTL, 0);
866d83de
AD		 5970
5971 cik_enable_gui_idle_interrupt(rdev, false);
5972
866d83de
AD		 5973 cik_wait_for_rlc_serdes(rdev);
5974}
5975
f6796cae
AD		 5976/**
5977 * cik_rlc_start - start the RLC ME
5978 *
5979 * @rdev: radeon_device pointer
5980 *
5981 * Unhalt the RLC ME (MicroEngine) (CIK).
5982 */
5983static void cik_rlc_start(struct radeon_device *rdev)
5984{
f6796cae 5985 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 5986
866d83de 5987 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 5988
f6796cae 5989 udelay(50);
6f2043ce
AD		 5990}
5991
5992/**
f6796cae 5993 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 5994 *
5995 * @rdev: radeon_device pointer
5996 *
f6796cae
AD		 5997 * Initialize the RLC registers, load the ucode,
5998 * and start the RLC (CIK).
5999 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 6000 */
f6796cae 6001static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 6002{
22c775ce 6003 u32 i, size, tmp;
cc066715 6004
f6796cae
AD		 6005 if (!rdev->rlc_fw)
6006 return -EINVAL;
cc066715 6007
cc066715
AD		 6008 cik_rlc_stop(rdev);
6009
22c775ce
AD		 6010 /* disable CG */
6011 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6012 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 6013
866d83de 6014 si_rlc_reset(rdev);
6f2043ce 6015
22c775ce 6016 cik_init_pg(rdev);
6f2043ce 6017
22c775ce 6018 cik_init_cg(rdev);
cc066715 6019
f6796cae
AD		 6020 WREG32(RLC_LB_CNTR_INIT, 0);
6021 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 6022
1c0a4625 6023 mutex_lock(&rdev->grbm_idx_mutex);
f6796cae
AD		 6024 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6025 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6026 WREG32(RLC_LB_PARAMS, 0x00600408);
6027 WREG32(RLC_LB_CNTL, 0x80000004);
1c0a4625 6028 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6029
f6796cae
AD		 6030 WREG32(RLC_MC_CNTL, 0);
6031 WREG32(RLC_UCODE_CNTL, 0);
cc066715 6032
f2c6b0f4
AD		 6033 if (rdev->new_fw) {
6034 const struct rlc_firmware_header_v1_0 *hdr =
6035 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6036 const __le32 *fw_data = (const __le32 *)
6037 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6038
6039 radeon_ucode_print_rlc_hdr(&hdr->header);
6040
6041 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6042 WREG32(RLC_GPM_UCODE_ADDR, 0);
6043 for (i = 0; i < size; i++)
6044 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
38aea071 6045 WREG32(RLC_GPM_UCODE_ADDR, le32_to_cpu(hdr->header.ucode_version));
f2c6b0f4
AD		 6046 } else {
6047 const __be32 *fw_data;
6048
6049 switch (rdev->family) {
6050 case CHIP_BONAIRE:
6051 case CHIP_HAWAII:
6052 default:
6053 size = BONAIRE_RLC_UCODE_SIZE;
6054 break;
6055 case CHIP_KAVERI:
6056 size = KV_RLC_UCODE_SIZE;
6057 break;
6058 case CHIP_KABINI:
6059 size = KB_RLC_UCODE_SIZE;
6060 break;
6061 case CHIP_MULLINS:
6062 size = ML_RLC_UCODE_SIZE;
6063 break;
6064 }
6065
6066 fw_data = (const __be32 *)rdev->rlc_fw->data;
6067 WREG32(RLC_GPM_UCODE_ADDR, 0);
6068 for (i = 0; i < size; i++)
6069 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6070 WREG32(RLC_GPM_UCODE_ADDR, 0);
6071 }
cc066715 6072
866d83de
AD		 6073 /* XXX - find out what chips support lbpw */
6074 cik_enable_lbpw(rdev, false);
cc066715 6075
22c775ce
AD		 6076 if (rdev->family == CHIP_BONAIRE)
6077 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 6078
f6796cae 6079 cik_rlc_start(rdev);
cc066715 6080
f6796cae
AD		 6081 return 0;
6082}
cc066715 6083
22c775ce
AD		 6084static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6085{
6086 u32 data, orig, tmp, tmp2;
cc066715 6087
22c775ce 6088 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 6089
473359bc 6090 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 6091 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6092
22c775ce 6093 tmp = cik_halt_rlc(rdev);
cc066715 6094
1c0a4625 6095 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6096 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6097 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6098 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6099 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6100 WREG32(RLC_SERDES_WR_CTRL, tmp2);
1c0a4625 6101 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6102
22c775ce 6103 cik_update_rlc(rdev, tmp);
cc066715 6104
22c775ce
AD		 6105 data |= CGCG_EN | CGLS_EN;
6106 } else {
ddc76ff6 6107 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 6108
22c775ce
AD		 6109 RREG32(CB_CGTT_SCLK_CTRL);
6110 RREG32(CB_CGTT_SCLK_CTRL);
6111 RREG32(CB_CGTT_SCLK_CTRL);
6112 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 6113
22c775ce 6114 data &= ~(CGCG_EN | CGLS_EN);
cc066715 6115 }
6f2043ce 6116
22c775ce
AD		 6117 if (orig != data)
6118 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 6119
6f2043ce
AD		 6120}
6121
22c775ce 6122static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 6123{
22c775ce
AD		 6124 u32 data, orig, tmp = 0;
6125
473359bc
AD		 6126 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6127 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6128 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6129 orig = data = RREG32(CP_MEM_SLP_CNTL);
6130 data |= CP_MEM_LS_EN;
6131 if (orig != data)
6132 WREG32(CP_MEM_SLP_CNTL, data);
6133 }
6134 }
cc066715 6135
22c775ce 6136 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4bb62c95 6137 data |= 0x00000001;
22c775ce
AD		 6138 data &= 0xfffffffd;
6139 if (orig != data)
6140 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6141
6142 tmp = cik_halt_rlc(rdev);
6143
1c0a4625 6144 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6145 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6146 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6147 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6148 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6149 WREG32(RLC_SERDES_WR_CTRL, data);
1c0a4625 6150 mutex_unlock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6151
6152 cik_update_rlc(rdev, tmp);
6153
473359bc
AD		 6154 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6155 orig = data = RREG32(CGTS_SM_CTRL_REG);
6156 data &= ~SM_MODE_MASK;
6157 data |= SM_MODE(0x2);
6158 data |= SM_MODE_ENABLE;
6159 data &= ~CGTS_OVERRIDE;
6160 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6161 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6162 data &= ~CGTS_LS_OVERRIDE;
6163 data &= ~ON_MONITOR_ADD_MASK;
6164 data |= ON_MONITOR_ADD_EN;
6165 data |= ON_MONITOR_ADD(0x96);
6166 if (orig != data)
6167 WREG32(CGTS_SM_CTRL_REG, data);
6168 }
22c775ce
AD		 6169 } else {
6170 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
4bb62c95 6171 data |= 0x00000003;
22c775ce
AD		 6172 if (orig != data)
6173 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6174
6175 data = RREG32(RLC_MEM_SLP_CNTL);
6176 if (data & RLC_MEM_LS_EN) {
6177 data &= ~RLC_MEM_LS_EN;
6178 WREG32(RLC_MEM_SLP_CNTL, data);
6179 }
6f2043ce 6180
22c775ce
AD		 6181 data = RREG32(CP_MEM_SLP_CNTL);
6182 if (data & CP_MEM_LS_EN) {
6183 data &= ~CP_MEM_LS_EN;
6184 WREG32(CP_MEM_SLP_CNTL, data);
6185 }
cc066715 6186
22c775ce
AD		 6187 orig = data = RREG32(CGTS_SM_CTRL_REG);
6188 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6189 if (orig != data)
6190 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 6191
22c775ce 6192 tmp = cik_halt_rlc(rdev);
cc066715 6193
1c0a4625 6194 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6195 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6196 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6197 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6198 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6199 WREG32(RLC_SERDES_WR_CTRL, data);
1c0a4625 6200 mutex_unlock(&rdev->grbm_idx_mutex);
cc066715 6201
22c775ce 6202 cik_update_rlc(rdev, tmp);
cc066715 6203 }
6f2043ce 6204}
1c49165d 6205
22c775ce 6206static const u32 mc_cg_registers[] =
21a93e13 6207{
22c775ce
AD		 6208 MC_HUB_MISC_HUB_CG,
6209 MC_HUB_MISC_SIP_CG,
6210 MC_HUB_MISC_VM_CG,
6211 MC_XPB_CLK_GAT,
6212 ATC_MISC_CG,
6213 MC_CITF_MISC_WR_CG,
6214 MC_CITF_MISC_RD_CG,
6215 MC_CITF_MISC_VM_CG,
6216 VM_L2_CG,
6217};
21a93e13 6218
22c775ce
AD		 6219static void cik_enable_mc_ls(struct radeon_device *rdev,
6220 bool enable)
1c49165d 6221{
22c775ce
AD		 6222 int i;
6223 u32 orig, data;
1c49165d 6224
22c775ce
AD		 6225 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6226 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6227 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 6228 data |= MC_LS_ENABLE;
6229 else
6230 data &= ~MC_LS_ENABLE;
6231 if (data != orig)
6232 WREG32(mc_cg_registers[i], data);
1c49165d 6233 }
22c775ce 6234}
1c49165d 6235
22c775ce
AD		 6236static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6237 bool enable)
6238{
6239 int i;
6240 u32 orig, data;
6241
6242 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6243 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6244 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 6245 data |= MC_CG_ENABLE;
6246 else
6247 data &= ~MC_CG_ENABLE;
6248 if (data != orig)
6249 WREG32(mc_cg_registers[i], data);
1c49165d 6250 }
1c49165d
AD		 6251}
6252
22c775ce
AD		 6253static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6254 bool enable)
1c49165d 6255{
22c775ce 6256 u32 orig, data;
1c49165d 6257
473359bc 6258 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 6259 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6260 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 6261 } else {
22c775ce
AD		 6262 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6263 data |= 0xff000000;
6264 if (data != orig)
6265 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6266
22c775ce
AD		 6267 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6268 data |= 0xff000000;
6269 if (data != orig)
6270 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6271 }
1c49165d
AD		 6272}
6273
22c775ce
AD		 6274static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6275 bool enable)
1c49165d 6276{
22c775ce
AD		 6277 u32 orig, data;
6278
473359bc 6279 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 6280 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6281 data |= 0x100;
6282 if (orig != data)
6283 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6284
6285 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6286 data |= 0x100;
6287 if (orig != data)
6288 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6289 } else {
6290 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6291 data &= ~0x100;
6292 if (orig != data)
6293 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6294
22c775ce
AD		 6295 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6296 data &= ~0x100;
6297 if (orig != data)
6298 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6299 }
1c49165d
AD		 6300}
6301
22c775ce
AD		 6302static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6303 bool enable)
1c49165d 6304{
22c775ce 6305 u32 orig, data;
1c49165d 6306
473359bc 6307 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 6308 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6309 data = 0xfff;
6310 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6311
22c775ce
AD		 6312 orig = data = RREG32(UVD_CGC_CTRL);
6313 data |= DCM;
6314 if (orig != data)
6315 WREG32(UVD_CGC_CTRL, data);
6316 } else {
6317 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6318 data &= ~0xfff;
6319 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6320
22c775ce
AD		 6321 orig = data = RREG32(UVD_CGC_CTRL);
6322 data &= ~DCM;
6323 if (orig != data)
6324 WREG32(UVD_CGC_CTRL, data);
1c49165d 6325 }
22c775ce 6326}
1c49165d 6327
473359bc
AD		 6328static void cik_enable_bif_mgls(struct radeon_device *rdev,
6329 bool enable)
6330{
6331 u32 orig, data;
1c49165d 6332
473359bc 6333 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 6334
473359bc
AD		 6335 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6336 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6337 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6338 else
6339 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6340 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 6341
473359bc
AD		 6342 if (orig != data)
6343 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6344}
1c49165d 6345
22c775ce
AD		 6346static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6347 bool enable)
6348{
6349 u32 orig, data;
1c49165d 6350
22c775ce 6351 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 6352
473359bc 6353 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 6354 data &= ~CLOCK_GATING_DIS;
6355 else
6356 data |= CLOCK_GATING_DIS;
6357
6358 if (orig != data)
6359 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 6360}
6361
22c775ce
AD		 6362static void cik_enable_hdp_ls(struct radeon_device *rdev,
6363 bool enable)
1c49165d 6364{
22c775ce
AD		 6365 u32 orig, data;
6366
6367 orig = data = RREG32(HDP_MEM_POWER_LS);
6368
473359bc 6369 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 6370 data |= HDP_LS_ENABLE;
6371 else
6372 data &= ~HDP_LS_ENABLE;
6373
6374 if (orig != data)
6375 WREG32(HDP_MEM_POWER_LS, data);
6376}
6377
6378void cik_update_cg(struct radeon_device *rdev,
6379 u32 block, bool enable)
6380{
4214faf6 6381
22c775ce 6382 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 6383 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 6384 /* order matters! */
6385 if (enable) {
6386 cik_enable_mgcg(rdev, true);
6387 cik_enable_cgcg(rdev, true);
6388 } else {
6389 cik_enable_cgcg(rdev, false);
6390 cik_enable_mgcg(rdev, false);
6391 }
4214faf6 6392 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 6393 }
6394
6395 if (block & RADEON_CG_BLOCK_MC) {
6396 if (!(rdev->flags & RADEON_IS_IGP)) {
6397 cik_enable_mc_mgcg(rdev, enable);
6398 cik_enable_mc_ls(rdev, enable);
6399 }
6400 }
6401
6402 if (block & RADEON_CG_BLOCK_SDMA) {
6403 cik_enable_sdma_mgcg(rdev, enable);
6404 cik_enable_sdma_mgls(rdev, enable);
6405 }
6406
473359bc
AD		 6407 if (block & RADEON_CG_BLOCK_BIF) {
6408 cik_enable_bif_mgls(rdev, enable);
6409 }
6410
22c775ce
AD		 6411 if (block & RADEON_CG_BLOCK_UVD) {
6412 if (rdev->has_uvd)
6413 cik_enable_uvd_mgcg(rdev, enable);
6414 }
6415
6416 if (block & RADEON_CG_BLOCK_HDP) {
6417 cik_enable_hdp_mgcg(rdev, enable);
6418 cik_enable_hdp_ls(rdev, enable);
6419 }
a1d6f97c
AD		 6420
6421 if (block & RADEON_CG_BLOCK_VCE) {
6422 vce_v2_0_enable_mgcg(rdev, enable);
6423 }
1c49165d
AD		 6424}
6425
22c775ce 6426static void cik_init_cg(struct radeon_device *rdev)
1c49165d 6427{
22c775ce 6428
ddc76ff6 6429 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 6430
6431 if (rdev->has_uvd)
6432 si_init_uvd_internal_cg(rdev);
6433
6434 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6435 RADEON_CG_BLOCK_SDMA |
473359bc 6436 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 6437 RADEON_CG_BLOCK_UVD |
6438 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 6439}
6440
473359bc 6441static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 6442{
473359bc
AD		 6443 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6444 RADEON_CG_BLOCK_SDMA |
6445 RADEON_CG_BLOCK_BIF |
6446 RADEON_CG_BLOCK_UVD |
6447 RADEON_CG_BLOCK_HDP), false);
6448
6449 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 6450}
6451
22c775ce
AD		 6452static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6453 bool enable)
1c49165d 6454{
22c775ce 6455 u32 data, orig;
1c49165d 6456
22c775ce 6457 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6458 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6459 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6460 else
6461 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6462 if (orig != data)
6463 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6464}
6465
22c775ce
AD		 6466static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6467 bool enable)
1c49165d 6468{
22c775ce
AD		 6469 u32 data, orig;
6470
6471 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6472 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6473 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6474 else
6475 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6476 if (orig != data)
6477 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6478}
6479
22c775ce 6480static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 6481{
22c775ce 6482 u32 data, orig;
3ec7d11b 6483
22c775ce 6484 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6485 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 6486 data &= ~DISABLE_CP_PG;
6487 else
6488 data |= DISABLE_CP_PG;
6489 if (orig != data)
6490 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 6491}
6492
22c775ce 6493static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 6494{
22c775ce 6495 u32 data, orig;
f96ab484 6496
22c775ce 6497 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6498 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 6499 data &= ~DISABLE_GDS_PG;
6500 else
6501 data |= DISABLE_GDS_PG;
6502 if (orig != data)
6503 WREG32(RLC_PG_CNTL, data);
6504}
6505
6506#define CP_ME_TABLE_SIZE 96
6507#define CP_ME_TABLE_OFFSET 2048
6508#define CP_MEC_TABLE_OFFSET 4096
6509
6510void cik_init_cp_pg_table(struct radeon_device *rdev)
6511{
22c775ce
AD		 6512 volatile u32 *dst_ptr;
6513 int me, i, max_me = 4;
6514 u32 bo_offset = 0;
f2c6b0f4 6515 u32 table_offset, table_size;
22c775ce
AD		 6516
6517 if (rdev->family == CHIP_KAVERI)
6518 max_me = 5;
6519
6520 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 6521 return;
6522
22c775ce
AD		 6523 /* write the cp table buffer */
6524 dst_ptr = rdev->rlc.cp_table_ptr;
6525 for (me = 0; me < max_me; me++) {
f2c6b0f4
AD		 6526 if (rdev->new_fw) {
6527 const __le32 *fw_data;
6528 const struct gfx_firmware_header_v1_0 *hdr;
6529
6530 if (me == 0) {
6531 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6532 fw_data = (const __le32 *)
6533 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6534 table_offset = le32_to_cpu(hdr->jt_offset);
6535 table_size = le32_to_cpu(hdr->jt_size);
6536 } else if (me == 1) {
6537 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6538 fw_data = (const __le32 *)
6539 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6540 table_offset = le32_to_cpu(hdr->jt_offset);
6541 table_size = le32_to_cpu(hdr->jt_size);
6542 } else if (me == 2) {
6543 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6544 fw_data = (const __le32 *)
6545 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6546 table_offset = le32_to_cpu(hdr->jt_offset);
6547 table_size = le32_to_cpu(hdr->jt_size);
6548 } else if (me == 3) {
6549 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6550 fw_data = (const __le32 *)
6551 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6552 table_offset = le32_to_cpu(hdr->jt_offset);
6553 table_size = le32_to_cpu(hdr->jt_size);
6554 } else {
6555 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6556 fw_data = (const __le32 *)
6557 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6558 table_offset = le32_to_cpu(hdr->jt_offset);
6559 table_size = le32_to_cpu(hdr->jt_size);
6560 }
6561
6562 for (i = 0; i < table_size; i ++) {
6563 dst_ptr[bo_offset + i] =
6564 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6565 }
6566 bo_offset += table_size;
22c775ce 6567 } else {
f2c6b0f4
AD		 6568 const __be32 *fw_data;
6569 table_size = CP_ME_TABLE_SIZE;
6570
6571 if (me == 0) {
6572 fw_data = (const __be32 *)rdev->ce_fw->data;
6573 table_offset = CP_ME_TABLE_OFFSET;
6574 } else if (me == 1) {
6575 fw_data = (const __be32 *)rdev->pfp_fw->data;
6576 table_offset = CP_ME_TABLE_OFFSET;
6577 } else if (me == 2) {
6578 fw_data = (const __be32 *)rdev->me_fw->data;
6579 table_offset = CP_ME_TABLE_OFFSET;
6580 } else {
6581 fw_data = (const __be32 *)rdev->mec_fw->data;
6582 table_offset = CP_MEC_TABLE_OFFSET;
6583 }
22c775ce 6584
f2c6b0f4
AD		 6585 for (i = 0; i < table_size; i ++) {
6586 dst_ptr[bo_offset + i] =
6587 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6588 }
6589 bo_offset += table_size;
22c775ce 6590 }
f96ab484 6591 }
22c775ce 6592}
f96ab484 6593
22c775ce
AD		 6594static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6595 bool enable)
6596{
6597 u32 data, orig;
6598
2b19d17f 6599 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 6600 orig = data = RREG32(RLC_PG_CNTL);
6601 data |= GFX_PG_ENABLE;
6602 if (orig != data)
6603 WREG32(RLC_PG_CNTL, data);
6604
6605 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6606 data |= AUTO_PG_EN;
6607 if (orig != data)
6608 WREG32(RLC_AUTO_PG_CTRL, data);
6609 } else {
6610 orig = data = RREG32(RLC_PG_CNTL);
6611 data &= ~GFX_PG_ENABLE;
6612 if (orig != data)
6613 WREG32(RLC_PG_CNTL, data);
f96ab484 6614
22c775ce
AD		 6615 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6616 data &= ~AUTO_PG_EN;
6617 if (orig != data)
6618 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 6619
22c775ce
AD		 6620 data = RREG32(DB_RENDER_CONTROL);
6621 }
6622}
f96ab484 6623
22c775ce
AD		 6624static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6625{
6626 u32 mask = 0, tmp, tmp1;
6627 int i;
f96ab484 6628
1c0a4625 6629 mutex_lock(&rdev->grbm_idx_mutex);
22c775ce
AD		 6630 cik_select_se_sh(rdev, se, sh);
6631 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6632 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6633 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
1c0a4625 6634 mutex_unlock(&rdev->grbm_idx_mutex);
f96ab484 6635
22c775ce 6636 tmp &= 0xffff0000;
f96ab484 6637
22c775ce
AD		 6638 tmp |= tmp1;
6639 tmp >>= 16;
6640
6641 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6642 mask <<= 1;
6643 mask |= 1;
b07fdd38 6644 }
22c775ce
AD		 6645
6646 return (~tmp) & mask;
f96ab484
AD		 6647}
6648
22c775ce 6649static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 6650{
22c775ce
AD		 6651 u32 i, j, k, active_cu_number = 0;
6652 u32 mask, counter, cu_bitmap;
6653 u32 tmp = 0;
d0e092d9 6654
22c775ce
AD		 6655 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6656 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6657 mask = 1;
6658 cu_bitmap = 0;
6659 counter = 0;
6660 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6661 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6662 if (counter < 2)
6663 cu_bitmap |= mask;
6664 counter ++;
d0e092d9 6665 }
22c775ce 6666 mask <<= 1;
d0e092d9 6667 }
d0e092d9 6668
22c775ce
AD		 6669 active_cu_number += counter;
6670 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 6671 }
d0e092d9 6672 }
22c775ce
AD		 6673
6674 WREG32(RLC_PG_AO_CU_MASK, tmp);
6675
6676 tmp = RREG32(RLC_MAX_PG_CU);
6677 tmp &= ~MAX_PU_CU_MASK;
6678 tmp |= MAX_PU_CU(active_cu_number);
6679 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 6680}
6681
22c775ce
AD		 6682static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6683 bool enable)
605de6b9 6684{
22c775ce 6685 u32 data, orig;
605de6b9 6686
22c775ce 6687 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6688 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 6689 data |= STATIC_PER_CU_PG_ENABLE;
6690 else
6691 data &= ~STATIC_PER_CU_PG_ENABLE;
6692 if (orig != data)
6693 WREG32(RLC_PG_CNTL, data);
6694}
6695
6696static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6697 bool enable)
6698{
6699 u32 data, orig;
605de6b9 6700
22c775ce 6701 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6702 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 6703 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 6704 else
22c775ce
AD		 6705 data &= ~DYN_PER_CU_PG_ENABLE;
6706 if (orig != data)
6707 WREG32(RLC_PG_CNTL, data);
6708}
605de6b9 6709
22c775ce
AD		 6710#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6711#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6712
6713static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6714{
6715 u32 data, orig;
6716 u32 i;
6717
6718 if (rdev->rlc.cs_data) {
6719 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6720 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 6721 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 6722 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 6723 } else {
22c775ce
AD		 6724 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6725 for (i = 0; i < 3; i++)
6726 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6727 }
6728 if (rdev->rlc.reg_list) {
6729 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6730 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6731 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 6732 }
605de6b9 6733
22c775ce
AD		 6734 orig = data = RREG32(RLC_PG_CNTL);
6735 data |= GFX_PG_SRC;
6736 if (orig != data)
6737 WREG32(RLC_PG_CNTL, data);
605de6b9 6738
22c775ce
AD		 6739 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6740 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 6741
22c775ce
AD		 6742 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6743 data &= ~IDLE_POLL_COUNT_MASK;
6744 data |= IDLE_POLL_COUNT(0x60);
6745 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 6746
22c775ce
AD		 6747 data = 0x10101010;
6748 WREG32(RLC_PG_DELAY, data);
605de6b9 6749
22c775ce
AD		 6750 data = RREG32(RLC_PG_DELAY_2);
6751 data &= ~0xff;
6752 data |= 0x3;
6753 WREG32(RLC_PG_DELAY_2, data);
605de6b9 6754
22c775ce
AD		 6755 data = RREG32(RLC_AUTO_PG_CTRL);
6756 data &= ~GRBM_REG_SGIT_MASK;
6757 data |= GRBM_REG_SGIT(0x700);
6758 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 6759
605de6b9
AD		 6760}
6761
22c775ce 6762static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 6763{
473359bc
AD		 6764 cik_enable_gfx_cgpg(rdev, enable);
6765 cik_enable_gfx_static_mgpg(rdev, enable);
6766 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 6767}
f6796cae 6768
a0f38609
AD		 6769u32 cik_get_csb_size(struct radeon_device *rdev)
6770{
6771 u32 count = 0;
6772 const struct cs_section_def *sect = NULL;
6773 const struct cs_extent_def *ext = NULL;
f6796cae 6774
a0f38609
AD		 6775 if (rdev->rlc.cs_data == NULL)
6776 return 0;
f6796cae 6777
a0f38609
AD		 6778 /* begin clear state */
6779 count += 2;
6780 /* context control state */
6781 count += 3;
6782
6783 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6784 for (ext = sect->section; ext->extent != NULL; ++ext) {
6785 if (sect->id == SECT_CONTEXT)
6786 count += 2 + ext->reg_count;
6787 else
6788 return 0;
f6796cae
AD		 6789 }
6790 }
a0f38609
AD		 6791 /* pa_sc_raster_config/pa_sc_raster_config1 */
6792 count += 4;
6793 /* end clear state */
6794 count += 2;
6795 /* clear state */
6796 count += 2;
f6796cae 6797
a0f38609 6798 return count;
f6796cae
AD		 6799}
6800
a0f38609 6801void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 6802{
a0f38609
AD		 6803 u32 count = 0, i;
6804 const struct cs_section_def *sect = NULL;
6805 const struct cs_extent_def *ext = NULL;
f6796cae 6806
a0f38609
AD		 6807 if (rdev->rlc.cs_data == NULL)
6808 return;
6809 if (buffer == NULL)
6810 return;
f6796cae 6811
6ba81e53
AD		 6812 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6813 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
a0f38609 6814
6ba81e53
AD		 6815 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6816 buffer[count++] = cpu_to_le32(0x80000000);
6817 buffer[count++] = cpu_to_le32(0x80000000);
a0f38609
AD		 6818
6819 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6820 for (ext = sect->section; ext->extent != NULL; ++ext) {
6821 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 6822 buffer[count++] =
6823 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6824 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
a0f38609 6825 for (i = 0; i < ext->reg_count; i++)
6ba81e53 6826 buffer[count++] = cpu_to_le32(ext->extent[i]);
a0f38609
AD		 6827 } else {
6828 return;
6829 }
6830 }
6831 }
f6796cae 6832
6ba81e53
AD		 6833 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6834 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
f6796cae
AD		 6835 switch (rdev->family) {
6836 case CHIP_BONAIRE:
6ba81e53
AD		 6837 buffer[count++] = cpu_to_le32(0x16000012);
6838 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6839 break;
6840 case CHIP_KAVERI:
6ba81e53
AD		 6841 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6842 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6843 break;
6844 case CHIP_KABINI:
f73a9e83 6845 case CHIP_MULLINS:
6ba81e53
AD		 6846 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6847 buffer[count++] = cpu_to_le32(0x00000000);
a0f38609 6848 break;
bbfe90bd 6849 case CHIP_HAWAII:
a8947f57
AD		 6850 buffer[count++] = cpu_to_le32(0x3a00161a);
6851 buffer[count++] = cpu_to_le32(0x0000002e);
bbfe90bd 6852 break;
a0f38609 6853 default:
6ba81e53
AD		 6854 buffer[count++] = cpu_to_le32(0x00000000);
6855 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 6856 break;
6857 }
6858
6ba81e53
AD		 6859 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6860 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
f6796cae 6861
6ba81e53
AD		 6862 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6863 buffer[count++] = cpu_to_le32(0);
a0f38609 6864}
f6796cae 6865
473359bc 6866static void cik_init_pg(struct radeon_device *rdev)
22c775ce 6867{
473359bc 6868 if (rdev->pg_flags) {
22c775ce
AD		 6869 cik_enable_sck_slowdown_on_pu(rdev, true);
6870 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 6871 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 6872 cik_init_gfx_cgpg(rdev);
6873 cik_enable_cp_pg(rdev, true);
6874 cik_enable_gds_pg(rdev, true);
6875 }
22c775ce
AD		 6876 cik_init_ao_cu_mask(rdev);
6877 cik_update_gfx_pg(rdev, true);
6878 }
6879}
f6796cae 6880
473359bc
AD		 6881static void cik_fini_pg(struct radeon_device *rdev)
6882{
6883 if (rdev->pg_flags) {
6884 cik_update_gfx_pg(rdev, false);
2b19d17f 6885 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 6886 cik_enable_cp_pg(rdev, false);
6887 cik_enable_gds_pg(rdev, false);
6888 }
6889 }
f6796cae 6890}
a59781bb
AD		 6891
6892/*
6893 * Interrupts
6894 * Starting with r6xx, interrupts are handled via a ring buffer.
6895 * Ring buffers are areas of GPU accessible memory that the GPU
6896 * writes interrupt vectors into and the host reads vectors out of.
6897 * There is a rptr (read pointer) that determines where the
6898 * host is currently reading, and a wptr (write pointer)
6899 * which determines where the GPU has written. When the
6900 * pointers are equal, the ring is idle. When the GPU
6901 * writes vectors to the ring buffer, it increments the
6902 * wptr. When there is an interrupt, the host then starts
6903 * fetching commands and processing them until the pointers are
6904 * equal again at which point it updates the rptr.
6905 */
6906
6907/**
6908 * cik_enable_interrupts - Enable the interrupt ring buffer
6909 *
6910 * @rdev: radeon_device pointer
6911 *
6912 * Enable the interrupt ring buffer (CIK).
6913 */
6914static void cik_enable_interrupts(struct radeon_device *rdev)
6915{
6916 u32 ih_cntl = RREG32(IH_CNTL);
6917 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6918
6919 ih_cntl |= ENABLE_INTR;
6920 ih_rb_cntl |= IH_RB_ENABLE;
6921 WREG32(IH_CNTL, ih_cntl);
6922 WREG32(IH_RB_CNTL, ih_rb_cntl);
6923 rdev->ih.enabled = true;
6924}
6925
6926/**
6927 * cik_disable_interrupts - Disable the interrupt ring buffer
6928 *
6929 * @rdev: radeon_device pointer
6930 *
6931 * Disable the interrupt ring buffer (CIK).
6932 */
6933static void cik_disable_interrupts(struct radeon_device *rdev)
6934{
6935 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6936 u32 ih_cntl = RREG32(IH_CNTL);
6937
6938 ih_rb_cntl &= ~IH_RB_ENABLE;
6939 ih_cntl &= ~ENABLE_INTR;
6940 WREG32(IH_RB_CNTL, ih_rb_cntl);
6941 WREG32(IH_CNTL, ih_cntl);
6942 /* set rptr, wptr to 0 */
6943 WREG32(IH_RB_RPTR, 0);
6944 WREG32(IH_RB_WPTR, 0);
6945 rdev->ih.enabled = false;
6946 rdev->ih.rptr = 0;
6947}
6948
6949/**
6950 * cik_disable_interrupt_state - Disable all interrupt sources
6951 *
6952 * @rdev: radeon_device pointer
6953 *
6954 * Clear all interrupt enable bits used by the driver (CIK).
6955 */
6956static void cik_disable_interrupt_state(struct radeon_device *rdev)
6957{
6958 u32 tmp;
6959
6960 /* gfx ring */
4214faf6
AD		 6961 tmp = RREG32(CP_INT_CNTL_RING0) &
6962 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6963 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 6964 /* sdma */
6965 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6966 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6967 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6968 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 6969 /* compute queues */
6970 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6971 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6972 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6973 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6974 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6975 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6976 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6977 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6978 /* grbm */
6979 WREG32(GRBM_INT_CNTL, 0);
dc12a3ec
LL		 6980 /* SRBM */
6981 WREG32(SRBM_INT_CNTL, 0);
a59781bb
AD		 6982 /* vline/vblank, etc. */
6983 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6984 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6985 if (rdev->num_crtc >= 4) {
6986 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6987 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6988 }
6989 if (rdev->num_crtc >= 6) {
6990 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6991 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6992 }
f5d636d2
CK		 6993 /* pflip */
6994 if (rdev->num_crtc >= 2) {
6995 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6996 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6997 }
6998 if (rdev->num_crtc >= 4) {
6999 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7000 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7001 }
7002 if (rdev->num_crtc >= 6) {
7003 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7004 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7005 }
a59781bb
AD		 7006
7007 /* dac hotplug */
7008 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7009
7010 /* digital hotplug */
7011 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7012 WREG32(DC_HPD1_INT_CONTROL, tmp);
7013 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7014 WREG32(DC_HPD2_INT_CONTROL, tmp);
7015 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7016 WREG32(DC_HPD3_INT_CONTROL, tmp);
7017 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7018 WREG32(DC_HPD4_INT_CONTROL, tmp);
7019 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7020 WREG32(DC_HPD5_INT_CONTROL, tmp);
7021 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7022 WREG32(DC_HPD6_INT_CONTROL, tmp);
7023
7024}
7025
7026/**
7027 * cik_irq_init - init and enable the interrupt ring
7028 *
7029 * @rdev: radeon_device pointer
7030 *
7031 * Allocate a ring buffer for the interrupt controller,
7032 * enable the RLC, disable interrupts, enable the IH
7033 * ring buffer and enable it (CIK).
7034 * Called at device load and reume.
7035 * Returns 0 for success, errors for failure.
7036 */
7037static int cik_irq_init(struct radeon_device *rdev)
7038{
7039 int ret = 0;
7040 int rb_bufsz;
7041 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7042
7043 /* allocate ring */
7044 ret = r600_ih_ring_alloc(rdev);
7045 if (ret)
7046 return ret;
7047
7048 /* disable irqs */
7049 cik_disable_interrupts(rdev);
7050
7051 /* init rlc */
7052 ret = cik_rlc_resume(rdev);
7053 if (ret) {
7054 r600_ih_ring_fini(rdev);
7055 return ret;
7056 }
7057
7058 /* setup interrupt control */
7059 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7060 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7061 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7062 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7063 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7064 */
7065 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7066 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7067 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7068 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7069
7070 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 7071 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 7072
7073 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7074 IH_WPTR_OVERFLOW_CLEAR |
7075 (rb_bufsz << 1));
7076
7077 if (rdev->wb.enabled)
7078 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7079
7080 /* set the writeback address whether it's enabled or not */
7081 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7082 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7083
7084 WREG32(IH_RB_CNTL, ih_rb_cntl);
7085
7086 /* set rptr, wptr to 0 */
7087 WREG32(IH_RB_RPTR, 0);
7088 WREG32(IH_RB_WPTR, 0);
7089
7090 /* Default settings for IH_CNTL (disabled at first) */
7091 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7092 /* RPTR_REARM only works if msi's are enabled */
7093 if (rdev->msi_enabled)
7094 ih_cntl |= RPTR_REARM;
7095 WREG32(IH_CNTL, ih_cntl);
7096
7097 /* force the active interrupt state to all disabled */
7098 cik_disable_interrupt_state(rdev);
7099
7100 pci_set_master(rdev->pdev);
7101
7102 /* enable irqs */
7103 cik_enable_interrupts(rdev);
7104
7105 return ret;
7106}
7107
7108/**
7109 * cik_irq_set - enable/disable interrupt sources
7110 *
7111 * @rdev: radeon_device pointer
7112 *
7113 * Enable interrupt sources on the GPU (vblanks, hpd,
7114 * etc.) (CIK).
7115 * Returns 0 for success, errors for failure.
7116 */
7117int cik_irq_set(struct radeon_device *rdev)
7118{
4214faf6 7119 u32 cp_int_cntl;
28b57b85 7120 u32 cp_m1p0;
a59781bb
AD		 7121 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7122 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7123 u32 grbm_int_cntl = 0;
21a93e13 7124 u32 dma_cntl, dma_cntl1;
a59781bb
AD		 7125
7126 if (!rdev->irq.installed) {
7127 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7128 return -EINVAL;
7129 }
7130 /* don't enable anything if the ih is disabled */
7131 if (!rdev->ih.enabled) {
7132 cik_disable_interrupts(rdev);
7133 /* force the active interrupt state to all disabled */
7134 cik_disable_interrupt_state(rdev);
7135 return 0;
7136 }
7137
4214faf6
AD		 7138 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7139 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7140 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7141
f6b355dd
AD		 7142 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7143 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7144 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7145 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7146 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
7147 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
a59781bb 7148
21a93e13
AD		 7149 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7150 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7151
2b0781a6 7152 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
2b0781a6 7153
a59781bb
AD		 7154 /* enable CP interrupts on all rings */
7155 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7156 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7157 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7158 }
2b0781a6
AD		 7159 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7160 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7161 DRM_DEBUG("si_irq_set: sw int cp1\n");
7162 if (ring->me == 1) {
7163 switch (ring->pipe) {
7164 case 0:
7165 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7166 break;
2b0781a6
AD		 7167 default:
7168 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7169 break;
7170 }
7171 } else {
7172 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7173 }
7174 }
7175 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7176 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7177 DRM_DEBUG("si_irq_set: sw int cp2\n");
7178 if (ring->me == 1) {
7179 switch (ring->pipe) {
7180 case 0:
7181 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7182 break;
2b0781a6
AD		 7183 default:
7184 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7185 break;
7186 }
7187 } else {
7188 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7189 }
7190 }
a59781bb 7191
21a93e13
AD		 7192 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7193 DRM_DEBUG("cik_irq_set: sw int dma\n");
7194 dma_cntl |= TRAP_ENABLE;
7195 }
7196
7197 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7198 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7199 dma_cntl1 |= TRAP_ENABLE;
7200 }
7201
a59781bb
AD		 7202 if (rdev->irq.crtc_vblank_int[0] ||
7203 atomic_read(&rdev->irq.pflip[0])) {
7204 DRM_DEBUG("cik_irq_set: vblank 0\n");
7205 crtc1 |= VBLANK_INTERRUPT_MASK;
7206 }
7207 if (rdev->irq.crtc_vblank_int[1] ||
7208 atomic_read(&rdev->irq.pflip[1])) {
7209 DRM_DEBUG("cik_irq_set: vblank 1\n");
7210 crtc2 |= VBLANK_INTERRUPT_MASK;
7211 }
7212 if (rdev->irq.crtc_vblank_int[2] ||
7213 atomic_read(&rdev->irq.pflip[2])) {
7214 DRM_DEBUG("cik_irq_set: vblank 2\n");
7215 crtc3 |= VBLANK_INTERRUPT_MASK;
7216 }
7217 if (rdev->irq.crtc_vblank_int[3] ||
7218 atomic_read(&rdev->irq.pflip[3])) {
7219 DRM_DEBUG("cik_irq_set: vblank 3\n");
7220 crtc4 |= VBLANK_INTERRUPT_MASK;
7221 }
7222 if (rdev->irq.crtc_vblank_int[4] ||
7223 atomic_read(&rdev->irq.pflip[4])) {
7224 DRM_DEBUG("cik_irq_set: vblank 4\n");
7225 crtc5 |= VBLANK_INTERRUPT_MASK;
7226 }
7227 if (rdev->irq.crtc_vblank_int[5] ||
7228 atomic_read(&rdev->irq.pflip[5])) {
7229 DRM_DEBUG("cik_irq_set: vblank 5\n");
7230 crtc6 |= VBLANK_INTERRUPT_MASK;
7231 }
7232 if (rdev->irq.hpd[0]) {
7233 DRM_DEBUG("cik_irq_set: hpd 1\n");
f6b355dd 7234 hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7235 }
7236 if (rdev->irq.hpd[1]) {
7237 DRM_DEBUG("cik_irq_set: hpd 2\n");
f6b355dd 7238 hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7239 }
7240 if (rdev->irq.hpd[2]) {
7241 DRM_DEBUG("cik_irq_set: hpd 3\n");
f6b355dd 7242 hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7243 }
7244 if (rdev->irq.hpd[3]) {
7245 DRM_DEBUG("cik_irq_set: hpd 4\n");
f6b355dd 7246 hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7247 }
7248 if (rdev->irq.hpd[4]) {
7249 DRM_DEBUG("cik_irq_set: hpd 5\n");
f6b355dd 7250 hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7251 }
7252 if (rdev->irq.hpd[5]) {
7253 DRM_DEBUG("cik_irq_set: hpd 6\n");
f6b355dd 7254 hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
a59781bb
AD		 7255 }
7256
7257 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7258
21a93e13
AD		 7259 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7260 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7261
2b0781a6 7262 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
2b0781a6 7263
a59781bb
AD		 7264 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7265
7266 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7267 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7268 if (rdev->num_crtc >= 4) {
7269 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7270 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7271 }
7272 if (rdev->num_crtc >= 6) {
7273 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7274 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7275 }
7276
f5d636d2
CK		 7277 if (rdev->num_crtc >= 2) {
7278 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7279 GRPH_PFLIP_INT_MASK);
7280 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7281 GRPH_PFLIP_INT_MASK);
7282 }
7283 if (rdev->num_crtc >= 4) {
7284 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7285 GRPH_PFLIP_INT_MASK);
7286 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7287 GRPH_PFLIP_INT_MASK);
7288 }
7289 if (rdev->num_crtc >= 6) {
7290 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7291 GRPH_PFLIP_INT_MASK);
7292 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7293 GRPH_PFLIP_INT_MASK);
7294 }
7295
a59781bb
AD		 7296 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7297 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7298 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7299 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7300 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7301 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7302
cffefd9b
AD		 7303 /* posting read */
7304 RREG32(SRBM_STATUS);
7305
a59781bb
AD		 7306 return 0;
7307}
7308
7309/**
7310 * cik_irq_ack - ack interrupt sources
7311 *
7312 * @rdev: radeon_device pointer
7313 *
7314 * Ack interrupt sources on the GPU (vblanks, hpd,
7315 * etc.) (CIK). Certain interrupts sources are sw
7316 * generated and do not require an explicit ack.
7317 */
7318static inline void cik_irq_ack(struct radeon_device *rdev)
7319{
7320 u32 tmp;
7321
7322 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7323 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7324 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7325 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7326 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7327 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7328 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7329
f5d636d2
CK		 7330 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7331 EVERGREEN_CRTC0_REGISTER_OFFSET);
7332 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7333 EVERGREEN_CRTC1_REGISTER_OFFSET);
7334 if (rdev->num_crtc >= 4) {
7335 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7336 EVERGREEN_CRTC2_REGISTER_OFFSET);
7337 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7338 EVERGREEN_CRTC3_REGISTER_OFFSET);
7339 }
7340 if (rdev->num_crtc >= 6) {
7341 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7342 EVERGREEN_CRTC4_REGISTER_OFFSET);
7343 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7344 EVERGREEN_CRTC5_REGISTER_OFFSET);
7345 }
7346
7347 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7348 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7349 GRPH_PFLIP_INT_CLEAR);
7350 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7351 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7352 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7353 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7354 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7355 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7356 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7357 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7358 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7359 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7360 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7361
7362 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 7363 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7364 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7365 GRPH_PFLIP_INT_CLEAR);
7366 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7367 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7368 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7369 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7370 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7371 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7372 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7373 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7374 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7375 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7376 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7377 }
7378
7379 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 7380 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7381 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7382 GRPH_PFLIP_INT_CLEAR);
7383 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7384 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7385 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7386 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7387 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7388 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7389 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7390 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7391 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7392 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7393 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7394 }
7395
7396 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7397 tmp = RREG32(DC_HPD1_INT_CONTROL);
7398 tmp |= DC_HPDx_INT_ACK;
7399 WREG32(DC_HPD1_INT_CONTROL, tmp);
7400 }
7401 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7402 tmp = RREG32(DC_HPD2_INT_CONTROL);
7403 tmp |= DC_HPDx_INT_ACK;
7404 WREG32(DC_HPD2_INT_CONTROL, tmp);
7405 }
7406 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7407 tmp = RREG32(DC_HPD3_INT_CONTROL);
7408 tmp |= DC_HPDx_INT_ACK;
7409 WREG32(DC_HPD3_INT_CONTROL, tmp);
7410 }
7411 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7412 tmp = RREG32(DC_HPD4_INT_CONTROL);
7413 tmp |= DC_HPDx_INT_ACK;
7414 WREG32(DC_HPD4_INT_CONTROL, tmp);
7415 }
7416 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7417 tmp = RREG32(DC_HPD5_INT_CONTROL);
7418 tmp |= DC_HPDx_INT_ACK;
7419 WREG32(DC_HPD5_INT_CONTROL, tmp);
7420 }
7421 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7422 tmp = RREG32(DC_HPD5_INT_CONTROL);
7423 tmp |= DC_HPDx_INT_ACK;
7424 WREG32(DC_HPD6_INT_CONTROL, tmp);
7425 }
f6b355dd
AD		 7426 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT) {
7427 tmp = RREG32(DC_HPD1_INT_CONTROL);
7428 tmp |= DC_HPDx_RX_INT_ACK;
7429 WREG32(DC_HPD1_INT_CONTROL, tmp);
7430 }
7431 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
7432 tmp = RREG32(DC_HPD2_INT_CONTROL);
7433 tmp |= DC_HPDx_RX_INT_ACK;
7434 WREG32(DC_HPD2_INT_CONTROL, tmp);
7435 }
7436 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
7437 tmp = RREG32(DC_HPD3_INT_CONTROL);
7438 tmp |= DC_HPDx_RX_INT_ACK;
7439 WREG32(DC_HPD3_INT_CONTROL, tmp);
7440 }
7441 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
7442 tmp = RREG32(DC_HPD4_INT_CONTROL);
7443 tmp |= DC_HPDx_RX_INT_ACK;
7444 WREG32(DC_HPD4_INT_CONTROL, tmp);
7445 }
7446 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
7447 tmp = RREG32(DC_HPD5_INT_CONTROL);
7448 tmp |= DC_HPDx_RX_INT_ACK;
7449 WREG32(DC_HPD5_INT_CONTROL, tmp);
7450 }
7451 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
7452 tmp = RREG32(DC_HPD5_INT_CONTROL);
7453 tmp |= DC_HPDx_RX_INT_ACK;
7454 WREG32(DC_HPD6_INT_CONTROL, tmp);
7455 }
a59781bb
AD		 7456}
7457
7458/**
7459 * cik_irq_disable - disable interrupts
7460 *
7461 * @rdev: radeon_device pointer
7462 *
7463 * Disable interrupts on the hw (CIK).
7464 */
7465static void cik_irq_disable(struct radeon_device *rdev)
7466{
7467 cik_disable_interrupts(rdev);
7468 /* Wait and acknowledge irq */
7469 mdelay(1);
7470 cik_irq_ack(rdev);
7471 cik_disable_interrupt_state(rdev);
7472}
7473
7474/**
7475 * cik_irq_disable - disable interrupts for suspend
7476 *
7477 * @rdev: radeon_device pointer
7478 *
7479 * Disable interrupts and stop the RLC (CIK).
7480 * Used for suspend.
7481 */
7482static void cik_irq_suspend(struct radeon_device *rdev)
7483{
7484 cik_irq_disable(rdev);
7485 cik_rlc_stop(rdev);
7486}
7487
7488/**
7489 * cik_irq_fini - tear down interrupt support
7490 *
7491 * @rdev: radeon_device pointer
7492 *
7493 * Disable interrupts on the hw and free the IH ring
7494 * buffer (CIK).
7495 * Used for driver unload.
7496 */
7497static void cik_irq_fini(struct radeon_device *rdev)
7498{
7499 cik_irq_suspend(rdev);
7500 r600_ih_ring_fini(rdev);
7501}
7502
7503/**
7504 * cik_get_ih_wptr - get the IH ring buffer wptr
7505 *
7506 * @rdev: radeon_device pointer
7507 *
7508 * Get the IH ring buffer wptr from either the register
7509 * or the writeback memory buffer (CIK). Also check for
7510 * ring buffer overflow and deal with it.
7511 * Used by cik_irq_process().
7512 * Returns the value of the wptr.
7513 */
7514static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7515{
7516 u32 wptr, tmp;
7517
7518 if (rdev->wb.enabled)
7519 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7520 else
7521 wptr = RREG32(IH_RB_WPTR);
7522
7523 if (wptr & RB_OVERFLOW) {
11bab0ae 7524 wptr &= ~RB_OVERFLOW;
a59781bb
AD		 7525 /* When a ring buffer overflow happen start parsing interrupt
7526 * from the last not overwritten vector (wptr + 16). Hopefully
7527 * this should allow us to catchup.
7528 */
6cc2fda2
MD		 7529 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
7530 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
a59781bb
AD		 7531 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7532 tmp = RREG32(IH_RB_CNTL);
7533 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7534 WREG32(IH_RB_CNTL, tmp);
7535 }
7536 return (wptr & rdev->ih.ptr_mask);
7537}
7538
7539/* CIK IV Ring
7540 * Each IV ring entry is 128 bits:
7541 * [7:0] - interrupt source id
7542 * [31:8] - reserved
7543 * [59:32] - interrupt source data
7544 * [63:60] - reserved
21a93e13
AD		 7545 * [71:64] - RINGID
7546 * CP:
7547 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 7548 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7549 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7550 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7551 * PIPE_ID - ME0 0=3D
7552 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 7553 * SDMA:
7554 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7555 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7556 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 7557 * [79:72] - VMID
7558 * [95:80] - PASID
7559 * [127:96] - reserved
7560 */
7561/**
7562 * cik_irq_process - interrupt handler
7563 *
7564 * @rdev: radeon_device pointer
7565 *
7566 * Interrupt hander (CIK). Walk the IH ring,
7567 * ack interrupts and schedule work to handle
7568 * interrupt events.
7569 * Returns irq process return code.
7570 */
7571int cik_irq_process(struct radeon_device *rdev)
7572{
2b0781a6
AD		 7573 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7574 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 7575 u32 wptr;
7576 u32 rptr;
7577 u32 src_id, src_data, ring_id;
7578 u8 me_id, pipe_id, queue_id;
7579 u32 ring_index;
7580 bool queue_hotplug = false;
f6b355dd 7581 bool queue_dp = false;
a59781bb 7582 bool queue_reset = false;
3ec7d11b 7583 u32 addr, status, mc_client;
41a524ab 7584 bool queue_thermal = false;
a59781bb
AD		 7585
7586 if (!rdev->ih.enabled || rdev->shutdown)
7587 return IRQ_NONE;
7588
7589 wptr = cik_get_ih_wptr(rdev);
7590
7591restart_ih:
7592 /* is somebody else already processing irqs? */
7593 if (atomic_xchg(&rdev->ih.lock, 1))
7594 return IRQ_NONE;
7595
7596 rptr = rdev->ih.rptr;
7597 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7598
7599 /* Order reading of wptr vs. reading of IH ring data */
7600 rmb();
7601
7602 /* display interrupts */
7603 cik_irq_ack(rdev);
7604
7605 while (rptr != wptr) {
7606 /* wptr/rptr are in bytes! */
7607 ring_index = rptr / 4;
e28740ec
OG		 7608
7609 radeon_kfd_interrupt(rdev,
7610 (const void *) &rdev->ih.ring[ring_index]);
7611
a59781bb
AD		 7612 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7613 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7614 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 7615
7616 switch (src_id) {
7617 case 1: /* D1 vblank/vline */
7618 switch (src_data) {
7619 case 0: /* D1 vblank */
07f18f0b
MK		 7620 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT))
7621 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7622
7623 if (rdev->irq.crtc_vblank_int[0]) {
7624 drm_handle_vblank(rdev->ddev, 0);
7625 rdev->pm.vblank_sync = true;
7626 wake_up(&rdev->irq.vblank_queue);
a59781bb 7627 }
07f18f0b
MK		 7628 if (atomic_read(&rdev->irq.pflip[0]))
7629 radeon_crtc_handle_vblank(rdev, 0);
7630 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7631 DRM_DEBUG("IH: D1 vblank\n");
7632
a59781bb
AD		 7633 break;
7634 case 1: /* D1 vline */
07f18f0b
MK		 7635 if (!(rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT))
7636 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7637
7638 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7639 DRM_DEBUG("IH: D1 vline\n");
7640
a59781bb
AD		 7641 break;
7642 default:
7643 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7644 break;
7645 }
7646 break;
7647 case 2: /* D2 vblank/vline */
7648 switch (src_data) {
7649 case 0: /* D2 vblank */
07f18f0b
MK		 7650 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
7651 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7652
7653 if (rdev->irq.crtc_vblank_int[1]) {
7654 drm_handle_vblank(rdev->ddev, 1);
7655 rdev->pm.vblank_sync = true;
7656 wake_up(&rdev->irq.vblank_queue);
a59781bb 7657 }
07f18f0b
MK		 7658 if (atomic_read(&rdev->irq.pflip[1]))
7659 radeon_crtc_handle_vblank(rdev, 1);
7660 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7661 DRM_DEBUG("IH: D2 vblank\n");
7662
a59781bb
AD		 7663 break;
7664 case 1: /* D2 vline */
07f18f0b
MK		 7665 if (!(rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT))
7666 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7667
7668 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7669 DRM_DEBUG("IH: D2 vline\n");
7670
a59781bb
AD		 7671 break;
7672 default:
7673 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7674 break;
7675 }
7676 break;
7677 case 3: /* D3 vblank/vline */
7678 switch (src_data) {
7679 case 0: /* D3 vblank */
07f18f0b
MK		 7680 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
7681 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7682
7683 if (rdev->irq.crtc_vblank_int[2]) {
7684 drm_handle_vblank(rdev->ddev, 2);
7685 rdev->pm.vblank_sync = true;
7686 wake_up(&rdev->irq.vblank_queue);
a59781bb 7687 }
07f18f0b
MK		 7688 if (atomic_read(&rdev->irq.pflip[2]))
7689 radeon_crtc_handle_vblank(rdev, 2);
7690 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7691 DRM_DEBUG("IH: D3 vblank\n");
7692
a59781bb
AD		 7693 break;
7694 case 1: /* D3 vline */
07f18f0b
MK		 7695 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
7696 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7697
7698 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7699 DRM_DEBUG("IH: D3 vline\n");
7700
a59781bb
AD		 7701 break;
7702 default:
7703 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7704 break;
7705 }
7706 break;
7707 case 4: /* D4 vblank/vline */
7708 switch (src_data) {
7709 case 0: /* D4 vblank */
07f18f0b
MK		 7710 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
7711 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7712
7713 if (rdev->irq.crtc_vblank_int[3]) {
7714 drm_handle_vblank(rdev->ddev, 3);
7715 rdev->pm.vblank_sync = true;
7716 wake_up(&rdev->irq.vblank_queue);
a59781bb 7717 }
07f18f0b
MK		 7718 if (atomic_read(&rdev->irq.pflip[3]))
7719 radeon_crtc_handle_vblank(rdev, 3);
7720 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7721 DRM_DEBUG("IH: D4 vblank\n");
7722
a59781bb
AD		 7723 break;
7724 case 1: /* D4 vline */
07f18f0b
MK		 7725 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
7726 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7727
7728 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7729 DRM_DEBUG("IH: D4 vline\n");
7730
a59781bb
AD		 7731 break;
7732 default:
7733 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7734 break;
7735 }
7736 break;
7737 case 5: /* D5 vblank/vline */
7738 switch (src_data) {
7739 case 0: /* D5 vblank */
07f18f0b
MK		 7740 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
7741 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7742
7743 if (rdev->irq.crtc_vblank_int[4]) {
7744 drm_handle_vblank(rdev->ddev, 4);
7745 rdev->pm.vblank_sync = true;
7746 wake_up(&rdev->irq.vblank_queue);
a59781bb 7747 }
07f18f0b
MK		 7748 if (atomic_read(&rdev->irq.pflip[4]))
7749 radeon_crtc_handle_vblank(rdev, 4);
7750 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7751 DRM_DEBUG("IH: D5 vblank\n");
7752
a59781bb
AD		 7753 break;
7754 case 1: /* D5 vline */
07f18f0b
MK		 7755 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
7756 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7757
7758 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7759 DRM_DEBUG("IH: D5 vline\n");
7760
a59781bb
AD		 7761 break;
7762 default:
7763 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7764 break;
7765 }
7766 break;
7767 case 6: /* D6 vblank/vline */
7768 switch (src_data) {
7769 case 0: /* D6 vblank */
07f18f0b
MK		 7770 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
7771 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7772
7773 if (rdev->irq.crtc_vblank_int[5]) {
7774 drm_handle_vblank(rdev->ddev, 5);
7775 rdev->pm.vblank_sync = true;
7776 wake_up(&rdev->irq.vblank_queue);
a59781bb 7777 }
07f18f0b
MK		 7778 if (atomic_read(&rdev->irq.pflip[5]))
7779 radeon_crtc_handle_vblank(rdev, 5);
7780 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7781 DRM_DEBUG("IH: D6 vblank\n");
7782
a59781bb
AD		 7783 break;
7784 case 1: /* D6 vline */
07f18f0b
MK		 7785 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
7786 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7787
7788 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7789 DRM_DEBUG("IH: D6 vline\n");
7790
a59781bb
AD		 7791 break;
7792 default:
7793 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7794 break;
7795 }
7796 break;
f5d636d2
CK		 7797 case 8: /* D1 page flip */
7798 case 10: /* D2 page flip */
7799 case 12: /* D3 page flip */
7800 case 14: /* D4 page flip */
7801 case 16: /* D5 page flip */
7802 case 18: /* D6 page flip */
7803 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 7804 if (radeon_use_pflipirq > 0)
7805 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 7806 break;
a59781bb
AD		 7807 case 42: /* HPD hotplug */
7808 switch (src_data) {
7809 case 0:
07f18f0b
MK		 7810 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT))
7811 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7812
7813 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7814 queue_hotplug = true;
7815 DRM_DEBUG("IH: HPD1\n");
7816
a59781bb
AD		 7817 break;
7818 case 1:
07f18f0b
MK		 7819 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT))
7820 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7821
7822 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7823 queue_hotplug = true;
7824 DRM_DEBUG("IH: HPD2\n");
7825
a59781bb
AD		 7826 break;
7827 case 2:
07f18f0b
MK		 7828 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT))
7829 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7830
7831 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7832 queue_hotplug = true;
7833 DRM_DEBUG("IH: HPD3\n");
7834
a59781bb
AD		 7835 break;
7836 case 3:
07f18f0b
MK		 7837 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT))
7838 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7839
7840 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7841 queue_hotplug = true;
7842 DRM_DEBUG("IH: HPD4\n");
7843
a59781bb
AD		 7844 break;
7845 case 4:
07f18f0b
MK		 7846 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT))
7847 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7848
7849 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7850 queue_hotplug = true;
7851 DRM_DEBUG("IH: HPD5\n");
7852
a59781bb
AD		 7853 break;
7854 case 5:
07f18f0b
MK		 7855 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT))
7856 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7857
7858 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7859 queue_hotplug = true;
7860 DRM_DEBUG("IH: HPD6\n");
7861
a59781bb 7862 break;
f6b355dd 7863 case 6:
07f18f0b
MK		 7864 if (!(rdev->irq.stat_regs.cik.disp_int & DC_HPD1_RX_INTERRUPT))
7865 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7866
7867 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_RX_INTERRUPT;
7868 queue_dp = true;
7869 DRM_DEBUG("IH: HPD_RX 1\n");
7870
f6b355dd
AD		 7871 break;
7872 case 7:
07f18f0b
MK		 7873 if (!(rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_RX_INTERRUPT))
7874 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7875
7876 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
7877 queue_dp = true;
7878 DRM_DEBUG("IH: HPD_RX 2\n");
7879
f6b355dd
AD		 7880 break;
7881 case 8:
07f18f0b
MK		 7882 if (!(rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
7883 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7884
7885 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
7886 queue_dp = true;
7887 DRM_DEBUG("IH: HPD_RX 3\n");
7888
f6b355dd
AD		 7889 break;
7890 case 9:
07f18f0b
MK		 7891 if (!(rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
7892 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7893
7894 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
7895 queue_dp = true;
7896 DRM_DEBUG("IH: HPD_RX 4\n");
7897
f6b355dd
AD		 7898 break;
7899 case 10:
07f18f0b
MK		 7900 if (!(rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
7901 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7902
7903 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
7904 queue_dp = true;
7905 DRM_DEBUG("IH: HPD_RX 5\n");
7906
f6b355dd
AD		 7907 break;
7908 case 11:
07f18f0b
MK		 7909 if (!(rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
7910 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
7911
7912 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
7913 queue_dp = true;
7914 DRM_DEBUG("IH: HPD_RX 6\n");
7915
f6b355dd 7916 break;
a59781bb
AD		 7917 default:
7918 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7919 break;
7920 }
7921 break;
dc12a3ec
LL		 7922 case 96:
7923 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
7924 WREG32(SRBM_INT_ACK, 0x1);
7925 break;
6a3808b8
CK		 7926 case 124: /* UVD */
7927 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7928 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7929 break;
9d97c99b
AD		 7930 case 146:
7931 case 147:
3ec7d11b
AD		 7932 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7933 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7934 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9b7d786b
CK		 7935 /* reset addr and status */
7936 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7937 if (addr == 0x0 && status == 0x0)
7938 break;
9d97c99b
AD		 7939 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7940 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 7941 addr);
9d97c99b 7942 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 7943 status);
7944 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b 7945 break;
d93f7937
CK		 7946 case 167: /* VCE */
7947 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
7948 switch (src_data) {
7949 case 0:
7950 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
7951 break;
7952 case 1:
7953 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
7954 break;
7955 default:
7956 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
7957 break;
7958 }
7959 break;
a59781bb
AD		 7960 case 176: /* GFX RB CP_INT */
7961 case 177: /* GFX IB CP_INT */
7962 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7963 break;
7964 case 181: /* CP EOP event */
7965 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 7966 /* XXX check the bitfield order! */
7967 me_id = (ring_id & 0x60) >> 5;
7968 pipe_id = (ring_id & 0x18) >> 3;
7969 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 7970 switch (me_id) {
7971 case 0:
7972 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7973 break;
7974 case 1:
a59781bb 7975 case 2:
2b0781a6
AD		 7976 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7977 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7978 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7979 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 7980 break;
7981 }
7982 break;
7983 case 184: /* CP Privileged reg access */
7984 DRM_ERROR("Illegal register access in command stream\n");
7985 /* XXX check the bitfield order! */
7986 me_id = (ring_id & 0x60) >> 5;
7987 pipe_id = (ring_id & 0x18) >> 3;
7988 queue_id = (ring_id & 0x7) >> 0;
7989 switch (me_id) {
7990 case 0:
7991 /* This results in a full GPU reset, but all we need to do is soft
7992 * reset the CP for gfx
7993 */
7994 queue_reset = true;
7995 break;
7996 case 1:
7997 /* XXX compute */
2b0781a6 7998 queue_reset = true;
a59781bb
AD		 7999 break;
8000 case 2:
8001 /* XXX compute */
2b0781a6 8002 queue_reset = true;
a59781bb
AD		 8003 break;
8004 }
8005 break;
8006 case 185: /* CP Privileged inst */
8007 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 8008 /* XXX check the bitfield order! */
8009 me_id = (ring_id & 0x60) >> 5;
8010 pipe_id = (ring_id & 0x18) >> 3;
8011 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 8012 switch (me_id) {
8013 case 0:
8014 /* This results in a full GPU reset, but all we need to do is soft
8015 * reset the CP for gfx
8016 */
8017 queue_reset = true;
8018 break;
8019 case 1:
8020 /* XXX compute */
2b0781a6 8021 queue_reset = true;
a59781bb
AD		 8022 break;
8023 case 2:
8024 /* XXX compute */
2b0781a6 8025 queue_reset = true;
a59781bb
AD		 8026 break;
8027 }
8028 break;
21a93e13
AD		 8029 case 224: /* SDMA trap event */
8030 /* XXX check the bitfield order! */
8031 me_id = (ring_id & 0x3) >> 0;
8032 queue_id = (ring_id & 0xc) >> 2;
8033 DRM_DEBUG("IH: SDMA trap\n");
8034 switch (me_id) {
8035 case 0:
8036 switch (queue_id) {
8037 case 0:
8038 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8039 break;
8040 case 1:
8041 /* XXX compute */
8042 break;
8043 case 2:
8044 /* XXX compute */
8045 break;
8046 }
8047 break;
8048 case 1:
8049 switch (queue_id) {
8050 case 0:
8051 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8052 break;
8053 case 1:
8054 /* XXX compute */
8055 break;
8056 case 2:
8057 /* XXX compute */
8058 break;
8059 }
8060 break;
8061 }
8062 break;
41a524ab
AD		 8063 case 230: /* thermal low to high */
8064 DRM_DEBUG("IH: thermal low to high\n");
8065 rdev->pm.dpm.thermal.high_to_low = false;
8066 queue_thermal = true;
8067 break;
8068 case 231: /* thermal high to low */
8069 DRM_DEBUG("IH: thermal high to low\n");
8070 rdev->pm.dpm.thermal.high_to_low = true;
8071 queue_thermal = true;
8072 break;
8073 case 233: /* GUI IDLE */
8074 DRM_DEBUG("IH: GUI idle\n");
8075 break;
21a93e13
AD		 8076 case 241: /* SDMA Privileged inst */
8077 case 247: /* SDMA Privileged inst */
8078 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8079 /* XXX check the bitfield order! */
8080 me_id = (ring_id & 0x3) >> 0;
8081 queue_id = (ring_id & 0xc) >> 2;
8082 switch (me_id) {
8083 case 0:
8084 switch (queue_id) {
8085 case 0:
8086 queue_reset = true;
8087 break;
8088 case 1:
8089 /* XXX compute */
8090 queue_reset = true;
8091 break;
8092 case 2:
8093 /* XXX compute */
8094 queue_reset = true;
8095 break;
8096 }
8097 break;
8098 case 1:
8099 switch (queue_id) {
8100 case 0:
8101 queue_reset = true;
8102 break;
8103 case 1:
8104 /* XXX compute */
8105 queue_reset = true;
8106 break;
8107 case 2:
8108 /* XXX compute */
8109 queue_reset = true;
8110 break;
8111 }
8112 break;
8113 }
8114 break;
a59781bb
AD		 8115 default:
8116 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8117 break;
8118 }
8119
8120 /* wptr/rptr are in bytes! */
8121 rptr += 16;
8122 rptr &= rdev->ih.ptr_mask;
f55e03b9 8123 WREG32(IH_RB_RPTR, rptr);
a59781bb 8124 }
f6b355dd
AD		 8125 if (queue_dp)
8126 schedule_work(&rdev->dp_work);
a59781bb 8127 if (queue_hotplug)
cb5d4166 8128 schedule_delayed_work(&rdev->hotplug_work, 0);
3c036389
CK		 8129 if (queue_reset) {
8130 rdev->needs_reset = true;
8131 wake_up_all(&rdev->fence_queue);
8132 }
41a524ab
AD		 8133 if (queue_thermal)
8134 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb 8135 rdev->ih.rptr = rptr;
a59781bb
AD		 8136 atomic_set(&rdev->ih.lock, 0);
8137
8138 /* make sure wptr hasn't changed while processing */
8139 wptr = cik_get_ih_wptr(rdev);
8140 if (wptr != rptr)
8141 goto restart_ih;
8142
8143 return IRQ_HANDLED;
8144}
7bf94a2c
AD		 8145
8146/*
8147 * startup/shutdown callbacks
8148 */
bc48a15a
JG		 8149static void cik_uvd_init(struct radeon_device *rdev)
8150{
8151 int r;
8152
8153 if (!rdev->has_uvd)
8154 return;
8155
8156 r = radeon_uvd_init(rdev);
8157 if (r) {
8158 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
8159 /*
8160 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
8161 * to early fails cik_uvd_start() and thus nothing happens
8162 * there. So it is pointless to try to go through that code
8163 * hence why we disable uvd here.
8164 */
8165 rdev->has_uvd = 0;
8166 return;
8167 }
8168 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
8169 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
8170}
8171
8172static void cik_uvd_start(struct radeon_device *rdev)
8173{
8174 int r;
8175
8176 if (!rdev->has_uvd)
8177 return;
8178
8179 r = radeon_uvd_resume(rdev);
8180 if (r) {
8181 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
8182 goto error;
8183 }
8184 r = uvd_v4_2_resume(rdev);
8185 if (r) {
8186 dev_err(rdev->dev, "failed UVD 4.2 resume (%d).\n", r);
8187 goto error;
8188 }
8189 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
8190 if (r) {
8191 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
8192 goto error;
8193 }
8194 return;
8195
8196error:
8197 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8198}
8199
8200static void cik_uvd_resume(struct radeon_device *rdev)
8201{
8202 struct radeon_ring *ring;
8203 int r;
8204
8205 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
8206 return;
8207
8208 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
70a033d2 8209 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
bc48a15a
JG		 8210 if (r) {
8211 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
8212 return;
8213 }
8214 r = uvd_v1_0_init(rdev);
8215 if (r) {
8216 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
8217 return;
8218 }
8219}
8220
cb25f7e0
JG		 8221static void cik_vce_init(struct radeon_device *rdev)
8222{
8223 int r;
8224
8225 if (!rdev->has_vce)
8226 return;
8227
8228 r = radeon_vce_init(rdev);
8229 if (r) {
8230 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
8231 /*
8232 * At this point rdev->vce.vcpu_bo is NULL which trickles down
8233 * to early fails cik_vce_start() and thus nothing happens
8234 * there. So it is pointless to try to go through that code
8235 * hence why we disable vce here.
8236 */
8237 rdev->has_vce = 0;
8238 return;
8239 }
8240 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
8241 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
8242 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
8243 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
8244}
8245
8246static void cik_vce_start(struct radeon_device *rdev)
8247{
8248 int r;
8249
8250 if (!rdev->has_vce)
8251 return;
8252
8253 r = radeon_vce_resume(rdev);
8254 if (r) {
8255 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8256 goto error;
8257 }
8258 r = vce_v2_0_resume(rdev);
8259 if (r) {
8260 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
8261 goto error;
8262 }
8263 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
8264 if (r) {
8265 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
8266 goto error;
8267 }
8268 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
8269 if (r) {
8270 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
8271 goto error;
8272 }
8273 return;
8274
8275error:
8276 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8277 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8278}
8279
8280static void cik_vce_resume(struct radeon_device *rdev)
8281{
8282 struct radeon_ring *ring;
8283 int r;
8284
8285 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
8286 return;
8287
8288 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8289 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8290 if (r) {
8291 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8292 return;
8293 }
8294 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8295 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
8296 if (r) {
8297 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
8298 return;
8299 }
8300 r = vce_v1_0_init(rdev);
8301 if (r) {
8302 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
8303 return;
8304 }
8305}
8306
7bf94a2c
AD		 8307/**
8308 * cik_startup - program the asic to a functional state
8309 *
8310 * @rdev: radeon_device pointer
8311 *
8312 * Programs the asic to a functional state (CIK).
8313 * Called by cik_init() and cik_resume().
8314 * Returns 0 for success, error for failure.
8315 */
8316static int cik_startup(struct radeon_device *rdev)
8317{
8318 struct radeon_ring *ring;
0e16e4cf 8319 u32 nop;
7bf94a2c
AD		 8320 int r;
8321
8a7cd276
AD		 8322 /* enable pcie gen2/3 link */
8323 cik_pcie_gen3_enable(rdev);
7235711a
AD		 8324 /* enable aspm */
8325 cik_program_aspm(rdev);
8a7cd276 8326
e5903d39
AD		 8327 /* scratch needs to be initialized before MC */
8328 r = r600_vram_scratch_init(rdev);
8329 if (r)
8330 return r;
8331
6fab3feb
AD		 8332 cik_mc_program(rdev);
8333
6c7bccea 8334 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7bf94a2c
AD		 8335 r = ci_mc_load_microcode(rdev);
8336 if (r) {
8337 DRM_ERROR("Failed to load MC firmware!\n");
8338 return r;
8339 }
8340 }
8341
7bf94a2c
AD		 8342 r = cik_pcie_gart_enable(rdev);
8343 if (r)
8344 return r;
8345 cik_gpu_init(rdev);
8346
8347 /* allocate rlc buffers */
22c775ce
AD		 8348 if (rdev->flags & RADEON_IS_IGP) {
8349 if (rdev->family == CHIP_KAVERI) {
8350 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8351 rdev->rlc.reg_list_size =
8352 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8353 } else {
8354 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8355 rdev->rlc.reg_list_size =
8356 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8357 }
8358 }
8359 rdev->rlc.cs_data = ci_cs_data;
e70a15f5 8360 rdev->rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */
9406d216 8361 rdev->rlc.cp_table_size += 64 * 1024; /* GDS */
1fd11777 8362 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 8363 if (r) {
8364 DRM_ERROR("Failed to init rlc BOs!\n");
8365 return r;
8366 }
8367
8368 /* allocate wb buffer */
8369 r = radeon_wb_init(rdev);
8370 if (r)
8371 return r;
8372
963e81f9
AD		 8373 /* allocate mec buffers */
8374 r = cik_mec_init(rdev);
8375 if (r) {
8376 DRM_ERROR("Failed to init MEC BOs!\n");
8377 return r;
8378 }
8379
7bf94a2c
AD		 8380 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8381 if (r) {
8382 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8383 return r;
8384 }
8385
963e81f9
AD		 8386 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8387 if (r) {
8388 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8389 return r;
8390 }
8391
8392 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8393 if (r) {
8394 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8395 return r;
8396 }
8397
7bf94a2c
AD		 8398 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8399 if (r) {
8400 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8401 return r;
8402 }
8403
8404 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8405 if (r) {
8406 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8407 return r;
8408 }
8409
bc48a15a 8410 cik_uvd_start(rdev);
cb25f7e0 8411 cik_vce_start(rdev);
d93f7937 8412
7bf94a2c
AD		 8413 /* Enable IRQ */
8414 if (!rdev->irq.installed) {
8415 r = radeon_irq_kms_init(rdev);
8416 if (r)
8417 return r;
8418 }
8419
8420 r = cik_irq_init(rdev);
8421 if (r) {
8422 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8423 radeon_irq_kms_fini(rdev);
8424 return r;
8425 }
8426 cik_irq_set(rdev);
8427
0e16e4cf 8428 if (rdev->family == CHIP_HAWAII) {
78cd3661
AD		 8429 if (rdev->new_fw)
8430 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8431 else
8432 nop = RADEON_CP_PACKET2;
0e16e4cf
AD		 8433 } else {
8434 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8435 }
8436
7bf94a2c
AD		 8437 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8438 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
0e16e4cf 8439 nop);
7bf94a2c
AD		 8440 if (r)
8441 return r;
8442
963e81f9 8443 /* set up the compute queues */
2615b53a 8444 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8445 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8446 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
0e16e4cf 8447 nop);
963e81f9
AD		 8448 if (r)
8449 return r;
8450 ring->me = 1; /* first MEC */
8451 ring->pipe = 0; /* first pipe */
8452 ring->queue = 0; /* first queue */
8453 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8454
2615b53a 8455 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8456 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8457 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
0e16e4cf 8458 nop);
963e81f9
AD		 8459 if (r)
8460 return r;
8461 /* dGPU only have 1 MEC */
8462 ring->me = 1; /* first MEC */
8463 ring->pipe = 0; /* first pipe */
8464 ring->queue = 1; /* second queue */
8465 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8466
7bf94a2c
AD		 8467 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8468 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 8469 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8470 if (r)
8471 return r;
8472
8473 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8474 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 8475 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8476 if (r)
8477 return r;
8478
8479 r = cik_cp_resume(rdev);
8480 if (r)
8481 return r;
8482
8483 r = cik_sdma_resume(rdev);
8484 if (r)
8485 return r;
8486
bc48a15a 8487 cik_uvd_resume(rdev);
cb25f7e0 8488 cik_vce_resume(rdev);
d93f7937 8489
7bf94a2c
AD		 8490 r = radeon_ib_pool_init(rdev);
8491 if (r) {
8492 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8493 return r;
8494 }
8495
8496 r = radeon_vm_manager_init(rdev);
8497 if (r) {
8498 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8499 return r;
8500 }
8501
bfc1f97d 8502 r = radeon_audio_init(rdev);
b530602f
AD		 8503 if (r)
8504 return r;
8505
e28740ec
OG		 8506 r = radeon_kfd_resume(rdev);
8507 if (r)
8508 return r;
8509
7bf94a2c
AD		 8510 return 0;
8511}
8512
8513/**
8514 * cik_resume - resume the asic to a functional state
8515 *
8516 * @rdev: radeon_device pointer
8517 *
8518 * Programs the asic to a functional state (CIK).
8519 * Called at resume.
8520 * Returns 0 for success, error for failure.
8521 */
8522int cik_resume(struct radeon_device *rdev)
8523{
8524 int r;
8525
8526 /* post card */
8527 atom_asic_init(rdev->mode_info.atom_context);
8528
0aafd313
AD		 8529 /* init golden registers */
8530 cik_init_golden_registers(rdev);
8531
bc6a6295
AD		 8532 if (rdev->pm.pm_method == PM_METHOD_DPM)
8533 radeon_pm_resume(rdev);
6c7bccea 8534
7bf94a2c
AD		 8535 rdev->accel_working = true;
8536 r = cik_startup(rdev);
8537 if (r) {
8538 DRM_ERROR("cik startup failed on resume\n");
8539 rdev->accel_working = false;
8540 return r;
8541 }
8542
8543 return r;
8544
8545}
8546
8547/**
8548 * cik_suspend - suspend the asic
8549 *
8550 * @rdev: radeon_device pointer
8551 *
8552 * Bring the chip into a state suitable for suspend (CIK).
8553 * Called at suspend.
8554 * Returns 0 for success.
8555 */
8556int cik_suspend(struct radeon_device *rdev)
8557{
e28740ec 8558 radeon_kfd_suspend(rdev);
6c7bccea 8559 radeon_pm_suspend(rdev);
7991d665 8560 radeon_audio_fini(rdev);
7bf94a2c
AD		 8561 radeon_vm_manager_fini(rdev);
8562 cik_cp_enable(rdev, false);
8563 cik_sdma_enable(rdev, false);
bc48a15a
JG		 8564 if (rdev->has_uvd) {
8565 uvd_v1_0_fini(rdev);
8566 radeon_uvd_suspend(rdev);
8567 }
cb25f7e0
JG		 8568 if (rdev->has_vce)
8569 radeon_vce_suspend(rdev);
473359bc
AD		 8570 cik_fini_pg(rdev);
8571 cik_fini_cg(rdev);
7bf94a2c
AD		 8572 cik_irq_suspend(rdev);
8573 radeon_wb_disable(rdev);
8574 cik_pcie_gart_disable(rdev);
8575 return 0;
8576}
8577
8578/* Plan is to move initialization in that function and use
8579 * helper function so that radeon_device_init pretty much
8580 * do nothing more than calling asic specific function. This
8581 * should also allow to remove a bunch of callback function
8582 * like vram_info.
8583 */
8584/**
8585 * cik_init - asic specific driver and hw init
8586 *
8587 * @rdev: radeon_device pointer
8588 *
8589 * Setup asic specific driver variables and program the hw
8590 * to a functional state (CIK).
8591 * Called at driver startup.
8592 * Returns 0 for success, errors for failure.
8593 */
8594int cik_init(struct radeon_device *rdev)
8595{
8596 struct radeon_ring *ring;
8597 int r;
8598
8599 /* Read BIOS */
8600 if (!radeon_get_bios(rdev)) {
8601 if (ASIC_IS_AVIVO(rdev))
8602 return -EINVAL;
8603 }
8604 /* Must be an ATOMBIOS */
8605 if (!rdev->is_atom_bios) {
8606 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8607 return -EINVAL;
8608 }
8609 r = radeon_atombios_init(rdev);
8610 if (r)
8611 return r;
8612
8613 /* Post card if necessary */
8614 if (!radeon_card_posted(rdev)) {
8615 if (!rdev->bios) {
8616 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8617 return -EINVAL;
8618 }
8619 DRM_INFO("GPU not posted. posting now...\n");
8620 atom_asic_init(rdev->mode_info.atom_context);
8621 }
0aafd313
AD		 8622 /* init golden registers */
8623 cik_init_golden_registers(rdev);
7bf94a2c
AD		 8624 /* Initialize scratch registers */
8625 cik_scratch_init(rdev);
8626 /* Initialize surface registers */
8627 radeon_surface_init(rdev);
8628 /* Initialize clocks */
8629 radeon_get_clock_info(rdev->ddev);
8630
8631 /* Fence driver */
8632 r = radeon_fence_driver_init(rdev);
8633 if (r)
8634 return r;
8635
8636 /* initialize memory controller */
8637 r = cik_mc_init(rdev);
8638 if (r)
8639 return r;
8640 /* Memory manager */
8641 r = radeon_bo_init(rdev);
8642 if (r)
8643 return r;
8644
01ac8794
AD		 8645 if (rdev->flags & RADEON_IS_IGP) {
8646 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8647 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8648 r = cik_init_microcode(rdev);
8649 if (r) {
8650 DRM_ERROR("Failed to load firmware!\n");
8651 return r;
8652 }
8653 }
8654 } else {
8655 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8656 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8657 !rdev->mc_fw) {
8658 r = cik_init_microcode(rdev);
8659 if (r) {
8660 DRM_ERROR("Failed to load firmware!\n");
8661 return r;
8662 }
8663 }
8664 }
8665
6c7bccea
AD		 8666 /* Initialize power management */
8667 radeon_pm_init(rdev);
8668
7bf94a2c
AD		 8669 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8670 ring->ring_obj = NULL;
8671 r600_ring_init(rdev, ring, 1024 * 1024);
8672
963e81f9
AD		 8673 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8674 ring->ring_obj = NULL;
8675 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8676 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8677 if (r)
8678 return r;
8679
8680 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8681 ring->ring_obj = NULL;
8682 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8683 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8684 if (r)
8685 return r;
8686
7bf94a2c
AD		 8687 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8688 ring->ring_obj = NULL;
8689 r600_ring_init(rdev, ring, 256 * 1024);
8690
8691 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8692 ring->ring_obj = NULL;
8693 r600_ring_init(rdev, ring, 256 * 1024);
8694
bc48a15a 8695 cik_uvd_init(rdev);
cb25f7e0 8696 cik_vce_init(rdev);
d93f7937 8697
7bf94a2c
AD		 8698 rdev->ih.ring_obj = NULL;
8699 r600_ih_ring_init(rdev, 64 * 1024);
8700
8701 r = r600_pcie_gart_init(rdev);
8702 if (r)
8703 return r;
8704
8705 rdev->accel_working = true;
8706 r = cik_startup(rdev);
8707 if (r) {
8708 dev_err(rdev->dev, "disabling GPU acceleration\n");
8709 cik_cp_fini(rdev);
8710 cik_sdma_fini(rdev);
8711 cik_irq_fini(rdev);
1fd11777 8712 sumo_rlc_fini(rdev);
963e81f9 8713 cik_mec_fini(rdev);
7bf94a2c
AD		 8714 radeon_wb_fini(rdev);
8715 radeon_ib_pool_fini(rdev);
8716 radeon_vm_manager_fini(rdev);
8717 radeon_irq_kms_fini(rdev);
8718 cik_pcie_gart_fini(rdev);
8719 rdev->accel_working = false;
8720 }
8721
8722 /* Don't start up if the MC ucode is missing.
8723 * The default clocks and voltages before the MC ucode
8724 * is loaded are not suffient for advanced operations.
8725 */
8726 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8727 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8728 return -EINVAL;
8729 }
8730
8731 return 0;
8732}
8733
8734/**
8735 * cik_fini - asic specific driver and hw fini
8736 *
8737 * @rdev: radeon_device pointer
8738 *
8739 * Tear down the asic specific driver variables and program the hw
8740 * to an idle state (CIK).
8741 * Called at driver unload.
8742 */
8743void cik_fini(struct radeon_device *rdev)
8744{
6c7bccea 8745 radeon_pm_fini(rdev);
7bf94a2c
AD		 8746 cik_cp_fini(rdev);
8747 cik_sdma_fini(rdev);
473359bc
AD		 8748 cik_fini_pg(rdev);
8749 cik_fini_cg(rdev);
7bf94a2c 8750 cik_irq_fini(rdev);
1fd11777 8751 sumo_rlc_fini(rdev);
963e81f9 8752 cik_mec_fini(rdev);
7bf94a2c
AD		 8753 radeon_wb_fini(rdev);
8754 radeon_vm_manager_fini(rdev);
8755 radeon_ib_pool_fini(rdev);
8756 radeon_irq_kms_fini(rdev);
e409b128 8757 uvd_v1_0_fini(rdev);
87167bb1 8758 radeon_uvd_fini(rdev);
d93f7937 8759 radeon_vce_fini(rdev);
7bf94a2c
AD		 8760 cik_pcie_gart_fini(rdev);
8761 r600_vram_scratch_fini(rdev);
8762 radeon_gem_fini(rdev);
8763 radeon_fence_driver_fini(rdev);
8764 radeon_bo_fini(rdev);
8765 radeon_atombios_fini(rdev);
8766 kfree(rdev->bios);
8767 rdev->bios = NULL;
8768}
cd84a27d 8769
134b480f
AD		 8770void dce8_program_fmt(struct drm_encoder *encoder)
8771{
8772 struct drm_device *dev = encoder->dev;
8773 struct radeon_device *rdev = dev->dev_private;
8774 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8775 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8776 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8777 int bpc = 0;
8778 u32 tmp = 0;
6214bb74 8779 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 8780
6214bb74
AD		 8781 if (connector) {
8782 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 8783 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 8784 dither = radeon_connector->dither;
8785 }
134b480f
AD		 8786
8787 /* LVDS/eDP FMT is set up by atom */
8788 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8789 return;
8790
8791 /* not needed for analog */
8792 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8793 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8794 return;
8795
8796 if (bpc == 0)
8797 return;
8798
8799 switch (bpc) {
8800 case 6:
6214bb74 8801 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8802 /* XXX sort out optimal dither settings */
8803 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8804 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8805 else
8806 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8807 break;
8808 case 8:
6214bb74 8809 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8810 /* XXX sort out optimal dither settings */
8811 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8812 FMT_RGB_RANDOM_ENABLE |
8813 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8814 else
8815 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8816 break;
8817 case 10:
6214bb74 8818 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8819 /* XXX sort out optimal dither settings */
8820 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8821 FMT_RGB_RANDOM_ENABLE |
8822 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8823 else
8824 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8825 break;
8826 default:
8827 /* not needed */
8828 break;
8829 }
8830
8831 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8832}
8833
cd84a27d
AD		 8834/* display watermark setup */
8835/**
8836 * dce8_line_buffer_adjust - Set up the line buffer
8837 *
8838 * @rdev: radeon_device pointer
8839 * @radeon_crtc: the selected display controller
8840 * @mode: the current display mode on the selected display
8841 * controller
8842 *
8843 * Setup up the line buffer allocation for
8844 * the selected display controller (CIK).
8845 * Returns the line buffer size in pixels.
8846 */
8847static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8848 struct radeon_crtc *radeon_crtc,
8849 struct drm_display_mode *mode)
8850{
bc01a8c7
AD		 8851 u32 tmp, buffer_alloc, i;
8852 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 8853 /*
8854 * Line Buffer Setup
8855 * There are 6 line buffers, one for each display controllers.
8856 * There are 3 partitions per LB. Select the number of partitions
8857 * to enable based on the display width. For display widths larger
8858 * than 4096, you need use to use 2 display controllers and combine
8859 * them using the stereo blender.
8860 */
8861 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 8862 if (mode->crtc_hdisplay < 1920) {
cd84a27d 8863 tmp = 1;
bc01a8c7
AD		 8864 buffer_alloc = 2;
8865 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 8866 tmp = 2;
bc01a8c7
AD		 8867 buffer_alloc = 2;
8868 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 8869 tmp = 0;
bc01a8c7
AD		 8870 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8871 } else {
cd84a27d
AD		 8872 DRM_DEBUG_KMS("Mode too big for LB!\n");
8873 tmp = 0;
bc01a8c7 8874 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 8875 }
bc01a8c7 8876 } else {
cd84a27d 8877 tmp = 1;
bc01a8c7
AD		 8878 buffer_alloc = 0;
8879 }
cd84a27d
AD		 8880
8881 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8882 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8883
bc01a8c7
AD		 8884 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8885 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8886 for (i = 0; i < rdev->usec_timeout; i++) {
8887 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8888 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8889 break;
8890 udelay(1);
8891 }
8892
cd84a27d
AD		 8893 if (radeon_crtc->base.enabled && mode) {
8894 switch (tmp) {
8895 case 0:
8896 default:
8897 return 4096 * 2;
8898 case 1:
8899 return 1920 * 2;
8900 case 2:
8901 return 2560 * 2;
8902 }
8903 }
8904
8905 /* controller not enabled, so no lb used */
8906 return 0;
8907}
8908
8909/**
8910 * cik_get_number_of_dram_channels - get the number of dram channels
8911 *
8912 * @rdev: radeon_device pointer
8913 *
8914 * Look up the number of video ram channels (CIK).
8915 * Used for display watermark bandwidth calculations
8916 * Returns the number of dram channels
8917 */
8918static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8919{
8920 u32 tmp = RREG32(MC_SHARED_CHMAP);
8921
8922 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8923 case 0:
8924 default:
8925 return 1;
8926 case 1:
8927 return 2;
8928 case 2:
8929 return 4;
8930 case 3:
8931 return 8;
8932 case 4:
8933 return 3;
8934 case 5:
8935 return 6;
8936 case 6:
8937 return 10;
8938 case 7:
8939 return 12;
8940 case 8:
8941 return 16;
8942 }
8943}
8944
8945struct dce8_wm_params {
8946 u32 dram_channels; /* number of dram channels */
8947 u32 yclk; /* bandwidth per dram data pin in kHz */
8948 u32 sclk; /* engine clock in kHz */
8949 u32 disp_clk; /* display clock in kHz */
8950 u32 src_width; /* viewport width */
8951 u32 active_time; /* active display time in ns */
8952 u32 blank_time; /* blank time in ns */
8953 bool interlaced; /* mode is interlaced */
8954 fixed20_12 vsc; /* vertical scale ratio */
8955 u32 num_heads; /* number of active crtcs */
8956 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8957 u32 lb_size; /* line buffer allocated to pipe */
8958 u32 vtaps; /* vertical scaler taps */
8959};
8960
8961/**
8962 * dce8_dram_bandwidth - get the dram bandwidth
8963 *
8964 * @wm: watermark calculation data
8965 *
8966 * Calculate the raw dram bandwidth (CIK).
8967 * Used for display watermark bandwidth calculations
8968 * Returns the dram bandwidth in MBytes/s
8969 */
8970static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8971{
8972 /* Calculate raw DRAM Bandwidth */
8973 fixed20_12 dram_efficiency; /* 0.7 */
8974 fixed20_12 yclk, dram_channels, bandwidth;
8975 fixed20_12 a;
8976
8977 a.full = dfixed_const(1000);
8978 yclk.full = dfixed_const(wm->yclk);
8979 yclk.full = dfixed_div(yclk, a);
8980 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8981 a.full = dfixed_const(10);
8982 dram_efficiency.full = dfixed_const(7);
8983 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8984 bandwidth.full = dfixed_mul(dram_channels, yclk);
8985 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8986
8987 return dfixed_trunc(bandwidth);
8988}
8989
8990/**
8991 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8992 *
8993 * @wm: watermark calculation data
8994 *
8995 * Calculate the dram bandwidth used for display (CIK).
8996 * Used for display watermark bandwidth calculations
8997 * Returns the dram bandwidth for display in MBytes/s
8998 */
8999static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9000{
9001 /* Calculate DRAM Bandwidth and the part allocated to display. */
9002 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9003 fixed20_12 yclk, dram_channels, bandwidth;
9004 fixed20_12 a;
9005
9006 a.full = dfixed_const(1000);
9007 yclk.full = dfixed_const(wm->yclk);
9008 yclk.full = dfixed_div(yclk, a);
9009 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9010 a.full = dfixed_const(10);
9011 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9012 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9013 bandwidth.full = dfixed_mul(dram_channels, yclk);
9014 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9015
9016 return dfixed_trunc(bandwidth);
9017}
9018
9019/**
9020 * dce8_data_return_bandwidth - get the data return bandwidth
9021 *
9022 * @wm: watermark calculation data
9023 *
9024 * Calculate the data return bandwidth used for display (CIK).
9025 * Used for display watermark bandwidth calculations
9026 * Returns the data return bandwidth in MBytes/s
9027 */
9028static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9029{
9030 /* Calculate the display Data return Bandwidth */
9031 fixed20_12 return_efficiency; /* 0.8 */
9032 fixed20_12 sclk, bandwidth;
9033 fixed20_12 a;
9034
9035 a.full = dfixed_const(1000);
9036 sclk.full = dfixed_const(wm->sclk);
9037 sclk.full = dfixed_div(sclk, a);
9038 a.full = dfixed_const(10);
9039 return_efficiency.full = dfixed_const(8);
9040 return_efficiency.full = dfixed_div(return_efficiency, a);
9041 a.full = dfixed_const(32);
9042 bandwidth.full = dfixed_mul(a, sclk);
9043 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9044
9045 return dfixed_trunc(bandwidth);
9046}
9047
9048/**
9049 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9050 *
9051 * @wm: watermark calculation data
9052 *
9053 * Calculate the dmif bandwidth used for display (CIK).
9054 * Used for display watermark bandwidth calculations
9055 * Returns the dmif bandwidth in MBytes/s
9056 */
9057static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9058{
9059 /* Calculate the DMIF Request Bandwidth */
9060 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9061 fixed20_12 disp_clk, bandwidth;
9062 fixed20_12 a, b;
9063
9064 a.full = dfixed_const(1000);
9065 disp_clk.full = dfixed_const(wm->disp_clk);
9066 disp_clk.full = dfixed_div(disp_clk, a);
9067 a.full = dfixed_const(32);
9068 b.full = dfixed_mul(a, disp_clk);
9069
9070 a.full = dfixed_const(10);
9071 disp_clk_request_efficiency.full = dfixed_const(8);
9072 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9073
9074 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9075
9076 return dfixed_trunc(bandwidth);
9077}
9078
9079/**
9080 * dce8_available_bandwidth - get the min available bandwidth
9081 *
9082 * @wm: watermark calculation data
9083 *
9084 * Calculate the min available bandwidth used for display (CIK).
9085 * Used for display watermark bandwidth calculations
9086 * Returns the min available bandwidth in MBytes/s
9087 */
9088static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9089{
9090 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9091 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9092 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9093 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9094
9095 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9096}
9097
9098/**
9099 * dce8_average_bandwidth - get the average available bandwidth
9100 *
9101 * @wm: watermark calculation data
9102 *
9103 * Calculate the average available bandwidth used for display (CIK).
9104 * Used for display watermark bandwidth calculations
9105 * Returns the average available bandwidth in MBytes/s
9106 */
9107static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9108{
9109 /* Calculate the display mode Average Bandwidth
9110 * DisplayMode should contain the source and destination dimensions,
9111 * timing, etc.
9112 */
9113 fixed20_12 bpp;
9114 fixed20_12 line_time;
9115 fixed20_12 src_width;
9116 fixed20_12 bandwidth;
9117 fixed20_12 a;
9118
9119 a.full = dfixed_const(1000);
9120 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9121 line_time.full = dfixed_div(line_time, a);
9122 bpp.full = dfixed_const(wm->bytes_per_pixel);
9123 src_width.full = dfixed_const(wm->src_width);
9124 bandwidth.full = dfixed_mul(src_width, bpp);
9125 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9126 bandwidth.full = dfixed_div(bandwidth, line_time);
9127
9128 return dfixed_trunc(bandwidth);
9129}
9130
9131/**
9132 * dce8_latency_watermark - get the latency watermark
9133 *
9134 * @wm: watermark calculation data
9135 *
9136 * Calculate the latency watermark (CIK).
9137 * Used for display watermark bandwidth calculations
9138 * Returns the latency watermark in ns
9139 */
9140static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9141{
9142 /* First calculate the latency in ns */
9143 u32 mc_latency = 2000; /* 2000 ns. */
9144 u32 available_bandwidth = dce8_available_bandwidth(wm);
9145 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9146 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9147 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9148 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9149 (wm->num_heads * cursor_line_pair_return_time);
9150 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9151 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9152 u32 tmp, dmif_size = 12288;
9153 fixed20_12 a, b, c;
9154
9155 if (wm->num_heads == 0)
9156 return 0;
9157
9158 a.full = dfixed_const(2);
9159 b.full = dfixed_const(1);
9160 if ((wm->vsc.full > a.full) ||
9161 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9162 (wm->vtaps >= 5) ||
9163 ((wm->vsc.full >= a.full) && wm->interlaced))
9164 max_src_lines_per_dst_line = 4;
9165 else
9166 max_src_lines_per_dst_line = 2;
9167
9168 a.full = dfixed_const(available_bandwidth);
9169 b.full = dfixed_const(wm->num_heads);
9170 a.full = dfixed_div(a, b);
9171
9172 b.full = dfixed_const(mc_latency + 512);
9173 c.full = dfixed_const(wm->disp_clk);
9174 b.full = dfixed_div(b, c);
9175
9176 c.full = dfixed_const(dmif_size);
9177 b.full = dfixed_div(c, b);
9178
9179 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9180
9181 b.full = dfixed_const(1000);
9182 c.full = dfixed_const(wm->disp_clk);
9183 b.full = dfixed_div(c, b);
9184 c.full = dfixed_const(wm->bytes_per_pixel);
9185 b.full = dfixed_mul(b, c);
9186
9187 lb_fill_bw = min(tmp, dfixed_trunc(b));
9188
9189 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9190 b.full = dfixed_const(1000);
9191 c.full = dfixed_const(lb_fill_bw);
9192 b.full = dfixed_div(c, b);
9193 a.full = dfixed_div(a, b);
9194 line_fill_time = dfixed_trunc(a);
9195
9196 if (line_fill_time < wm->active_time)
9197 return latency;
9198 else
9199 return latency + (line_fill_time - wm->active_time);
9200
9201}
9202
9203/**
9204 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9205 * average and available dram bandwidth
9206 *
9207 * @wm: watermark calculation data
9208 *
9209 * Check if the display average bandwidth fits in the display
9210 * dram bandwidth (CIK).
9211 * Used for display watermark bandwidth calculations
9212 * Returns true if the display fits, false if not.
9213 */
9214static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9215{
9216 if (dce8_average_bandwidth(wm) <=
9217 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9218 return true;
9219 else
9220 return false;
9221}
9222
9223/**
9224 * dce8_average_bandwidth_vs_available_bandwidth - check
9225 * average and available bandwidth
9226 *
9227 * @wm: watermark calculation data
9228 *
9229 * Check if the display average bandwidth fits in the display
9230 * available bandwidth (CIK).
9231 * Used for display watermark bandwidth calculations
9232 * Returns true if the display fits, false if not.
9233 */
9234static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9235{
9236 if (dce8_average_bandwidth(wm) <=
9237 (dce8_available_bandwidth(wm) / wm->num_heads))
9238 return true;
9239 else
9240 return false;
9241}
9242
9243/**
9244 * dce8_check_latency_hiding - check latency hiding
9245 *
9246 * @wm: watermark calculation data
9247 *
9248 * Check latency hiding (CIK).
9249 * Used for display watermark bandwidth calculations
9250 * Returns true if the display fits, false if not.
9251 */
9252static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9253{
9254 u32 lb_partitions = wm->lb_size / wm->src_width;
9255 u32 line_time = wm->active_time + wm->blank_time;
9256 u32 latency_tolerant_lines;
9257 u32 latency_hiding;
9258 fixed20_12 a;
9259
9260 a.full = dfixed_const(1);
9261 if (wm->vsc.full > a.full)
9262 latency_tolerant_lines = 1;
9263 else {
9264 if (lb_partitions <= (wm->vtaps + 1))
9265 latency_tolerant_lines = 1;
9266 else
9267 latency_tolerant_lines = 2;
9268 }
9269
9270 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9271
9272 if (dce8_latency_watermark(wm) <= latency_hiding)
9273 return true;
9274 else
9275 return false;
9276}
9277
9278/**
9279 * dce8_program_watermarks - program display watermarks
9280 *
9281 * @rdev: radeon_device pointer
9282 * @radeon_crtc: the selected display controller
9283 * @lb_size: line buffer size
9284 * @num_heads: number of display controllers in use
9285 *
9286 * Calculate and program the display watermarks for the
9287 * selected display controller (CIK).
9288 */
9289static void dce8_program_watermarks(struct radeon_device *rdev,
9290 struct radeon_crtc *radeon_crtc,
9291 u32 lb_size, u32 num_heads)
9292{
9293 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 9294 struct dce8_wm_params wm_low, wm_high;
cd84a27d
AD		 9295 u32 pixel_period;
9296 u32 line_time = 0;
9297 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9298 u32 tmp, wm_mask;
9299
9300 if (radeon_crtc->base.enabled && num_heads && mode) {
9301 pixel_period = 1000000 / (u32)mode->clock;
9302 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9303
58ea2dea
AD		 9304 /* watermark for high clocks */
9305 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9306 rdev->pm.dpm_enabled) {
9307 wm_high.yclk =
9308 radeon_dpm_get_mclk(rdev, false) * 10;
9309 wm_high.sclk =
9310 radeon_dpm_get_sclk(rdev, false) * 10;
9311 } else {
9312 wm_high.yclk = rdev->pm.current_mclk * 10;
9313 wm_high.sclk = rdev->pm.current_sclk * 10;
9314 }
9315
9316 wm_high.disp_clk = mode->clock;
9317 wm_high.src_width = mode->crtc_hdisplay;
9318 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9319 wm_high.blank_time = line_time - wm_high.active_time;
9320 wm_high.interlaced = false;
cd84a27d 9321 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 9322 wm_high.interlaced = true;
9323 wm_high.vsc = radeon_crtc->vsc;
9324 wm_high.vtaps = 1;
cd84a27d 9325 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 9326 wm_high.vtaps = 2;
9327 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9328 wm_high.lb_size = lb_size;
9329 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9330 wm_high.num_heads = num_heads;
cd84a27d
AD		 9331
9332 /* set for high clocks */
58ea2dea
AD		 9333 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9334
9335 /* possibly force display priority to high */
9336 /* should really do this at mode validation time... */
9337 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9338 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9339 !dce8_check_latency_hiding(&wm_high) ||
9340 (rdev->disp_priority == 2)) {
9341 DRM_DEBUG_KMS("force priority to high\n");
9342 }
9343
9344 /* watermark for low clocks */
9345 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9346 rdev->pm.dpm_enabled) {
9347 wm_low.yclk =
9348 radeon_dpm_get_mclk(rdev, true) * 10;
9349 wm_low.sclk =
9350 radeon_dpm_get_sclk(rdev, true) * 10;
9351 } else {
9352 wm_low.yclk = rdev->pm.current_mclk * 10;
9353 wm_low.sclk = rdev->pm.current_sclk * 10;
9354 }
9355
9356 wm_low.disp_clk = mode->clock;
9357 wm_low.src_width = mode->crtc_hdisplay;
9358 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9359 wm_low.blank_time = line_time - wm_low.active_time;
9360 wm_low.interlaced = false;
9361 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9362 wm_low.interlaced = true;
9363 wm_low.vsc = radeon_crtc->vsc;
9364 wm_low.vtaps = 1;
9365 if (radeon_crtc->rmx_type != RMX_OFF)
9366 wm_low.vtaps = 2;
9367 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9368 wm_low.lb_size = lb_size;
9369 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9370 wm_low.num_heads = num_heads;
9371
cd84a27d 9372 /* set for low clocks */
58ea2dea 9373 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 9374
9375 /* possibly force display priority to high */
9376 /* should really do this at mode validation time... */
58ea2dea
AD		 9377 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9378 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9379 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 9380 (rdev->disp_priority == 2)) {
9381 DRM_DEBUG_KMS("force priority to high\n");
9382 }
5b5561b3
MK		 9383
9384 /* Save number of lines the linebuffer leads before the scanout */
9385 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
cd84a27d
AD		 9386 }
9387
9388 /* select wm A */
9389 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9390 tmp = wm_mask;
9391 tmp &= ~LATENCY_WATERMARK_MASK(3);
9392 tmp |= LATENCY_WATERMARK_MASK(1);
9393 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9394 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9395 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9396 LATENCY_HIGH_WATERMARK(line_time)));
9397 /* select wm B */
9398 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9399 tmp &= ~LATENCY_WATERMARK_MASK(3);
9400 tmp |= LATENCY_WATERMARK_MASK(2);
9401 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9402 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9403 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9404 LATENCY_HIGH_WATERMARK(line_time)));
9405 /* restore original selection */
9406 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 9407
9408 /* save values for DPM */
9409 radeon_crtc->line_time = line_time;
9410 radeon_crtc->wm_high = latency_watermark_a;
9411 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 9412}
9413
9414/**
9415 * dce8_bandwidth_update - program display watermarks
9416 *
9417 * @rdev: radeon_device pointer
9418 *
9419 * Calculate and program the display watermarks and line
9420 * buffer allocation (CIK).
9421 */
9422void dce8_bandwidth_update(struct radeon_device *rdev)
9423{
9424 struct drm_display_mode *mode = NULL;
9425 u32 num_heads = 0, lb_size;
9426 int i;
9427
8efe82ca
AD		 9428 if (!rdev->mode_info.mode_config_initialized)
9429 return;
9430
cd84a27d
AD		 9431 radeon_update_display_priority(rdev);
9432
9433 for (i = 0; i < rdev->num_crtc; i++) {
9434 if (rdev->mode_info.crtcs[i]->base.enabled)
9435 num_heads++;
9436 }
9437 for (i = 0; i < rdev->num_crtc; i++) {
9438 mode = &rdev->mode_info.crtcs[i]->base.mode;
9439 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9440 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9441 }
9442}
44fa346f
AD		 9443
9444/**
9445 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9446 *
9447 * @rdev: radeon_device pointer
9448 *
9449 * Fetches a GPU clock counter snapshot (SI).
9450 * Returns the 64 bit clock counter snapshot.
9451 */
9452uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9453{
9454 uint64_t clock;
9455
9456 mutex_lock(&rdev->gpu_clock_mutex);
9457 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9458 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
3cf8bb1a 9459 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
44fa346f
AD		 9460 mutex_unlock(&rdev->gpu_clock_mutex);
9461 return clock;
9462}
9463
87167bb1 9464static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
3cf8bb1a 9465 u32 cntl_reg, u32 status_reg)
87167bb1
CK		 9466{
9467 int r, i;
9468 struct atom_clock_dividers dividers;
9469 uint32_t tmp;
9470
9471 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9472 clock, false, &dividers);
9473 if (r)
9474 return r;
9475
9476 tmp = RREG32_SMC(cntl_reg);
9477 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9478 tmp |= dividers.post_divider;
9479 WREG32_SMC(cntl_reg, tmp);
9480
9481 for (i = 0; i < 100; i++) {
9482 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9483 break;
9484 mdelay(10);
9485 }
9486 if (i == 100)
9487 return -ETIMEDOUT;
9488
9489 return 0;
9490}
9491
9492int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9493{
9494 int r = 0;
9495
9496 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9497 if (r)
9498 return r;
9499
9500 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9501 return r;
9502}
9503
5ad6bf91
AD		 9504int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9505{
9506 int r, i;
9507 struct atom_clock_dividers dividers;
9508 u32 tmp;
9509
9510 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9511 ecclk, false, &dividers);
9512 if (r)
9513 return r;
9514
9515 for (i = 0; i < 100; i++) {
9516 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9517 break;
9518 mdelay(10);
9519 }
9520 if (i == 100)
9521 return -ETIMEDOUT;
9522
9523 tmp = RREG32_SMC(CG_ECLK_CNTL);
9524 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9525 tmp |= dividers.post_divider;
9526 WREG32_SMC(CG_ECLK_CNTL, tmp);
9527
9528 for (i = 0; i < 100; i++) {
9529 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9530 break;
9531 mdelay(10);
9532 }
9533 if (i == 100)
9534 return -ETIMEDOUT;
9535
9536 return 0;
9537}
9538
8a7cd276 9539static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 9540{
8a7cd276
AD		 9541 struct pci_dev *root = rdev->pdev->bus->self;
9542 int bridge_pos, gpu_pos;
9543 u32 speed_cntl, mask, current_data_rate;
9544 int ret, i;
9545 u16 tmp16;
87167bb1 9546
0bd252de
AW		 9547 if (pci_is_root_bus(rdev->pdev->bus))
9548 return;
9549
8a7cd276
AD		 9550 if (radeon_pcie_gen2 == 0)
9551 return;
87167bb1 9552
8a7cd276
AD		 9553 if (rdev->flags & RADEON_IS_IGP)
9554 return;
87167bb1 9555
8a7cd276
AD		 9556 if (!(rdev->flags & RADEON_IS_PCIE))
9557 return;
87167bb1 9558
8a7cd276
AD		 9559 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9560 if (ret != 0)
9561 return;
87167bb1 9562
8a7cd276
AD		 9563 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9564 return;
87167bb1 9565
8a7cd276
AD		 9566 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9567 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9568 LC_CURRENT_DATA_RATE_SHIFT;
9569 if (mask & DRM_PCIE_SPEED_80) {
9570 if (current_data_rate == 2) {
9571 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9572 return;
9573 }
9574 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9575 } else if (mask & DRM_PCIE_SPEED_50) {
9576 if (current_data_rate == 1) {
9577 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9578 return;
9579 }
9580 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9581 }
87167bb1 9582
8a7cd276
AD		 9583 bridge_pos = pci_pcie_cap(root);
9584 if (!bridge_pos)
9585 return;
9586
9587 gpu_pos = pci_pcie_cap(rdev->pdev);
9588 if (!gpu_pos)
9589 return;
9590
9591 if (mask & DRM_PCIE_SPEED_80) {
9592 /* re-try equalization if gen3 is not already enabled */
9593 if (current_data_rate != 2) {
9594 u16 bridge_cfg, gpu_cfg;
9595 u16 bridge_cfg2, gpu_cfg2;
9596 u32 max_lw, current_lw, tmp;
9597
9598 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9599 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9600
9601 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9602 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9603
9604 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9605 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9606
9607 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9608 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9609 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9610
9611 if (current_lw < max_lw) {
9612 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9613 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9614 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9615 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9616 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9617 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9618 }
9619 }
9620
9621 for (i = 0; i < 10; i++) {
9622 /* check status */
9623 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9624 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9625 break;
9626
9627 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9628 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9629
9630 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9631 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9632
9633 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9634 tmp |= LC_SET_QUIESCE;
9635 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9636
9637 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9638 tmp |= LC_REDO_EQ;
9639 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9640
9641 mdelay(100);
9642
9643 /* linkctl */
9644 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9645 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9646 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9647 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9648
9649 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9650 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9651 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9652 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9653
9654 /* linkctl2 */
9655 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9656 tmp16 &= ~((1 << 4) | (7 << 9));
9657 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9658 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9659
9660 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9661 tmp16 &= ~((1 << 4) | (7 << 9));
9662 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9663 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9664
9665 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9666 tmp &= ~LC_SET_QUIESCE;
9667 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9668 }
9669 }
9670 }
9671
9672 /* set the link speed */
9673 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9674 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9675 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9676
9677 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9678 tmp16 &= ~0xf;
9679 if (mask & DRM_PCIE_SPEED_80)
9680 tmp16 |= 3; /* gen3 */
9681 else if (mask & DRM_PCIE_SPEED_50)
9682 tmp16 |= 2; /* gen2 */
9683 else
9684 tmp16 |= 1; /* gen1 */
9685 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9686
9687 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9688 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9689 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9690
9691 for (i = 0; i < rdev->usec_timeout; i++) {
9692 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9693 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9694 break;
9695 udelay(1);
9696 }
9697}
7235711a
AD		 9698
9699static void cik_program_aspm(struct radeon_device *rdev)
9700{
9701 u32 data, orig;
9702 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9703 bool disable_clkreq = false;
9704
9705 if (radeon_aspm == 0)
9706 return;
9707
9708 /* XXX double check IGPs */
9709 if (rdev->flags & RADEON_IS_IGP)
9710 return;
9711
9712 if (!(rdev->flags & RADEON_IS_PCIE))
9713 return;
9714
9715 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9716 data &= ~LC_XMIT_N_FTS_MASK;
9717 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9718 if (orig != data)
9719 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9720
9721 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9722 data |= LC_GO_TO_RECOVERY;
9723 if (orig != data)
9724 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9725
9726 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9727 data |= P_IGNORE_EDB_ERR;
9728 if (orig != data)
9729 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9730
9731 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9732 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9733 data |= LC_PMI_TO_L1_DIS;
9734 if (!disable_l0s)
9735 data |= LC_L0S_INACTIVITY(7);
9736
9737 if (!disable_l1) {
9738 data |= LC_L1_INACTIVITY(7);
9739 data &= ~LC_PMI_TO_L1_DIS;
9740 if (orig != data)
9741 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9742
9743 if (!disable_plloff_in_l1) {
9744 bool clk_req_support;
9745
9746 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9747 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9748 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9749 if (orig != data)
9750 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9751
9752 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9753 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9754 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9755 if (orig != data)
9756 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9757
9758 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9759 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9760 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9761 if (orig != data)
9762 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9763
9764 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9765 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9766 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9767 if (orig != data)
9768 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9769
9770 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9771 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9772 data |= LC_DYN_LANES_PWR_STATE(3);
9773 if (orig != data)
9774 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9775
0bd252de
AW		 9776 if (!disable_clkreq &&
9777 !pci_is_root_bus(rdev->pdev->bus)) {
7235711a
AD		 9778 struct pci_dev *root = rdev->pdev->bus->self;
9779 u32 lnkcap;
9780
9781 clk_req_support = false;
9782 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9783 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9784 clk_req_support = true;
9785 } else {
9786 clk_req_support = false;
9787 }
9788
9789 if (clk_req_support) {
9790 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9791 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9792 if (orig != data)
9793 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9794
9795 orig = data = RREG32_SMC(THM_CLK_CNTL);
9796 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9797 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9798 if (orig != data)
9799 WREG32_SMC(THM_CLK_CNTL, data);
9800
9801 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9802 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9803 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9804 if (orig != data)
9805 WREG32_SMC(MISC_CLK_CTRL, data);
9806
9807 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9808 data &= ~BCLK_AS_XCLK;
9809 if (orig != data)
9810 WREG32_SMC(CG_CLKPIN_CNTL, data);
9811
9812 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9813 data &= ~FORCE_BIF_REFCLK_EN;
9814 if (orig != data)
9815 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9816
9817 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9818 data &= ~MPLL_CLKOUT_SEL_MASK;
9819 data |= MPLL_CLKOUT_SEL(4);
9820 if (orig != data)
9821 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9822 }
9823 }
9824 } else {
9825 if (orig != data)
9826 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9827 }
9828
9829 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9830 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9831 if (orig != data)
9832 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9833
9834 if (!disable_l0s) {
9835 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9836 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9837 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9838 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9839 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9840 data &= ~LC_L0S_INACTIVITY_MASK;
9841 if (orig != data)
9842 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9843 }
9844 }
9845 }
87167bb1 9846}
Linux 6.x block layer and io_uring tree(s)