projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'drm-intel-fixes-2014-08-21' of git://anongit.freedesktop.org/drm-intel
[linux-2.6-block.git] / drivers / gpu / drm / radeon / cik.c
CommitLineData
8cc1a532
AD		 1/*
2 * Copyright 2012 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 * Authors: Alex Deucher
23 */
24#include <linux/firmware.h>
8cc1a532
AD		 25#include <linux/slab.h>
26#include <linux/module.h>
27#include "drmP.h"
28#include "radeon.h"
6f2043ce 29#include "radeon_asic.h"
8cc1a532
AD		 30#include "cikd.h"
31#include "atom.h"
841cf442 32#include "cik_blit_shaders.h"
8c68e393 33#include "radeon_ucode.h"
22c775ce 34#include "clearstate_ci.h"
02c81327
AD		 35
36MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
277babc3 41MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
02c81327 42MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
21a93e13 43MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
cc8dbbb4 44MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
f2c6b0f4
AD		 45
46MODULE_FIRMWARE("radeon/bonaire_pfp.bin");
47MODULE_FIRMWARE("radeon/bonaire_me.bin");
48MODULE_FIRMWARE("radeon/bonaire_ce.bin");
49MODULE_FIRMWARE("radeon/bonaire_mec.bin");
50MODULE_FIRMWARE("radeon/bonaire_mc.bin");
51MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
52MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
53MODULE_FIRMWARE("radeon/bonaire_smc.bin");
54
d4775655
AD		 55MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
56MODULE_FIRMWARE("radeon/HAWAII_me.bin");
57MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
58MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
59MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
277babc3 60MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
d4775655
AD		 61MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
62MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
63MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
f2c6b0f4
AD		 64
65MODULE_FIRMWARE("radeon/hawaii_pfp.bin");
66MODULE_FIRMWARE("radeon/hawaii_me.bin");
67MODULE_FIRMWARE("radeon/hawaii_ce.bin");
68MODULE_FIRMWARE("radeon/hawaii_mec.bin");
69MODULE_FIRMWARE("radeon/hawaii_mc.bin");
70MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
71MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
72MODULE_FIRMWARE("radeon/hawaii_smc.bin");
73
02c81327
AD		 74MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
75MODULE_FIRMWARE("radeon/KAVERI_me.bin");
76MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
77MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
78MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
21a93e13 79MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
f2c6b0f4
AD		 80
81MODULE_FIRMWARE("radeon/kaveri_pfp.bin");
82MODULE_FIRMWARE("radeon/kaveri_me.bin");
83MODULE_FIRMWARE("radeon/kaveri_ce.bin");
84MODULE_FIRMWARE("radeon/kaveri_mec.bin");
85MODULE_FIRMWARE("radeon/kaveri_mec2.bin");
86MODULE_FIRMWARE("radeon/kaveri_rlc.bin");
87MODULE_FIRMWARE("radeon/kaveri_sdma.bin");
88
02c81327
AD		 89MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
90MODULE_FIRMWARE("radeon/KABINI_me.bin");
91MODULE_FIRMWARE("radeon/KABINI_ce.bin");
92MODULE_FIRMWARE("radeon/KABINI_mec.bin");
93MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
21a93e13 94MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
f2c6b0f4
AD		 95
96MODULE_FIRMWARE("radeon/kabini_pfp.bin");
97MODULE_FIRMWARE("radeon/kabini_me.bin");
98MODULE_FIRMWARE("radeon/kabini_ce.bin");
99MODULE_FIRMWARE("radeon/kabini_mec.bin");
100MODULE_FIRMWARE("radeon/kabini_rlc.bin");
101MODULE_FIRMWARE("radeon/kabini_sdma.bin");
102
f73a9e83
SL		 103MODULE_FIRMWARE("radeon/MULLINS_pfp.bin");
104MODULE_FIRMWARE("radeon/MULLINS_me.bin");
105MODULE_FIRMWARE("radeon/MULLINS_ce.bin");
106MODULE_FIRMWARE("radeon/MULLINS_mec.bin");
107MODULE_FIRMWARE("radeon/MULLINS_rlc.bin");
108MODULE_FIRMWARE("radeon/MULLINS_sdma.bin");
02c81327 109
f2c6b0f4
AD		 110MODULE_FIRMWARE("radeon/mullins_pfp.bin");
111MODULE_FIRMWARE("radeon/mullins_me.bin");
112MODULE_FIRMWARE("radeon/mullins_ce.bin");
113MODULE_FIRMWARE("radeon/mullins_mec.bin");
114MODULE_FIRMWARE("radeon/mullins_rlc.bin");
115MODULE_FIRMWARE("radeon/mullins_sdma.bin");
116
a59781bb
AD		 117extern int r600_ih_ring_alloc(struct radeon_device *rdev);
118extern void r600_ih_ring_fini(struct radeon_device *rdev);
6f2043ce
AD		 119extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
120extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
cc066715 121extern bool evergreen_is_display_hung(struct radeon_device *rdev);
1fd11777
AD		 122extern void sumo_rlc_fini(struct radeon_device *rdev);
123extern int sumo_rlc_init(struct radeon_device *rdev);
1c49165d 124extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
866d83de 125extern void si_rlc_reset(struct radeon_device *rdev);
22c775ce 126extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
65fcf668 127static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
2483b4ea
CK		 128extern int cik_sdma_resume(struct radeon_device *rdev);
129extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
130extern void cik_sdma_fini(struct radeon_device *rdev);
a1d6f97c 131extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
cc066715 132static void cik_rlc_stop(struct radeon_device *rdev);
8a7cd276 133static void cik_pcie_gen3_enable(struct radeon_device *rdev);
7235711a 134static void cik_program_aspm(struct radeon_device *rdev);
22c775ce
AD		 135static void cik_init_pg(struct radeon_device *rdev);
136static void cik_init_cg(struct radeon_device *rdev);
fb2c7f4d
AD		 137static void cik_fini_pg(struct radeon_device *rdev);
138static void cik_fini_cg(struct radeon_device *rdev);
4214faf6
AD		 139static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
140 bool enable);
6f2043ce 141
286d9cc6
AD		 142/* get temperature in millidegrees */
143int ci_get_temp(struct radeon_device *rdev)
144{
145 u32 temp;
146 int actual_temp = 0;
147
148 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
149 CTF_TEMP_SHIFT;
150
151 if (temp & 0x200)
152 actual_temp = 255;
153 else
154 actual_temp = temp & 0x1ff;
155
156 actual_temp = actual_temp * 1000;
157
158 return actual_temp;
159}
160
161/* get temperature in millidegrees */
162int kv_get_temp(struct radeon_device *rdev)
163{
164 u32 temp;
165 int actual_temp = 0;
166
167 temp = RREG32_SMC(0xC0300E0C);
168
169 if (temp)
170 actual_temp = (temp / 8) - 49;
171 else
172 actual_temp = 0;
173
174 actual_temp = actual_temp * 1000;
175
176 return actual_temp;
177}
6f2043ce 178
6e2c3c0a
AD		 179/*
180 * Indirect registers accessor
181 */
182u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
183{
0a5b7b0b 184 unsigned long flags;
6e2c3c0a
AD		 185 u32 r;
186
0a5b7b0b 187 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 188 WREG32(PCIE_INDEX, reg);
189 (void)RREG32(PCIE_INDEX);
190 r = RREG32(PCIE_DATA);
0a5b7b0b 191 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 192 return r;
193}
194
195void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
196{
0a5b7b0b
AD		 197 unsigned long flags;
198
199 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 200 WREG32(PCIE_INDEX, reg);
201 (void)RREG32(PCIE_INDEX);
202 WREG32(PCIE_DATA, v);
203 (void)RREG32(PCIE_DATA);
0a5b7b0b 204 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
6e2c3c0a
AD		 205}
206
22c775ce
AD		 207static const u32 spectre_rlc_save_restore_register_list[] =
208{
209 (0x0e00 << 16) | (0xc12c >> 2),
210 0x00000000,
211 (0x0e00 << 16) | (0xc140 >> 2),
212 0x00000000,
213 (0x0e00 << 16) | (0xc150 >> 2),
214 0x00000000,
215 (0x0e00 << 16) | (0xc15c >> 2),
216 0x00000000,
217 (0x0e00 << 16) | (0xc168 >> 2),
218 0x00000000,
219 (0x0e00 << 16) | (0xc170 >> 2),
220 0x00000000,
221 (0x0e00 << 16) | (0xc178 >> 2),
222 0x00000000,
223 (0x0e00 << 16) | (0xc204 >> 2),
224 0x00000000,
225 (0x0e00 << 16) | (0xc2b4 >> 2),
226 0x00000000,
227 (0x0e00 << 16) | (0xc2b8 >> 2),
228 0x00000000,
229 (0x0e00 << 16) | (0xc2bc >> 2),
230 0x00000000,
231 (0x0e00 << 16) | (0xc2c0 >> 2),
232 0x00000000,
233 (0x0e00 << 16) | (0x8228 >> 2),
234 0x00000000,
235 (0x0e00 << 16) | (0x829c >> 2),
236 0x00000000,
237 (0x0e00 << 16) | (0x869c >> 2),
238 0x00000000,
239 (0x0600 << 16) | (0x98f4 >> 2),
240 0x00000000,
241 (0x0e00 << 16) | (0x98f8 >> 2),
242 0x00000000,
243 (0x0e00 << 16) | (0x9900 >> 2),
244 0x00000000,
245 (0x0e00 << 16) | (0xc260 >> 2),
246 0x00000000,
247 (0x0e00 << 16) | (0x90e8 >> 2),
248 0x00000000,
249 (0x0e00 << 16) | (0x3c000 >> 2),
250 0x00000000,
251 (0x0e00 << 16) | (0x3c00c >> 2),
252 0x00000000,
253 (0x0e00 << 16) | (0x8c1c >> 2),
254 0x00000000,
255 (0x0e00 << 16) | (0x9700 >> 2),
256 0x00000000,
257 (0x0e00 << 16) | (0xcd20 >> 2),
258 0x00000000,
259 (0x4e00 << 16) | (0xcd20 >> 2),
260 0x00000000,
261 (0x5e00 << 16) | (0xcd20 >> 2),
262 0x00000000,
263 (0x6e00 << 16) | (0xcd20 >> 2),
264 0x00000000,
265 (0x7e00 << 16) | (0xcd20 >> 2),
266 0x00000000,
267 (0x8e00 << 16) | (0xcd20 >> 2),
268 0x00000000,
269 (0x9e00 << 16) | (0xcd20 >> 2),
270 0x00000000,
271 (0xae00 << 16) | (0xcd20 >> 2),
272 0x00000000,
273 (0xbe00 << 16) | (0xcd20 >> 2),
274 0x00000000,
275 (0x0e00 << 16) | (0x89bc >> 2),
276 0x00000000,
277 (0x0e00 << 16) | (0x8900 >> 2),
278 0x00000000,
279 0x3,
280 (0x0e00 << 16) | (0xc130 >> 2),
281 0x00000000,
282 (0x0e00 << 16) | (0xc134 >> 2),
283 0x00000000,
284 (0x0e00 << 16) | (0xc1fc >> 2),
285 0x00000000,
286 (0x0e00 << 16) | (0xc208 >> 2),
287 0x00000000,
288 (0x0e00 << 16) | (0xc264 >> 2),
289 0x00000000,
290 (0x0e00 << 16) | (0xc268 >> 2),
291 0x00000000,
292 (0x0e00 << 16) | (0xc26c >> 2),
293 0x00000000,
294 (0x0e00 << 16) | (0xc270 >> 2),
295 0x00000000,
296 (0x0e00 << 16) | (0xc274 >> 2),
297 0x00000000,
298 (0x0e00 << 16) | (0xc278 >> 2),
299 0x00000000,
300 (0x0e00 << 16) | (0xc27c >> 2),
301 0x00000000,
302 (0x0e00 << 16) | (0xc280 >> 2),
303 0x00000000,
304 (0x0e00 << 16) | (0xc284 >> 2),
305 0x00000000,
306 (0x0e00 << 16) | (0xc288 >> 2),
307 0x00000000,
308 (0x0e00 << 16) | (0xc28c >> 2),
309 0x00000000,
310 (0x0e00 << 16) | (0xc290 >> 2),
311 0x00000000,
312 (0x0e00 << 16) | (0xc294 >> 2),
313 0x00000000,
314 (0x0e00 << 16) | (0xc298 >> 2),
315 0x00000000,
316 (0x0e00 << 16) | (0xc29c >> 2),
317 0x00000000,
318 (0x0e00 << 16) | (0xc2a0 >> 2),
319 0x00000000,
320 (0x0e00 << 16) | (0xc2a4 >> 2),
321 0x00000000,
322 (0x0e00 << 16) | (0xc2a8 >> 2),
323 0x00000000,
324 (0x0e00 << 16) | (0xc2ac >> 2),
325 0x00000000,
326 (0x0e00 << 16) | (0xc2b0 >> 2),
327 0x00000000,
328 (0x0e00 << 16) | (0x301d0 >> 2),
329 0x00000000,
330 (0x0e00 << 16) | (0x30238 >> 2),
331 0x00000000,
332 (0x0e00 << 16) | (0x30250 >> 2),
333 0x00000000,
334 (0x0e00 << 16) | (0x30254 >> 2),
335 0x00000000,
336 (0x0e00 << 16) | (0x30258 >> 2),
337 0x00000000,
338 (0x0e00 << 16) | (0x3025c >> 2),
339 0x00000000,
340 (0x4e00 << 16) | (0xc900 >> 2),
341 0x00000000,
342 (0x5e00 << 16) | (0xc900 >> 2),
343 0x00000000,
344 (0x6e00 << 16) | (0xc900 >> 2),
345 0x00000000,
346 (0x7e00 << 16) | (0xc900 >> 2),
347 0x00000000,
348 (0x8e00 << 16) | (0xc900 >> 2),
349 0x00000000,
350 (0x9e00 << 16) | (0xc900 >> 2),
351 0x00000000,
352 (0xae00 << 16) | (0xc900 >> 2),
353 0x00000000,
354 (0xbe00 << 16) | (0xc900 >> 2),
355 0x00000000,
356 (0x4e00 << 16) | (0xc904 >> 2),
357 0x00000000,
358 (0x5e00 << 16) | (0xc904 >> 2),
359 0x00000000,
360 (0x6e00 << 16) | (0xc904 >> 2),
361 0x00000000,
362 (0x7e00 << 16) | (0xc904 >> 2),
363 0x00000000,
364 (0x8e00 << 16) | (0xc904 >> 2),
365 0x00000000,
366 (0x9e00 << 16) | (0xc904 >> 2),
367 0x00000000,
368 (0xae00 << 16) | (0xc904 >> 2),
369 0x00000000,
370 (0xbe00 << 16) | (0xc904 >> 2),
371 0x00000000,
372 (0x4e00 << 16) | (0xc908 >> 2),
373 0x00000000,
374 (0x5e00 << 16) | (0xc908 >> 2),
375 0x00000000,
376 (0x6e00 << 16) | (0xc908 >> 2),
377 0x00000000,
378 (0x7e00 << 16) | (0xc908 >> 2),
379 0x00000000,
380 (0x8e00 << 16) | (0xc908 >> 2),
381 0x00000000,
382 (0x9e00 << 16) | (0xc908 >> 2),
383 0x00000000,
384 (0xae00 << 16) | (0xc908 >> 2),
385 0x00000000,
386 (0xbe00 << 16) | (0xc908 >> 2),
387 0x00000000,
388 (0x4e00 << 16) | (0xc90c >> 2),
389 0x00000000,
390 (0x5e00 << 16) | (0xc90c >> 2),
391 0x00000000,
392 (0x6e00 << 16) | (0xc90c >> 2),
393 0x00000000,
394 (0x7e00 << 16) | (0xc90c >> 2),
395 0x00000000,
396 (0x8e00 << 16) | (0xc90c >> 2),
397 0x00000000,
398 (0x9e00 << 16) | (0xc90c >> 2),
399 0x00000000,
400 (0xae00 << 16) | (0xc90c >> 2),
401 0x00000000,
402 (0xbe00 << 16) | (0xc90c >> 2),
403 0x00000000,
404 (0x4e00 << 16) | (0xc910 >> 2),
405 0x00000000,
406 (0x5e00 << 16) | (0xc910 >> 2),
407 0x00000000,
408 (0x6e00 << 16) | (0xc910 >> 2),
409 0x00000000,
410 (0x7e00 << 16) | (0xc910 >> 2),
411 0x00000000,
412 (0x8e00 << 16) | (0xc910 >> 2),
413 0x00000000,
414 (0x9e00 << 16) | (0xc910 >> 2),
415 0x00000000,
416 (0xae00 << 16) | (0xc910 >> 2),
417 0x00000000,
418 (0xbe00 << 16) | (0xc910 >> 2),
419 0x00000000,
420 (0x0e00 << 16) | (0xc99c >> 2),
421 0x00000000,
422 (0x0e00 << 16) | (0x9834 >> 2),
423 0x00000000,
424 (0x0000 << 16) | (0x30f00 >> 2),
425 0x00000000,
426 (0x0001 << 16) | (0x30f00 >> 2),
427 0x00000000,
428 (0x0000 << 16) | (0x30f04 >> 2),
429 0x00000000,
430 (0x0001 << 16) | (0x30f04 >> 2),
431 0x00000000,
432 (0x0000 << 16) | (0x30f08 >> 2),
433 0x00000000,
434 (0x0001 << 16) | (0x30f08 >> 2),
435 0x00000000,
436 (0x0000 << 16) | (0x30f0c >> 2),
437 0x00000000,
438 (0x0001 << 16) | (0x30f0c >> 2),
439 0x00000000,
440 (0x0600 << 16) | (0x9b7c >> 2),
441 0x00000000,
442 (0x0e00 << 16) | (0x8a14 >> 2),
443 0x00000000,
444 (0x0e00 << 16) | (0x8a18 >> 2),
445 0x00000000,
446 (0x0600 << 16) | (0x30a00 >> 2),
447 0x00000000,
448 (0x0e00 << 16) | (0x8bf0 >> 2),
449 0x00000000,
450 (0x0e00 << 16) | (0x8bcc >> 2),
451 0x00000000,
452 (0x0e00 << 16) | (0x8b24 >> 2),
453 0x00000000,
454 (0x0e00 << 16) | (0x30a04 >> 2),
455 0x00000000,
456 (0x0600 << 16) | (0x30a10 >> 2),
457 0x00000000,
458 (0x0600 << 16) | (0x30a14 >> 2),
459 0x00000000,
460 (0x0600 << 16) | (0x30a18 >> 2),
461 0x00000000,
462 (0x0600 << 16) | (0x30a2c >> 2),
463 0x00000000,
464 (0x0e00 << 16) | (0xc700 >> 2),
465 0x00000000,
466 (0x0e00 << 16) | (0xc704 >> 2),
467 0x00000000,
468 (0x0e00 << 16) | (0xc708 >> 2),
469 0x00000000,
470 (0x0e00 << 16) | (0xc768 >> 2),
471 0x00000000,
472 (0x0400 << 16) | (0xc770 >> 2),
473 0x00000000,
474 (0x0400 << 16) | (0xc774 >> 2),
475 0x00000000,
476 (0x0400 << 16) | (0xc778 >> 2),
477 0x00000000,
478 (0x0400 << 16) | (0xc77c >> 2),
479 0x00000000,
480 (0x0400 << 16) | (0xc780 >> 2),
481 0x00000000,
482 (0x0400 << 16) | (0xc784 >> 2),
483 0x00000000,
484 (0x0400 << 16) | (0xc788 >> 2),
485 0x00000000,
486 (0x0400 << 16) | (0xc78c >> 2),
487 0x00000000,
488 (0x0400 << 16) | (0xc798 >> 2),
489 0x00000000,
490 (0x0400 << 16) | (0xc79c >> 2),
491 0x00000000,
492 (0x0400 << 16) | (0xc7a0 >> 2),
493 0x00000000,
494 (0x0400 << 16) | (0xc7a4 >> 2),
495 0x00000000,
496 (0x0400 << 16) | (0xc7a8 >> 2),
497 0x00000000,
498 (0x0400 << 16) | (0xc7ac >> 2),
499 0x00000000,
500 (0x0400 << 16) | (0xc7b0 >> 2),
501 0x00000000,
502 (0x0400 << 16) | (0xc7b4 >> 2),
503 0x00000000,
504 (0x0e00 << 16) | (0x9100 >> 2),
505 0x00000000,
506 (0x0e00 << 16) | (0x3c010 >> 2),
507 0x00000000,
508 (0x0e00 << 16) | (0x92a8 >> 2),
509 0x00000000,
510 (0x0e00 << 16) | (0x92ac >> 2),
511 0x00000000,
512 (0x0e00 << 16) | (0x92b4 >> 2),
513 0x00000000,
514 (0x0e00 << 16) | (0x92b8 >> 2),
515 0x00000000,
516 (0x0e00 << 16) | (0x92bc >> 2),
517 0x00000000,
518 (0x0e00 << 16) | (0x92c0 >> 2),
519 0x00000000,
520 (0x0e00 << 16) | (0x92c4 >> 2),
521 0x00000000,
522 (0x0e00 << 16) | (0x92c8 >> 2),
523 0x00000000,
524 (0x0e00 << 16) | (0x92cc >> 2),
525 0x00000000,
526 (0x0e00 << 16) | (0x92d0 >> 2),
527 0x00000000,
528 (0x0e00 << 16) | (0x8c00 >> 2),
529 0x00000000,
530 (0x0e00 << 16) | (0x8c04 >> 2),
531 0x00000000,
532 (0x0e00 << 16) | (0x8c20 >> 2),
533 0x00000000,
534 (0x0e00 << 16) | (0x8c38 >> 2),
535 0x00000000,
536 (0x0e00 << 16) | (0x8c3c >> 2),
537 0x00000000,
538 (0x0e00 << 16) | (0xae00 >> 2),
539 0x00000000,
540 (0x0e00 << 16) | (0x9604 >> 2),
541 0x00000000,
542 (0x0e00 << 16) | (0xac08 >> 2),
543 0x00000000,
544 (0x0e00 << 16) | (0xac0c >> 2),
545 0x00000000,
546 (0x0e00 << 16) | (0xac10 >> 2),
547 0x00000000,
548 (0x0e00 << 16) | (0xac14 >> 2),
549 0x00000000,
550 (0x0e00 << 16) | (0xac58 >> 2),
551 0x00000000,
552 (0x0e00 << 16) | (0xac68 >> 2),
553 0x00000000,
554 (0x0e00 << 16) | (0xac6c >> 2),
555 0x00000000,
556 (0x0e00 << 16) | (0xac70 >> 2),
557 0x00000000,
558 (0x0e00 << 16) | (0xac74 >> 2),
559 0x00000000,
560 (0x0e00 << 16) | (0xac78 >> 2),
561 0x00000000,
562 (0x0e00 << 16) | (0xac7c >> 2),
563 0x00000000,
564 (0x0e00 << 16) | (0xac80 >> 2),
565 0x00000000,
566 (0x0e00 << 16) | (0xac84 >> 2),
567 0x00000000,
568 (0x0e00 << 16) | (0xac88 >> 2),
569 0x00000000,
570 (0x0e00 << 16) | (0xac8c >> 2),
571 0x00000000,
572 (0x0e00 << 16) | (0x970c >> 2),
573 0x00000000,
574 (0x0e00 << 16) | (0x9714 >> 2),
575 0x00000000,
576 (0x0e00 << 16) | (0x9718 >> 2),
577 0x00000000,
578 (0x0e00 << 16) | (0x971c >> 2),
579 0x00000000,
580 (0x0e00 << 16) | (0x31068 >> 2),
581 0x00000000,
582 (0x4e00 << 16) | (0x31068 >> 2),
583 0x00000000,
584 (0x5e00 << 16) | (0x31068 >> 2),
585 0x00000000,
586 (0x6e00 << 16) | (0x31068 >> 2),
587 0x00000000,
588 (0x7e00 << 16) | (0x31068 >> 2),
589 0x00000000,
590 (0x8e00 << 16) | (0x31068 >> 2),
591 0x00000000,
592 (0x9e00 << 16) | (0x31068 >> 2),
593 0x00000000,
594 (0xae00 << 16) | (0x31068 >> 2),
595 0x00000000,
596 (0xbe00 << 16) | (0x31068 >> 2),
597 0x00000000,
598 (0x0e00 << 16) | (0xcd10 >> 2),
599 0x00000000,
600 (0x0e00 << 16) | (0xcd14 >> 2),
601 0x00000000,
602 (0x0e00 << 16) | (0x88b0 >> 2),
603 0x00000000,
604 (0x0e00 << 16) | (0x88b4 >> 2),
605 0x00000000,
606 (0x0e00 << 16) | (0x88b8 >> 2),
607 0x00000000,
608 (0x0e00 << 16) | (0x88bc >> 2),
609 0x00000000,
610 (0x0400 << 16) | (0x89c0 >> 2),
611 0x00000000,
612 (0x0e00 << 16) | (0x88c4 >> 2),
613 0x00000000,
614 (0x0e00 << 16) | (0x88c8 >> 2),
615 0x00000000,
616 (0x0e00 << 16) | (0x88d0 >> 2),
617 0x00000000,
618 (0x0e00 << 16) | (0x88d4 >> 2),
619 0x00000000,
620 (0x0e00 << 16) | (0x88d8 >> 2),
621 0x00000000,
622 (0x0e00 << 16) | (0x8980 >> 2),
623 0x00000000,
624 (0x0e00 << 16) | (0x30938 >> 2),
625 0x00000000,
626 (0x0e00 << 16) | (0x3093c >> 2),
627 0x00000000,
628 (0x0e00 << 16) | (0x30940 >> 2),
629 0x00000000,
630 (0x0e00 << 16) | (0x89a0 >> 2),
631 0x00000000,
632 (0x0e00 << 16) | (0x30900 >> 2),
633 0x00000000,
634 (0x0e00 << 16) | (0x30904 >> 2),
635 0x00000000,
636 (0x0e00 << 16) | (0x89b4 >> 2),
637 0x00000000,
638 (0x0e00 << 16) | (0x3c210 >> 2),
639 0x00000000,
640 (0x0e00 << 16) | (0x3c214 >> 2),
641 0x00000000,
642 (0x0e00 << 16) | (0x3c218 >> 2),
643 0x00000000,
644 (0x0e00 << 16) | (0x8904 >> 2),
645 0x00000000,
646 0x5,
647 (0x0e00 << 16) | (0x8c28 >> 2),
648 (0x0e00 << 16) | (0x8c2c >> 2),
649 (0x0e00 << 16) | (0x8c30 >> 2),
650 (0x0e00 << 16) | (0x8c34 >> 2),
651 (0x0e00 << 16) | (0x9600 >> 2),
652};
653
654static const u32 kalindi_rlc_save_restore_register_list[] =
655{
656 (0x0e00 << 16) | (0xc12c >> 2),
657 0x00000000,
658 (0x0e00 << 16) | (0xc140 >> 2),
659 0x00000000,
660 (0x0e00 << 16) | (0xc150 >> 2),
661 0x00000000,
662 (0x0e00 << 16) | (0xc15c >> 2),
663 0x00000000,
664 (0x0e00 << 16) | (0xc168 >> 2),
665 0x00000000,
666 (0x0e00 << 16) | (0xc170 >> 2),
667 0x00000000,
668 (0x0e00 << 16) | (0xc204 >> 2),
669 0x00000000,
670 (0x0e00 << 16) | (0xc2b4 >> 2),
671 0x00000000,
672 (0x0e00 << 16) | (0xc2b8 >> 2),
673 0x00000000,
674 (0x0e00 << 16) | (0xc2bc >> 2),
675 0x00000000,
676 (0x0e00 << 16) | (0xc2c0 >> 2),
677 0x00000000,
678 (0x0e00 << 16) | (0x8228 >> 2),
679 0x00000000,
680 (0x0e00 << 16) | (0x829c >> 2),
681 0x00000000,
682 (0x0e00 << 16) | (0x869c >> 2),
683 0x00000000,
684 (0x0600 << 16) | (0x98f4 >> 2),
685 0x00000000,
686 (0x0e00 << 16) | (0x98f8 >> 2),
687 0x00000000,
688 (0x0e00 << 16) | (0x9900 >> 2),
689 0x00000000,
690 (0x0e00 << 16) | (0xc260 >> 2),
691 0x00000000,
692 (0x0e00 << 16) | (0x90e8 >> 2),
693 0x00000000,
694 (0x0e00 << 16) | (0x3c000 >> 2),
695 0x00000000,
696 (0x0e00 << 16) | (0x3c00c >> 2),
697 0x00000000,
698 (0x0e00 << 16) | (0x8c1c >> 2),
699 0x00000000,
700 (0x0e00 << 16) | (0x9700 >> 2),
701 0x00000000,
702 (0x0e00 << 16) | (0xcd20 >> 2),
703 0x00000000,
704 (0x4e00 << 16) | (0xcd20 >> 2),
705 0x00000000,
706 (0x5e00 << 16) | (0xcd20 >> 2),
707 0x00000000,
708 (0x6e00 << 16) | (0xcd20 >> 2),
709 0x00000000,
710 (0x7e00 << 16) | (0xcd20 >> 2),
711 0x00000000,
712 (0x0e00 << 16) | (0x89bc >> 2),
713 0x00000000,
714 (0x0e00 << 16) | (0x8900 >> 2),
715 0x00000000,
716 0x3,
717 (0x0e00 << 16) | (0xc130 >> 2),
718 0x00000000,
719 (0x0e00 << 16) | (0xc134 >> 2),
720 0x00000000,
721 (0x0e00 << 16) | (0xc1fc >> 2),
722 0x00000000,
723 (0x0e00 << 16) | (0xc208 >> 2),
724 0x00000000,
725 (0x0e00 << 16) | (0xc264 >> 2),
726 0x00000000,
727 (0x0e00 << 16) | (0xc268 >> 2),
728 0x00000000,
729 (0x0e00 << 16) | (0xc26c >> 2),
730 0x00000000,
731 (0x0e00 << 16) | (0xc270 >> 2),
732 0x00000000,
733 (0x0e00 << 16) | (0xc274 >> 2),
734 0x00000000,
735 (0x0e00 << 16) | (0xc28c >> 2),
736 0x00000000,
737 (0x0e00 << 16) | (0xc290 >> 2),
738 0x00000000,
739 (0x0e00 << 16) | (0xc294 >> 2),
740 0x00000000,
741 (0x0e00 << 16) | (0xc298 >> 2),
742 0x00000000,
743 (0x0e00 << 16) | (0xc2a0 >> 2),
744 0x00000000,
745 (0x0e00 << 16) | (0xc2a4 >> 2),
746 0x00000000,
747 (0x0e00 << 16) | (0xc2a8 >> 2),
748 0x00000000,
749 (0x0e00 << 16) | (0xc2ac >> 2),
750 0x00000000,
751 (0x0e00 << 16) | (0x301d0 >> 2),
752 0x00000000,
753 (0x0e00 << 16) | (0x30238 >> 2),
754 0x00000000,
755 (0x0e00 << 16) | (0x30250 >> 2),
756 0x00000000,
757 (0x0e00 << 16) | (0x30254 >> 2),
758 0x00000000,
759 (0x0e00 << 16) | (0x30258 >> 2),
760 0x00000000,
761 (0x0e00 << 16) | (0x3025c >> 2),
762 0x00000000,
763 (0x4e00 << 16) | (0xc900 >> 2),
764 0x00000000,
765 (0x5e00 << 16) | (0xc900 >> 2),
766 0x00000000,
767 (0x6e00 << 16) | (0xc900 >> 2),
768 0x00000000,
769 (0x7e00 << 16) | (0xc900 >> 2),
770 0x00000000,
771 (0x4e00 << 16) | (0xc904 >> 2),
772 0x00000000,
773 (0x5e00 << 16) | (0xc904 >> 2),
774 0x00000000,
775 (0x6e00 << 16) | (0xc904 >> 2),
776 0x00000000,
777 (0x7e00 << 16) | (0xc904 >> 2),
778 0x00000000,
779 (0x4e00 << 16) | (0xc908 >> 2),
780 0x00000000,
781 (0x5e00 << 16) | (0xc908 >> 2),
782 0x00000000,
783 (0x6e00 << 16) | (0xc908 >> 2),
784 0x00000000,
785 (0x7e00 << 16) | (0xc908 >> 2),
786 0x00000000,
787 (0x4e00 << 16) | (0xc90c >> 2),
788 0x00000000,
789 (0x5e00 << 16) | (0xc90c >> 2),
790 0x00000000,
791 (0x6e00 << 16) | (0xc90c >> 2),
792 0x00000000,
793 (0x7e00 << 16) | (0xc90c >> 2),
794 0x00000000,
795 (0x4e00 << 16) | (0xc910 >> 2),
796 0x00000000,
797 (0x5e00 << 16) | (0xc910 >> 2),
798 0x00000000,
799 (0x6e00 << 16) | (0xc910 >> 2),
800 0x00000000,
801 (0x7e00 << 16) | (0xc910 >> 2),
802 0x00000000,
803 (0x0e00 << 16) | (0xc99c >> 2),
804 0x00000000,
805 (0x0e00 << 16) | (0x9834 >> 2),
806 0x00000000,
807 (0x0000 << 16) | (0x30f00 >> 2),
808 0x00000000,
809 (0x0000 << 16) | (0x30f04 >> 2),
810 0x00000000,
811 (0x0000 << 16) | (0x30f08 >> 2),
812 0x00000000,
813 (0x0000 << 16) | (0x30f0c >> 2),
814 0x00000000,
815 (0x0600 << 16) | (0x9b7c >> 2),
816 0x00000000,
817 (0x0e00 << 16) | (0x8a14 >> 2),
818 0x00000000,
819 (0x0e00 << 16) | (0x8a18 >> 2),
820 0x00000000,
821 (0x0600 << 16) | (0x30a00 >> 2),
822 0x00000000,
823 (0x0e00 << 16) | (0x8bf0 >> 2),
824 0x00000000,
825 (0x0e00 << 16) | (0x8bcc >> 2),
826 0x00000000,
827 (0x0e00 << 16) | (0x8b24 >> 2),
828 0x00000000,
829 (0x0e00 << 16) | (0x30a04 >> 2),
830 0x00000000,
831 (0x0600 << 16) | (0x30a10 >> 2),
832 0x00000000,
833 (0x0600 << 16) | (0x30a14 >> 2),
834 0x00000000,
835 (0x0600 << 16) | (0x30a18 >> 2),
836 0x00000000,
837 (0x0600 << 16) | (0x30a2c >> 2),
838 0x00000000,
839 (0x0e00 << 16) | (0xc700 >> 2),
840 0x00000000,
841 (0x0e00 << 16) | (0xc704 >> 2),
842 0x00000000,
843 (0x0e00 << 16) | (0xc708 >> 2),
844 0x00000000,
845 (0x0e00 << 16) | (0xc768 >> 2),
846 0x00000000,
847 (0x0400 << 16) | (0xc770 >> 2),
848 0x00000000,
849 (0x0400 << 16) | (0xc774 >> 2),
850 0x00000000,
851 (0x0400 << 16) | (0xc798 >> 2),
852 0x00000000,
853 (0x0400 << 16) | (0xc79c >> 2),
854 0x00000000,
855 (0x0e00 << 16) | (0x9100 >> 2),
856 0x00000000,
857 (0x0e00 << 16) | (0x3c010 >> 2),
858 0x00000000,
859 (0x0e00 << 16) | (0x8c00 >> 2),
860 0x00000000,
861 (0x0e00 << 16) | (0x8c04 >> 2),
862 0x00000000,
863 (0x0e00 << 16) | (0x8c20 >> 2),
864 0x00000000,
865 (0x0e00 << 16) | (0x8c38 >> 2),
866 0x00000000,
867 (0x0e00 << 16) | (0x8c3c >> 2),
868 0x00000000,
869 (0x0e00 << 16) | (0xae00 >> 2),
870 0x00000000,
871 (0x0e00 << 16) | (0x9604 >> 2),
872 0x00000000,
873 (0x0e00 << 16) | (0xac08 >> 2),
874 0x00000000,
875 (0x0e00 << 16) | (0xac0c >> 2),
876 0x00000000,
877 (0x0e00 << 16) | (0xac10 >> 2),
878 0x00000000,
879 (0x0e00 << 16) | (0xac14 >> 2),
880 0x00000000,
881 (0x0e00 << 16) | (0xac58 >> 2),
882 0x00000000,
883 (0x0e00 << 16) | (0xac68 >> 2),
884 0x00000000,
885 (0x0e00 << 16) | (0xac6c >> 2),
886 0x00000000,
887 (0x0e00 << 16) | (0xac70 >> 2),
888 0x00000000,
889 (0x0e00 << 16) | (0xac74 >> 2),
890 0x00000000,
891 (0x0e00 << 16) | (0xac78 >> 2),
892 0x00000000,
893 (0x0e00 << 16) | (0xac7c >> 2),
894 0x00000000,
895 (0x0e00 << 16) | (0xac80 >> 2),
896 0x00000000,
897 (0x0e00 << 16) | (0xac84 >> 2),
898 0x00000000,
899 (0x0e00 << 16) | (0xac88 >> 2),
900 0x00000000,
901 (0x0e00 << 16) | (0xac8c >> 2),
902 0x00000000,
903 (0x0e00 << 16) | (0x970c >> 2),
904 0x00000000,
905 (0x0e00 << 16) | (0x9714 >> 2),
906 0x00000000,
907 (0x0e00 << 16) | (0x9718 >> 2),
908 0x00000000,
909 (0x0e00 << 16) | (0x971c >> 2),
910 0x00000000,
911 (0x0e00 << 16) | (0x31068 >> 2),
912 0x00000000,
913 (0x4e00 << 16) | (0x31068 >> 2),
914 0x00000000,
915 (0x5e00 << 16) | (0x31068 >> 2),
916 0x00000000,
917 (0x6e00 << 16) | (0x31068 >> 2),
918 0x00000000,
919 (0x7e00 << 16) | (0x31068 >> 2),
920 0x00000000,
921 (0x0e00 << 16) | (0xcd10 >> 2),
922 0x00000000,
923 (0x0e00 << 16) | (0xcd14 >> 2),
924 0x00000000,
925 (0x0e00 << 16) | (0x88b0 >> 2),
926 0x00000000,
927 (0x0e00 << 16) | (0x88b4 >> 2),
928 0x00000000,
929 (0x0e00 << 16) | (0x88b8 >> 2),
930 0x00000000,
931 (0x0e00 << 16) | (0x88bc >> 2),
932 0x00000000,
933 (0x0400 << 16) | (0x89c0 >> 2),
934 0x00000000,
935 (0x0e00 << 16) | (0x88c4 >> 2),
936 0x00000000,
937 (0x0e00 << 16) | (0x88c8 >> 2),
938 0x00000000,
939 (0x0e00 << 16) | (0x88d0 >> 2),
940 0x00000000,
941 (0x0e00 << 16) | (0x88d4 >> 2),
942 0x00000000,
943 (0x0e00 << 16) | (0x88d8 >> 2),
944 0x00000000,
945 (0x0e00 << 16) | (0x8980 >> 2),
946 0x00000000,
947 (0x0e00 << 16) | (0x30938 >> 2),
948 0x00000000,
949 (0x0e00 << 16) | (0x3093c >> 2),
950 0x00000000,
951 (0x0e00 << 16) | (0x30940 >> 2),
952 0x00000000,
953 (0x0e00 << 16) | (0x89a0 >> 2),
954 0x00000000,
955 (0x0e00 << 16) | (0x30900 >> 2),
956 0x00000000,
957 (0x0e00 << 16) | (0x30904 >> 2),
958 0x00000000,
959 (0x0e00 << 16) | (0x89b4 >> 2),
960 0x00000000,
961 (0x0e00 << 16) | (0x3e1fc >> 2),
962 0x00000000,
963 (0x0e00 << 16) | (0x3c210 >> 2),
964 0x00000000,
965 (0x0e00 << 16) | (0x3c214 >> 2),
966 0x00000000,
967 (0x0e00 << 16) | (0x3c218 >> 2),
968 0x00000000,
969 (0x0e00 << 16) | (0x8904 >> 2),
970 0x00000000,
971 0x5,
972 (0x0e00 << 16) | (0x8c28 >> 2),
973 (0x0e00 << 16) | (0x8c2c >> 2),
974 (0x0e00 << 16) | (0x8c30 >> 2),
975 (0x0e00 << 16) | (0x8c34 >> 2),
976 (0x0e00 << 16) | (0x9600 >> 2),
977};
978
0aafd313
AD		 979static const u32 bonaire_golden_spm_registers[] =
980{
981 0x30800, 0xe0ffffff, 0xe0000000
982};
983
984static const u32 bonaire_golden_common_registers[] =
985{
986 0xc770, 0xffffffff, 0x00000800,
987 0xc774, 0xffffffff, 0x00000800,
988 0xc798, 0xffffffff, 0x00007fbf,
989 0xc79c, 0xffffffff, 0x00007faf
990};
991
992static const u32 bonaire_golden_registers[] =
993{
994 0x3354, 0x00000333, 0x00000333,
995 0x3350, 0x000c0fc0, 0x00040200,
996 0x9a10, 0x00010000, 0x00058208,
997 0x3c000, 0xffff1fff, 0x00140000,
998 0x3c200, 0xfdfc0fff, 0x00000100,
999 0x3c234, 0x40000000, 0x40000200,
1000 0x9830, 0xffffffff, 0x00000000,
1001 0x9834, 0xf00fffff, 0x00000400,
1002 0x9838, 0x0002021c, 0x00020200,
1003 0xc78, 0x00000080, 0x00000000,
1004 0x5bb0, 0x000000f0, 0x00000070,
1005 0x5bc0, 0xf0311fff, 0x80300000,
1006 0x98f8, 0x73773777, 0x12010001,
1007 0x350c, 0x00810000, 0x408af000,
1008 0x7030, 0x31000111, 0x00000011,
1009 0x2f48, 0x73773777, 0x12010001,
1010 0x220c, 0x00007fb6, 0x0021a1b1,
1011 0x2210, 0x00007fb6, 0x002021b1,
1012 0x2180, 0x00007fb6, 0x00002191,
1013 0x2218, 0x00007fb6, 0x002121b1,
1014 0x221c, 0x00007fb6, 0x002021b1,
1015 0x21dc, 0x00007fb6, 0x00002191,
1016 0x21e0, 0x00007fb6, 0x00002191,
1017 0x3628, 0x0000003f, 0x0000000a,
1018 0x362c, 0x0000003f, 0x0000000a,
1019 0x2ae4, 0x00073ffe, 0x000022a2,
1020 0x240c, 0x000007ff, 0x00000000,
1021 0x8a14, 0xf000003f, 0x00000007,
1022 0x8bf0, 0x00002001, 0x00000001,
1023 0x8b24, 0xffffffff, 0x00ffffff,
1024 0x30a04, 0x0000ff0f, 0x00000000,
1025 0x28a4c, 0x07ffffff, 0x06000000,
1026 0x4d8, 0x00000fff, 0x00000100,
1027 0x3e78, 0x00000001, 0x00000002,
1028 0x9100, 0x03000000, 0x0362c688,
1029 0x8c00, 0x000000ff, 0x00000001,
1030 0xe40, 0x00001fff, 0x00001fff,
1031 0x9060, 0x0000007f, 0x00000020,
1032 0x9508, 0x00010000, 0x00010000,
1033 0xac14, 0x000003ff, 0x000000f3,
1034 0xac0c, 0xffffffff, 0x00001032
1035};
1036
1037static const u32 bonaire_mgcg_cgcg_init[] =
1038{
1039 0xc420, 0xffffffff, 0xfffffffc,
1040 0x30800, 0xffffffff, 0xe0000000,
1041 0x3c2a0, 0xffffffff, 0x00000100,
1042 0x3c208, 0xffffffff, 0x00000100,
1043 0x3c2c0, 0xffffffff, 0xc0000100,
1044 0x3c2c8, 0xffffffff, 0xc0000100,
1045 0x3c2c4, 0xffffffff, 0xc0000100,
1046 0x55e4, 0xffffffff, 0x00600100,
1047 0x3c280, 0xffffffff, 0x00000100,
1048 0x3c214, 0xffffffff, 0x06000100,
1049 0x3c220, 0xffffffff, 0x00000100,
1050 0x3c218, 0xffffffff, 0x06000100,
1051 0x3c204, 0xffffffff, 0x00000100,
1052 0x3c2e0, 0xffffffff, 0x00000100,
1053 0x3c224, 0xffffffff, 0x00000100,
1054 0x3c200, 0xffffffff, 0x00000100,
1055 0x3c230, 0xffffffff, 0x00000100,
1056 0x3c234, 0xffffffff, 0x00000100,
1057 0x3c250, 0xffffffff, 0x00000100,
1058 0x3c254, 0xffffffff, 0x00000100,
1059 0x3c258, 0xffffffff, 0x00000100,
1060 0x3c25c, 0xffffffff, 0x00000100,
1061 0x3c260, 0xffffffff, 0x00000100,
1062 0x3c27c, 0xffffffff, 0x00000100,
1063 0x3c278, 0xffffffff, 0x00000100,
1064 0x3c210, 0xffffffff, 0x06000100,
1065 0x3c290, 0xffffffff, 0x00000100,
1066 0x3c274, 0xffffffff, 0x00000100,
1067 0x3c2b4, 0xffffffff, 0x00000100,
1068 0x3c2b0, 0xffffffff, 0x00000100,
1069 0x3c270, 0xffffffff, 0x00000100,
1070 0x30800, 0xffffffff, 0xe0000000,
1071 0x3c020, 0xffffffff, 0x00010000,
1072 0x3c024, 0xffffffff, 0x00030002,
1073 0x3c028, 0xffffffff, 0x00040007,
1074 0x3c02c, 0xffffffff, 0x00060005,
1075 0x3c030, 0xffffffff, 0x00090008,
1076 0x3c034, 0xffffffff, 0x00010000,
1077 0x3c038, 0xffffffff, 0x00030002,
1078 0x3c03c, 0xffffffff, 0x00040007,
1079 0x3c040, 0xffffffff, 0x00060005,
1080 0x3c044, 0xffffffff, 0x00090008,
1081 0x3c048, 0xffffffff, 0x00010000,
1082 0x3c04c, 0xffffffff, 0x00030002,
1083 0x3c050, 0xffffffff, 0x00040007,
1084 0x3c054, 0xffffffff, 0x00060005,
1085 0x3c058, 0xffffffff, 0x00090008,
1086 0x3c05c, 0xffffffff, 0x00010000,
1087 0x3c060, 0xffffffff, 0x00030002,
1088 0x3c064, 0xffffffff, 0x00040007,
1089 0x3c068, 0xffffffff, 0x00060005,
1090 0x3c06c, 0xffffffff, 0x00090008,
1091 0x3c070, 0xffffffff, 0x00010000,
1092 0x3c074, 0xffffffff, 0x00030002,
1093 0x3c078, 0xffffffff, 0x00040007,
1094 0x3c07c, 0xffffffff, 0x00060005,
1095 0x3c080, 0xffffffff, 0x00090008,
1096 0x3c084, 0xffffffff, 0x00010000,
1097 0x3c088, 0xffffffff, 0x00030002,
1098 0x3c08c, 0xffffffff, 0x00040007,
1099 0x3c090, 0xffffffff, 0x00060005,
1100 0x3c094, 0xffffffff, 0x00090008,
1101 0x3c098, 0xffffffff, 0x00010000,
1102 0x3c09c, 0xffffffff, 0x00030002,
1103 0x3c0a0, 0xffffffff, 0x00040007,
1104 0x3c0a4, 0xffffffff, 0x00060005,
1105 0x3c0a8, 0xffffffff, 0x00090008,
1106 0x3c000, 0xffffffff, 0x96e00200,
1107 0x8708, 0xffffffff, 0x00900100,
1108 0xc424, 0xffffffff, 0x0020003f,
1109 0x38, 0xffffffff, 0x0140001c,
1110 0x3c, 0x000f0000, 0x000f0000,
1111 0x220, 0xffffffff, 0xC060000C,
1112 0x224, 0xc0000fff, 0x00000100,
1113 0xf90, 0xffffffff, 0x00000100,
1114 0xf98, 0x00000101, 0x00000000,
1115 0x20a8, 0xffffffff, 0x00000104,
1116 0x55e4, 0xff000fff, 0x00000100,
1117 0x30cc, 0xc0000fff, 0x00000104,
1118 0xc1e4, 0x00000001, 0x00000001,
1119 0xd00c, 0xff000ff0, 0x00000100,
1120 0xd80c, 0xff000ff0, 0x00000100
1121};
1122
1123static const u32 spectre_golden_spm_registers[] =
1124{
1125 0x30800, 0xe0ffffff, 0xe0000000
1126};
1127
1128static const u32 spectre_golden_common_registers[] =
1129{
1130 0xc770, 0xffffffff, 0x00000800,
1131 0xc774, 0xffffffff, 0x00000800,
1132 0xc798, 0xffffffff, 0x00007fbf,
1133 0xc79c, 0xffffffff, 0x00007faf
1134};
1135
1136static const u32 spectre_golden_registers[] =
1137{
1138 0x3c000, 0xffff1fff, 0x96940200,
1139 0x3c00c, 0xffff0001, 0xff000000,
1140 0x3c200, 0xfffc0fff, 0x00000100,
1141 0x6ed8, 0x00010101, 0x00010000,
1142 0x9834, 0xf00fffff, 0x00000400,
1143 0x9838, 0xfffffffc, 0x00020200,
1144 0x5bb0, 0x000000f0, 0x00000070,
1145 0x5bc0, 0xf0311fff, 0x80300000,
1146 0x98f8, 0x73773777, 0x12010001,
1147 0x9b7c, 0x00ff0000, 0x00fc0000,
1148 0x2f48, 0x73773777, 0x12010001,
1149 0x8a14, 0xf000003f, 0x00000007,
1150 0x8b24, 0xffffffff, 0x00ffffff,
1151 0x28350, 0x3f3f3fff, 0x00000082,
f1553174 1152 0x28354, 0x0000003f, 0x00000000,
0aafd313
AD		 1153 0x3e78, 0x00000001, 0x00000002,
1154 0x913c, 0xffff03df, 0x00000004,
1155 0xc768, 0x00000008, 0x00000008,
1156 0x8c00, 0x000008ff, 0x00000800,
1157 0x9508, 0x00010000, 0x00010000,
1158 0xac0c, 0xffffffff, 0x54763210,
1159 0x214f8, 0x01ff01ff, 0x00000002,
1160 0x21498, 0x007ff800, 0x00200000,
1161 0x2015c, 0xffffffff, 0x00000f40,
1162 0x30934, 0xffffffff, 0x00000001
1163};
1164
1165static const u32 spectre_mgcg_cgcg_init[] =
1166{
1167 0xc420, 0xffffffff, 0xfffffffc,
1168 0x30800, 0xffffffff, 0xe0000000,
1169 0x3c2a0, 0xffffffff, 0x00000100,
1170 0x3c208, 0xffffffff, 0x00000100,
1171 0x3c2c0, 0xffffffff, 0x00000100,
1172 0x3c2c8, 0xffffffff, 0x00000100,
1173 0x3c2c4, 0xffffffff, 0x00000100,
1174 0x55e4, 0xffffffff, 0x00600100,
1175 0x3c280, 0xffffffff, 0x00000100,
1176 0x3c214, 0xffffffff, 0x06000100,
1177 0x3c220, 0xffffffff, 0x00000100,
1178 0x3c218, 0xffffffff, 0x06000100,
1179 0x3c204, 0xffffffff, 0x00000100,
1180 0x3c2e0, 0xffffffff, 0x00000100,
1181 0x3c224, 0xffffffff, 0x00000100,
1182 0x3c200, 0xffffffff, 0x00000100,
1183 0x3c230, 0xffffffff, 0x00000100,
1184 0x3c234, 0xffffffff, 0x00000100,
1185 0x3c250, 0xffffffff, 0x00000100,
1186 0x3c254, 0xffffffff, 0x00000100,
1187 0x3c258, 0xffffffff, 0x00000100,
1188 0x3c25c, 0xffffffff, 0x00000100,
1189 0x3c260, 0xffffffff, 0x00000100,
1190 0x3c27c, 0xffffffff, 0x00000100,
1191 0x3c278, 0xffffffff, 0x00000100,
1192 0x3c210, 0xffffffff, 0x06000100,
1193 0x3c290, 0xffffffff, 0x00000100,
1194 0x3c274, 0xffffffff, 0x00000100,
1195 0x3c2b4, 0xffffffff, 0x00000100,
1196 0x3c2b0, 0xffffffff, 0x00000100,
1197 0x3c270, 0xffffffff, 0x00000100,
1198 0x30800, 0xffffffff, 0xe0000000,
1199 0x3c020, 0xffffffff, 0x00010000,
1200 0x3c024, 0xffffffff, 0x00030002,
1201 0x3c028, 0xffffffff, 0x00040007,
1202 0x3c02c, 0xffffffff, 0x00060005,
1203 0x3c030, 0xffffffff, 0x00090008,
1204 0x3c034, 0xffffffff, 0x00010000,
1205 0x3c038, 0xffffffff, 0x00030002,
1206 0x3c03c, 0xffffffff, 0x00040007,
1207 0x3c040, 0xffffffff, 0x00060005,
1208 0x3c044, 0xffffffff, 0x00090008,
1209 0x3c048, 0xffffffff, 0x00010000,
1210 0x3c04c, 0xffffffff, 0x00030002,
1211 0x3c050, 0xffffffff, 0x00040007,
1212 0x3c054, 0xffffffff, 0x00060005,
1213 0x3c058, 0xffffffff, 0x00090008,
1214 0x3c05c, 0xffffffff, 0x00010000,
1215 0x3c060, 0xffffffff, 0x00030002,
1216 0x3c064, 0xffffffff, 0x00040007,
1217 0x3c068, 0xffffffff, 0x00060005,
1218 0x3c06c, 0xffffffff, 0x00090008,
1219 0x3c070, 0xffffffff, 0x00010000,
1220 0x3c074, 0xffffffff, 0x00030002,
1221 0x3c078, 0xffffffff, 0x00040007,
1222 0x3c07c, 0xffffffff, 0x00060005,
1223 0x3c080, 0xffffffff, 0x00090008,
1224 0x3c084, 0xffffffff, 0x00010000,
1225 0x3c088, 0xffffffff, 0x00030002,
1226 0x3c08c, 0xffffffff, 0x00040007,
1227 0x3c090, 0xffffffff, 0x00060005,
1228 0x3c094, 0xffffffff, 0x00090008,
1229 0x3c098, 0xffffffff, 0x00010000,
1230 0x3c09c, 0xffffffff, 0x00030002,
1231 0x3c0a0, 0xffffffff, 0x00040007,
1232 0x3c0a4, 0xffffffff, 0x00060005,
1233 0x3c0a8, 0xffffffff, 0x00090008,
1234 0x3c0ac, 0xffffffff, 0x00010000,
1235 0x3c0b0, 0xffffffff, 0x00030002,
1236 0x3c0b4, 0xffffffff, 0x00040007,
1237 0x3c0b8, 0xffffffff, 0x00060005,
1238 0x3c0bc, 0xffffffff, 0x00090008,
1239 0x3c000, 0xffffffff, 0x96e00200,
1240 0x8708, 0xffffffff, 0x00900100,
1241 0xc424, 0xffffffff, 0x0020003f,
1242 0x38, 0xffffffff, 0x0140001c,
1243 0x3c, 0x000f0000, 0x000f0000,
1244 0x220, 0xffffffff, 0xC060000C,
1245 0x224, 0xc0000fff, 0x00000100,
1246 0xf90, 0xffffffff, 0x00000100,
1247 0xf98, 0x00000101, 0x00000000,
1248 0x20a8, 0xffffffff, 0x00000104,
1249 0x55e4, 0xff000fff, 0x00000100,
1250 0x30cc, 0xc0000fff, 0x00000104,
1251 0xc1e4, 0x00000001, 0x00000001,
1252 0xd00c, 0xff000ff0, 0x00000100,
1253 0xd80c, 0xff000ff0, 0x00000100
1254};
1255
1256static const u32 kalindi_golden_spm_registers[] =
1257{
1258 0x30800, 0xe0ffffff, 0xe0000000
1259};
1260
1261static const u32 kalindi_golden_common_registers[] =
1262{
1263 0xc770, 0xffffffff, 0x00000800,
1264 0xc774, 0xffffffff, 0x00000800,
1265 0xc798, 0xffffffff, 0x00007fbf,
1266 0xc79c, 0xffffffff, 0x00007faf
1267};
1268
1269static const u32 kalindi_golden_registers[] =
1270{
1271 0x3c000, 0xffffdfff, 0x6e944040,
1272 0x55e4, 0xff607fff, 0xfc000100,
1273 0x3c220, 0xff000fff, 0x00000100,
1274 0x3c224, 0xff000fff, 0x00000100,
1275 0x3c200, 0xfffc0fff, 0x00000100,
1276 0x6ed8, 0x00010101, 0x00010000,
1277 0x9830, 0xffffffff, 0x00000000,
1278 0x9834, 0xf00fffff, 0x00000400,
1279 0x5bb0, 0x000000f0, 0x00000070,
1280 0x5bc0, 0xf0311fff, 0x80300000,
1281 0x98f8, 0x73773777, 0x12010001,
1282 0x98fc, 0xffffffff, 0x00000010,
1283 0x9b7c, 0x00ff0000, 0x00fc0000,
1284 0x8030, 0x00001f0f, 0x0000100a,
1285 0x2f48, 0x73773777, 0x12010001,
1286 0x2408, 0x000fffff, 0x000c007f,
1287 0x8a14, 0xf000003f, 0x00000007,
1288 0x8b24, 0x3fff3fff, 0x00ffcfff,
1289 0x30a04, 0x0000ff0f, 0x00000000,
1290 0x28a4c, 0x07ffffff, 0x06000000,
1291 0x4d8, 0x00000fff, 0x00000100,
1292 0x3e78, 0x00000001, 0x00000002,
1293 0xc768, 0x00000008, 0x00000008,
1294 0x8c00, 0x000000ff, 0x00000003,
1295 0x214f8, 0x01ff01ff, 0x00000002,
1296 0x21498, 0x007ff800, 0x00200000,
1297 0x2015c, 0xffffffff, 0x00000f40,
1298 0x88c4, 0x001f3ae3, 0x00000082,
1299 0x88d4, 0x0000001f, 0x00000010,
1300 0x30934, 0xffffffff, 0x00000000
1301};
1302
1303static const u32 kalindi_mgcg_cgcg_init[] =
1304{
1305 0xc420, 0xffffffff, 0xfffffffc,
1306 0x30800, 0xffffffff, 0xe0000000,
1307 0x3c2a0, 0xffffffff, 0x00000100,
1308 0x3c208, 0xffffffff, 0x00000100,
1309 0x3c2c0, 0xffffffff, 0x00000100,
1310 0x3c2c8, 0xffffffff, 0x00000100,
1311 0x3c2c4, 0xffffffff, 0x00000100,
1312 0x55e4, 0xffffffff, 0x00600100,
1313 0x3c280, 0xffffffff, 0x00000100,
1314 0x3c214, 0xffffffff, 0x06000100,
1315 0x3c220, 0xffffffff, 0x00000100,
1316 0x3c218, 0xffffffff, 0x06000100,
1317 0x3c204, 0xffffffff, 0x00000100,
1318 0x3c2e0, 0xffffffff, 0x00000100,
1319 0x3c224, 0xffffffff, 0x00000100,
1320 0x3c200, 0xffffffff, 0x00000100,
1321 0x3c230, 0xffffffff, 0x00000100,
1322 0x3c234, 0xffffffff, 0x00000100,
1323 0x3c250, 0xffffffff, 0x00000100,
1324 0x3c254, 0xffffffff, 0x00000100,
1325 0x3c258, 0xffffffff, 0x00000100,
1326 0x3c25c, 0xffffffff, 0x00000100,
1327 0x3c260, 0xffffffff, 0x00000100,
1328 0x3c27c, 0xffffffff, 0x00000100,
1329 0x3c278, 0xffffffff, 0x00000100,
1330 0x3c210, 0xffffffff, 0x06000100,
1331 0x3c290, 0xffffffff, 0x00000100,
1332 0x3c274, 0xffffffff, 0x00000100,
1333 0x3c2b4, 0xffffffff, 0x00000100,
1334 0x3c2b0, 0xffffffff, 0x00000100,
1335 0x3c270, 0xffffffff, 0x00000100,
1336 0x30800, 0xffffffff, 0xe0000000,
1337 0x3c020, 0xffffffff, 0x00010000,
1338 0x3c024, 0xffffffff, 0x00030002,
1339 0x3c028, 0xffffffff, 0x00040007,
1340 0x3c02c, 0xffffffff, 0x00060005,
1341 0x3c030, 0xffffffff, 0x00090008,
1342 0x3c034, 0xffffffff, 0x00010000,
1343 0x3c038, 0xffffffff, 0x00030002,
1344 0x3c03c, 0xffffffff, 0x00040007,
1345 0x3c040, 0xffffffff, 0x00060005,
1346 0x3c044, 0xffffffff, 0x00090008,
1347 0x3c000, 0xffffffff, 0x96e00200,
1348 0x8708, 0xffffffff, 0x00900100,
1349 0xc424, 0xffffffff, 0x0020003f,
1350 0x38, 0xffffffff, 0x0140001c,
1351 0x3c, 0x000f0000, 0x000f0000,
1352 0x220, 0xffffffff, 0xC060000C,
1353 0x224, 0xc0000fff, 0x00000100,
1354 0x20a8, 0xffffffff, 0x00000104,
1355 0x55e4, 0xff000fff, 0x00000100,
1356 0x30cc, 0xc0000fff, 0x00000104,
1357 0xc1e4, 0x00000001, 0x00000001,
1358 0xd00c, 0xff000ff0, 0x00000100,
1359 0xd80c, 0xff000ff0, 0x00000100
1360};
1361
8efff337
AD		 1362static const u32 hawaii_golden_spm_registers[] =
1363{
1364 0x30800, 0xe0ffffff, 0xe0000000
1365};
1366
1367static const u32 hawaii_golden_common_registers[] =
1368{
1369 0x30800, 0xffffffff, 0xe0000000,
1370 0x28350, 0xffffffff, 0x3a00161a,
1371 0x28354, 0xffffffff, 0x0000002e,
1372 0x9a10, 0xffffffff, 0x00018208,
1373 0x98f8, 0xffffffff, 0x12011003
1374};
1375
1376static const u32 hawaii_golden_registers[] =
1377{
1378 0x3354, 0x00000333, 0x00000333,
1379 0x9a10, 0x00010000, 0x00058208,
1380 0x9830, 0xffffffff, 0x00000000,
1381 0x9834, 0xf00fffff, 0x00000400,
1382 0x9838, 0x0002021c, 0x00020200,
1383 0xc78, 0x00000080, 0x00000000,
1384 0x5bb0, 0x000000f0, 0x00000070,
1385 0x5bc0, 0xf0311fff, 0x80300000,
1386 0x350c, 0x00810000, 0x408af000,
1387 0x7030, 0x31000111, 0x00000011,
1388 0x2f48, 0x73773777, 0x12010001,
1389 0x2120, 0x0000007f, 0x0000001b,
1390 0x21dc, 0x00007fb6, 0x00002191,
1391 0x3628, 0x0000003f, 0x0000000a,
1392 0x362c, 0x0000003f, 0x0000000a,
1393 0x2ae4, 0x00073ffe, 0x000022a2,
1394 0x240c, 0x000007ff, 0x00000000,
1395 0x8bf0, 0x00002001, 0x00000001,
1396 0x8b24, 0xffffffff, 0x00ffffff,
1397 0x30a04, 0x0000ff0f, 0x00000000,
1398 0x28a4c, 0x07ffffff, 0x06000000,
1399 0x3e78, 0x00000001, 0x00000002,
1400 0xc768, 0x00000008, 0x00000008,
1401 0xc770, 0x00000f00, 0x00000800,
1402 0xc774, 0x00000f00, 0x00000800,
1403 0xc798, 0x00ffffff, 0x00ff7fbf,
1404 0xc79c, 0x00ffffff, 0x00ff7faf,
1405 0x8c00, 0x000000ff, 0x00000800,
1406 0xe40, 0x00001fff, 0x00001fff,
1407 0x9060, 0x0000007f, 0x00000020,
1408 0x9508, 0x00010000, 0x00010000,
1409 0xae00, 0x00100000, 0x000ff07c,
1410 0xac14, 0x000003ff, 0x0000000f,
1411 0xac10, 0xffffffff, 0x7564fdec,
1412 0xac0c, 0xffffffff, 0x3120b9a8,
1413 0xac08, 0x20000000, 0x0f9c0000
1414};
1415
1416static const u32 hawaii_mgcg_cgcg_init[] =
1417{
1418 0xc420, 0xffffffff, 0xfffffffd,
1419 0x30800, 0xffffffff, 0xe0000000,
1420 0x3c2a0, 0xffffffff, 0x00000100,
1421 0x3c208, 0xffffffff, 0x00000100,
1422 0x3c2c0, 0xffffffff, 0x00000100,
1423 0x3c2c8, 0xffffffff, 0x00000100,
1424 0x3c2c4, 0xffffffff, 0x00000100,
1425 0x55e4, 0xffffffff, 0x00200100,
1426 0x3c280, 0xffffffff, 0x00000100,
1427 0x3c214, 0xffffffff, 0x06000100,
1428 0x3c220, 0xffffffff, 0x00000100,
1429 0x3c218, 0xffffffff, 0x06000100,
1430 0x3c204, 0xffffffff, 0x00000100,
1431 0x3c2e0, 0xffffffff, 0x00000100,
1432 0x3c224, 0xffffffff, 0x00000100,
1433 0x3c200, 0xffffffff, 0x00000100,
1434 0x3c230, 0xffffffff, 0x00000100,
1435 0x3c234, 0xffffffff, 0x00000100,
1436 0x3c250, 0xffffffff, 0x00000100,
1437 0x3c254, 0xffffffff, 0x00000100,
1438 0x3c258, 0xffffffff, 0x00000100,
1439 0x3c25c, 0xffffffff, 0x00000100,
1440 0x3c260, 0xffffffff, 0x00000100,
1441 0x3c27c, 0xffffffff, 0x00000100,
1442 0x3c278, 0xffffffff, 0x00000100,
1443 0x3c210, 0xffffffff, 0x06000100,
1444 0x3c290, 0xffffffff, 0x00000100,
1445 0x3c274, 0xffffffff, 0x00000100,
1446 0x3c2b4, 0xffffffff, 0x00000100,
1447 0x3c2b0, 0xffffffff, 0x00000100,
1448 0x3c270, 0xffffffff, 0x00000100,
1449 0x30800, 0xffffffff, 0xe0000000,
1450 0x3c020, 0xffffffff, 0x00010000,
1451 0x3c024, 0xffffffff, 0x00030002,
1452 0x3c028, 0xffffffff, 0x00040007,
1453 0x3c02c, 0xffffffff, 0x00060005,
1454 0x3c030, 0xffffffff, 0x00090008,
1455 0x3c034, 0xffffffff, 0x00010000,
1456 0x3c038, 0xffffffff, 0x00030002,
1457 0x3c03c, 0xffffffff, 0x00040007,
1458 0x3c040, 0xffffffff, 0x00060005,
1459 0x3c044, 0xffffffff, 0x00090008,
1460 0x3c048, 0xffffffff, 0x00010000,
1461 0x3c04c, 0xffffffff, 0x00030002,
1462 0x3c050, 0xffffffff, 0x00040007,
1463 0x3c054, 0xffffffff, 0x00060005,
1464 0x3c058, 0xffffffff, 0x00090008,
1465 0x3c05c, 0xffffffff, 0x00010000,
1466 0x3c060, 0xffffffff, 0x00030002,
1467 0x3c064, 0xffffffff, 0x00040007,
1468 0x3c068, 0xffffffff, 0x00060005,
1469 0x3c06c, 0xffffffff, 0x00090008,
1470 0x3c070, 0xffffffff, 0x00010000,
1471 0x3c074, 0xffffffff, 0x00030002,
1472 0x3c078, 0xffffffff, 0x00040007,
1473 0x3c07c, 0xffffffff, 0x00060005,
1474 0x3c080, 0xffffffff, 0x00090008,
1475 0x3c084, 0xffffffff, 0x00010000,
1476 0x3c088, 0xffffffff, 0x00030002,
1477 0x3c08c, 0xffffffff, 0x00040007,
1478 0x3c090, 0xffffffff, 0x00060005,
1479 0x3c094, 0xffffffff, 0x00090008,
1480 0x3c098, 0xffffffff, 0x00010000,
1481 0x3c09c, 0xffffffff, 0x00030002,
1482 0x3c0a0, 0xffffffff, 0x00040007,
1483 0x3c0a4, 0xffffffff, 0x00060005,
1484 0x3c0a8, 0xffffffff, 0x00090008,
1485 0x3c0ac, 0xffffffff, 0x00010000,
1486 0x3c0b0, 0xffffffff, 0x00030002,
1487 0x3c0b4, 0xffffffff, 0x00040007,
1488 0x3c0b8, 0xffffffff, 0x00060005,
1489 0x3c0bc, 0xffffffff, 0x00090008,
1490 0x3c0c0, 0xffffffff, 0x00010000,
1491 0x3c0c4, 0xffffffff, 0x00030002,
1492 0x3c0c8, 0xffffffff, 0x00040007,
1493 0x3c0cc, 0xffffffff, 0x00060005,
1494 0x3c0d0, 0xffffffff, 0x00090008,
1495 0x3c0d4, 0xffffffff, 0x00010000,
1496 0x3c0d8, 0xffffffff, 0x00030002,
1497 0x3c0dc, 0xffffffff, 0x00040007,
1498 0x3c0e0, 0xffffffff, 0x00060005,
1499 0x3c0e4, 0xffffffff, 0x00090008,
1500 0x3c0e8, 0xffffffff, 0x00010000,
1501 0x3c0ec, 0xffffffff, 0x00030002,
1502 0x3c0f0, 0xffffffff, 0x00040007,
1503 0x3c0f4, 0xffffffff, 0x00060005,
1504 0x3c0f8, 0xffffffff, 0x00090008,
1505 0xc318, 0xffffffff, 0x00020200,
1506 0x3350, 0xffffffff, 0x00000200,
1507 0x15c0, 0xffffffff, 0x00000400,
1508 0x55e8, 0xffffffff, 0x00000000,
1509 0x2f50, 0xffffffff, 0x00000902,
1510 0x3c000, 0xffffffff, 0x96940200,
1511 0x8708, 0xffffffff, 0x00900100,
1512 0xc424, 0xffffffff, 0x0020003f,
1513 0x38, 0xffffffff, 0x0140001c,
1514 0x3c, 0x000f0000, 0x000f0000,
1515 0x220, 0xffffffff, 0xc060000c,
1516 0x224, 0xc0000fff, 0x00000100,
1517 0xf90, 0xffffffff, 0x00000100,
1518 0xf98, 0x00000101, 0x00000000,
1519 0x20a8, 0xffffffff, 0x00000104,
1520 0x55e4, 0xff000fff, 0x00000100,
1521 0x30cc, 0xc0000fff, 0x00000104,
1522 0xc1e4, 0x00000001, 0x00000001,
1523 0xd00c, 0xff000ff0, 0x00000100,
1524 0xd80c, 0xff000ff0, 0x00000100
1525};
1526
f73a9e83
SL		 1527static const u32 godavari_golden_registers[] =
1528{
1529 0x55e4, 0xff607fff, 0xfc000100,
1530 0x6ed8, 0x00010101, 0x00010000,
1531 0x9830, 0xffffffff, 0x00000000,
1532 0x98302, 0xf00fffff, 0x00000400,
1533 0x6130, 0xffffffff, 0x00010000,
1534 0x5bb0, 0x000000f0, 0x00000070,
1535 0x5bc0, 0xf0311fff, 0x80300000,
1536 0x98f8, 0x73773777, 0x12010001,
1537 0x98fc, 0xffffffff, 0x00000010,
1538 0x8030, 0x00001f0f, 0x0000100a,
1539 0x2f48, 0x73773777, 0x12010001,
1540 0x2408, 0x000fffff, 0x000c007f,
1541 0x8a14, 0xf000003f, 0x00000007,
1542 0x8b24, 0xffffffff, 0x00ff0fff,
1543 0x30a04, 0x0000ff0f, 0x00000000,
1544 0x28a4c, 0x07ffffff, 0x06000000,
1545 0x4d8, 0x00000fff, 0x00000100,
1546 0xd014, 0x00010000, 0x00810001,
1547 0xd814, 0x00010000, 0x00810001,
1548 0x3e78, 0x00000001, 0x00000002,
1549 0xc768, 0x00000008, 0x00000008,
1550 0xc770, 0x00000f00, 0x00000800,
1551 0xc774, 0x00000f00, 0x00000800,
1552 0xc798, 0x00ffffff, 0x00ff7fbf,
1553 0xc79c, 0x00ffffff, 0x00ff7faf,
1554 0x8c00, 0x000000ff, 0x00000001,
1555 0x214f8, 0x01ff01ff, 0x00000002,
1556 0x21498, 0x007ff800, 0x00200000,
1557 0x2015c, 0xffffffff, 0x00000f40,
1558 0x88c4, 0x001f3ae3, 0x00000082,
1559 0x88d4, 0x0000001f, 0x00000010,
1560 0x30934, 0xffffffff, 0x00000000
1561};
1562
1563
0aafd313
AD		 1564static void cik_init_golden_registers(struct radeon_device *rdev)
1565{
1566 switch (rdev->family) {
1567 case CHIP_BONAIRE:
1568 radeon_program_register_sequence(rdev,
1569 bonaire_mgcg_cgcg_init,
1570 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1571 radeon_program_register_sequence(rdev,
1572 bonaire_golden_registers,
1573 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1574 radeon_program_register_sequence(rdev,
1575 bonaire_golden_common_registers,
1576 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1577 radeon_program_register_sequence(rdev,
1578 bonaire_golden_spm_registers,
1579 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1580 break;
1581 case CHIP_KABINI:
1582 radeon_program_register_sequence(rdev,
1583 kalindi_mgcg_cgcg_init,
1584 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1585 radeon_program_register_sequence(rdev,
1586 kalindi_golden_registers,
1587 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1588 radeon_program_register_sequence(rdev,
1589 kalindi_golden_common_registers,
1590 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1591 radeon_program_register_sequence(rdev,
1592 kalindi_golden_spm_registers,
1593 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1594 break;
f73a9e83
SL		 1595 case CHIP_MULLINS:
1596 radeon_program_register_sequence(rdev,
1597 kalindi_mgcg_cgcg_init,
1598 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1599 radeon_program_register_sequence(rdev,
1600 godavari_golden_registers,
1601 (const u32)ARRAY_SIZE(godavari_golden_registers));
1602 radeon_program_register_sequence(rdev,
1603 kalindi_golden_common_registers,
1604 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1605 radeon_program_register_sequence(rdev,
1606 kalindi_golden_spm_registers,
1607 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1608 break;
0aafd313
AD		 1609 case CHIP_KAVERI:
1610 radeon_program_register_sequence(rdev,
1611 spectre_mgcg_cgcg_init,
1612 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1613 radeon_program_register_sequence(rdev,
1614 spectre_golden_registers,
1615 (const u32)ARRAY_SIZE(spectre_golden_registers));
1616 radeon_program_register_sequence(rdev,
1617 spectre_golden_common_registers,
1618 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1619 radeon_program_register_sequence(rdev,
1620 spectre_golden_spm_registers,
1621 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1622 break;
8efff337
AD		 1623 case CHIP_HAWAII:
1624 radeon_program_register_sequence(rdev,
1625 hawaii_mgcg_cgcg_init,
1626 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1627 radeon_program_register_sequence(rdev,
1628 hawaii_golden_registers,
1629 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1630 radeon_program_register_sequence(rdev,
1631 hawaii_golden_common_registers,
1632 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1633 radeon_program_register_sequence(rdev,
1634 hawaii_golden_spm_registers,
1635 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1636 break;
0aafd313
AD		 1637 default:
1638 break;
1639 }
1640}
1641
2c67912c
AD		 1642/**
1643 * cik_get_xclk - get the xclk
1644 *
1645 * @rdev: radeon_device pointer
1646 *
1647 * Returns the reference clock used by the gfx engine
1648 * (CIK).
1649 */
1650u32 cik_get_xclk(struct radeon_device *rdev)
1651{
1652 u32 reference_clock = rdev->clock.spll.reference_freq;
1653
1654 if (rdev->flags & RADEON_IS_IGP) {
1655 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1656 return reference_clock / 2;
1657 } else {
1658 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1659 return reference_clock / 4;
1660 }
1661 return reference_clock;
1662}
1663
75efdee1
AD		 1664/**
1665 * cik_mm_rdoorbell - read a doorbell dword
1666 *
1667 * @rdev: radeon_device pointer
d5754ab8 1668 * @index: doorbell index
75efdee1
AD		 1669 *
1670 * Returns the value in the doorbell aperture at the
d5754ab8 1671 * requested doorbell index (CIK).
75efdee1 1672 */
d5754ab8 1673u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
75efdee1 1674{
d5754ab8
AL		 1675 if (index < rdev->doorbell.num_doorbells) {
1676 return readl(rdev->doorbell.ptr + index);
75efdee1 1677 } else {
d5754ab8 1678 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1679 return 0;
1680 }
1681}
1682
1683/**
1684 * cik_mm_wdoorbell - write a doorbell dword
1685 *
1686 * @rdev: radeon_device pointer
d5754ab8 1687 * @index: doorbell index
75efdee1
AD		 1688 * @v: value to write
1689 *
1690 * Writes @v to the doorbell aperture at the
d5754ab8 1691 * requested doorbell index (CIK).
75efdee1 1692 */
d5754ab8 1693void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
75efdee1 1694{
d5754ab8
AL		 1695 if (index < rdev->doorbell.num_doorbells) {
1696 writel(v, rdev->doorbell.ptr + index);
75efdee1 1697 } else {
d5754ab8 1698 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
75efdee1
AD		 1699 }
1700}
1701
bc8273fe
AD		 1702#define BONAIRE_IO_MC_REGS_SIZE 36
1703
1704static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1705{
1706 {0x00000070, 0x04400000},
1707 {0x00000071, 0x80c01803},
1708 {0x00000072, 0x00004004},
1709 {0x00000073, 0x00000100},
1710 {0x00000074, 0x00ff0000},
1711 {0x00000075, 0x34000000},
1712 {0x00000076, 0x08000014},
1713 {0x00000077, 0x00cc08ec},
1714 {0x00000078, 0x00000400},
1715 {0x00000079, 0x00000000},
1716 {0x0000007a, 0x04090000},
1717 {0x0000007c, 0x00000000},
1718 {0x0000007e, 0x4408a8e8},
1719 {0x0000007f, 0x00000304},
1720 {0x00000080, 0x00000000},
1721 {0x00000082, 0x00000001},
1722 {0x00000083, 0x00000002},
1723 {0x00000084, 0xf3e4f400},
1724 {0x00000085, 0x052024e3},
1725 {0x00000087, 0x00000000},
1726 {0x00000088, 0x01000000},
1727 {0x0000008a, 0x1c0a0000},
1728 {0x0000008b, 0xff010000},
1729 {0x0000008d, 0xffffefff},
1730 {0x0000008e, 0xfff3efff},
1731 {0x0000008f, 0xfff3efbf},
1732 {0x00000092, 0xf7ffffff},
1733 {0x00000093, 0xffffff7f},
1734 {0x00000095, 0x00101101},
1735 {0x00000096, 0x00000fff},
1736 {0x00000097, 0x00116fff},
1737 {0x00000098, 0x60010000},
1738 {0x00000099, 0x10010000},
1739 {0x0000009a, 0x00006000},
1740 {0x0000009b, 0x00001000},
1741 {0x0000009f, 0x00b48000}
1742};
1743
d4775655
AD		 1744#define HAWAII_IO_MC_REGS_SIZE 22
1745
1746static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1747{
1748 {0x0000007d, 0x40000000},
1749 {0x0000007e, 0x40180304},
1750 {0x0000007f, 0x0000ff00},
1751 {0x00000081, 0x00000000},
1752 {0x00000083, 0x00000800},
1753 {0x00000086, 0x00000000},
1754 {0x00000087, 0x00000100},
1755 {0x00000088, 0x00020100},
1756 {0x00000089, 0x00000000},
1757 {0x0000008b, 0x00040000},
1758 {0x0000008c, 0x00000100},
1759 {0x0000008e, 0xff010000},
1760 {0x00000090, 0xffffefff},
1761 {0x00000091, 0xfff3efff},
1762 {0x00000092, 0xfff3efbf},
1763 {0x00000093, 0xf7ffffff},
1764 {0x00000094, 0xffffff7f},
1765 {0x00000095, 0x00000fff},
1766 {0x00000096, 0x00116fff},
1767 {0x00000097, 0x60010000},
1768 {0x00000098, 0x10010000},
1769 {0x0000009f, 0x00c79000}
1770};
1771
1772
b556b12e
AD		 1773/**
1774 * cik_srbm_select - select specific register instances
1775 *
1776 * @rdev: radeon_device pointer
1777 * @me: selected ME (micro engine)
1778 * @pipe: pipe
1779 * @queue: queue
1780 * @vmid: VMID
1781 *
1782 * Switches the currently active registers instances. Some
1783 * registers are instanced per VMID, others are instanced per
1784 * me/pipe/queue combination.
1785 */
1786static void cik_srbm_select(struct radeon_device *rdev,
1787 u32 me, u32 pipe, u32 queue, u32 vmid)
1788{
1789 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1790 MEID(me & 0x3) |
1791 VMID(vmid & 0xf) |
1792 QUEUEID(queue & 0x7));
1793 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1794}
1795
bc8273fe
AD		 1796/* ucode loading */
1797/**
1798 * ci_mc_load_microcode - load MC ucode into the hw
1799 *
1800 * @rdev: radeon_device pointer
1801 *
1802 * Load the GDDR MC ucode into the hw (CIK).
1803 * Returns 0 on success, error on failure.
1804 */
6c7bccea 1805int ci_mc_load_microcode(struct radeon_device *rdev)
bc8273fe 1806{
f2c6b0f4
AD		 1807 const __be32 *fw_data = NULL;
1808 const __le32 *new_fw_data = NULL;
bc8273fe 1809 u32 running, blackout = 0;
f2c6b0f4
AD		 1810 u32 *io_mc_regs = NULL;
1811 const __le32 *new_io_mc_regs = NULL;
bcddee29 1812 int i, regs_size, ucode_size;
bc8273fe
AD		 1813
1814 if (!rdev->mc_fw)
1815 return -EINVAL;
1816
f2c6b0f4
AD		 1817 if (rdev->new_fw) {
1818 const struct mc_firmware_header_v1_0 *hdr =
1819 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
bcddee29 1820
f2c6b0f4
AD		 1821 radeon_ucode_print_mc_hdr(&hdr->header);
1822
1823 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1824 new_io_mc_regs = (const __le32 *)
1825 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1826 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1827 new_fw_data = (const __le32 *)
1828 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1829 } else {
1830 ucode_size = rdev->mc_fw->size / 4;
1831
1832 switch (rdev->family) {
1833 case CHIP_BONAIRE:
1834 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1835 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1836 break;
1837 case CHIP_HAWAII:
1838 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1839 regs_size = HAWAII_IO_MC_REGS_SIZE;
1840 break;
1841 default:
1842 return -EINVAL;
1843 }
1844 fw_data = (const __be32 *)rdev->mc_fw->data;
bc8273fe
AD		 1845 }
1846
1847 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1848
1849 if (running == 0) {
1850 if (running) {
1851 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1852 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1853 }
1854
1855 /* reset the engine and set to writable */
1856 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1857 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1858
1859 /* load mc io regs */
1860 for (i = 0; i < regs_size; i++) {
f2c6b0f4
AD		 1861 if (rdev->new_fw) {
1862 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1863 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1864 } else {
1865 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1866 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1867 }
bc8273fe
AD		 1868 }
1869 /* load the MC ucode */
f2c6b0f4
AD		 1870 for (i = 0; i < ucode_size; i++) {
1871 if (rdev->new_fw)
1872 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1873 else
1874 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1875 }
bc8273fe
AD		 1876
1877 /* put the engine back into the active state */
1878 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1879 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1880 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1881
1882 /* wait for training to complete */
1883 for (i = 0; i < rdev->usec_timeout; i++) {
1884 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1885 break;
1886 udelay(1);
1887 }
1888 for (i = 0; i < rdev->usec_timeout; i++) {
1889 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1890 break;
1891 udelay(1);
1892 }
1893
1894 if (running)
1895 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1896 }
1897
1898 return 0;
1899}
1900
02c81327
AD		 1901/**
1902 * cik_init_microcode - load ucode images from disk
1903 *
1904 * @rdev: radeon_device pointer
1905 *
1906 * Use the firmware interface to load the ucode images into
1907 * the driver (not loaded into hw).
1908 * Returns 0 on success, error on failure.
1909 */
1910static int cik_init_microcode(struct radeon_device *rdev)
1911{
02c81327 1912 const char *chip_name;
f2c6b0f4 1913 const char *new_chip_name;
02c81327 1914 size_t pfp_req_size, me_req_size, ce_req_size,
d4775655 1915 mec_req_size, rlc_req_size, mc_req_size = 0,
277babc3 1916 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
02c81327 1917 char fw_name[30];
f2c6b0f4 1918 int new_fw = 0;
02c81327 1919 int err;
f2c6b0f4 1920 int num_fw;
02c81327
AD		 1921
1922 DRM_DEBUG("\n");
1923
02c81327
AD		 1924 switch (rdev->family) {
1925 case CHIP_BONAIRE:
1926 chip_name = "BONAIRE";
f2c6b0f4 1927 new_chip_name = "bonaire";
02c81327
AD		 1928 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1929 me_req_size = CIK_ME_UCODE_SIZE * 4;
1930 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1931 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1932 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
277babc3
AD		 1933 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1934 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
21a93e13 1935 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
cc8dbbb4 1936 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
f2c6b0f4 1937 num_fw = 8;
02c81327 1938 break;
d4775655
AD		 1939 case CHIP_HAWAII:
1940 chip_name = "HAWAII";
f2c6b0f4 1941 new_chip_name = "hawaii";
d4775655
AD		 1942 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1943 me_req_size = CIK_ME_UCODE_SIZE * 4;
1944 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1945 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1946 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1947 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
277babc3 1948 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
d4775655
AD		 1949 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1950 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
f2c6b0f4 1951 num_fw = 8;
d4775655 1952 break;
02c81327
AD		 1953 case CHIP_KAVERI:
1954 chip_name = "KAVERI";
f2c6b0f4 1955 new_chip_name = "kaveri";
02c81327
AD		 1956 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1957 me_req_size = CIK_ME_UCODE_SIZE * 4;
1958 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1959 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1960 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
21a93e13 1961 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 1962 num_fw = 7;
02c81327
AD		 1963 break;
1964 case CHIP_KABINI:
1965 chip_name = "KABINI";
f2c6b0f4 1966 new_chip_name = "kabini";
02c81327
AD		 1967 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1968 me_req_size = CIK_ME_UCODE_SIZE * 4;
1969 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1970 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1971 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
21a93e13 1972 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 1973 num_fw = 6;
02c81327 1974 break;
f73a9e83
SL		 1975 case CHIP_MULLINS:
1976 chip_name = "MULLINS";
f2c6b0f4 1977 new_chip_name = "mullins";
f73a9e83
SL		 1978 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1979 me_req_size = CIK_ME_UCODE_SIZE * 4;
1980 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1981 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1982 rlc_req_size = ML_RLC_UCODE_SIZE * 4;
1983 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
f2c6b0f4 1984 num_fw = 6;
f73a9e83 1985 break;
02c81327
AD		 1986 default: BUG();
1987 }
1988
f2c6b0f4 1989 DRM_INFO("Loading %s Microcode\n", new_chip_name);
02c81327 1990
f2c6b0f4 1991 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
0a168933 1992 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 1993 if (err) {
1994 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1995 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1996 if (err)
1997 goto out;
1998 if (rdev->pfp_fw->size != pfp_req_size) {
1999 printk(KERN_ERR
2000 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2001 rdev->pfp_fw->size, fw_name);
2002 err = -EINVAL;
2003 goto out;
2004 }
2005 } else {
2006 err = radeon_ucode_validate(rdev->pfp_fw);
2007 if (err) {
2008 printk(KERN_ERR
2009 "cik_fw: validation failed for firmware \"%s\"\n",
2010 fw_name);
2011 goto out;
2012 } else {
2013 new_fw++;
2014 }
02c81327
AD		 2015 }
2016
f2c6b0f4 2017 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
0a168933 2018 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2019 if (err) {
2020 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
2021 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
2022 if (err)
2023 goto out;
2024 if (rdev->me_fw->size != me_req_size) {
2025 printk(KERN_ERR
2026 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2027 rdev->me_fw->size, fw_name);
2028 err = -EINVAL;
2029 }
2030 } else {
2031 err = radeon_ucode_validate(rdev->me_fw);
2032 if (err) {
2033 printk(KERN_ERR
2034 "cik_fw: validation failed for firmware \"%s\"\n",
2035 fw_name);
2036 goto out;
2037 } else {
2038 new_fw++;
2039 }
02c81327
AD		 2040 }
2041
f2c6b0f4 2042 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
0a168933 2043 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2044 if (err) {
2045 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
2046 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
2047 if (err)
2048 goto out;
2049 if (rdev->ce_fw->size != ce_req_size) {
2050 printk(KERN_ERR
2051 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2052 rdev->ce_fw->size, fw_name);
2053 err = -EINVAL;
2054 }
2055 } else {
2056 err = radeon_ucode_validate(rdev->ce_fw);
2057 if (err) {
2058 printk(KERN_ERR
2059 "cik_fw: validation failed for firmware \"%s\"\n",
2060 fw_name);
2061 goto out;
2062 } else {
2063 new_fw++;
2064 }
02c81327
AD		 2065 }
2066
f2c6b0f4 2067 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", new_chip_name);
0a168933 2068 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2069 if (err) {
2070 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
2071 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
2072 if (err)
2073 goto out;
2074 if (rdev->mec_fw->size != mec_req_size) {
2075 printk(KERN_ERR
2076 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
2077 rdev->mec_fw->size, fw_name);
2078 err = -EINVAL;
2079 }
2080 } else {
2081 err = radeon_ucode_validate(rdev->mec_fw);
2082 if (err) {
2083 printk(KERN_ERR
2084 "cik_fw: validation failed for firmware \"%s\"\n",
2085 fw_name);
2086 goto out;
2087 } else {
2088 new_fw++;
2089 }
2090 }
2091
2092 if (rdev->family == CHIP_KAVERI) {
2093 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec2.bin", new_chip_name);
2094 err = request_firmware(&rdev->mec2_fw, fw_name, rdev->dev);
2095 if (err) {
2096 goto out;
2097 } else {
2098 err = radeon_ucode_validate(rdev->mec2_fw);
2099 if (err) {
2100 goto out;
2101 } else {
2102 new_fw++;
2103 }
2104 }
02c81327
AD		 2105 }
2106
f2c6b0f4 2107 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
0a168933 2108 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2109 if (err) {
2110 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
2111 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
2112 if (err)
2113 goto out;
2114 if (rdev->rlc_fw->size != rlc_req_size) {
2115 printk(KERN_ERR
2116 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
2117 rdev->rlc_fw->size, fw_name);
2118 err = -EINVAL;
2119 }
2120 } else {
2121 err = radeon_ucode_validate(rdev->rlc_fw);
2122 if (err) {
2123 printk(KERN_ERR
2124 "cik_fw: validation failed for firmware \"%s\"\n",
2125 fw_name);
2126 goto out;
2127 } else {
2128 new_fw++;
2129 }
02c81327
AD		 2130 }
2131
f2c6b0f4 2132 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", new_chip_name);
0a168933 2133 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2134 if (err) {
2135 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
2136 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
2137 if (err)
2138 goto out;
2139 if (rdev->sdma_fw->size != sdma_req_size) {
2140 printk(KERN_ERR
2141 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
2142 rdev->sdma_fw->size, fw_name);
2143 err = -EINVAL;
2144 }
2145 } else {
2146 err = radeon_ucode_validate(rdev->sdma_fw);
2147 if (err) {
2148 printk(KERN_ERR
2149 "cik_fw: validation failed for firmware \"%s\"\n",
2150 fw_name);
2151 goto out;
2152 } else {
2153 new_fw++;
2154 }
21a93e13
AD		 2155 }
2156
cc8dbbb4 2157 /* No SMC, MC ucode on APUs */
02c81327 2158 if (!(rdev->flags & RADEON_IS_IGP)) {
f2c6b0f4 2159 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
0a168933 2160 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
277babc3 2161 if (err) {
f2c6b0f4 2162 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
277babc3 2163 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
f2c6b0f4
AD		 2164 if (err) {
2165 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
2166 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
2167 if (err)
2168 goto out;
2169 }
2170 if ((rdev->mc_fw->size != mc_req_size) &&
2171 (rdev->mc_fw->size != mc2_req_size)){
2172 printk(KERN_ERR
2173 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
2174 rdev->mc_fw->size, fw_name);
2175 err = -EINVAL;
2176 }
2177 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
2178 } else {
2179 err = radeon_ucode_validate(rdev->mc_fw);
2180 if (err) {
2181 printk(KERN_ERR
2182 "cik_fw: validation failed for firmware \"%s\"\n",
2183 fw_name);
277babc3 2184 goto out;
f2c6b0f4
AD		 2185 } else {
2186 new_fw++;
2187 }
277babc3 2188 }
cc8dbbb4 2189
f2c6b0f4 2190 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
cc8dbbb4
AD		 2191 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2192 if (err) {
f2c6b0f4
AD		 2193 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
2194 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
2195 if (err) {
2196 printk(KERN_ERR
2197 "smc: error loading firmware \"%s\"\n",
2198 fw_name);
2199 release_firmware(rdev->smc_fw);
2200 rdev->smc_fw = NULL;
2201 err = 0;
2202 } else if (rdev->smc_fw->size != smc_req_size) {
2203 printk(KERN_ERR
2204 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
2205 rdev->smc_fw->size, fw_name);
2206 err = -EINVAL;
2207 }
2208 } else {
2209 err = radeon_ucode_validate(rdev->smc_fw);
2210 if (err) {
2211 printk(KERN_ERR
2212 "cik_fw: validation failed for firmware \"%s\"\n",
2213 fw_name);
2214 goto out;
2215 } else {
2216 new_fw++;
2217 }
cc8dbbb4 2218 }
02c81327
AD		 2219 }
2220
f2c6b0f4
AD		 2221 if (new_fw == 0) {
2222 rdev->new_fw = false;
2223 } else if (new_fw < num_fw) {
2224 printk(KERN_ERR "ci_fw: mixing new and old firmware!\n");
2225 err = -EINVAL;
2226 } else {
2227 rdev->new_fw = true;
2228 }
2229
02c81327 2230out:
02c81327
AD		 2231 if (err) {
2232 if (err != -EINVAL)
2233 printk(KERN_ERR
2234 "cik_cp: Failed to load firmware \"%s\"\n",
2235 fw_name);
2236 release_firmware(rdev->pfp_fw);
2237 rdev->pfp_fw = NULL;
2238 release_firmware(rdev->me_fw);
2239 rdev->me_fw = NULL;
2240 release_firmware(rdev->ce_fw);
2241 rdev->ce_fw = NULL;
f2c6b0f4
AD		 2242 release_firmware(rdev->mec_fw);
2243 rdev->mec_fw = NULL;
2244 release_firmware(rdev->mec2_fw);
2245 rdev->mec2_fw = NULL;
02c81327
AD		 2246 release_firmware(rdev->rlc_fw);
2247 rdev->rlc_fw = NULL;
f2c6b0f4
AD		 2248 release_firmware(rdev->sdma_fw);
2249 rdev->sdma_fw = NULL;
02c81327
AD		 2250 release_firmware(rdev->mc_fw);
2251 rdev->mc_fw = NULL;
cc8dbbb4
AD		 2252 release_firmware(rdev->smc_fw);
2253 rdev->smc_fw = NULL;
02c81327
AD		 2254 }
2255 return err;
2256}
2257
8cc1a532
AD		 2258/*
2259 * Core functions
2260 */
2261/**
2262 * cik_tiling_mode_table_init - init the hw tiling table
2263 *
2264 * @rdev: radeon_device pointer
2265 *
2266 * Starting with SI, the tiling setup is done globally in a
2267 * set of 32 tiling modes. Rather than selecting each set of
2268 * parameters per surface as on older asics, we just select
2269 * which index in the tiling table we want to use, and the
2270 * surface uses those parameters (CIK).
2271 */
2272static void cik_tiling_mode_table_init(struct radeon_device *rdev)
2273{
2274 const u32 num_tile_mode_states = 32;
2275 const u32 num_secondary_tile_mode_states = 16;
2276 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2277 u32 num_pipe_configs;
2278 u32 num_rbs = rdev->config.cik.max_backends_per_se *
2279 rdev->config.cik.max_shader_engines;
2280
2281 switch (rdev->config.cik.mem_row_size_in_kb) {
2282 case 1:
2283 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2284 break;
2285 case 2:
2286 default:
2287 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2288 break;
2289 case 4:
2290 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2291 break;
2292 }
2293
2294 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2295 if (num_pipe_configs > 8)
21e438af 2296 num_pipe_configs = 16;
8cc1a532 2297
21e438af
AD		 2298 if (num_pipe_configs == 16) {
2299 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2300 switch (reg_offset) {
2301 case 0:
2302 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2303 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2304 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2305 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2306 break;
2307 case 1:
2308 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2309 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2310 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2311 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2312 break;
2313 case 2:
2314 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2315 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2316 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2317 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2318 break;
2319 case 3:
2320 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2321 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2322 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2323 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2324 break;
2325 case 4:
2326 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2327 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2328 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2329 TILE_SPLIT(split_equal_to_row_size));
2330 break;
2331 case 5:
2332 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2333 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2334 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2335 break;
2336 case 6:
2337 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2338 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2339 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2340 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2341 break;
2342 case 7:
2343 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2344 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2345 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2346 TILE_SPLIT(split_equal_to_row_size));
2347 break;
2348 case 8:
2349 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2350 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2351 break;
2352 case 9:
2353 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2354 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2355 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2356 break;
2357 case 10:
2358 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2359 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2360 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2361 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2362 break;
2363 case 11:
2364 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2365 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2366 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2367 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2368 break;
2369 case 12:
2370 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2371 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2372 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2373 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2374 break;
2375 case 13:
2376 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2377 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2378 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2379 break;
2380 case 14:
2381 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2382 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2383 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2384 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2385 break;
2386 case 16:
2387 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2388 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2389 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2390 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2391 break;
2392 case 17:
2393 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2394 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2395 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2396 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2397 break;
2398 case 27:
2399 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2400 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
21e438af
AD		 2401 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2402 break;
2403 case 28:
2404 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2405 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2406 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2407 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2408 break;
2409 case 29:
2410 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2411 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2412 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2413 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2414 break;
2415 case 30:
2416 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2417 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2418 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2419 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2420 break;
2421 default:
2422 gb_tile_moden = 0;
2423 break;
2424 }
2425 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2426 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2427 }
2428 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2429 switch (reg_offset) {
2430 case 0:
2431 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2432 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2433 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2434 NUM_BANKS(ADDR_SURF_16_BANK));
2435 break;
2436 case 1:
2437 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2438 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2439 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2440 NUM_BANKS(ADDR_SURF_16_BANK));
2441 break;
2442 case 2:
2443 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2444 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2445 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2446 NUM_BANKS(ADDR_SURF_16_BANK));
2447 break;
2448 case 3:
2449 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2450 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2451 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2452 NUM_BANKS(ADDR_SURF_16_BANK));
2453 break;
2454 case 4:
2455 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2456 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2457 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2458 NUM_BANKS(ADDR_SURF_8_BANK));
2459 break;
2460 case 5:
2461 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2462 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2463 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2464 NUM_BANKS(ADDR_SURF_4_BANK));
2465 break;
2466 case 6:
2467 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2468 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2469 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2470 NUM_BANKS(ADDR_SURF_2_BANK));
2471 break;
2472 case 8:
2473 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2474 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2475 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2476 NUM_BANKS(ADDR_SURF_16_BANK));
2477 break;
2478 case 9:
2479 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2480 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2481 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2482 NUM_BANKS(ADDR_SURF_16_BANK));
2483 break;
2484 case 10:
2485 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2486 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2487 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2488 NUM_BANKS(ADDR_SURF_16_BANK));
2489 break;
2490 case 11:
2491 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2492 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2493 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2494 NUM_BANKS(ADDR_SURF_8_BANK));
2495 break;
2496 case 12:
2497 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2498 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2499 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2500 NUM_BANKS(ADDR_SURF_4_BANK));
2501 break;
2502 case 13:
2503 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2504 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2505 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2506 NUM_BANKS(ADDR_SURF_2_BANK));
2507 break;
2508 case 14:
2509 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2510 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2511 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2512 NUM_BANKS(ADDR_SURF_2_BANK));
2513 break;
2514 default:
2515 gb_tile_moden = 0;
2516 break;
2517 }
1b2c4869 2518 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
21e438af
AD		 2519 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2520 }
2521 } else if (num_pipe_configs == 8) {
8cc1a532
AD		 2522 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2523 switch (reg_offset) {
2524 case 0:
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2526 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2528 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2529 break;
2530 case 1:
2531 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2532 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2533 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2534 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2535 break;
2536 case 2:
2537 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2538 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2539 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2540 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2541 break;
2542 case 3:
2543 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2544 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2545 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2546 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2547 break;
2548 case 4:
2549 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2550 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2551 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2552 TILE_SPLIT(split_equal_to_row_size));
2553 break;
2554 case 5:
2555 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2556 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2557 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2558 break;
2559 case 6:
2560 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2561 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2562 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2563 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2564 break;
2565 case 7:
2566 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2567 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2568 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2569 TILE_SPLIT(split_equal_to_row_size));
2570 break;
2571 case 8:
2572 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2573 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2574 break;
2575 case 9:
2576 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2577 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2578 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2579 break;
2580 case 10:
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2583 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2584 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2585 break;
2586 case 11:
2587 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2588 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2590 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2591 break;
2592 case 12:
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2594 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2595 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2596 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2597 break;
2598 case 13:
2599 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2600 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2601 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2602 break;
2603 case 14:
2604 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2607 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2608 break;
2609 case 16:
2610 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2611 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2612 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2613 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2614 break;
2615 case 17:
2616 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2617 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2618 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2619 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2620 break;
2621 case 27:
2622 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2623 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
8cc1a532
AD		 2624 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2625 break;
2626 case 28:
2627 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2628 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2629 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2630 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2631 break;
2632 case 29:
2633 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2634 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2635 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2636 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2637 break;
2638 case 30:
2639 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2640 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2641 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2642 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2643 break;
2644 default:
2645 gb_tile_moden = 0;
2646 break;
2647 }
39aee490 2648 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2649 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2650 }
2651 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2652 switch (reg_offset) {
2653 case 0:
2654 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2656 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2657 NUM_BANKS(ADDR_SURF_16_BANK));
2658 break;
2659 case 1:
2660 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2661 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2662 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2663 NUM_BANKS(ADDR_SURF_16_BANK));
2664 break;
2665 case 2:
2666 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2667 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2668 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2669 NUM_BANKS(ADDR_SURF_16_BANK));
2670 break;
2671 case 3:
2672 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2673 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2674 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2675 NUM_BANKS(ADDR_SURF_16_BANK));
2676 break;
2677 case 4:
2678 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2679 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2680 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2681 NUM_BANKS(ADDR_SURF_8_BANK));
2682 break;
2683 case 5:
2684 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2685 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2686 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2687 NUM_BANKS(ADDR_SURF_4_BANK));
2688 break;
2689 case 6:
2690 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2693 NUM_BANKS(ADDR_SURF_2_BANK));
2694 break;
2695 case 8:
2696 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2697 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2698 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2699 NUM_BANKS(ADDR_SURF_16_BANK));
2700 break;
2701 case 9:
2702 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2703 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2704 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2705 NUM_BANKS(ADDR_SURF_16_BANK));
2706 break;
2707 case 10:
2708 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2709 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2710 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2711 NUM_BANKS(ADDR_SURF_16_BANK));
2712 break;
2713 case 11:
2714 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2715 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2716 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2717 NUM_BANKS(ADDR_SURF_16_BANK));
2718 break;
2719 case 12:
2720 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2722 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2723 NUM_BANKS(ADDR_SURF_8_BANK));
2724 break;
2725 case 13:
2726 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2727 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2728 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2729 NUM_BANKS(ADDR_SURF_4_BANK));
2730 break;
2731 case 14:
2732 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2733 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2734 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2735 NUM_BANKS(ADDR_SURF_2_BANK));
2736 break;
2737 default:
2738 gb_tile_moden = 0;
2739 break;
2740 }
32f79a8a 2741 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2742 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2743 }
2744 } else if (num_pipe_configs == 4) {
2745 if (num_rbs == 4) {
2746 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2747 switch (reg_offset) {
2748 case 0:
2749 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2750 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2751 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2752 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2753 break;
2754 case 1:
2755 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2756 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2757 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2758 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2759 break;
2760 case 2:
2761 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2762 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2763 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2764 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2765 break;
2766 case 3:
2767 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2768 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2769 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2770 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2771 break;
2772 case 4:
2773 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2774 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2775 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2776 TILE_SPLIT(split_equal_to_row_size));
2777 break;
2778 case 5:
2779 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2780 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2781 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2782 break;
2783 case 6:
2784 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2785 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2786 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2787 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2788 break;
2789 case 7:
2790 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2791 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2792 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2793 TILE_SPLIT(split_equal_to_row_size));
2794 break;
2795 case 8:
2796 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2797 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2798 break;
2799 case 9:
2800 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2801 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2802 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2803 break;
2804 case 10:
2805 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2806 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2807 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2808 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2809 break;
2810 case 11:
2811 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2812 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2813 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2814 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2815 break;
2816 case 12:
2817 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2818 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2819 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2820 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2821 break;
2822 case 13:
2823 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2824 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2825 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2826 break;
2827 case 14:
2828 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2829 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2830 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2831 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2832 break;
2833 case 16:
2834 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2835 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2836 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2837 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2838 break;
2839 case 17:
2840 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2841 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2842 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2843 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2844 break;
2845 case 27:
2846 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2847 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
8cc1a532
AD		 2848 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2849 break;
2850 case 28:
2851 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2852 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2853 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2854 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2855 break;
2856 case 29:
2857 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2858 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2859 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2860 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2861 break;
2862 case 30:
2863 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2864 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2865 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2866 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2867 break;
2868 default:
2869 gb_tile_moden = 0;
2870 break;
2871 }
39aee490 2872 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 2873 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2874 }
2875 } else if (num_rbs < 4) {
2876 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2877 switch (reg_offset) {
2878 case 0:
2879 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2880 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2881 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2882 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2883 break;
2884 case 1:
2885 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2886 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2887 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2888 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2889 break;
2890 case 2:
2891 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2894 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2895 break;
2896 case 3:
2897 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2898 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2899 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2900 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2901 break;
2902 case 4:
2903 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2904 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2905 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2906 TILE_SPLIT(split_equal_to_row_size));
2907 break;
2908 case 5:
2909 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2910 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2911 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2912 break;
2913 case 6:
2914 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2915 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2916 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2917 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2918 break;
2919 case 7:
2920 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2921 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2922 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2923 TILE_SPLIT(split_equal_to_row_size));
2924 break;
2925 case 8:
2926 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2927 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2928 break;
2929 case 9:
2930 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2931 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2932 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2933 break;
2934 case 10:
2935 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2936 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2937 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2938 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2939 break;
2940 case 11:
2941 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2942 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2943 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2944 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2945 break;
2946 case 12:
2947 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2948 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2949 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2950 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2951 break;
2952 case 13:
2953 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2954 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2955 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2956 break;
2957 case 14:
2958 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2959 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2960 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2961 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2962 break;
2963 case 16:
2964 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2965 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2966 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2967 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2968 break;
2969 case 17:
2970 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2971 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2972 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2973 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2974 break;
2975 case 27:
2976 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 2977 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
8cc1a532
AD		 2978 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2979 break;
2980 case 28:
2981 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2982 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2983 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2984 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2985 break;
2986 case 29:
2987 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2988 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2989 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2990 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2991 break;
2992 case 30:
2993 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2994 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2995 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2996 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2997 break;
2998 default:
2999 gb_tile_moden = 0;
3000 break;
3001 }
39aee490 3002 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3003 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3004 }
3005 }
3006 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3007 switch (reg_offset) {
3008 case 0:
3009 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3010 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3011 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3012 NUM_BANKS(ADDR_SURF_16_BANK));
3013 break;
3014 case 1:
3015 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3016 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3017 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3018 NUM_BANKS(ADDR_SURF_16_BANK));
3019 break;
3020 case 2:
3021 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3022 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3023 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3024 NUM_BANKS(ADDR_SURF_16_BANK));
3025 break;
3026 case 3:
3027 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3028 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3029 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3030 NUM_BANKS(ADDR_SURF_16_BANK));
3031 break;
3032 case 4:
3033 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3034 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3035 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3036 NUM_BANKS(ADDR_SURF_16_BANK));
3037 break;
3038 case 5:
3039 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3040 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3041 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3042 NUM_BANKS(ADDR_SURF_8_BANK));
3043 break;
3044 case 6:
3045 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3046 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3047 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3048 NUM_BANKS(ADDR_SURF_4_BANK));
3049 break;
3050 case 8:
3051 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3052 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3053 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3054 NUM_BANKS(ADDR_SURF_16_BANK));
3055 break;
3056 case 9:
3057 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3058 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3059 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3060 NUM_BANKS(ADDR_SURF_16_BANK));
3061 break;
3062 case 10:
3063 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3064 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3065 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3066 NUM_BANKS(ADDR_SURF_16_BANK));
3067 break;
3068 case 11:
3069 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3070 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3071 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3072 NUM_BANKS(ADDR_SURF_16_BANK));
3073 break;
3074 case 12:
3075 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3076 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3077 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3078 NUM_BANKS(ADDR_SURF_16_BANK));
3079 break;
3080 case 13:
3081 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3082 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3083 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3084 NUM_BANKS(ADDR_SURF_8_BANK));
3085 break;
3086 case 14:
3087 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3088 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3089 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3090 NUM_BANKS(ADDR_SURF_4_BANK));
3091 break;
3092 default:
3093 gb_tile_moden = 0;
3094 break;
3095 }
32f79a8a 3096 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3097 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3098 }
3099 } else if (num_pipe_configs == 2) {
3100 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
3101 switch (reg_offset) {
3102 case 0:
3103 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3104 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3105 PIPE_CONFIG(ADDR_SURF_P2) |
3106 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
3107 break;
3108 case 1:
3109 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3110 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3111 PIPE_CONFIG(ADDR_SURF_P2) |
3112 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
3113 break;
3114 case 2:
3115 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3116 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3117 PIPE_CONFIG(ADDR_SURF_P2) |
3118 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3119 break;
3120 case 3:
3121 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3122 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3123 PIPE_CONFIG(ADDR_SURF_P2) |
3124 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
3125 break;
3126 case 4:
3127 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3128 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3129 PIPE_CONFIG(ADDR_SURF_P2) |
3130 TILE_SPLIT(split_equal_to_row_size));
3131 break;
3132 case 5:
3133 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3134 PIPE_CONFIG(ADDR_SURF_P2) |
8cc1a532
AD		 3135 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3136 break;
3137 case 6:
3138 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3139 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3140 PIPE_CONFIG(ADDR_SURF_P2) |
3141 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
3142 break;
3143 case 7:
3144 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3145 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
3146 PIPE_CONFIG(ADDR_SURF_P2) |
3147 TILE_SPLIT(split_equal_to_row_size));
3148 break;
3149 case 8:
020ff546
MO		 3150 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3151 PIPE_CONFIG(ADDR_SURF_P2);
8cc1a532
AD		 3152 break;
3153 case 9:
3154 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546
MO		 3155 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3156 PIPE_CONFIG(ADDR_SURF_P2));
8cc1a532
AD		 3157 break;
3158 case 10:
3159 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3160 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3161 PIPE_CONFIG(ADDR_SURF_P2) |
3162 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3163 break;
3164 case 11:
3165 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3166 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3167 PIPE_CONFIG(ADDR_SURF_P2) |
3168 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3169 break;
3170 case 12:
3171 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3172 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3173 PIPE_CONFIG(ADDR_SURF_P2) |
3174 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3175 break;
3176 case 13:
3177 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546 3178 PIPE_CONFIG(ADDR_SURF_P2) |
8cc1a532
AD		 3179 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
3180 break;
3181 case 14:
3182 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3183 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3184 PIPE_CONFIG(ADDR_SURF_P2) |
3185 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3186 break;
3187 case 16:
3188 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3189 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3190 PIPE_CONFIG(ADDR_SURF_P2) |
3191 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3192 break;
3193 case 17:
3194 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3195 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3196 PIPE_CONFIG(ADDR_SURF_P2) |
3197 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3198 break;
3199 case 27:
3200 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
020ff546
MO		 3201 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3202 PIPE_CONFIG(ADDR_SURF_P2));
8cc1a532
AD		 3203 break;
3204 case 28:
3205 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3206 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3207 PIPE_CONFIG(ADDR_SURF_P2) |
3208 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3209 break;
3210 case 29:
3211 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3212 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3213 PIPE_CONFIG(ADDR_SURF_P2) |
3214 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3215 break;
3216 case 30:
3217 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
3218 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3219 PIPE_CONFIG(ADDR_SURF_P2) |
3220 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3221 break;
3222 default:
3223 gb_tile_moden = 0;
3224 break;
3225 }
39aee490 3226 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3227 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3228 }
3229 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
3230 switch (reg_offset) {
3231 case 0:
3232 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3233 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3234 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3235 NUM_BANKS(ADDR_SURF_16_BANK));
3236 break;
3237 case 1:
3238 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3239 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3240 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3241 NUM_BANKS(ADDR_SURF_16_BANK));
3242 break;
3243 case 2:
3244 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3245 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3246 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3247 NUM_BANKS(ADDR_SURF_16_BANK));
3248 break;
3249 case 3:
3250 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3251 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3252 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3253 NUM_BANKS(ADDR_SURF_16_BANK));
3254 break;
3255 case 4:
3256 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3257 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3258 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3259 NUM_BANKS(ADDR_SURF_16_BANK));
3260 break;
3261 case 5:
3262 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3263 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3264 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3265 NUM_BANKS(ADDR_SURF_16_BANK));
3266 break;
3267 case 6:
3268 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3269 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3270 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3271 NUM_BANKS(ADDR_SURF_8_BANK));
3272 break;
3273 case 8:
3274 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3275 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3276 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3277 NUM_BANKS(ADDR_SURF_16_BANK));
3278 break;
3279 case 9:
3280 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3281 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3282 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3283 NUM_BANKS(ADDR_SURF_16_BANK));
3284 break;
3285 case 10:
3286 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3287 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3288 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3289 NUM_BANKS(ADDR_SURF_16_BANK));
3290 break;
3291 case 11:
3292 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3293 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3294 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3295 NUM_BANKS(ADDR_SURF_16_BANK));
3296 break;
3297 case 12:
3298 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3299 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3300 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3301 NUM_BANKS(ADDR_SURF_16_BANK));
3302 break;
3303 case 13:
3304 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3305 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3306 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3307 NUM_BANKS(ADDR_SURF_16_BANK));
3308 break;
3309 case 14:
3310 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3311 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3312 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3313 NUM_BANKS(ADDR_SURF_8_BANK));
3314 break;
3315 default:
3316 gb_tile_moden = 0;
3317 break;
3318 }
32f79a8a 3319 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
8cc1a532
AD		 3320 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3321 }
3322 } else
3323 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3324}
3325
3326/**
3327 * cik_select_se_sh - select which SE, SH to address
3328 *
3329 * @rdev: radeon_device pointer
3330 * @se_num: shader engine to address
3331 * @sh_num: sh block to address
3332 *
3333 * Select which SE, SH combinations to address. Certain
3334 * registers are instanced per SE or SH. 0xffffffff means
3335 * broadcast to all SEs or SHs (CIK).
3336 */
3337static void cik_select_se_sh(struct radeon_device *rdev,
3338 u32 se_num, u32 sh_num)
3339{
3340 u32 data = INSTANCE_BROADCAST_WRITES;
3341
3342 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
b0fe3d39 3343 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
8cc1a532
AD		 3344 else if (se_num == 0xffffffff)
3345 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3346 else if (sh_num == 0xffffffff)
3347 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3348 else
3349 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3350 WREG32(GRBM_GFX_INDEX, data);
3351}
3352
3353/**
3354 * cik_create_bitmask - create a bitmask
3355 *
3356 * @bit_width: length of the mask
3357 *
3358 * create a variable length bit mask (CIK).
3359 * Returns the bitmask.
3360 */
3361static u32 cik_create_bitmask(u32 bit_width)
3362{
3363 u32 i, mask = 0;
3364
3365 for (i = 0; i < bit_width; i++) {
3366 mask <<= 1;
3367 mask |= 1;
3368 }
3369 return mask;
3370}
3371
3372/**
972c5ddb 3373 * cik_get_rb_disabled - computes the mask of disabled RBs
8cc1a532
AD		 3374 *
3375 * @rdev: radeon_device pointer
3376 * @max_rb_num: max RBs (render backends) for the asic
3377 * @se_num: number of SEs (shader engines) for the asic
3378 * @sh_per_se: number of SH blocks per SE for the asic
3379 *
3380 * Calculates the bitmask of disabled RBs (CIK).
3381 * Returns the disabled RB bitmask.
3382 */
3383static u32 cik_get_rb_disabled(struct radeon_device *rdev,
9fadb352 3384 u32 max_rb_num_per_se,
8cc1a532
AD		 3385 u32 sh_per_se)
3386{
3387 u32 data, mask;
3388
3389 data = RREG32(CC_RB_BACKEND_DISABLE);
3390 if (data & 1)
3391 data &= BACKEND_DISABLE_MASK;
3392 else
3393 data = 0;
3394 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3395
3396 data >>= BACKEND_DISABLE_SHIFT;
3397
9fadb352 3398 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
8cc1a532
AD		 3399
3400 return data & mask;
3401}
3402
3403/**
3404 * cik_setup_rb - setup the RBs on the asic
3405 *
3406 * @rdev: radeon_device pointer
3407 * @se_num: number of SEs (shader engines) for the asic
3408 * @sh_per_se: number of SH blocks per SE for the asic
3409 * @max_rb_num: max RBs (render backends) for the asic
3410 *
3411 * Configures per-SE/SH RB registers (CIK).
3412 */
3413static void cik_setup_rb(struct radeon_device *rdev,
3414 u32 se_num, u32 sh_per_se,
9fadb352 3415 u32 max_rb_num_per_se)
8cc1a532
AD		 3416{
3417 int i, j;
3418 u32 data, mask;
3419 u32 disabled_rbs = 0;
3420 u32 enabled_rbs = 0;
3421
3422 for (i = 0; i < se_num; i++) {
3423 for (j = 0; j < sh_per_se; j++) {
3424 cik_select_se_sh(rdev, i, j);
9fadb352 3425 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
fc821b70
AD		 3426 if (rdev->family == CHIP_HAWAII)
3427 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3428 else
3429 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
8cc1a532
AD		 3430 }
3431 }
3432 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3433
3434 mask = 1;
9fadb352 3435 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
8cc1a532
AD		 3436 if (!(disabled_rbs & mask))
3437 enabled_rbs |= mask;
3438 mask <<= 1;
3439 }
3440
439a1cff
MO		 3441 rdev->config.cik.backend_enable_mask = enabled_rbs;
3442
8cc1a532
AD		 3443 for (i = 0; i < se_num; i++) {
3444 cik_select_se_sh(rdev, i, 0xffffffff);
3445 data = 0;
3446 for (j = 0; j < sh_per_se; j++) {
3447 switch (enabled_rbs & 3) {
fc821b70
AD		 3448 case 0:
3449 if (j == 0)
3450 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3451 else
3452 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3453 break;
8cc1a532
AD		 3454 case 1:
3455 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3456 break;
3457 case 2:
3458 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3459 break;
3460 case 3:
3461 default:
3462 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3463 break;
3464 }
3465 enabled_rbs >>= 2;
3466 }
3467 WREG32(PA_SC_RASTER_CONFIG, data);
3468 }
3469 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3470}
3471
3472/**
3473 * cik_gpu_init - setup the 3D engine
3474 *
3475 * @rdev: radeon_device pointer
3476 *
3477 * Configures the 3D engine and tiling configuration
3478 * registers so that the 3D engine is usable.
3479 */
3480static void cik_gpu_init(struct radeon_device *rdev)
3481{
3482 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3483 u32 mc_shared_chmap, mc_arb_ramcfg;
3484 u32 hdp_host_path_cntl;
3485 u32 tmp;
6101b3ae 3486 int i, j;
8cc1a532
AD		 3487
3488 switch (rdev->family) {
3489 case CHIP_BONAIRE:
3490 rdev->config.cik.max_shader_engines = 2;
3491 rdev->config.cik.max_tile_pipes = 4;
3492 rdev->config.cik.max_cu_per_sh = 7;
3493 rdev->config.cik.max_sh_per_se = 1;
3494 rdev->config.cik.max_backends_per_se = 2;
3495 rdev->config.cik.max_texture_channel_caches = 4;
3496 rdev->config.cik.max_gprs = 256;
3497 rdev->config.cik.max_gs_threads = 32;
3498 rdev->config.cik.max_hw_contexts = 8;
3499
3500 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3501 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3502 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3503 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3504 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3505 break;
b496038b
AD		 3506 case CHIP_HAWAII:
3507 rdev->config.cik.max_shader_engines = 4;
3508 rdev->config.cik.max_tile_pipes = 16;
3509 rdev->config.cik.max_cu_per_sh = 11;
3510 rdev->config.cik.max_sh_per_se = 1;
3511 rdev->config.cik.max_backends_per_se = 4;
3512 rdev->config.cik.max_texture_channel_caches = 16;
3513 rdev->config.cik.max_gprs = 256;
3514 rdev->config.cik.max_gs_threads = 32;
3515 rdev->config.cik.max_hw_contexts = 8;
3516
3517 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3518 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3519 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3520 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3521 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3522 break;
8cc1a532 3523 case CHIP_KAVERI:
b2e4c70a
AD		 3524 rdev->config.cik.max_shader_engines = 1;
3525 rdev->config.cik.max_tile_pipes = 4;
3526 if ((rdev->pdev->device == 0x1304) ||
3527 (rdev->pdev->device == 0x1305) ||
3528 (rdev->pdev->device == 0x130C) ||
3529 (rdev->pdev->device == 0x130F) ||
3530 (rdev->pdev->device == 0x1310) ||
3531 (rdev->pdev->device == 0x1311) ||
3532 (rdev->pdev->device == 0x131C)) {
3533 rdev->config.cik.max_cu_per_sh = 8;
3534 rdev->config.cik.max_backends_per_se = 2;
3535 } else if ((rdev->pdev->device == 0x1309) ||
3536 (rdev->pdev->device == 0x130A) ||
3537 (rdev->pdev->device == 0x130D) ||
7c4622d5
AD		 3538 (rdev->pdev->device == 0x1313) ||
3539 (rdev->pdev->device == 0x131D)) {
b2e4c70a
AD		 3540 rdev->config.cik.max_cu_per_sh = 6;
3541 rdev->config.cik.max_backends_per_se = 2;
3542 } else if ((rdev->pdev->device == 0x1306) ||
3543 (rdev->pdev->device == 0x1307) ||
3544 (rdev->pdev->device == 0x130B) ||
3545 (rdev->pdev->device == 0x130E) ||
3546 (rdev->pdev->device == 0x1315) ||
3547 (rdev->pdev->device == 0x131B)) {
3548 rdev->config.cik.max_cu_per_sh = 4;
3549 rdev->config.cik.max_backends_per_se = 1;
3550 } else {
3551 rdev->config.cik.max_cu_per_sh = 3;
3552 rdev->config.cik.max_backends_per_se = 1;
3553 }
3554 rdev->config.cik.max_sh_per_se = 1;
3555 rdev->config.cik.max_texture_channel_caches = 4;
3556 rdev->config.cik.max_gprs = 256;
3557 rdev->config.cik.max_gs_threads = 16;
3558 rdev->config.cik.max_hw_contexts = 8;
3559
3560 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3561 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3562 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3563 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3564 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
8cc1a532
AD		 3565 break;
3566 case CHIP_KABINI:
f73a9e83 3567 case CHIP_MULLINS:
8cc1a532
AD		 3568 default:
3569 rdev->config.cik.max_shader_engines = 1;
3570 rdev->config.cik.max_tile_pipes = 2;
3571 rdev->config.cik.max_cu_per_sh = 2;
3572 rdev->config.cik.max_sh_per_se = 1;
3573 rdev->config.cik.max_backends_per_se = 1;
3574 rdev->config.cik.max_texture_channel_caches = 2;
3575 rdev->config.cik.max_gprs = 256;
3576 rdev->config.cik.max_gs_threads = 16;
3577 rdev->config.cik.max_hw_contexts = 8;
3578
3579 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3580 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3581 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3582 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3583 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3584 break;
3585 }
3586
3587 /* Initialize HDP */
3588 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3589 WREG32((0x2c14 + j), 0x00000000);
3590 WREG32((0x2c18 + j), 0x00000000);
3591 WREG32((0x2c1c + j), 0x00000000);
3592 WREG32((0x2c20 + j), 0x00000000);
3593 WREG32((0x2c24 + j), 0x00000000);
3594 }
3595
3596 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3597
3598 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3599
3600 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3601 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3602
3603 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3604 rdev->config.cik.mem_max_burst_length_bytes = 256;
3605 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3606 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3607 if (rdev->config.cik.mem_row_size_in_kb > 4)
3608 rdev->config.cik.mem_row_size_in_kb = 4;
3609 /* XXX use MC settings? */
3610 rdev->config.cik.shader_engine_tile_size = 32;
3611 rdev->config.cik.num_gpus = 1;
3612 rdev->config.cik.multi_gpu_tile_size = 64;
3613
3614 /* fix up row size */
3615 gb_addr_config &= ~ROW_SIZE_MASK;
3616 switch (rdev->config.cik.mem_row_size_in_kb) {
3617 case 1:
3618 default:
3619 gb_addr_config |= ROW_SIZE(0);
3620 break;
3621 case 2:
3622 gb_addr_config |= ROW_SIZE(1);
3623 break;
3624 case 4:
3625 gb_addr_config |= ROW_SIZE(2);
3626 break;
3627 }
3628
3629 /* setup tiling info dword. gb_addr_config is not adequate since it does
3630 * not have bank info, so create a custom tiling dword.
3631 * bits 3:0 num_pipes
3632 * bits 7:4 num_banks
3633 * bits 11:8 group_size
3634 * bits 15:12 row_size
3635 */
3636 rdev->config.cik.tile_config = 0;
3637 switch (rdev->config.cik.num_tile_pipes) {
3638 case 1:
3639 rdev->config.cik.tile_config |= (0 << 0);
3640 break;
3641 case 2:
3642 rdev->config.cik.tile_config |= (1 << 0);
3643 break;
3644 case 4:
3645 rdev->config.cik.tile_config |= (2 << 0);
3646 break;
3647 case 8:
3648 default:
3649 /* XXX what about 12? */
3650 rdev->config.cik.tile_config |= (3 << 0);
3651 break;
3652 }
a537314e
MD		 3653 rdev->config.cik.tile_config |=
3654 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
8cc1a532
AD		 3655 rdev->config.cik.tile_config |=
3656 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3657 rdev->config.cik.tile_config |=
3658 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3659
3660 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3661 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3662 WREG32(DMIF_ADDR_CALC, gb_addr_config);
21a93e13
AD		 3663 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3664 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
87167bb1
CK		 3665 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3666 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3667 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
8cc1a532
AD		 3668
3669 cik_tiling_mode_table_init(rdev);
3670
3671 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3672 rdev->config.cik.max_sh_per_se,
3673 rdev->config.cik.max_backends_per_se);
3674
52da51f0 3675 rdev->config.cik.active_cus = 0;
65fcf668
AD		 3676 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
3677 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6101b3ae
AD		 3678 rdev->config.cik.active_cus +=
3679 hweight32(cik_get_cu_active_bitmap(rdev, i, j));
65fcf668
AD		 3680 }
3681 }
3682
8cc1a532
AD		 3683 /* set HW defaults for 3D engine */
3684 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3685
3686 WREG32(SX_DEBUG_1, 0x20);
3687
3688 WREG32(TA_CNTL_AUX, 0x00010000);
3689
3690 tmp = RREG32(SPI_CONFIG_CNTL);
3691 tmp |= 0x03000000;
3692 WREG32(SPI_CONFIG_CNTL, tmp);
3693
3694 WREG32(SQ_CONFIG, 1);
3695
3696 WREG32(DB_DEBUG, 0);
3697
3698 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3699 tmp |= 0x00000400;
3700 WREG32(DB_DEBUG2, tmp);
3701
3702 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3703 tmp |= 0x00020200;
3704 WREG32(DB_DEBUG3, tmp);
3705
3706 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3707 tmp |= 0x00018208;
3708 WREG32(CB_HW_CONTROL, tmp);
3709
3710 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3711
3712 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3713 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3714 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3715 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3716
3717 WREG32(VGT_NUM_INSTANCES, 1);
3718
3719 WREG32(CP_PERFMON_CNTL, 0);
3720
3721 WREG32(SQ_CONFIG, 0);
3722
3723 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3724 FORCE_EOV_MAX_REZ_CNT(255)));
3725
3726 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3727 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3728
3729 WREG32(VGT_GS_VERTEX_REUSE, 16);
3730 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3731
3732 tmp = RREG32(HDP_MISC_CNTL);
3733 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3734 WREG32(HDP_MISC_CNTL, tmp);
3735
3736 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3737 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3738
3739 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3740 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3741
3742 udelay(50);
3743}
3744
2cae3bc3
AD		 3745/*
3746 * GPU scratch registers helpers function.
3747 */
3748/**
3749 * cik_scratch_init - setup driver info for CP scratch regs
3750 *
3751 * @rdev: radeon_device pointer
3752 *
3753 * Set up the number and offset of the CP scratch registers.
3754 * NOTE: use of CP scratch registers is a legacy inferface and
3755 * is not used by default on newer asics (r6xx+). On newer asics,
3756 * memory buffers are used for fences rather than scratch regs.
3757 */
3758static void cik_scratch_init(struct radeon_device *rdev)
3759{
3760 int i;
3761
3762 rdev->scratch.num_reg = 7;
3763 rdev->scratch.reg_base = SCRATCH_REG0;
3764 for (i = 0; i < rdev->scratch.num_reg; i++) {
3765 rdev->scratch.free[i] = true;
3766 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3767 }
3768}
3769
fbc832c7
AD		 3770/**
3771 * cik_ring_test - basic gfx ring test
3772 *
3773 * @rdev: radeon_device pointer
3774 * @ring: radeon_ring structure holding ring information
3775 *
3776 * Allocate a scratch register and write to it using the gfx ring (CIK).
3777 * Provides a basic gfx ring test to verify that the ring is working.
3778 * Used by cik_cp_gfx_resume();
3779 * Returns 0 on success, error on failure.
3780 */
3781int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3782{
3783 uint32_t scratch;
3784 uint32_t tmp = 0;
3785 unsigned i;
3786 int r;
3787
3788 r = radeon_scratch_get(rdev, &scratch);
3789 if (r) {
3790 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3791 return r;
3792 }
3793 WREG32(scratch, 0xCAFEDEAD);
3794 r = radeon_ring_lock(rdev, ring, 3);
3795 if (r) {
3796 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3797 radeon_scratch_free(rdev, scratch);
3798 return r;
3799 }
3800 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3801 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3802 radeon_ring_write(ring, 0xDEADBEEF);
1538a9e0 3803 radeon_ring_unlock_commit(rdev, ring, false);
963e81f9 3804
fbc832c7
AD		 3805 for (i = 0; i < rdev->usec_timeout; i++) {
3806 tmp = RREG32(scratch);
3807 if (tmp == 0xDEADBEEF)
3808 break;
3809 DRM_UDELAY(1);
3810 }
3811 if (i < rdev->usec_timeout) {
3812 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3813 } else {
3814 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3815 ring->idx, scratch, tmp);
3816 r = -EINVAL;
3817 }
3818 radeon_scratch_free(rdev, scratch);
3819 return r;
3820}
3821
780f5ddd
AD		 3822/**
3823 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3824 *
3825 * @rdev: radeon_device pointer
3826 * @ridx: radeon ring index
3827 *
3828 * Emits an hdp flush on the cp.
3829 */
3830static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3831 int ridx)
3832{
3833 struct radeon_ring *ring = &rdev->ring[ridx];
5d259067 3834 u32 ref_and_mask;
780f5ddd 3835
5d259067
AD		 3836 switch (ring->idx) {
3837 case CAYMAN_RING_TYPE_CP1_INDEX:
3838 case CAYMAN_RING_TYPE_CP2_INDEX:
3839 default:
3840 switch (ring->me) {
3841 case 0:
3842 ref_and_mask = CP2 << ring->pipe;
3843 break;
3844 case 1:
3845 ref_and_mask = CP6 << ring->pipe;
3846 break;
3847 default:
3848 return;
3849 }
3850 break;
3851 case RADEON_RING_TYPE_GFX_INDEX:
3852 ref_and_mask = CP0;
3853 break;
3854 }
3855
3856 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3857 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3858 WAIT_REG_MEM_FUNCTION(3) | /* == */
3859 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3860 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3861 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3862 radeon_ring_write(ring, ref_and_mask);
3863 radeon_ring_write(ring, ref_and_mask);
3864 radeon_ring_write(ring, 0x20); /* poll interval */
780f5ddd
AD		 3865}
3866
2cae3bc3 3867/**
b07fdd38 3868 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
2cae3bc3
AD		 3869 *
3870 * @rdev: radeon_device pointer
3871 * @fence: radeon fence object
3872 *
3873 * Emits a fence sequnce number on the gfx ring and flushes
3874 * GPU caches.
3875 */
b07fdd38
AD		 3876void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3877 struct radeon_fence *fence)
2cae3bc3
AD		 3878{
3879 struct radeon_ring *ring = &rdev->ring[fence->ring];
3880 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3881
3882 /* EVENT_WRITE_EOP - flush caches, send int */
3883 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3884 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3885 EOP_TC_ACTION_EN |
3886 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3887 EVENT_INDEX(5)));
3888 radeon_ring_write(ring, addr & 0xfffffffc);
3889 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3890 radeon_ring_write(ring, fence->seq);
3891 radeon_ring_write(ring, 0);
2cae3bc3
AD		 3892}
3893
b07fdd38
AD		 3894/**
3895 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3896 *
3897 * @rdev: radeon_device pointer
3898 * @fence: radeon fence object
3899 *
3900 * Emits a fence sequnce number on the compute ring and flushes
3901 * GPU caches.
3902 */
3903void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3904 struct radeon_fence *fence)
3905{
3906 struct radeon_ring *ring = &rdev->ring[fence->ring];
3907 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3908
3909 /* RELEASE_MEM - flush caches, send int */
3910 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3911 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3912 EOP_TC_ACTION_EN |
3913 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3914 EVENT_INDEX(5)));
3915 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3916 radeon_ring_write(ring, addr & 0xfffffffc);
3917 radeon_ring_write(ring, upper_32_bits(addr));
3918 radeon_ring_write(ring, fence->seq);
3919 radeon_ring_write(ring, 0);
b07fdd38
AD		 3920}
3921
86302eea
CK		 3922/**
3923 * cik_semaphore_ring_emit - emit a semaphore on the CP ring
3924 *
3925 * @rdev: radeon_device pointer
3926 * @ring: radeon ring buffer object
3927 * @semaphore: radeon semaphore object
3928 * @emit_wait: Is this a sempahore wait?
3929 *
3930 * Emits a semaphore signal/wait packet to the CP ring and prevents the PFP
3931 * from running ahead of semaphore waits.
3932 */
1654b817 3933bool cik_semaphore_ring_emit(struct radeon_device *rdev,
2cae3bc3
AD		 3934 struct radeon_ring *ring,
3935 struct radeon_semaphore *semaphore,
3936 bool emit_wait)
3937{
3938 uint64_t addr = semaphore->gpu_addr;
3939 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3940
3941 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
5e167cdb 3942 radeon_ring_write(ring, lower_32_bits(addr));
2cae3bc3 3943 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
1654b817 3944
86302eea
CK		 3945 if (emit_wait && ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
3946 /* Prevent the PFP from running ahead of the semaphore wait */
3947 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
3948 radeon_ring_write(ring, 0x0);
3949 }
3950
1654b817 3951 return true;
2cae3bc3
AD		 3952}
3953
c9dbd705
AD		 3954/**
3955 * cik_copy_cpdma - copy pages using the CP DMA engine
3956 *
3957 * @rdev: radeon_device pointer
3958 * @src_offset: src GPU address
3959 * @dst_offset: dst GPU address
3960 * @num_gpu_pages: number of GPU pages to xfer
3961 * @fence: radeon fence object
3962 *
3963 * Copy GPU paging using the CP DMA engine (CIK+).
3964 * Used by the radeon ttm implementation to move pages if
3965 * registered as the asic copy callback.
3966 */
3967int cik_copy_cpdma(struct radeon_device *rdev,
3968 uint64_t src_offset, uint64_t dst_offset,
3969 unsigned num_gpu_pages,
3970 struct radeon_fence **fence)
3971{
3972 struct radeon_semaphore *sem = NULL;
3973 int ring_index = rdev->asic->copy.blit_ring_index;
3974 struct radeon_ring *ring = &rdev->ring[ring_index];
3975 u32 size_in_bytes, cur_size_in_bytes, control;
3976 int i, num_loops;
3977 int r = 0;
3978
3979 r = radeon_semaphore_create(rdev, &sem);
3980 if (r) {
3981 DRM_ERROR("radeon: moving bo (%d).\n", r);
3982 return r;
3983 }
3984
3985 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3986 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3987 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3988 if (r) {
3989 DRM_ERROR("radeon: moving bo (%d).\n", r);
3990 radeon_semaphore_free(rdev, &sem, NULL);
3991 return r;
3992 }
3993
1654b817
CK		 3994 radeon_semaphore_sync_to(sem, *fence);
3995 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
c9dbd705
AD		 3996
3997 for (i = 0; i < num_loops; i++) {
3998 cur_size_in_bytes = size_in_bytes;
3999 if (cur_size_in_bytes > 0x1fffff)
4000 cur_size_in_bytes = 0x1fffff;
4001 size_in_bytes -= cur_size_in_bytes;
4002 control = 0;
4003 if (size_in_bytes == 0)
4004 control |= PACKET3_DMA_DATA_CP_SYNC;
4005 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4006 radeon_ring_write(ring, control);
4007 radeon_ring_write(ring, lower_32_bits(src_offset));
4008 radeon_ring_write(ring, upper_32_bits(src_offset));
4009 radeon_ring_write(ring, lower_32_bits(dst_offset));
4010 radeon_ring_write(ring, upper_32_bits(dst_offset));
4011 radeon_ring_write(ring, cur_size_in_bytes);
4012 src_offset += cur_size_in_bytes;
4013 dst_offset += cur_size_in_bytes;
4014 }
4015
4016 r = radeon_fence_emit(rdev, fence, ring->idx);
4017 if (r) {
4018 radeon_ring_unlock_undo(rdev, ring);
aa4c8b36 4019 radeon_semaphore_free(rdev, &sem, NULL);
c9dbd705
AD		 4020 return r;
4021 }
4022
1538a9e0 4023 radeon_ring_unlock_commit(rdev, ring, false);
c9dbd705
AD		 4024 radeon_semaphore_free(rdev, &sem, *fence);
4025
4026 return r;
4027}
4028
2cae3bc3
AD		 4029/*
4030 * IB stuff
4031 */
4032/**
4033 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
4034 *
4035 * @rdev: radeon_device pointer
4036 * @ib: radeon indirect buffer object
4037 *
4038 * Emits an DE (drawing engine) or CE (constant engine) IB
4039 * on the gfx ring. IBs are usually generated by userspace
4040 * acceleration drivers and submitted to the kernel for
4041 * sheduling on the ring. This function schedules the IB
4042 * on the gfx ring for execution by the GPU.
4043 */
4044void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
4045{
4046 struct radeon_ring *ring = &rdev->ring[ib->ring];
4047 u32 header, control = INDIRECT_BUFFER_VALID;
4048
4049 if (ib->is_const_ib) {
4050 /* set switch buffer packet before const IB */
4051 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
4052 radeon_ring_write(ring, 0);
4053
4054 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
4055 } else {
4056 u32 next_rptr;
4057 if (ring->rptr_save_reg) {
4058 next_rptr = ring->wptr + 3 + 4;
4059 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
4060 radeon_ring_write(ring, ((ring->rptr_save_reg -
4061 PACKET3_SET_UCONFIG_REG_START) >> 2));
4062 radeon_ring_write(ring, next_rptr);
4063 } else if (rdev->wb.enabled) {
4064 next_rptr = ring->wptr + 5 + 4;
4065 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4066 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
4067 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
5e167cdb 4068 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
2cae3bc3
AD		 4069 radeon_ring_write(ring, next_rptr);
4070 }
4071
4072 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
4073 }
4074
4075 control |= ib->length_dw |
4076 (ib->vm ? (ib->vm->id << 24) : 0);
4077
4078 radeon_ring_write(ring, header);
4079 radeon_ring_write(ring,
4080#ifdef __BIG_ENDIAN
4081 (2 << 0) |
4082#endif
4083 (ib->gpu_addr & 0xFFFFFFFC));
4084 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
4085 radeon_ring_write(ring, control);
4086}
4087
fbc832c7
AD		 4088/**
4089 * cik_ib_test - basic gfx ring IB test
4090 *
4091 * @rdev: radeon_device pointer
4092 * @ring: radeon_ring structure holding ring information
4093 *
4094 * Allocate an IB and execute it on the gfx ring (CIK).
4095 * Provides a basic gfx ring test to verify that IBs are working.
4096 * Returns 0 on success, error on failure.
4097 */
4098int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
4099{
4100 struct radeon_ib ib;
4101 uint32_t scratch;
4102 uint32_t tmp = 0;
4103 unsigned i;
4104 int r;
4105
4106 r = radeon_scratch_get(rdev, &scratch);
4107 if (r) {
4108 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
4109 return r;
4110 }
4111 WREG32(scratch, 0xCAFEDEAD);
4112 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
4113 if (r) {
4114 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
5510f124 4115 radeon_scratch_free(rdev, scratch);
fbc832c7
AD		 4116 return r;
4117 }
4118 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
4119 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
4120 ib.ptr[2] = 0xDEADBEEF;
4121 ib.length_dw = 3;
1538a9e0 4122 r = radeon_ib_schedule(rdev, &ib, NULL, false);
fbc832c7
AD		 4123 if (r) {
4124 radeon_scratch_free(rdev, scratch);
4125 radeon_ib_free(rdev, &ib);
4126 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
4127 return r;
4128 }
4129 r = radeon_fence_wait(ib.fence, false);
4130 if (r) {
4131 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
5510f124
CK		 4132 radeon_scratch_free(rdev, scratch);
4133 radeon_ib_free(rdev, &ib);
fbc832c7
AD		 4134 return r;
4135 }
4136 for (i = 0; i < rdev->usec_timeout; i++) {
4137 tmp = RREG32(scratch);
4138 if (tmp == 0xDEADBEEF)
4139 break;
4140 DRM_UDELAY(1);
4141 }
4142 if (i < rdev->usec_timeout) {
4143 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
4144 } else {
4145 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
4146 scratch, tmp);
4147 r = -EINVAL;
4148 }
4149 radeon_scratch_free(rdev, scratch);
4150 radeon_ib_free(rdev, &ib);
4151 return r;
4152}
4153
841cf442
AD		 4154/*
4155 * CP.
4156 * On CIK, gfx and compute now have independant command processors.
4157 *
4158 * GFX
4159 * Gfx consists of a single ring and can process both gfx jobs and
4160 * compute jobs. The gfx CP consists of three microengines (ME):
4161 * PFP - Pre-Fetch Parser
4162 * ME - Micro Engine
4163 * CE - Constant Engine
4164 * The PFP and ME make up what is considered the Drawing Engine (DE).
4165 * The CE is an asynchronous engine used for updating buffer desciptors
4166 * used by the DE so that they can be loaded into cache in parallel
4167 * while the DE is processing state update packets.
4168 *
4169 * Compute
4170 * The compute CP consists of two microengines (ME):
4171 * MEC1 - Compute MicroEngine 1
4172 * MEC2 - Compute MicroEngine 2
4173 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
4174 * The queues are exposed to userspace and are programmed directly
4175 * by the compute runtime.
4176 */
4177/**
4178 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
4179 *
4180 * @rdev: radeon_device pointer
4181 * @enable: enable or disable the MEs
4182 *
4183 * Halts or unhalts the gfx MEs.
4184 */
4185static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
4186{
4187 if (enable)
4188 WREG32(CP_ME_CNTL, 0);
4189 else {
50efa51a
AD		 4190 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4191 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
841cf442
AD		 4192 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
4193 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4194 }
4195 udelay(50);
4196}
4197
4198/**
4199 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
4200 *
4201 * @rdev: radeon_device pointer
4202 *
4203 * Loads the gfx PFP, ME, and CE ucode.
4204 * Returns 0 for success, -EINVAL if the ucode is not available.
4205 */
4206static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
4207{
841cf442
AD		 4208 int i;
4209
4210 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
4211 return -EINVAL;
4212
4213 cik_cp_gfx_enable(rdev, false);
4214
f2c6b0f4
AD		 4215 if (rdev->new_fw) {
4216 const struct gfx_firmware_header_v1_0 *pfp_hdr =
4217 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
4218 const struct gfx_firmware_header_v1_0 *ce_hdr =
4219 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
4220 const struct gfx_firmware_header_v1_0 *me_hdr =
4221 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
4222 const __le32 *fw_data;
4223 u32 fw_size;
4224
4225 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
4226 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
4227 radeon_ucode_print_gfx_hdr(&me_hdr->header);
4228
4229 /* PFP */
4230 fw_data = (const __le32 *)
4231 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
4232 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
4233 WREG32(CP_PFP_UCODE_ADDR, 0);
4234 for (i = 0; i < fw_size; i++)
4235 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
4236 WREG32(CP_PFP_UCODE_ADDR, 0);
4237
4238 /* CE */
4239 fw_data = (const __le32 *)
4240 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
4241 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
4242 WREG32(CP_CE_UCODE_ADDR, 0);
4243 for (i = 0; i < fw_size; i++)
4244 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
4245 WREG32(CP_CE_UCODE_ADDR, 0);
4246
4247 /* ME */
4248 fw_data = (const __be32 *)
4249 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
4250 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
4251 WREG32(CP_ME_RAM_WADDR, 0);
4252 for (i = 0; i < fw_size; i++)
4253 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
4254 WREG32(CP_ME_RAM_WADDR, 0);
4255 } else {
4256 const __be32 *fw_data;
4257
4258 /* PFP */
4259 fw_data = (const __be32 *)rdev->pfp_fw->data;
4260 WREG32(CP_PFP_UCODE_ADDR, 0);
4261 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
4262 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
4263 WREG32(CP_PFP_UCODE_ADDR, 0);
4264
4265 /* CE */
4266 fw_data = (const __be32 *)rdev->ce_fw->data;
4267 WREG32(CP_CE_UCODE_ADDR, 0);
4268 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
4269 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
4270 WREG32(CP_CE_UCODE_ADDR, 0);
4271
4272 /* ME */
4273 fw_data = (const __be32 *)rdev->me_fw->data;
4274 WREG32(CP_ME_RAM_WADDR, 0);
4275 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
4276 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
4277 WREG32(CP_ME_RAM_WADDR, 0);
4278 }
841cf442
AD		 4279
4280 WREG32(CP_PFP_UCODE_ADDR, 0);
4281 WREG32(CP_CE_UCODE_ADDR, 0);
4282 WREG32(CP_ME_RAM_WADDR, 0);
4283 WREG32(CP_ME_RAM_RADDR, 0);
4284 return 0;
4285}
4286
4287/**
4288 * cik_cp_gfx_start - start the gfx ring
4289 *
4290 * @rdev: radeon_device pointer
4291 *
4292 * Enables the ring and loads the clear state context and other
4293 * packets required to init the ring.
4294 * Returns 0 for success, error for failure.
4295 */
4296static int cik_cp_gfx_start(struct radeon_device *rdev)
4297{
4298 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4299 int r, i;
4300
4301 /* init the CP */
4302 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
4303 WREG32(CP_ENDIAN_SWAP, 0);
4304 WREG32(CP_DEVICE_ID, 1);
4305
4306 cik_cp_gfx_enable(rdev, true);
4307
4308 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
4309 if (r) {
4310 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
4311 return r;
4312 }
4313
4314 /* init the CE partitions. CE only used for gfx on CIK */
4315 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4316 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4317 radeon_ring_write(ring, 0xc000);
4318 radeon_ring_write(ring, 0xc000);
4319
4320 /* setup clear context state */
4321 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4322 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4323
4324 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4325 radeon_ring_write(ring, 0x80000000);
4326 radeon_ring_write(ring, 0x80000000);
4327
4328 for (i = 0; i < cik_default_size; i++)
4329 radeon_ring_write(ring, cik_default_state[i]);
4330
4331 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4332 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4333
4334 /* set clear context state */
4335 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4336 radeon_ring_write(ring, 0);
4337
4338 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4339 radeon_ring_write(ring, 0x00000316);
4340 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
4341 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
4342
1538a9e0 4343 radeon_ring_unlock_commit(rdev, ring, false);
841cf442
AD		 4344
4345 return 0;
4346}
4347
4348/**
4349 * cik_cp_gfx_fini - stop the gfx ring
4350 *
4351 * @rdev: radeon_device pointer
4352 *
4353 * Stop the gfx ring and tear down the driver ring
4354 * info.
4355 */
4356static void cik_cp_gfx_fini(struct radeon_device *rdev)
4357{
4358 cik_cp_gfx_enable(rdev, false);
4359 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4360}
4361
4362/**
4363 * cik_cp_gfx_resume - setup the gfx ring buffer registers
4364 *
4365 * @rdev: radeon_device pointer
4366 *
4367 * Program the location and size of the gfx ring buffer
4368 * and test it to make sure it's working.
4369 * Returns 0 for success, error for failure.
4370 */
4371static int cik_cp_gfx_resume(struct radeon_device *rdev)
4372{
4373 struct radeon_ring *ring;
4374 u32 tmp;
4375 u32 rb_bufsz;
4376 u64 rb_addr;
4377 int r;
4378
4379 WREG32(CP_SEM_WAIT_TIMER, 0x0);
939c0d3c
AD		 4380 if (rdev->family != CHIP_HAWAII)
4381 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
841cf442
AD		 4382
4383 /* Set the write pointer delay */
4384 WREG32(CP_RB_WPTR_DELAY, 0);
4385
4386 /* set the RB to use vmid 0 */
4387 WREG32(CP_RB_VMID, 0);
4388
4389 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4390
4391 /* ring 0 - compute and gfx */
4392 /* Set ring buffer size */
4393 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
b72a8925
DV		 4394 rb_bufsz = order_base_2(ring->ring_size / 8);
4395 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
841cf442
AD		 4396#ifdef __BIG_ENDIAN
4397 tmp |= BUF_SWAP_32BIT;
4398#endif
4399 WREG32(CP_RB0_CNTL, tmp);
4400
4401 /* Initialize the ring buffer's read and write pointers */
4402 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4403 ring->wptr = 0;
4404 WREG32(CP_RB0_WPTR, ring->wptr);
4405
4406 /* set the wb address wether it's enabled or not */
4407 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4408 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4409
4410 /* scratch register shadowing is no longer supported */
4411 WREG32(SCRATCH_UMSK, 0);
4412
4413 if (!rdev->wb.enabled)
4414 tmp |= RB_NO_UPDATE;
4415
4416 mdelay(1);
4417 WREG32(CP_RB0_CNTL, tmp);
4418
4419 rb_addr = ring->gpu_addr >> 8;
4420 WREG32(CP_RB0_BASE, rb_addr);
4421 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4422
841cf442
AD		 4423 /* start the ring */
4424 cik_cp_gfx_start(rdev);
4425 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4426 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4427 if (r) {
4428 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4429 return r;
4430 }
50efa51a
AD		 4431
4432 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4433 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4434
841cf442
AD		 4435 return 0;
4436}
4437
ea31bf69
AD		 4438u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4439 struct radeon_ring *ring)
963e81f9
AD		 4440{
4441 u32 rptr;
4442
ea31bf69
AD		 4443 if (rdev->wb.enabled)
4444 rptr = rdev->wb.wb[ring->rptr_offs/4];
4445 else
4446 rptr = RREG32(CP_RB0_RPTR);
4447
4448 return rptr;
4449}
4450
4451u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4452 struct radeon_ring *ring)
4453{
4454 u32 wptr;
4455
4456 wptr = RREG32(CP_RB0_WPTR);
963e81f9 4457
ea31bf69
AD		 4458 return wptr;
4459}
4460
4461void cik_gfx_set_wptr(struct radeon_device *rdev,
4462 struct radeon_ring *ring)
4463{
4464 WREG32(CP_RB0_WPTR, ring->wptr);
4465 (void)RREG32(CP_RB0_WPTR);
4466}
4467
4468u32 cik_compute_get_rptr(struct radeon_device *rdev,
4469 struct radeon_ring *ring)
4470{
4471 u32 rptr;
963e81f9
AD		 4472
4473 if (rdev->wb.enabled) {
ea31bf69 4474 rptr = rdev->wb.wb[ring->rptr_offs/4];
963e81f9 4475 } else {
f61d5b46 4476 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4477 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4478 rptr = RREG32(CP_HQD_PQ_RPTR);
4479 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4480 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4481 }
963e81f9
AD		 4482
4483 return rptr;
4484}
4485
ea31bf69
AD		 4486u32 cik_compute_get_wptr(struct radeon_device *rdev,
4487 struct radeon_ring *ring)
963e81f9
AD		 4488{
4489 u32 wptr;
4490
4491 if (rdev->wb.enabled) {
ea31bf69
AD		 4492 /* XXX check if swapping is necessary on BE */
4493 wptr = rdev->wb.wb[ring->wptr_offs/4];
963e81f9 4494 } else {
f61d5b46 4495 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4496 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4497 wptr = RREG32(CP_HQD_PQ_WPTR);
4498 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4499 mutex_unlock(&rdev->srbm_mutex);
963e81f9 4500 }
963e81f9
AD		 4501
4502 return wptr;
4503}
4504
ea31bf69
AD		 4505void cik_compute_set_wptr(struct radeon_device *rdev,
4506 struct radeon_ring *ring)
963e81f9 4507{
ea31bf69
AD		 4508 /* XXX check if swapping is necessary on BE */
4509 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
d5754ab8 4510 WDOORBELL32(ring->doorbell_index, ring->wptr);
963e81f9
AD		 4511}
4512
841cf442
AD		 4513/**
4514 * cik_cp_compute_enable - enable/disable the compute CP MEs
4515 *
4516 * @rdev: radeon_device pointer
4517 * @enable: enable or disable the MEs
4518 *
4519 * Halts or unhalts the compute MEs.
4520 */
4521static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4522{
4523 if (enable)
4524 WREG32(CP_MEC_CNTL, 0);
b2b3d8d9 4525 else {
841cf442 4526 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
b2b3d8d9
AD		 4527 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4528 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4529 }
841cf442
AD		 4530 udelay(50);
4531}
4532
4533/**
4534 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4535 *
4536 * @rdev: radeon_device pointer
4537 *
4538 * Loads the compute MEC1&2 ucode.
4539 * Returns 0 for success, -EINVAL if the ucode is not available.
4540 */
4541static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4542{
841cf442
AD		 4543 int i;
4544
4545 if (!rdev->mec_fw)
4546 return -EINVAL;
4547
4548 cik_cp_compute_enable(rdev, false);
4549
f2c6b0f4
AD		 4550 if (rdev->new_fw) {
4551 const struct gfx_firmware_header_v1_0 *mec_hdr =
4552 (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
4553 const __le32 *fw_data;
4554 u32 fw_size;
4555
4556 radeon_ucode_print_gfx_hdr(&mec_hdr->header);
4557
4558 /* MEC1 */
4559 fw_data = (const __le32 *)
4560 (rdev->mec_fw->data + le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
4561 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
4562 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4563 for (i = 0; i < fw_size; i++)
4564 WREG32(CP_MEC_ME1_UCODE_DATA, le32_to_cpup(fw_data++));
4565 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
841cf442 4566
841cf442 4567 /* MEC2 */
f2c6b0f4
AD		 4568 if (rdev->family == CHIP_KAVERI) {
4569 const struct gfx_firmware_header_v1_0 *mec2_hdr =
4570 (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
4571
4572 fw_data = (const __le32 *)
4573 (rdev->mec2_fw->data +
4574 le32_to_cpu(mec2_hdr->header.ucode_array_offset_bytes));
4575 fw_size = le32_to_cpu(mec2_hdr->header.ucode_size_bytes) / 4;
4576 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4577 for (i = 0; i < fw_size; i++)
4578 WREG32(CP_MEC_ME2_UCODE_DATA, le32_to_cpup(fw_data++));
4579 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4580 }
4581 } else {
4582 const __be32 *fw_data;
4583
4584 /* MEC1 */
841cf442 4585 fw_data = (const __be32 *)rdev->mec_fw->data;
f2c6b0f4 4586 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
841cf442 4587 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
f2c6b0f4
AD		 4588 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4589 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4590
4591 if (rdev->family == CHIP_KAVERI) {
4592 /* MEC2 */
4593 fw_data = (const __be32 *)rdev->mec_fw->data;
4594 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4595 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4596 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4597 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4598 }
841cf442
AD		 4599 }
4600
4601 return 0;
4602}
4603
4604/**
4605 * cik_cp_compute_start - start the compute queues
4606 *
4607 * @rdev: radeon_device pointer
4608 *
4609 * Enable the compute queues.
4610 * Returns 0 for success, error for failure.
4611 */
4612static int cik_cp_compute_start(struct radeon_device *rdev)
4613{
963e81f9
AD		 4614 cik_cp_compute_enable(rdev, true);
4615
841cf442
AD		 4616 return 0;
4617}
4618
4619/**
4620 * cik_cp_compute_fini - stop the compute queues
4621 *
4622 * @rdev: radeon_device pointer
4623 *
4624 * Stop the compute queues and tear down the driver queue
4625 * info.
4626 */
4627static void cik_cp_compute_fini(struct radeon_device *rdev)
4628{
963e81f9
AD		 4629 int i, idx, r;
4630
841cf442 4631 cik_cp_compute_enable(rdev, false);
963e81f9
AD		 4632
4633 for (i = 0; i < 2; i++) {
4634 if (i == 0)
4635 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4636 else
4637 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4638
4639 if (rdev->ring[idx].mqd_obj) {
4640 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4641 if (unlikely(r != 0))
4642 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4643
4644 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4645 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4646
4647 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4648 rdev->ring[idx].mqd_obj = NULL;
4649 }
4650 }
841cf442
AD		 4651}
4652
963e81f9
AD		 4653static void cik_mec_fini(struct radeon_device *rdev)
4654{
4655 int r;
4656
4657 if (rdev->mec.hpd_eop_obj) {
4658 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4659 if (unlikely(r != 0))
4660 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4661 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4662 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4663
4664 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4665 rdev->mec.hpd_eop_obj = NULL;
4666 }
4667}
4668
4669#define MEC_HPD_SIZE 2048
4670
4671static int cik_mec_init(struct radeon_device *rdev)
4672{
4673 int r;
4674 u32 *hpd;
4675
4676 /*
4677 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4678 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4679 */
4680 if (rdev->family == CHIP_KAVERI)
4681 rdev->mec.num_mec = 2;
4682 else
4683 rdev->mec.num_mec = 1;
4684 rdev->mec.num_pipe = 4;
4685 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4686
4687 if (rdev->mec.hpd_eop_obj == NULL) {
4688 r = radeon_bo_create(rdev,
4689 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4690 PAGE_SIZE, true,
02376d82 4691 RADEON_GEM_DOMAIN_GTT, 0, NULL,
963e81f9
AD		 4692 &rdev->mec.hpd_eop_obj);
4693 if (r) {
4694 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4695 return r;
4696 }
4697 }
4698
4699 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4700 if (unlikely(r != 0)) {
4701 cik_mec_fini(rdev);
4702 return r;
4703 }
4704 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4705 &rdev->mec.hpd_eop_gpu_addr);
4706 if (r) {
4707 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4708 cik_mec_fini(rdev);
4709 return r;
4710 }
4711 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4712 if (r) {
4713 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4714 cik_mec_fini(rdev);
4715 return r;
4716 }
4717
4718 /* clear memory. Not sure if this is required or not */
4719 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4720
4721 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4722 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4723
4724 return 0;
4725}
4726
4727struct hqd_registers
4728{
4729 u32 cp_mqd_base_addr;
4730 u32 cp_mqd_base_addr_hi;
4731 u32 cp_hqd_active;
4732 u32 cp_hqd_vmid;
4733 u32 cp_hqd_persistent_state;
4734 u32 cp_hqd_pipe_priority;
4735 u32 cp_hqd_queue_priority;
4736 u32 cp_hqd_quantum;
4737 u32 cp_hqd_pq_base;
4738 u32 cp_hqd_pq_base_hi;
4739 u32 cp_hqd_pq_rptr;
4740 u32 cp_hqd_pq_rptr_report_addr;
4741 u32 cp_hqd_pq_rptr_report_addr_hi;
4742 u32 cp_hqd_pq_wptr_poll_addr;
4743 u32 cp_hqd_pq_wptr_poll_addr_hi;
4744 u32 cp_hqd_pq_doorbell_control;
4745 u32 cp_hqd_pq_wptr;
4746 u32 cp_hqd_pq_control;
4747 u32 cp_hqd_ib_base_addr;
4748 u32 cp_hqd_ib_base_addr_hi;
4749 u32 cp_hqd_ib_rptr;
4750 u32 cp_hqd_ib_control;
4751 u32 cp_hqd_iq_timer;
4752 u32 cp_hqd_iq_rptr;
4753 u32 cp_hqd_dequeue_request;
4754 u32 cp_hqd_dma_offload;
4755 u32 cp_hqd_sema_cmd;
4756 u32 cp_hqd_msg_type;
4757 u32 cp_hqd_atomic0_preop_lo;
4758 u32 cp_hqd_atomic0_preop_hi;
4759 u32 cp_hqd_atomic1_preop_lo;
4760 u32 cp_hqd_atomic1_preop_hi;
4761 u32 cp_hqd_hq_scheduler0;
4762 u32 cp_hqd_hq_scheduler1;
4763 u32 cp_mqd_control;
4764};
4765
4766struct bonaire_mqd
4767{
4768 u32 header;
4769 u32 dispatch_initiator;
4770 u32 dimensions[3];
4771 u32 start_idx[3];
4772 u32 num_threads[3];
4773 u32 pipeline_stat_enable;
4774 u32 perf_counter_enable;
4775 u32 pgm[2];
4776 u32 tba[2];
4777 u32 tma[2];
4778 u32 pgm_rsrc[2];
4779 u32 vmid;
4780 u32 resource_limits;
4781 u32 static_thread_mgmt01[2];
4782 u32 tmp_ring_size;
4783 u32 static_thread_mgmt23[2];
4784 u32 restart[3];
4785 u32 thread_trace_enable;
4786 u32 reserved1;
4787 u32 user_data[16];
4788 u32 vgtcs_invoke_count[2];
4789 struct hqd_registers queue_state;
4790 u32 dequeue_cntr;
4791 u32 interrupt_queue[64];
4792};
4793
841cf442
AD		 4794/**
4795 * cik_cp_compute_resume - setup the compute queue registers
4796 *
4797 * @rdev: radeon_device pointer
4798 *
4799 * Program the compute queues and test them to make sure they
4800 * are working.
4801 * Returns 0 for success, error for failure.
4802 */
4803static int cik_cp_compute_resume(struct radeon_device *rdev)
4804{
963e81f9
AD		 4805 int r, i, idx;
4806 u32 tmp;
4807 bool use_doorbell = true;
4808 u64 hqd_gpu_addr;
4809 u64 mqd_gpu_addr;
4810 u64 eop_gpu_addr;
4811 u64 wb_gpu_addr;
4812 u32 *buf;
4813 struct bonaire_mqd *mqd;
841cf442 4814
841cf442
AD		 4815 r = cik_cp_compute_start(rdev);
4816 if (r)
4817 return r;
963e81f9
AD		 4818
4819 /* fix up chicken bits */
4820 tmp = RREG32(CP_CPF_DEBUG);
4821 tmp |= (1 << 23);
4822 WREG32(CP_CPF_DEBUG, tmp);
4823
4824 /* init the pipes */
f61d5b46 4825 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4826 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4827 int me = (i < 4) ? 1 : 2;
4828 int pipe = (i < 4) ? i : (i - 4);
4829
4830 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4831
4832 cik_srbm_select(rdev, me, pipe, 0, 0);
4833
4834 /* write the EOP addr */
4835 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4836 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4837
4838 /* set the VMID assigned */
4839 WREG32(CP_HPD_EOP_VMID, 0);
4840
4841 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4842 tmp = RREG32(CP_HPD_EOP_CONTROL);
4843 tmp &= ~EOP_SIZE_MASK;
b72a8925 4844 tmp |= order_base_2(MEC_HPD_SIZE / 8);
963e81f9
AD		 4845 WREG32(CP_HPD_EOP_CONTROL, tmp);
4846 }
4847 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 4848 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 4849
4850 /* init the queues. Just two for now. */
4851 for (i = 0; i < 2; i++) {
4852 if (i == 0)
4853 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4854 else
4855 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4856
4857 if (rdev->ring[idx].mqd_obj == NULL) {
4858 r = radeon_bo_create(rdev,
4859 sizeof(struct bonaire_mqd),
4860 PAGE_SIZE, true,
02376d82 4861 RADEON_GEM_DOMAIN_GTT, 0, NULL,
963e81f9
AD		 4862 &rdev->ring[idx].mqd_obj);
4863 if (r) {
4864 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4865 return r;
4866 }
4867 }
4868
4869 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4870 if (unlikely(r != 0)) {
4871 cik_cp_compute_fini(rdev);
4872 return r;
4873 }
4874 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4875 &mqd_gpu_addr);
4876 if (r) {
4877 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4878 cik_cp_compute_fini(rdev);
4879 return r;
4880 }
4881 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4882 if (r) {
4883 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4884 cik_cp_compute_fini(rdev);
4885 return r;
4886 }
4887
963e81f9
AD		 4888 /* init the mqd struct */
4889 memset(buf, 0, sizeof(struct bonaire_mqd));
4890
4891 mqd = (struct bonaire_mqd *)buf;
4892 mqd->header = 0xC0310800;
4893 mqd->static_thread_mgmt01[0] = 0xffffffff;
4894 mqd->static_thread_mgmt01[1] = 0xffffffff;
4895 mqd->static_thread_mgmt23[0] = 0xffffffff;
4896 mqd->static_thread_mgmt23[1] = 0xffffffff;
4897
f61d5b46 4898 mutex_lock(&rdev->srbm_mutex);
963e81f9
AD		 4899 cik_srbm_select(rdev, rdev->ring[idx].me,
4900 rdev->ring[idx].pipe,
4901 rdev->ring[idx].queue, 0);
4902
4903 /* disable wptr polling */
4904 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4905 tmp &= ~WPTR_POLL_EN;
4906 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4907
4908 /* enable doorbell? */
4909 mqd->queue_state.cp_hqd_pq_doorbell_control =
4910 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4911 if (use_doorbell)
4912 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4913 else
4914 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4915 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4916 mqd->queue_state.cp_hqd_pq_doorbell_control);
4917
4918 /* disable the queue if it's active */
4919 mqd->queue_state.cp_hqd_dequeue_request = 0;
4920 mqd->queue_state.cp_hqd_pq_rptr = 0;
4921 mqd->queue_state.cp_hqd_pq_wptr= 0;
4922 if (RREG32(CP_HQD_ACTIVE) & 1) {
4923 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4924 for (i = 0; i < rdev->usec_timeout; i++) {
4925 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4926 break;
4927 udelay(1);
4928 }
4929 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4930 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4931 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4932 }
4933
4934 /* set the pointer to the MQD */
4935 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4936 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4937 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4938 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4939 /* set MQD vmid to 0 */
4940 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4941 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4942 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4943
4944 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4945 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4946 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4947 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4948 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4949 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4950
4951 /* set up the HQD, this is similar to CP_RB0_CNTL */
4952 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4953 mqd->queue_state.cp_hqd_pq_control &=
4954 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4955
4956 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4957 order_base_2(rdev->ring[idx].ring_size / 8);
963e81f9 4958 mqd->queue_state.cp_hqd_pq_control |=
b72a8925 4959 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
963e81f9
AD		 4960#ifdef __BIG_ENDIAN
4961 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4962#endif
4963 mqd->queue_state.cp_hqd_pq_control &=
4964 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4965 mqd->queue_state.cp_hqd_pq_control |=
4966 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4967 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4968
4969 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4970 if (i == 0)
4971 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4972 else
4973 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4974 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4975 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4976 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4977 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4978 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4979
4980 /* set the wb address wether it's enabled or not */
4981 if (i == 0)
4982 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4983 else
4984 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4985 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4986 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4987 upper_32_bits(wb_gpu_addr) & 0xffff;
4988 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4989 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4990 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4991 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4992
4993 /* enable the doorbell if requested */
4994 if (use_doorbell) {
4995 mqd->queue_state.cp_hqd_pq_doorbell_control =
4996 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4997 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4998 mqd->queue_state.cp_hqd_pq_doorbell_control |=
d5754ab8 4999 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
963e81f9
AD		 5000 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
5001 mqd->queue_state.cp_hqd_pq_doorbell_control &=
5002 ~(DOORBELL_SOURCE | DOORBELL_HIT);
5003
5004 } else {
5005 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
5006 }
5007 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
5008 mqd->queue_state.cp_hqd_pq_doorbell_control);
5009
5010 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
5011 rdev->ring[idx].wptr = 0;
5012 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
5013 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
ff212f25 5014 mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
963e81f9
AD		 5015
5016 /* set the vmid for the queue */
5017 mqd->queue_state.cp_hqd_vmid = 0;
5018 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
5019
5020 /* activate the queue */
5021 mqd->queue_state.cp_hqd_active = 1;
5022 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
5023
5024 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5025 mutex_unlock(&rdev->srbm_mutex);
963e81f9
AD		 5026
5027 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
5028 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
5029
5030 rdev->ring[idx].ready = true;
5031 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
5032 if (r)
5033 rdev->ring[idx].ready = false;
5034 }
5035
841cf442
AD		 5036 return 0;
5037}
5038
841cf442
AD		 5039static void cik_cp_enable(struct radeon_device *rdev, bool enable)
5040{
5041 cik_cp_gfx_enable(rdev, enable);
5042 cik_cp_compute_enable(rdev, enable);
5043}
5044
841cf442
AD		 5045static int cik_cp_load_microcode(struct radeon_device *rdev)
5046{
5047 int r;
5048
5049 r = cik_cp_gfx_load_microcode(rdev);
5050 if (r)
5051 return r;
5052 r = cik_cp_compute_load_microcode(rdev);
5053 if (r)
5054 return r;
5055
5056 return 0;
5057}
5058
841cf442
AD		 5059static void cik_cp_fini(struct radeon_device *rdev)
5060{
5061 cik_cp_gfx_fini(rdev);
5062 cik_cp_compute_fini(rdev);
5063}
5064
841cf442
AD		 5065static int cik_cp_resume(struct radeon_device *rdev)
5066{
5067 int r;
5068
4214faf6
AD		 5069 cik_enable_gui_idle_interrupt(rdev, false);
5070
841cf442
AD		 5071 r = cik_cp_load_microcode(rdev);
5072 if (r)
5073 return r;
5074
5075 r = cik_cp_gfx_resume(rdev);
5076 if (r)
5077 return r;
5078 r = cik_cp_compute_resume(rdev);
5079 if (r)
5080 return r;
5081
4214faf6
AD		 5082 cik_enable_gui_idle_interrupt(rdev, true);
5083
841cf442
AD		 5084 return 0;
5085}
5086
cc066715 5087static void cik_print_gpu_status_regs(struct radeon_device *rdev)
6f2043ce 5088{
6f2043ce
AD		 5089 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
5090 RREG32(GRBM_STATUS));
5091 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
5092 RREG32(GRBM_STATUS2));
5093 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
5094 RREG32(GRBM_STATUS_SE0));
5095 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
5096 RREG32(GRBM_STATUS_SE1));
5097 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
5098 RREG32(GRBM_STATUS_SE2));
5099 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
5100 RREG32(GRBM_STATUS_SE3));
5101 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
5102 RREG32(SRBM_STATUS));
5103 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
5104 RREG32(SRBM_STATUS2));
cc066715
AD		 5105 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
5106 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
5107 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
5108 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
963e81f9
AD		 5109 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
5110 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
5111 RREG32(CP_STALLED_STAT1));
5112 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
5113 RREG32(CP_STALLED_STAT2));
5114 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
5115 RREG32(CP_STALLED_STAT3));
5116 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
5117 RREG32(CP_CPF_BUSY_STAT));
5118 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
5119 RREG32(CP_CPF_STALLED_STAT1));
5120 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
5121 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
5122 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
5123 RREG32(CP_CPC_STALLED_STAT1));
5124 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
cc066715 5125}
6f2043ce 5126
21a93e13 5127/**
cc066715 5128 * cik_gpu_check_soft_reset - check which blocks are busy
21a93e13
AD		 5129 *
5130 * @rdev: radeon_device pointer
21a93e13 5131 *
cc066715
AD		 5132 * Check which blocks are busy and return the relevant reset
5133 * mask to be used by cik_gpu_soft_reset().
5134 * Returns a mask of the blocks to be reset.
21a93e13 5135 */
2483b4ea 5136u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
21a93e13 5137{
cc066715
AD		 5138 u32 reset_mask = 0;
5139 u32 tmp;
21a93e13 5140
cc066715
AD		 5141 /* GRBM_STATUS */
5142 tmp = RREG32(GRBM_STATUS);
5143 if (tmp & (PA_BUSY | SC_BUSY |
5144 BCI_BUSY | SX_BUSY |
5145 TA_BUSY | VGT_BUSY |
5146 DB_BUSY | CB_BUSY |
5147 GDS_BUSY | SPI_BUSY |
5148 IA_BUSY | IA_BUSY_NO_DMA))
5149 reset_mask |= RADEON_RESET_GFX;
21a93e13 5150
cc066715
AD		 5151 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
5152 reset_mask |= RADEON_RESET_CP;
21a93e13 5153
cc066715
AD		 5154 /* GRBM_STATUS2 */
5155 tmp = RREG32(GRBM_STATUS2);
5156 if (tmp & RLC_BUSY)
5157 reset_mask |= RADEON_RESET_RLC;
21a93e13 5158
cc066715
AD		 5159 /* SDMA0_STATUS_REG */
5160 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
5161 if (!(tmp & SDMA_IDLE))
5162 reset_mask |= RADEON_RESET_DMA;
21a93e13 5163
cc066715
AD		 5164 /* SDMA1_STATUS_REG */
5165 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
5166 if (!(tmp & SDMA_IDLE))
5167 reset_mask |= RADEON_RESET_DMA1;
21a93e13 5168
cc066715
AD		 5169 /* SRBM_STATUS2 */
5170 tmp = RREG32(SRBM_STATUS2);
5171 if (tmp & SDMA_BUSY)
5172 reset_mask |= RADEON_RESET_DMA;
21a93e13 5173
cc066715
AD		 5174 if (tmp & SDMA1_BUSY)
5175 reset_mask |= RADEON_RESET_DMA1;
21a93e13 5176
cc066715
AD		 5177 /* SRBM_STATUS */
5178 tmp = RREG32(SRBM_STATUS);
21a93e13 5179
cc066715
AD		 5180 if (tmp & IH_BUSY)
5181 reset_mask |= RADEON_RESET_IH;
21a93e13 5182
cc066715
AD		 5183 if (tmp & SEM_BUSY)
5184 reset_mask |= RADEON_RESET_SEM;
21a93e13 5185
cc066715
AD		 5186 if (tmp & GRBM_RQ_PENDING)
5187 reset_mask |= RADEON_RESET_GRBM;
21a93e13 5188
cc066715
AD		 5189 if (tmp & VMC_BUSY)
5190 reset_mask |= RADEON_RESET_VMC;
21a93e13 5191
cc066715
AD		 5192 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
5193 MCC_BUSY | MCD_BUSY))
5194 reset_mask |= RADEON_RESET_MC;
21a93e13 5195
cc066715
AD		 5196 if (evergreen_is_display_hung(rdev))
5197 reset_mask |= RADEON_RESET_DISPLAY;
5198
5199 /* Skip MC reset as it's mostly likely not hung, just busy */
5200 if (reset_mask & RADEON_RESET_MC) {
5201 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
5202 reset_mask &= ~RADEON_RESET_MC;
21a93e13 5203 }
cc066715
AD		 5204
5205 return reset_mask;
21a93e13
AD		 5206}
5207
5208/**
cc066715 5209 * cik_gpu_soft_reset - soft reset GPU
21a93e13
AD		 5210 *
5211 * @rdev: radeon_device pointer
cc066715 5212 * @reset_mask: mask of which blocks to reset
21a93e13 5213 *
cc066715 5214 * Soft reset the blocks specified in @reset_mask.
21a93e13 5215 */
cc066715 5216static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
21a93e13 5217{
6f2043ce 5218 struct evergreen_mc_save save;
cc066715
AD		 5219 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5220 u32 tmp;
21a93e13 5221
cc066715
AD		 5222 if (reset_mask == 0)
5223 return;
21a93e13 5224
cc066715 5225 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
21a93e13 5226
cc066715
AD		 5227 cik_print_gpu_status_regs(rdev);
5228 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5229 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
5230 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5231 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
21a93e13 5232
fb2c7f4d
AD		 5233 /* disable CG/PG */
5234 cik_fini_pg(rdev);
5235 cik_fini_cg(rdev);
5236
cc066715
AD		 5237 /* stop the rlc */
5238 cik_rlc_stop(rdev);
21a93e13 5239
cc066715
AD		 5240 /* Disable GFX parsing/prefetching */
5241 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
21a93e13 5242
cc066715
AD		 5243 /* Disable MEC parsing/prefetching */
5244 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
21a93e13 5245
cc066715
AD		 5246 if (reset_mask & RADEON_RESET_DMA) {
5247 /* sdma0 */
5248 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5249 tmp |= SDMA_HALT;
5250 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5251 }
5252 if (reset_mask & RADEON_RESET_DMA1) {
5253 /* sdma1 */
5254 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5255 tmp |= SDMA_HALT;
5256 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5257 }
21a93e13 5258
6f2043ce 5259 evergreen_mc_stop(rdev, &save);
cc066715 5260 if (evergreen_mc_wait_for_idle(rdev)) {
6f2043ce
AD		 5261 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5262 }
21a93e13 5263
cc066715
AD		 5264 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
5265 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
21a93e13 5266
cc066715
AD		 5267 if (reset_mask & RADEON_RESET_CP) {
5268 grbm_soft_reset |= SOFT_RESET_CP;
21a93e13 5269
cc066715
AD		 5270 srbm_soft_reset |= SOFT_RESET_GRBM;
5271 }
21a93e13 5272
cc066715
AD		 5273 if (reset_mask & RADEON_RESET_DMA)
5274 srbm_soft_reset |= SOFT_RESET_SDMA;
21a93e13 5275
cc066715
AD		 5276 if (reset_mask & RADEON_RESET_DMA1)
5277 srbm_soft_reset |= SOFT_RESET_SDMA1;
5278
5279 if (reset_mask & RADEON_RESET_DISPLAY)
5280 srbm_soft_reset |= SOFT_RESET_DC;
5281
5282 if (reset_mask & RADEON_RESET_RLC)
5283 grbm_soft_reset |= SOFT_RESET_RLC;
5284
5285 if (reset_mask & RADEON_RESET_SEM)
5286 srbm_soft_reset |= SOFT_RESET_SEM;
5287
5288 if (reset_mask & RADEON_RESET_IH)
5289 srbm_soft_reset |= SOFT_RESET_IH;
5290
5291 if (reset_mask & RADEON_RESET_GRBM)
5292 srbm_soft_reset |= SOFT_RESET_GRBM;
5293
5294 if (reset_mask & RADEON_RESET_VMC)
5295 srbm_soft_reset |= SOFT_RESET_VMC;
5296
5297 if (!(rdev->flags & RADEON_IS_IGP)) {
5298 if (reset_mask & RADEON_RESET_MC)
5299 srbm_soft_reset |= SOFT_RESET_MC;
21a93e13
AD		 5300 }
5301
cc066715
AD		 5302 if (grbm_soft_reset) {
5303 tmp = RREG32(GRBM_SOFT_RESET);
5304 tmp |= grbm_soft_reset;
5305 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5306 WREG32(GRBM_SOFT_RESET, tmp);
5307 tmp = RREG32(GRBM_SOFT_RESET);
21a93e13 5308
cc066715 5309 udelay(50);
21a93e13 5310
cc066715
AD		 5311 tmp &= ~grbm_soft_reset;
5312 WREG32(GRBM_SOFT_RESET, tmp);
5313 tmp = RREG32(GRBM_SOFT_RESET);
5314 }
21a93e13 5315
cc066715
AD		 5316 if (srbm_soft_reset) {
5317 tmp = RREG32(SRBM_SOFT_RESET);
5318 tmp |= srbm_soft_reset;
5319 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5320 WREG32(SRBM_SOFT_RESET, tmp);
5321 tmp = RREG32(SRBM_SOFT_RESET);
21a93e13 5322
cc066715 5323 udelay(50);
21a93e13 5324
cc066715
AD		 5325 tmp &= ~srbm_soft_reset;
5326 WREG32(SRBM_SOFT_RESET, tmp);
5327 tmp = RREG32(SRBM_SOFT_RESET);
5328 }
21a93e13 5329
6f2043ce
AD		 5330 /* Wait a little for things to settle down */
5331 udelay(50);
21a93e13 5332
6f2043ce 5333 evergreen_mc_resume(rdev, &save);
cc066715
AD		 5334 udelay(50);
5335
5336 cik_print_gpu_status_regs(rdev);
21a93e13
AD		 5337}
5338
0279ed19
AD		 5339struct kv_reset_save_regs {
5340 u32 gmcon_reng_execute;
5341 u32 gmcon_misc;
5342 u32 gmcon_misc3;
5343};
5344
5345static void kv_save_regs_for_reset(struct radeon_device *rdev,
5346 struct kv_reset_save_regs *save)
5347{
5348 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
5349 save->gmcon_misc = RREG32(GMCON_MISC);
5350 save->gmcon_misc3 = RREG32(GMCON_MISC3);
5351
5352 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
5353 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
5354 STCTRL_STUTTER_EN));
5355}
5356
5357static void kv_restore_regs_for_reset(struct radeon_device *rdev,
5358 struct kv_reset_save_regs *save)
5359{
5360 int i;
5361
5362 WREG32(GMCON_PGFSM_WRITE, 0);
5363 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
5364
5365 for (i = 0; i < 5; i++)
5366 WREG32(GMCON_PGFSM_WRITE, 0);
5367
5368 WREG32(GMCON_PGFSM_WRITE, 0);
5369 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
5370
5371 for (i = 0; i < 5; i++)
5372 WREG32(GMCON_PGFSM_WRITE, 0);
5373
5374 WREG32(GMCON_PGFSM_WRITE, 0x210000);
5375 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
5376
5377 for (i = 0; i < 5; i++)
5378 WREG32(GMCON_PGFSM_WRITE, 0);
5379
5380 WREG32(GMCON_PGFSM_WRITE, 0x21003);
5381 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
5382
5383 for (i = 0; i < 5; i++)
5384 WREG32(GMCON_PGFSM_WRITE, 0);
5385
5386 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
5387 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
5388
5389 for (i = 0; i < 5; i++)
5390 WREG32(GMCON_PGFSM_WRITE, 0);
5391
5392 WREG32(GMCON_PGFSM_WRITE, 0);
5393 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
5394
5395 for (i = 0; i < 5; i++)
5396 WREG32(GMCON_PGFSM_WRITE, 0);
5397
5398 WREG32(GMCON_PGFSM_WRITE, 0x420000);
5399 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
5400
5401 for (i = 0; i < 5; i++)
5402 WREG32(GMCON_PGFSM_WRITE, 0);
5403
5404 WREG32(GMCON_PGFSM_WRITE, 0x120202);
5405 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
5406
5407 for (i = 0; i < 5; i++)
5408 WREG32(GMCON_PGFSM_WRITE, 0);
5409
5410 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5411 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5412
5413 for (i = 0; i < 5; i++)
5414 WREG32(GMCON_PGFSM_WRITE, 0);
5415
5416 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5417 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5418
5419 for (i = 0; i < 5; i++)
5420 WREG32(GMCON_PGFSM_WRITE, 0);
5421
5422 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5423 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5424
5425 WREG32(GMCON_MISC3, save->gmcon_misc3);
5426 WREG32(GMCON_MISC, save->gmcon_misc);
5427 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5428}
5429
5430static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5431{
5432 struct evergreen_mc_save save;
5433 struct kv_reset_save_regs kv_save = { 0 };
5434 u32 tmp, i;
5435
5436 dev_info(rdev->dev, "GPU pci config reset\n");
5437
5438 /* disable dpm? */
5439
5440 /* disable cg/pg */
5441 cik_fini_pg(rdev);
5442 cik_fini_cg(rdev);
5443
5444 /* Disable GFX parsing/prefetching */
5445 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5446
5447 /* Disable MEC parsing/prefetching */
5448 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5449
5450 /* sdma0 */
5451 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5452 tmp |= SDMA_HALT;
5453 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5454 /* sdma1 */
5455 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5456 tmp |= SDMA_HALT;
5457 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5458 /* XXX other engines? */
5459
5460 /* halt the rlc, disable cp internal ints */
5461 cik_rlc_stop(rdev);
5462
5463 udelay(50);
5464
5465 /* disable mem access */
5466 evergreen_mc_stop(rdev, &save);
5467 if (evergreen_mc_wait_for_idle(rdev)) {
5468 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5469 }
5470
5471 if (rdev->flags & RADEON_IS_IGP)
5472 kv_save_regs_for_reset(rdev, &kv_save);
5473
5474 /* disable BM */
5475 pci_clear_master(rdev->pdev);
5476 /* reset */
5477 radeon_pci_config_reset(rdev);
5478
5479 udelay(100);
5480
5481 /* wait for asic to come out of reset */
5482 for (i = 0; i < rdev->usec_timeout; i++) {
5483 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5484 break;
5485 udelay(1);
5486 }
5487
5488 /* does asic init need to be run first??? */
5489 if (rdev->flags & RADEON_IS_IGP)
5490 kv_restore_regs_for_reset(rdev, &kv_save);
5491}
5492
21a93e13 5493/**
cc066715 5494 * cik_asic_reset - soft reset GPU
21a93e13
AD		 5495 *
5496 * @rdev: radeon_device pointer
5497 *
cc066715
AD		 5498 * Look up which blocks are hung and attempt
5499 * to reset them.
6f2043ce 5500 * Returns 0 for success.
21a93e13 5501 */
6f2043ce 5502int cik_asic_reset(struct radeon_device *rdev)
21a93e13 5503{
cc066715 5504 u32 reset_mask;
21a93e13 5505
cc066715 5506 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5507
cc066715
AD		 5508 if (reset_mask)
5509 r600_set_bios_scratch_engine_hung(rdev, true);
21a93e13 5510
0279ed19 5511 /* try soft reset */
cc066715 5512 cik_gpu_soft_reset(rdev, reset_mask);
21a93e13 5513
cc066715
AD		 5514 reset_mask = cik_gpu_check_soft_reset(rdev);
5515
0279ed19
AD		 5516 /* try pci config reset */
5517 if (reset_mask && radeon_hard_reset)
5518 cik_gpu_pci_config_reset(rdev);
5519
5520 reset_mask = cik_gpu_check_soft_reset(rdev);
5521
cc066715
AD		 5522 if (!reset_mask)
5523 r600_set_bios_scratch_engine_hung(rdev, false);
21a93e13
AD		 5524
5525 return 0;
5526}
5527
5528/**
cc066715 5529 * cik_gfx_is_lockup - check if the 3D engine is locked up
21a93e13
AD		 5530 *
5531 * @rdev: radeon_device pointer
cc066715 5532 * @ring: radeon_ring structure holding ring information
21a93e13 5533 *
cc066715
AD		 5534 * Check if the 3D engine is locked up (CIK).
5535 * Returns true if the engine is locked, false if not.
21a93e13 5536 */
cc066715 5537bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
21a93e13 5538{
cc066715 5539 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
21a93e13 5540
cc066715
AD		 5541 if (!(reset_mask & (RADEON_RESET_GFX |
5542 RADEON_RESET_COMPUTE |
5543 RADEON_RESET_CP))) {
ff212f25 5544 radeon_ring_lockup_update(rdev, ring);
cc066715 5545 return false;
21a93e13 5546 }
cc066715 5547 return radeon_ring_test_lockup(rdev, ring);
21a93e13
AD		 5548}
5549
1c49165d 5550/* MC */
21a93e13 5551/**
1c49165d 5552 * cik_mc_program - program the GPU memory controller
21a93e13
AD		 5553 *
5554 * @rdev: radeon_device pointer
21a93e13 5555 *
1c49165d
AD		 5556 * Set the location of vram, gart, and AGP in the GPU's
5557 * physical address space (CIK).
21a93e13 5558 */
1c49165d 5559static void cik_mc_program(struct radeon_device *rdev)
21a93e13 5560{
1c49165d 5561 struct evergreen_mc_save save;
21a93e13 5562 u32 tmp;
1c49165d 5563 int i, j;
21a93e13 5564
1c49165d
AD		 5565 /* Initialize HDP */
5566 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5567 WREG32((0x2c14 + j), 0x00000000);
5568 WREG32((0x2c18 + j), 0x00000000);
5569 WREG32((0x2c1c + j), 0x00000000);
5570 WREG32((0x2c20 + j), 0x00000000);
5571 WREG32((0x2c24 + j), 0x00000000);
21a93e13 5572 }
1c49165d 5573 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
21a93e13 5574
1c49165d
AD		 5575 evergreen_mc_stop(rdev, &save);
5576 if (radeon_mc_wait_for_idle(rdev)) {
5577 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5578 }
1c49165d
AD		 5579 /* Lockout access through VGA aperture*/
5580 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5581 /* Update configuration */
5582 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5583 rdev->mc.vram_start >> 12);
5584 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5585 rdev->mc.vram_end >> 12);
5586 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5587 rdev->vram_scratch.gpu_addr >> 12);
5588 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5589 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5590 WREG32(MC_VM_FB_LOCATION, tmp);
5591 /* XXX double check these! */
5592 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5593 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5594 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5595 WREG32(MC_VM_AGP_BASE, 0);
5596 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5597 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5598 if (radeon_mc_wait_for_idle(rdev)) {
5599 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
21a93e13 5600 }
1c49165d
AD		 5601 evergreen_mc_resume(rdev, &save);
5602 /* we need to own VRAM, so turn off the VGA renderer here
5603 * to stop it overwriting our objects */
5604 rv515_vga_render_disable(rdev);
21a93e13
AD		 5605}
5606
5607/**
1c49165d 5608 * cik_mc_init - initialize the memory controller driver params
21a93e13
AD		 5609 *
5610 * @rdev: radeon_device pointer
21a93e13 5611 *
1c49165d
AD		 5612 * Look up the amount of vram, vram width, and decide how to place
5613 * vram and gart within the GPU's physical address space (CIK).
5614 * Returns 0 for success.
21a93e13 5615 */
1c49165d 5616static int cik_mc_init(struct radeon_device *rdev)
21a93e13 5617{
1c49165d
AD		 5618 u32 tmp;
5619 int chansize, numchan;
21a93e13 5620
1c49165d
AD		 5621 /* Get VRAM informations */
5622 rdev->mc.vram_is_ddr = true;
5623 tmp = RREG32(MC_ARB_RAMCFG);
5624 if (tmp & CHANSIZE_MASK) {
5625 chansize = 64;
21a93e13 5626 } else {
1c49165d 5627 chansize = 32;
21a93e13 5628 }
1c49165d
AD		 5629 tmp = RREG32(MC_SHARED_CHMAP);
5630 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5631 case 0:
5632 default:
5633 numchan = 1;
5634 break;
5635 case 1:
5636 numchan = 2;
5637 break;
5638 case 2:
5639 numchan = 4;
5640 break;
5641 case 3:
5642 numchan = 8;
5643 break;
5644 case 4:
5645 numchan = 3;
5646 break;
5647 case 5:
5648 numchan = 6;
5649 break;
5650 case 6:
5651 numchan = 10;
5652 break;
5653 case 7:
5654 numchan = 12;
5655 break;
5656 case 8:
5657 numchan = 16;
5658 break;
5659 }
5660 rdev->mc.vram_width = numchan * chansize;
5661 /* Could aper size report 0 ? */
5662 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5663 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5664 /* size in MB on si */
13c5bfda
AD		 5665 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5666 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
1c49165d
AD		 5667 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5668 si_vram_gtt_location(rdev, &rdev->mc);
5669 radeon_update_bandwidth_info(rdev);
5670
5671 return 0;
5672}
5673
5674/*
5675 * GART
5676 * VMID 0 is the physical GPU addresses as used by the kernel.
5677 * VMIDs 1-15 are used for userspace clients and are handled
5678 * by the radeon vm/hsa code.
5679 */
5680/**
5681 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5682 *
5683 * @rdev: radeon_device pointer
5684 *
5685 * Flush the TLB for the VMID 0 page table (CIK).
5686 */
5687void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5688{
5689 /* flush hdp cache */
5690 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5691
5692 /* bits 0-15 are the VM contexts0-15 */
5693 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5694}
5695
5696/**
5697 * cik_pcie_gart_enable - gart enable
5698 *
5699 * @rdev: radeon_device pointer
5700 *
5701 * This sets up the TLBs, programs the page tables for VMID0,
5702 * sets up the hw for VMIDs 1-15 which are allocated on
5703 * demand, and sets up the global locations for the LDS, GDS,
5704 * and GPUVM for FSA64 clients (CIK).
5705 * Returns 0 for success, errors for failure.
5706 */
5707static int cik_pcie_gart_enable(struct radeon_device *rdev)
5708{
5709 int r, i;
5710
5711 if (rdev->gart.robj == NULL) {
5712 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5713 return -EINVAL;
5714 }
5715 r = radeon_gart_table_vram_pin(rdev);
5716 if (r)
5717 return r;
1c49165d
AD		 5718 /* Setup TLB control */
5719 WREG32(MC_VM_MX_L1_TLB_CNTL,
5720 (0xA << 7) |
5721 ENABLE_L1_TLB |
ec3dbbcb 5722 ENABLE_L1_FRAGMENT_PROCESSING |
1c49165d
AD		 5723 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5724 ENABLE_ADVANCED_DRIVER_MODEL |
5725 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5726 /* Setup L2 cache */
5727 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5728 ENABLE_L2_FRAGMENT_PROCESSING |
5729 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5730 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5731 EFFECTIVE_L2_QUEUE_SIZE(7) |
5732 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5733 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5734 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
ec3dbbcb
CK		 5735 BANK_SELECT(4) |
5736 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
1c49165d
AD		 5737 /* setup context0 */
5738 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5739 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5740 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5741 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5742 (u32)(rdev->dummy_page.addr >> 12));
5743 WREG32(VM_CONTEXT0_CNTL2, 0);
5744 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5745 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5746
5747 WREG32(0x15D4, 0);
5748 WREG32(0x15D8, 0);
5749 WREG32(0x15DC, 0);
5750
5751 /* empty context1-15 */
5752 /* FIXME start with 4G, once using 2 level pt switch to full
5753 * vm size space
5754 */
5755 /* set vm size, must be a multiple of 4 */
5756 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5757 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5758 for (i = 1; i < 16; i++) {
5759 if (i < 8)
5760 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5761 rdev->gart.table_addr >> 12);
5762 else
5763 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5764 rdev->gart.table_addr >> 12);
5765 }
5766
5767 /* enable context1-15 */
5768 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5769 (u32)(rdev->dummy_page.addr >> 12));
a00024b0 5770 WREG32(VM_CONTEXT1_CNTL2, 4);
1c49165d 5771 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4510fb98 5772 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
a00024b0
AD		 5773 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5774 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5775 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5776 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5777 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5778 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5779 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5780 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5781 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5782 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5783 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5784 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1c49165d 5785
1c49165d
AD		 5786 if (rdev->family == CHIP_KAVERI) {
5787 u32 tmp = RREG32(CHUB_CONTROL);
5788 tmp &= ~BYPASS_VM;
5789 WREG32(CHUB_CONTROL, tmp);
5790 }
5791
5792 /* XXX SH_MEM regs */
5793 /* where to put LDS, scratch, GPUVM in FSA64 space */
f61d5b46 5794 mutex_lock(&rdev->srbm_mutex);
1c49165d 5795 for (i = 0; i < 16; i++) {
b556b12e 5796 cik_srbm_select(rdev, 0, 0, 0, i);
21a93e13 5797 /* CP and shaders */
1c49165d
AD		 5798 WREG32(SH_MEM_CONFIG, 0);
5799 WREG32(SH_MEM_APE1_BASE, 1);
5800 WREG32(SH_MEM_APE1_LIMIT, 0);
5801 WREG32(SH_MEM_BASES, 0);
21a93e13
AD		 5802 /* SDMA GFX */
5803 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5804 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5805 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5806 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5807 /* XXX SDMA RLC - todo */
1c49165d 5808 }
b556b12e 5809 cik_srbm_select(rdev, 0, 0, 0, 0);
f61d5b46 5810 mutex_unlock(&rdev->srbm_mutex);
1c49165d
AD		 5811
5812 cik_pcie_gart_tlb_flush(rdev);
5813 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5814 (unsigned)(rdev->mc.gtt_size >> 20),
5815 (unsigned long long)rdev->gart.table_addr);
5816 rdev->gart.ready = true;
5817 return 0;
5818}
5819
5820/**
5821 * cik_pcie_gart_disable - gart disable
5822 *
5823 * @rdev: radeon_device pointer
5824 *
5825 * This disables all VM page table (CIK).
5826 */
5827static void cik_pcie_gart_disable(struct radeon_device *rdev)
5828{
5829 /* Disable all tables */
5830 WREG32(VM_CONTEXT0_CNTL, 0);
5831 WREG32(VM_CONTEXT1_CNTL, 0);
5832 /* Setup TLB control */
5833 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5834 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5835 /* Setup L2 cache */
5836 WREG32(VM_L2_CNTL,
5837 ENABLE_L2_FRAGMENT_PROCESSING |
5838 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5839 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5840 EFFECTIVE_L2_QUEUE_SIZE(7) |
5841 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5842 WREG32(VM_L2_CNTL2, 0);
5843 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5844 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5845 radeon_gart_table_vram_unpin(rdev);
5846}
5847
5848/**
5849 * cik_pcie_gart_fini - vm fini callback
5850 *
5851 * @rdev: radeon_device pointer
5852 *
5853 * Tears down the driver GART/VM setup (CIK).
5854 */
5855static void cik_pcie_gart_fini(struct radeon_device *rdev)
5856{
5857 cik_pcie_gart_disable(rdev);
5858 radeon_gart_table_vram_free(rdev);
5859 radeon_gart_fini(rdev);
5860}
5861
5862/* vm parser */
5863/**
5864 * cik_ib_parse - vm ib_parse callback
5865 *
5866 * @rdev: radeon_device pointer
5867 * @ib: indirect buffer pointer
5868 *
5869 * CIK uses hw IB checking so this is a nop (CIK).
5870 */
5871int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5872{
5873 return 0;
5874}
5875
5876/*
5877 * vm
5878 * VMID 0 is the physical GPU addresses as used by the kernel.
5879 * VMIDs 1-15 are used for userspace clients and are handled
5880 * by the radeon vm/hsa code.
5881 */
5882/**
5883 * cik_vm_init - cik vm init callback
5884 *
5885 * @rdev: radeon_device pointer
5886 *
5887 * Inits cik specific vm parameters (number of VMs, base of vram for
5888 * VMIDs 1-15) (CIK).
5889 * Returns 0 for success.
5890 */
5891int cik_vm_init(struct radeon_device *rdev)
5892{
5893 /* number of VMs */
5894 rdev->vm_manager.nvm = 16;
5895 /* base offset of vram pages */
5896 if (rdev->flags & RADEON_IS_IGP) {
5897 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5898 tmp <<= 22;
5899 rdev->vm_manager.vram_base_offset = tmp;
5900 } else
5901 rdev->vm_manager.vram_base_offset = 0;
5902
5903 return 0;
5904}
5905
5906/**
5907 * cik_vm_fini - cik vm fini callback
5908 *
5909 * @rdev: radeon_device pointer
5910 *
5911 * Tear down any asic specific VM setup (CIK).
5912 */
5913void cik_vm_fini(struct radeon_device *rdev)
5914{
5915}
5916
3ec7d11b
AD		 5917/**
5918 * cik_vm_decode_fault - print human readable fault info
5919 *
5920 * @rdev: radeon_device pointer
5921 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5922 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5923 *
5924 * Print human readable fault information (CIK).
5925 */
5926static void cik_vm_decode_fault(struct radeon_device *rdev,
5927 u32 status, u32 addr, u32 mc_client)
5928{
939c0d3c 5929 u32 mc_id;
3ec7d11b
AD		 5930 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5931 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
328a50c7
MD		 5932 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5933 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
3ec7d11b 5934
939c0d3c
AD		 5935 if (rdev->family == CHIP_HAWAII)
5936 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5937 else
5938 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5939
328a50c7 5940 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
3ec7d11b
AD		 5941 protections, vmid, addr,
5942 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
328a50c7 5943 block, mc_client, mc_id);
3ec7d11b
AD		 5944}
5945
f96ab484
AD		 5946/**
5947 * cik_vm_flush - cik vm flush using the CP
5948 *
5949 * @rdev: radeon_device pointer
5950 *
5951 * Update the page table base and flush the VM TLB
5952 * using the CP (CIK).
5953 */
5954void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5955{
5956 struct radeon_ring *ring = &rdev->ring[ridx];
f1d2a26b 5957 int usepfp = (ridx == RADEON_RING_TYPE_GFX_INDEX);
f96ab484
AD		 5958
5959 if (vm == NULL)
5960 return;
5961
5962 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
f1d2a26b 5963 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5964 WRITE_DATA_DST_SEL(0)));
5965 if (vm->id < 8) {
5966 radeon_ring_write(ring,
5967 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5968 } else {
5969 radeon_ring_write(ring,
5970 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5971 }
5972 radeon_ring_write(ring, 0);
5973 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5974
5975 /* update SH_MEM_* regs */
5976 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5977 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5978 WRITE_DATA_DST_SEL(0)));
5979 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5980 radeon_ring_write(ring, 0);
5981 radeon_ring_write(ring, VMID(vm->id));
5982
5983 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
4fb0bbd5 5984 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5985 WRITE_DATA_DST_SEL(0)));
5986 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5987 radeon_ring_write(ring, 0);
5988
5989 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5990 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5991 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5992 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
21a93e13 5993
f96ab484 5994 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 5995 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 5996 WRITE_DATA_DST_SEL(0)));
5997 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5998 radeon_ring_write(ring, 0);
5999 radeon_ring_write(ring, VMID(0));
6f2043ce 6000
f96ab484 6001 /* HDP flush */
780f5ddd 6002 cik_hdp_flush_cp_ring_emit(rdev, ridx);
f96ab484
AD		 6003
6004 /* bits 0-15 are the VM contexts0-15 */
6005 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4fb0bbd5 6006 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
f96ab484
AD		 6007 WRITE_DATA_DST_SEL(0)));
6008 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
6009 radeon_ring_write(ring, 0);
6010 radeon_ring_write(ring, 1 << vm->id);
6011
b07fdd38 6012 /* compute doesn't have PFP */
f1d2a26b 6013 if (usepfp) {
b07fdd38
AD		 6014 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6015 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6016 radeon_ring_write(ring, 0x0);
6017 }
cc066715 6018}
6f2043ce 6019
f6796cae
AD		 6020/*
6021 * RLC
6022 * The RLC is a multi-purpose microengine that handles a
6023 * variety of functions, the most important of which is
6024 * the interrupt controller.
6025 */
866d83de
AD		 6026static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
6027 bool enable)
f6796cae 6028{
866d83de 6029 u32 tmp = RREG32(CP_INT_CNTL_RING0);
f6796cae 6030
866d83de
AD		 6031 if (enable)
6032 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6033 else
6034 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
f6796cae 6035 WREG32(CP_INT_CNTL_RING0, tmp);
866d83de 6036}
f6796cae 6037
866d83de 6038static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
cc066715 6039{
cc066715 6040 u32 tmp;
6f2043ce 6041
866d83de
AD		 6042 tmp = RREG32(RLC_LB_CNTL);
6043 if (enable)
6044 tmp |= LOAD_BALANCE_ENABLE;
6045 else
6046 tmp &= ~LOAD_BALANCE_ENABLE;
6047 WREG32(RLC_LB_CNTL, tmp);
6048}
cc066715 6049
866d83de
AD		 6050static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
6051{
6052 u32 i, j, k;
6053 u32 mask;
cc066715 6054
f6796cae
AD		 6055 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6056 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6057 cik_select_se_sh(rdev, i, j);
6058 for (k = 0; k < rdev->usec_timeout; k++) {
6059 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
6060 break;
6061 udelay(1);
6062 }
6063 }
6064 }
6065 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
cc066715 6066
f6796cae
AD		 6067 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
6068 for (k = 0; k < rdev->usec_timeout; k++) {
6069 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
6070 break;
6071 udelay(1);
6072 }
6073}
cc066715 6074
22c775ce
AD		 6075static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
6076{
6077 u32 tmp;
cc066715 6078
22c775ce
AD		 6079 tmp = RREG32(RLC_CNTL);
6080 if (tmp != rlc)
6081 WREG32(RLC_CNTL, rlc);
6082}
cc066715 6083
22c775ce
AD		 6084static u32 cik_halt_rlc(struct radeon_device *rdev)
6085{
6086 u32 data, orig;
cc066715 6087
22c775ce 6088 orig = data = RREG32(RLC_CNTL);
cc066715 6089
22c775ce
AD		 6090 if (data & RLC_ENABLE) {
6091 u32 i;
cc066715 6092
22c775ce
AD		 6093 data &= ~RLC_ENABLE;
6094 WREG32(RLC_CNTL, data);
cc066715 6095
22c775ce
AD		 6096 for (i = 0; i < rdev->usec_timeout; i++) {
6097 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
6098 break;
6099 udelay(1);
6100 }
cc066715 6101
22c775ce
AD		 6102 cik_wait_for_rlc_serdes(rdev);
6103 }
cc066715 6104
22c775ce
AD		 6105 return orig;
6106}
cc066715 6107
a412fce0
AD		 6108void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
6109{
6110 u32 tmp, i, mask;
6111
6112 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
6113 WREG32(RLC_GPR_REG2, tmp);
6114
6115 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
6116 for (i = 0; i < rdev->usec_timeout; i++) {
6117 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
6118 break;
6119 udelay(1);
6120 }
6121
6122 for (i = 0; i < rdev->usec_timeout; i++) {
6123 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
6124 break;
6125 udelay(1);
6126 }
6127}
6128
6129void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
6130{
6131 u32 tmp;
6132
6133 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
6134 WREG32(RLC_GPR_REG2, tmp);
6135}
6136
866d83de
AD		 6137/**
6138 * cik_rlc_stop - stop the RLC ME
6139 *
6140 * @rdev: radeon_device pointer
6141 *
6142 * Halt the RLC ME (MicroEngine) (CIK).
6143 */
6144static void cik_rlc_stop(struct radeon_device *rdev)
6145{
22c775ce 6146 WREG32(RLC_CNTL, 0);
866d83de
AD		 6147
6148 cik_enable_gui_idle_interrupt(rdev, false);
6149
866d83de
AD		 6150 cik_wait_for_rlc_serdes(rdev);
6151}
6152
f6796cae
AD		 6153/**
6154 * cik_rlc_start - start the RLC ME
6155 *
6156 * @rdev: radeon_device pointer
6157 *
6158 * Unhalt the RLC ME (MicroEngine) (CIK).
6159 */
6160static void cik_rlc_start(struct radeon_device *rdev)
6161{
f6796cae 6162 WREG32(RLC_CNTL, RLC_ENABLE);
cc066715 6163
866d83de 6164 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6165
f6796cae 6166 udelay(50);
6f2043ce
AD		 6167}
6168
6169/**
f6796cae 6170 * cik_rlc_resume - setup the RLC hw
6f2043ce
AD		 6171 *
6172 * @rdev: radeon_device pointer
6173 *
f6796cae
AD		 6174 * Initialize the RLC registers, load the ucode,
6175 * and start the RLC (CIK).
6176 * Returns 0 for success, -EINVAL if the ucode is not available.
6f2043ce 6177 */
f6796cae 6178static int cik_rlc_resume(struct radeon_device *rdev)
6f2043ce 6179{
22c775ce 6180 u32 i, size, tmp;
cc066715 6181
f6796cae
AD		 6182 if (!rdev->rlc_fw)
6183 return -EINVAL;
cc066715 6184
cc066715
AD		 6185 cik_rlc_stop(rdev);
6186
22c775ce
AD		 6187 /* disable CG */
6188 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
6189 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
cc066715 6190
866d83de 6191 si_rlc_reset(rdev);
6f2043ce 6192
22c775ce 6193 cik_init_pg(rdev);
6f2043ce 6194
22c775ce 6195 cik_init_cg(rdev);
cc066715 6196
f6796cae
AD		 6197 WREG32(RLC_LB_CNTR_INIT, 0);
6198 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
cc066715 6199
f6796cae
AD		 6200 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6201 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
6202 WREG32(RLC_LB_PARAMS, 0x00600408);
6203 WREG32(RLC_LB_CNTL, 0x80000004);
cc066715 6204
f6796cae
AD		 6205 WREG32(RLC_MC_CNTL, 0);
6206 WREG32(RLC_UCODE_CNTL, 0);
cc066715 6207
f2c6b0f4
AD		 6208 if (rdev->new_fw) {
6209 const struct rlc_firmware_header_v1_0 *hdr =
6210 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
6211 const __le32 *fw_data = (const __le32 *)
6212 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6213
6214 radeon_ucode_print_rlc_hdr(&hdr->header);
6215
6216 size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
6217 WREG32(RLC_GPM_UCODE_ADDR, 0);
6218 for (i = 0; i < size; i++)
6219 WREG32(RLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
f6796cae 6220 WREG32(RLC_GPM_UCODE_ADDR, 0);
f2c6b0f4
AD		 6221 } else {
6222 const __be32 *fw_data;
6223
6224 switch (rdev->family) {
6225 case CHIP_BONAIRE:
6226 case CHIP_HAWAII:
6227 default:
6228 size = BONAIRE_RLC_UCODE_SIZE;
6229 break;
6230 case CHIP_KAVERI:
6231 size = KV_RLC_UCODE_SIZE;
6232 break;
6233 case CHIP_KABINI:
6234 size = KB_RLC_UCODE_SIZE;
6235 break;
6236 case CHIP_MULLINS:
6237 size = ML_RLC_UCODE_SIZE;
6238 break;
6239 }
6240
6241 fw_data = (const __be32 *)rdev->rlc_fw->data;
6242 WREG32(RLC_GPM_UCODE_ADDR, 0);
6243 for (i = 0; i < size; i++)
6244 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
6245 WREG32(RLC_GPM_UCODE_ADDR, 0);
6246 }
cc066715 6247
866d83de
AD		 6248 /* XXX - find out what chips support lbpw */
6249 cik_enable_lbpw(rdev, false);
cc066715 6250
22c775ce
AD		 6251 if (rdev->family == CHIP_BONAIRE)
6252 WREG32(RLC_DRIVER_DMA_STATUS, 0);
cc066715 6253
f6796cae 6254 cik_rlc_start(rdev);
cc066715 6255
f6796cae
AD		 6256 return 0;
6257}
cc066715 6258
22c775ce
AD		 6259static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
6260{
6261 u32 data, orig, tmp, tmp2;
cc066715 6262
22c775ce 6263 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
cc066715 6264
473359bc 6265 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
ddc76ff6 6266 cik_enable_gui_idle_interrupt(rdev, true);
cc066715 6267
22c775ce 6268 tmp = cik_halt_rlc(rdev);
cc066715 6269
22c775ce
AD		 6270 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6271 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6272 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6273 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
6274 WREG32(RLC_SERDES_WR_CTRL, tmp2);
cc066715 6275
22c775ce 6276 cik_update_rlc(rdev, tmp);
cc066715 6277
22c775ce
AD		 6278 data |= CGCG_EN | CGLS_EN;
6279 } else {
ddc76ff6 6280 cik_enable_gui_idle_interrupt(rdev, false);
cc066715 6281
22c775ce
AD		 6282 RREG32(CB_CGTT_SCLK_CTRL);
6283 RREG32(CB_CGTT_SCLK_CTRL);
6284 RREG32(CB_CGTT_SCLK_CTRL);
6285 RREG32(CB_CGTT_SCLK_CTRL);
cc066715 6286
22c775ce 6287 data &= ~(CGCG_EN | CGLS_EN);
cc066715 6288 }
6f2043ce 6289
22c775ce
AD		 6290 if (orig != data)
6291 WREG32(RLC_CGCG_CGLS_CTRL, data);
cc066715 6292
6f2043ce
AD		 6293}
6294
22c775ce 6295static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
6f2043ce 6296{
22c775ce
AD		 6297 u32 data, orig, tmp = 0;
6298
473359bc
AD		 6299 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
6300 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
6301 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
6302 orig = data = RREG32(CP_MEM_SLP_CNTL);
6303 data |= CP_MEM_LS_EN;
6304 if (orig != data)
6305 WREG32(CP_MEM_SLP_CNTL, data);
6306 }
6307 }
cc066715 6308
22c775ce
AD		 6309 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6310 data &= 0xfffffffd;
6311 if (orig != data)
6312 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6313
6314 tmp = cik_halt_rlc(rdev);
6315
6316 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6317 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6318 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6319 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
6320 WREG32(RLC_SERDES_WR_CTRL, data);
6321
6322 cik_update_rlc(rdev, tmp);
6323
473359bc
AD		 6324 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
6325 orig = data = RREG32(CGTS_SM_CTRL_REG);
6326 data &= ~SM_MODE_MASK;
6327 data |= SM_MODE(0x2);
6328 data |= SM_MODE_ENABLE;
6329 data &= ~CGTS_OVERRIDE;
6330 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
6331 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
6332 data &= ~CGTS_LS_OVERRIDE;
6333 data &= ~ON_MONITOR_ADD_MASK;
6334 data |= ON_MONITOR_ADD_EN;
6335 data |= ON_MONITOR_ADD(0x96);
6336 if (orig != data)
6337 WREG32(CGTS_SM_CTRL_REG, data);
6338 }
22c775ce
AD		 6339 } else {
6340 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
6341 data |= 0x00000002;
6342 if (orig != data)
6343 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
6344
6345 data = RREG32(RLC_MEM_SLP_CNTL);
6346 if (data & RLC_MEM_LS_EN) {
6347 data &= ~RLC_MEM_LS_EN;
6348 WREG32(RLC_MEM_SLP_CNTL, data);
6349 }
6f2043ce 6350
22c775ce
AD		 6351 data = RREG32(CP_MEM_SLP_CNTL);
6352 if (data & CP_MEM_LS_EN) {
6353 data &= ~CP_MEM_LS_EN;
6354 WREG32(CP_MEM_SLP_CNTL, data);
6355 }
cc066715 6356
22c775ce
AD		 6357 orig = data = RREG32(CGTS_SM_CTRL_REG);
6358 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
6359 if (orig != data)
6360 WREG32(CGTS_SM_CTRL_REG, data);
cc066715 6361
22c775ce 6362 tmp = cik_halt_rlc(rdev);
cc066715 6363
22c775ce
AD		 6364 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6365 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
6366 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
6367 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
6368 WREG32(RLC_SERDES_WR_CTRL, data);
cc066715 6369
22c775ce 6370 cik_update_rlc(rdev, tmp);
cc066715 6371 }
6f2043ce 6372}
1c49165d 6373
22c775ce 6374static const u32 mc_cg_registers[] =
21a93e13 6375{
22c775ce
AD		 6376 MC_HUB_MISC_HUB_CG,
6377 MC_HUB_MISC_SIP_CG,
6378 MC_HUB_MISC_VM_CG,
6379 MC_XPB_CLK_GAT,
6380 ATC_MISC_CG,
6381 MC_CITF_MISC_WR_CG,
6382 MC_CITF_MISC_RD_CG,
6383 MC_CITF_MISC_VM_CG,
6384 VM_L2_CG,
6385};
21a93e13 6386
22c775ce
AD		 6387static void cik_enable_mc_ls(struct radeon_device *rdev,
6388 bool enable)
1c49165d 6389{
22c775ce
AD		 6390 int i;
6391 u32 orig, data;
1c49165d 6392
22c775ce
AD		 6393 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6394 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6395 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
22c775ce
AD		 6396 data |= MC_LS_ENABLE;
6397 else
6398 data &= ~MC_LS_ENABLE;
6399 if (data != orig)
6400 WREG32(mc_cg_registers[i], data);
1c49165d 6401 }
22c775ce 6402}
1c49165d 6403
22c775ce
AD		 6404static void cik_enable_mc_mgcg(struct radeon_device *rdev,
6405 bool enable)
6406{
6407 int i;
6408 u32 orig, data;
6409
6410 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
6411 orig = data = RREG32(mc_cg_registers[i]);
473359bc 6412 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
22c775ce
AD		 6413 data |= MC_CG_ENABLE;
6414 else
6415 data &= ~MC_CG_ENABLE;
6416 if (data != orig)
6417 WREG32(mc_cg_registers[i], data);
1c49165d 6418 }
1c49165d
AD		 6419}
6420
22c775ce
AD		 6421static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
6422 bool enable)
1c49165d 6423{
22c775ce 6424 u32 orig, data;
1c49165d 6425
473359bc 6426 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
22c775ce
AD		 6427 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6428 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
1c49165d 6429 } else {
22c775ce
AD		 6430 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6431 data |= 0xff000000;
6432 if (data != orig)
6433 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6434
22c775ce
AD		 6435 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6436 data |= 0xff000000;
6437 if (data != orig)
6438 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6439 }
1c49165d
AD		 6440}
6441
22c775ce
AD		 6442static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6443 bool enable)
1c49165d 6444{
22c775ce
AD		 6445 u32 orig, data;
6446
473359bc 6447 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
22c775ce
AD		 6448 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6449 data |= 0x100;
6450 if (orig != data)
6451 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6452
6453 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6454 data |= 0x100;
6455 if (orig != data)
6456 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6457 } else {
6458 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6459 data &= ~0x100;
6460 if (orig != data)
6461 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
1c49165d 6462
22c775ce
AD		 6463 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6464 data &= ~0x100;
6465 if (orig != data)
6466 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6467 }
1c49165d
AD		 6468}
6469
22c775ce
AD		 6470static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6471 bool enable)
1c49165d 6472{
22c775ce 6473 u32 orig, data;
1c49165d 6474
473359bc 6475 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
22c775ce
AD		 6476 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6477 data = 0xfff;
6478 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6479
22c775ce
AD		 6480 orig = data = RREG32(UVD_CGC_CTRL);
6481 data |= DCM;
6482 if (orig != data)
6483 WREG32(UVD_CGC_CTRL, data);
6484 } else {
6485 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6486 data &= ~0xfff;
6487 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
1c49165d 6488
22c775ce
AD		 6489 orig = data = RREG32(UVD_CGC_CTRL);
6490 data &= ~DCM;
6491 if (orig != data)
6492 WREG32(UVD_CGC_CTRL, data);
1c49165d 6493 }
22c775ce 6494}
1c49165d 6495
473359bc
AD		 6496static void cik_enable_bif_mgls(struct radeon_device *rdev,
6497 bool enable)
6498{
6499 u32 orig, data;
1c49165d 6500
473359bc 6501 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
1c49165d 6502
473359bc
AD		 6503 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6504 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6505 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6506 else
6507 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6508 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
1c49165d 6509
473359bc
AD		 6510 if (orig != data)
6511 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6512}
1c49165d 6513
22c775ce
AD		 6514static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6515 bool enable)
6516{
6517 u32 orig, data;
1c49165d 6518
22c775ce 6519 orig = data = RREG32(HDP_HOST_PATH_CNTL);
1c49165d 6520
473359bc 6521 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
22c775ce
AD		 6522 data &= ~CLOCK_GATING_DIS;
6523 else
6524 data |= CLOCK_GATING_DIS;
6525
6526 if (orig != data)
6527 WREG32(HDP_HOST_PATH_CNTL, data);
1c49165d
AD		 6528}
6529
22c775ce
AD		 6530static void cik_enable_hdp_ls(struct radeon_device *rdev,
6531 bool enable)
1c49165d 6532{
22c775ce
AD		 6533 u32 orig, data;
6534
6535 orig = data = RREG32(HDP_MEM_POWER_LS);
6536
473359bc 6537 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
22c775ce
AD		 6538 data |= HDP_LS_ENABLE;
6539 else
6540 data &= ~HDP_LS_ENABLE;
6541
6542 if (orig != data)
6543 WREG32(HDP_MEM_POWER_LS, data);
6544}
6545
6546void cik_update_cg(struct radeon_device *rdev,
6547 u32 block, bool enable)
6548{
4214faf6 6549
22c775ce 6550 if (block & RADEON_CG_BLOCK_GFX) {
4214faf6 6551 cik_enable_gui_idle_interrupt(rdev, false);
22c775ce
AD		 6552 /* order matters! */
6553 if (enable) {
6554 cik_enable_mgcg(rdev, true);
6555 cik_enable_cgcg(rdev, true);
6556 } else {
6557 cik_enable_cgcg(rdev, false);
6558 cik_enable_mgcg(rdev, false);
6559 }
4214faf6 6560 cik_enable_gui_idle_interrupt(rdev, true);
22c775ce
AD		 6561 }
6562
6563 if (block & RADEON_CG_BLOCK_MC) {
6564 if (!(rdev->flags & RADEON_IS_IGP)) {
6565 cik_enable_mc_mgcg(rdev, enable);
6566 cik_enable_mc_ls(rdev, enable);
6567 }
6568 }
6569
6570 if (block & RADEON_CG_BLOCK_SDMA) {
6571 cik_enable_sdma_mgcg(rdev, enable);
6572 cik_enable_sdma_mgls(rdev, enable);
6573 }
6574
473359bc
AD		 6575 if (block & RADEON_CG_BLOCK_BIF) {
6576 cik_enable_bif_mgls(rdev, enable);
6577 }
6578
22c775ce
AD		 6579 if (block & RADEON_CG_BLOCK_UVD) {
6580 if (rdev->has_uvd)
6581 cik_enable_uvd_mgcg(rdev, enable);
6582 }
6583
6584 if (block & RADEON_CG_BLOCK_HDP) {
6585 cik_enable_hdp_mgcg(rdev, enable);
6586 cik_enable_hdp_ls(rdev, enable);
6587 }
a1d6f97c
AD		 6588
6589 if (block & RADEON_CG_BLOCK_VCE) {
6590 vce_v2_0_enable_mgcg(rdev, enable);
6591 }
1c49165d
AD		 6592}
6593
22c775ce 6594static void cik_init_cg(struct radeon_device *rdev)
1c49165d 6595{
22c775ce 6596
ddc76ff6 6597 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
22c775ce
AD		 6598
6599 if (rdev->has_uvd)
6600 si_init_uvd_internal_cg(rdev);
6601
6602 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6603 RADEON_CG_BLOCK_SDMA |
473359bc 6604 RADEON_CG_BLOCK_BIF |
22c775ce
AD		 6605 RADEON_CG_BLOCK_UVD |
6606 RADEON_CG_BLOCK_HDP), true);
1c49165d
AD		 6607}
6608
473359bc 6609static void cik_fini_cg(struct radeon_device *rdev)
1c49165d 6610{
473359bc
AD		 6611 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6612 RADEON_CG_BLOCK_SDMA |
6613 RADEON_CG_BLOCK_BIF |
6614 RADEON_CG_BLOCK_UVD |
6615 RADEON_CG_BLOCK_HDP), false);
6616
6617 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
1c49165d
AD		 6618}
6619
22c775ce
AD		 6620static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6621 bool enable)
1c49165d 6622{
22c775ce 6623 u32 data, orig;
1c49165d 6624
22c775ce 6625 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6626 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6627 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6628 else
6629 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6630 if (orig != data)
6631 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6632}
6633
22c775ce
AD		 6634static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6635 bool enable)
1c49165d 6636{
22c775ce
AD		 6637 u32 data, orig;
6638
6639 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6640 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
22c775ce
AD		 6641 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6642 else
6643 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6644 if (orig != data)
6645 WREG32(RLC_PG_CNTL, data);
1c49165d
AD		 6646}
6647
22c775ce 6648static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
3ec7d11b 6649{
22c775ce 6650 u32 data, orig;
3ec7d11b 6651
22c775ce 6652 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6653 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
22c775ce
AD		 6654 data &= ~DISABLE_CP_PG;
6655 else
6656 data |= DISABLE_CP_PG;
6657 if (orig != data)
6658 WREG32(RLC_PG_CNTL, data);
3ec7d11b
AD		 6659}
6660
22c775ce 6661static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
f96ab484 6662{
22c775ce 6663 u32 data, orig;
f96ab484 6664
22c775ce 6665 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6666 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
22c775ce
AD		 6667 data &= ~DISABLE_GDS_PG;
6668 else
6669 data |= DISABLE_GDS_PG;
6670 if (orig != data)
6671 WREG32(RLC_PG_CNTL, data);
6672}
6673
6674#define CP_ME_TABLE_SIZE 96
6675#define CP_ME_TABLE_OFFSET 2048
6676#define CP_MEC_TABLE_OFFSET 4096
6677
6678void cik_init_cp_pg_table(struct radeon_device *rdev)
6679{
22c775ce
AD		 6680 volatile u32 *dst_ptr;
6681 int me, i, max_me = 4;
6682 u32 bo_offset = 0;
f2c6b0f4 6683 u32 table_offset, table_size;
22c775ce
AD		 6684
6685 if (rdev->family == CHIP_KAVERI)
6686 max_me = 5;
6687
6688 if (rdev->rlc.cp_table_ptr == NULL)
f96ab484
AD		 6689 return;
6690
22c775ce
AD		 6691 /* write the cp table buffer */
6692 dst_ptr = rdev->rlc.cp_table_ptr;
6693 for (me = 0; me < max_me; me++) {
f2c6b0f4
AD		 6694 if (rdev->new_fw) {
6695 const __le32 *fw_data;
6696 const struct gfx_firmware_header_v1_0 *hdr;
6697
6698 if (me == 0) {
6699 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
6700 fw_data = (const __le32 *)
6701 (rdev->ce_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6702 table_offset = le32_to_cpu(hdr->jt_offset);
6703 table_size = le32_to_cpu(hdr->jt_size);
6704 } else if (me == 1) {
6705 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
6706 fw_data = (const __le32 *)
6707 (rdev->pfp_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6708 table_offset = le32_to_cpu(hdr->jt_offset);
6709 table_size = le32_to_cpu(hdr->jt_size);
6710 } else if (me == 2) {
6711 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
6712 fw_data = (const __le32 *)
6713 (rdev->me_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6714 table_offset = le32_to_cpu(hdr->jt_offset);
6715 table_size = le32_to_cpu(hdr->jt_size);
6716 } else if (me == 3) {
6717 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec_fw->data;
6718 fw_data = (const __le32 *)
6719 (rdev->mec_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6720 table_offset = le32_to_cpu(hdr->jt_offset);
6721 table_size = le32_to_cpu(hdr->jt_size);
6722 } else {
6723 hdr = (const struct gfx_firmware_header_v1_0 *)rdev->mec2_fw->data;
6724 fw_data = (const __le32 *)
6725 (rdev->mec2_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
6726 table_offset = le32_to_cpu(hdr->jt_offset);
6727 table_size = le32_to_cpu(hdr->jt_size);
6728 }
6729
6730 for (i = 0; i < table_size; i ++) {
6731 dst_ptr[bo_offset + i] =
6732 cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
6733 }
6734 bo_offset += table_size;
22c775ce 6735 } else {
f2c6b0f4
AD		 6736 const __be32 *fw_data;
6737 table_size = CP_ME_TABLE_SIZE;
6738
6739 if (me == 0) {
6740 fw_data = (const __be32 *)rdev->ce_fw->data;
6741 table_offset = CP_ME_TABLE_OFFSET;
6742 } else if (me == 1) {
6743 fw_data = (const __be32 *)rdev->pfp_fw->data;
6744 table_offset = CP_ME_TABLE_OFFSET;
6745 } else if (me == 2) {
6746 fw_data = (const __be32 *)rdev->me_fw->data;
6747 table_offset = CP_ME_TABLE_OFFSET;
6748 } else {
6749 fw_data = (const __be32 *)rdev->mec_fw->data;
6750 table_offset = CP_MEC_TABLE_OFFSET;
6751 }
22c775ce 6752
f2c6b0f4
AD		 6753 for (i = 0; i < table_size; i ++) {
6754 dst_ptr[bo_offset + i] =
6755 cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6756 }
6757 bo_offset += table_size;
22c775ce 6758 }
f96ab484 6759 }
22c775ce 6760}
f96ab484 6761
22c775ce
AD		 6762static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6763 bool enable)
6764{
6765 u32 data, orig;
6766
2b19d17f 6767 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
22c775ce
AD		 6768 orig = data = RREG32(RLC_PG_CNTL);
6769 data |= GFX_PG_ENABLE;
6770 if (orig != data)
6771 WREG32(RLC_PG_CNTL, data);
6772
6773 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6774 data |= AUTO_PG_EN;
6775 if (orig != data)
6776 WREG32(RLC_AUTO_PG_CTRL, data);
6777 } else {
6778 orig = data = RREG32(RLC_PG_CNTL);
6779 data &= ~GFX_PG_ENABLE;
6780 if (orig != data)
6781 WREG32(RLC_PG_CNTL, data);
f96ab484 6782
22c775ce
AD		 6783 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6784 data &= ~AUTO_PG_EN;
6785 if (orig != data)
6786 WREG32(RLC_AUTO_PG_CTRL, data);
f96ab484 6787
22c775ce
AD		 6788 data = RREG32(DB_RENDER_CONTROL);
6789 }
6790}
f96ab484 6791
22c775ce
AD		 6792static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6793{
6794 u32 mask = 0, tmp, tmp1;
6795 int i;
f96ab484 6796
22c775ce
AD		 6797 cik_select_se_sh(rdev, se, sh);
6798 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6799 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6800 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
f96ab484 6801
22c775ce 6802 tmp &= 0xffff0000;
f96ab484 6803
22c775ce
AD		 6804 tmp |= tmp1;
6805 tmp >>= 16;
6806
6807 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6808 mask <<= 1;
6809 mask |= 1;
b07fdd38 6810 }
22c775ce
AD		 6811
6812 return (~tmp) & mask;
f96ab484
AD		 6813}
6814
22c775ce 6815static void cik_init_ao_cu_mask(struct radeon_device *rdev)
d0e092d9 6816{
22c775ce
AD		 6817 u32 i, j, k, active_cu_number = 0;
6818 u32 mask, counter, cu_bitmap;
6819 u32 tmp = 0;
d0e092d9 6820
22c775ce
AD		 6821 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6822 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6823 mask = 1;
6824 cu_bitmap = 0;
6825 counter = 0;
6826 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6827 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6828 if (counter < 2)
6829 cu_bitmap |= mask;
6830 counter ++;
d0e092d9 6831 }
22c775ce 6832 mask <<= 1;
d0e092d9 6833 }
d0e092d9 6834
22c775ce
AD		 6835 active_cu_number += counter;
6836 tmp |= (cu_bitmap << (i * 16 + j * 8));
d0e092d9 6837 }
d0e092d9 6838 }
22c775ce
AD		 6839
6840 WREG32(RLC_PG_AO_CU_MASK, tmp);
6841
6842 tmp = RREG32(RLC_MAX_PG_CU);
6843 tmp &= ~MAX_PU_CU_MASK;
6844 tmp |= MAX_PU_CU(active_cu_number);
6845 WREG32(RLC_MAX_PG_CU, tmp);
d0e092d9
AD		 6846}
6847
22c775ce
AD		 6848static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6849 bool enable)
605de6b9 6850{
22c775ce 6851 u32 data, orig;
605de6b9 6852
22c775ce 6853 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6854 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
22c775ce
AD		 6855 data |= STATIC_PER_CU_PG_ENABLE;
6856 else
6857 data &= ~STATIC_PER_CU_PG_ENABLE;
6858 if (orig != data)
6859 WREG32(RLC_PG_CNTL, data);
6860}
6861
6862static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6863 bool enable)
6864{
6865 u32 data, orig;
605de6b9 6866
22c775ce 6867 orig = data = RREG32(RLC_PG_CNTL);
473359bc 6868 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
22c775ce 6869 data |= DYN_PER_CU_PG_ENABLE;
605de6b9 6870 else
22c775ce
AD		 6871 data &= ~DYN_PER_CU_PG_ENABLE;
6872 if (orig != data)
6873 WREG32(RLC_PG_CNTL, data);
6874}
605de6b9 6875
22c775ce
AD		 6876#define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6877#define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6878
6879static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6880{
6881 u32 data, orig;
6882 u32 i;
6883
6884 if (rdev->rlc.cs_data) {
6885 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6886 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
a0f38609 6887 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
22c775ce 6888 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
605de6b9 6889 } else {
22c775ce
AD		 6890 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6891 for (i = 0; i < 3; i++)
6892 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6893 }
6894 if (rdev->rlc.reg_list) {
6895 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6896 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6897 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
605de6b9 6898 }
605de6b9 6899
22c775ce
AD		 6900 orig = data = RREG32(RLC_PG_CNTL);
6901 data |= GFX_PG_SRC;
6902 if (orig != data)
6903 WREG32(RLC_PG_CNTL, data);
605de6b9 6904
22c775ce
AD		 6905 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6906 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
605de6b9 6907
22c775ce
AD		 6908 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6909 data &= ~IDLE_POLL_COUNT_MASK;
6910 data |= IDLE_POLL_COUNT(0x60);
6911 WREG32(CP_RB_WPTR_POLL_CNTL, data);
605de6b9 6912
22c775ce
AD		 6913 data = 0x10101010;
6914 WREG32(RLC_PG_DELAY, data);
605de6b9 6915
22c775ce
AD		 6916 data = RREG32(RLC_PG_DELAY_2);
6917 data &= ~0xff;
6918 data |= 0x3;
6919 WREG32(RLC_PG_DELAY_2, data);
605de6b9 6920
22c775ce
AD		 6921 data = RREG32(RLC_AUTO_PG_CTRL);
6922 data &= ~GRBM_REG_SGIT_MASK;
6923 data |= GRBM_REG_SGIT(0x700);
6924 WREG32(RLC_AUTO_PG_CTRL, data);
605de6b9 6925
605de6b9
AD		 6926}
6927
22c775ce 6928static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
f6796cae 6929{
473359bc
AD		 6930 cik_enable_gfx_cgpg(rdev, enable);
6931 cik_enable_gfx_static_mgpg(rdev, enable);
6932 cik_enable_gfx_dynamic_mgpg(rdev, enable);
22c775ce 6933}
f6796cae 6934
a0f38609
AD		 6935u32 cik_get_csb_size(struct radeon_device *rdev)
6936{
6937 u32 count = 0;
6938 const struct cs_section_def *sect = NULL;
6939 const struct cs_extent_def *ext = NULL;
f6796cae 6940
a0f38609
AD		 6941 if (rdev->rlc.cs_data == NULL)
6942 return 0;
f6796cae 6943
a0f38609
AD		 6944 /* begin clear state */
6945 count += 2;
6946 /* context control state */
6947 count += 3;
6948
6949 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6950 for (ext = sect->section; ext->extent != NULL; ++ext) {
6951 if (sect->id == SECT_CONTEXT)
6952 count += 2 + ext->reg_count;
6953 else
6954 return 0;
f6796cae
AD		 6955 }
6956 }
a0f38609
AD		 6957 /* pa_sc_raster_config/pa_sc_raster_config1 */
6958 count += 4;
6959 /* end clear state */
6960 count += 2;
6961 /* clear state */
6962 count += 2;
f6796cae 6963
a0f38609 6964 return count;
f6796cae
AD		 6965}
6966
a0f38609 6967void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
f6796cae 6968{
a0f38609
AD		 6969 u32 count = 0, i;
6970 const struct cs_section_def *sect = NULL;
6971 const struct cs_extent_def *ext = NULL;
f6796cae 6972
a0f38609
AD		 6973 if (rdev->rlc.cs_data == NULL)
6974 return;
6975 if (buffer == NULL)
6976 return;
f6796cae 6977
6ba81e53
AD		 6978 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6979 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
a0f38609 6980
6ba81e53
AD		 6981 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6982 buffer[count++] = cpu_to_le32(0x80000000);
6983 buffer[count++] = cpu_to_le32(0x80000000);
a0f38609
AD		 6984
6985 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6986 for (ext = sect->section; ext->extent != NULL; ++ext) {
6987 if (sect->id == SECT_CONTEXT) {
6ba81e53
AD		 6988 buffer[count++] =
6989 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6990 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
a0f38609 6991 for (i = 0; i < ext->reg_count; i++)
6ba81e53 6992 buffer[count++] = cpu_to_le32(ext->extent[i]);
a0f38609
AD		 6993 } else {
6994 return;
6995 }
6996 }
6997 }
f6796cae 6998
6ba81e53
AD		 6999 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
7000 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
f6796cae
AD		 7001 switch (rdev->family) {
7002 case CHIP_BONAIRE:
6ba81e53
AD		 7003 buffer[count++] = cpu_to_le32(0x16000012);
7004 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7005 break;
7006 case CHIP_KAVERI:
6ba81e53
AD		 7007 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7008 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7009 break;
7010 case CHIP_KABINI:
f73a9e83 7011 case CHIP_MULLINS:
6ba81e53
AD		 7012 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
7013 buffer[count++] = cpu_to_le32(0x00000000);
a0f38609 7014 break;
bbfe90bd 7015 case CHIP_HAWAII:
a8947f57
AD		 7016 buffer[count++] = cpu_to_le32(0x3a00161a);
7017 buffer[count++] = cpu_to_le32(0x0000002e);
bbfe90bd 7018 break;
a0f38609 7019 default:
6ba81e53
AD		 7020 buffer[count++] = cpu_to_le32(0x00000000);
7021 buffer[count++] = cpu_to_le32(0x00000000);
f6796cae
AD		 7022 break;
7023 }
7024
6ba81e53
AD		 7025 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
7026 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
f6796cae 7027
6ba81e53
AD		 7028 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
7029 buffer[count++] = cpu_to_le32(0);
a0f38609 7030}
f6796cae 7031
473359bc 7032static void cik_init_pg(struct radeon_device *rdev)
22c775ce 7033{
473359bc 7034 if (rdev->pg_flags) {
22c775ce
AD		 7035 cik_enable_sck_slowdown_on_pu(rdev, true);
7036 cik_enable_sck_slowdown_on_pd(rdev, true);
2b19d17f 7037 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 7038 cik_init_gfx_cgpg(rdev);
7039 cik_enable_cp_pg(rdev, true);
7040 cik_enable_gds_pg(rdev, true);
7041 }
22c775ce
AD		 7042 cik_init_ao_cu_mask(rdev);
7043 cik_update_gfx_pg(rdev, true);
7044 }
7045}
f6796cae 7046
473359bc
AD		 7047static void cik_fini_pg(struct radeon_device *rdev)
7048{
7049 if (rdev->pg_flags) {
7050 cik_update_gfx_pg(rdev, false);
2b19d17f 7051 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
473359bc
AD		 7052 cik_enable_cp_pg(rdev, false);
7053 cik_enable_gds_pg(rdev, false);
7054 }
7055 }
f6796cae 7056}
a59781bb
AD		 7057
7058/*
7059 * Interrupts
7060 * Starting with r6xx, interrupts are handled via a ring buffer.
7061 * Ring buffers are areas of GPU accessible memory that the GPU
7062 * writes interrupt vectors into and the host reads vectors out of.
7063 * There is a rptr (read pointer) that determines where the
7064 * host is currently reading, and a wptr (write pointer)
7065 * which determines where the GPU has written. When the
7066 * pointers are equal, the ring is idle. When the GPU
7067 * writes vectors to the ring buffer, it increments the
7068 * wptr. When there is an interrupt, the host then starts
7069 * fetching commands and processing them until the pointers are
7070 * equal again at which point it updates the rptr.
7071 */
7072
7073/**
7074 * cik_enable_interrupts - Enable the interrupt ring buffer
7075 *
7076 * @rdev: radeon_device pointer
7077 *
7078 * Enable the interrupt ring buffer (CIK).
7079 */
7080static void cik_enable_interrupts(struct radeon_device *rdev)
7081{
7082 u32 ih_cntl = RREG32(IH_CNTL);
7083 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7084
7085 ih_cntl |= ENABLE_INTR;
7086 ih_rb_cntl |= IH_RB_ENABLE;
7087 WREG32(IH_CNTL, ih_cntl);
7088 WREG32(IH_RB_CNTL, ih_rb_cntl);
7089 rdev->ih.enabled = true;
7090}
7091
7092/**
7093 * cik_disable_interrupts - Disable the interrupt ring buffer
7094 *
7095 * @rdev: radeon_device pointer
7096 *
7097 * Disable the interrupt ring buffer (CIK).
7098 */
7099static void cik_disable_interrupts(struct radeon_device *rdev)
7100{
7101 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
7102 u32 ih_cntl = RREG32(IH_CNTL);
7103
7104 ih_rb_cntl &= ~IH_RB_ENABLE;
7105 ih_cntl &= ~ENABLE_INTR;
7106 WREG32(IH_RB_CNTL, ih_rb_cntl);
7107 WREG32(IH_CNTL, ih_cntl);
7108 /* set rptr, wptr to 0 */
7109 WREG32(IH_RB_RPTR, 0);
7110 WREG32(IH_RB_WPTR, 0);
7111 rdev->ih.enabled = false;
7112 rdev->ih.rptr = 0;
7113}
7114
7115/**
7116 * cik_disable_interrupt_state - Disable all interrupt sources
7117 *
7118 * @rdev: radeon_device pointer
7119 *
7120 * Clear all interrupt enable bits used by the driver (CIK).
7121 */
7122static void cik_disable_interrupt_state(struct radeon_device *rdev)
7123{
7124 u32 tmp;
7125
7126 /* gfx ring */
4214faf6
AD		 7127 tmp = RREG32(CP_INT_CNTL_RING0) &
7128 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7129 WREG32(CP_INT_CNTL_RING0, tmp);
21a93e13
AD		 7130 /* sdma */
7131 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7132 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
7133 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7134 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
a59781bb
AD		 7135 /* compute queues */
7136 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
7137 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
7138 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
7139 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
7140 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
7141 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
7142 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
7143 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
7144 /* grbm */
7145 WREG32(GRBM_INT_CNTL, 0);
7146 /* vline/vblank, etc. */
7147 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7148 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7149 if (rdev->num_crtc >= 4) {
7150 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7151 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7152 }
7153 if (rdev->num_crtc >= 6) {
7154 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7155 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7156 }
f5d636d2
CK		 7157 /* pflip */
7158 if (rdev->num_crtc >= 2) {
7159 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
7160 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
7161 }
7162 if (rdev->num_crtc >= 4) {
7163 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
7164 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
7165 }
7166 if (rdev->num_crtc >= 6) {
7167 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
7168 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
7169 }
a59781bb
AD		 7170
7171 /* dac hotplug */
7172 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
7173
7174 /* digital hotplug */
7175 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7176 WREG32(DC_HPD1_INT_CONTROL, tmp);
7177 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7178 WREG32(DC_HPD2_INT_CONTROL, tmp);
7179 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7180 WREG32(DC_HPD3_INT_CONTROL, tmp);
7181 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7182 WREG32(DC_HPD4_INT_CONTROL, tmp);
7183 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7184 WREG32(DC_HPD5_INT_CONTROL, tmp);
7185 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
7186 WREG32(DC_HPD6_INT_CONTROL, tmp);
7187
7188}
7189
7190/**
7191 * cik_irq_init - init and enable the interrupt ring
7192 *
7193 * @rdev: radeon_device pointer
7194 *
7195 * Allocate a ring buffer for the interrupt controller,
7196 * enable the RLC, disable interrupts, enable the IH
7197 * ring buffer and enable it (CIK).
7198 * Called at device load and reume.
7199 * Returns 0 for success, errors for failure.
7200 */
7201static int cik_irq_init(struct radeon_device *rdev)
7202{
7203 int ret = 0;
7204 int rb_bufsz;
7205 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
7206
7207 /* allocate ring */
7208 ret = r600_ih_ring_alloc(rdev);
7209 if (ret)
7210 return ret;
7211
7212 /* disable irqs */
7213 cik_disable_interrupts(rdev);
7214
7215 /* init rlc */
7216 ret = cik_rlc_resume(rdev);
7217 if (ret) {
7218 r600_ih_ring_fini(rdev);
7219 return ret;
7220 }
7221
7222 /* setup interrupt control */
7223 /* XXX this should actually be a bus address, not an MC address. same on older asics */
7224 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
7225 interrupt_cntl = RREG32(INTERRUPT_CNTL);
7226 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
7227 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
7228 */
7229 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
7230 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
7231 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
7232 WREG32(INTERRUPT_CNTL, interrupt_cntl);
7233
7234 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
b72a8925 7235 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
a59781bb
AD		 7236
7237 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
7238 IH_WPTR_OVERFLOW_CLEAR |
7239 (rb_bufsz << 1));
7240
7241 if (rdev->wb.enabled)
7242 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
7243
7244 /* set the writeback address whether it's enabled or not */
7245 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
7246 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
7247
7248 WREG32(IH_RB_CNTL, ih_rb_cntl);
7249
7250 /* set rptr, wptr to 0 */
7251 WREG32(IH_RB_RPTR, 0);
7252 WREG32(IH_RB_WPTR, 0);
7253
7254 /* Default settings for IH_CNTL (disabled at first) */
7255 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
7256 /* RPTR_REARM only works if msi's are enabled */
7257 if (rdev->msi_enabled)
7258 ih_cntl |= RPTR_REARM;
7259 WREG32(IH_CNTL, ih_cntl);
7260
7261 /* force the active interrupt state to all disabled */
7262 cik_disable_interrupt_state(rdev);
7263
7264 pci_set_master(rdev->pdev);
7265
7266 /* enable irqs */
7267 cik_enable_interrupts(rdev);
7268
7269 return ret;
7270}
7271
7272/**
7273 * cik_irq_set - enable/disable interrupt sources
7274 *
7275 * @rdev: radeon_device pointer
7276 *
7277 * Enable interrupt sources on the GPU (vblanks, hpd,
7278 * etc.) (CIK).
7279 * Returns 0 for success, errors for failure.
7280 */
7281int cik_irq_set(struct radeon_device *rdev)
7282{
4214faf6 7283 u32 cp_int_cntl;
2b0781a6
AD		 7284 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
7285 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
a59781bb
AD		 7286 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
7287 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
7288 u32 grbm_int_cntl = 0;
21a93e13 7289 u32 dma_cntl, dma_cntl1;
41a524ab 7290 u32 thermal_int;
a59781bb
AD		 7291
7292 if (!rdev->irq.installed) {
7293 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
7294 return -EINVAL;
7295 }
7296 /* don't enable anything if the ih is disabled */
7297 if (!rdev->ih.enabled) {
7298 cik_disable_interrupts(rdev);
7299 /* force the active interrupt state to all disabled */
7300 cik_disable_interrupt_state(rdev);
7301 return 0;
7302 }
7303
4214faf6
AD		 7304 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
7305 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
7306 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
7307
a59781bb
AD		 7308 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
7309 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
7310 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
7311 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
7312 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
7313 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
7314
21a93e13
AD		 7315 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
7316 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
7317
2b0781a6
AD		 7318 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7319 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7320 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7321 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7322 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7323 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7324 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7325 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
7326
cc8dbbb4
AD		 7327 if (rdev->flags & RADEON_IS_IGP)
7328 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
7329 ~(THERM_INTH_MASK | THERM_INTL_MASK);
7330 else
7331 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
7332 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
41a524ab 7333
a59781bb
AD		 7334 /* enable CP interrupts on all rings */
7335 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
7336 DRM_DEBUG("cik_irq_set: sw int gfx\n");
7337 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
7338 }
2b0781a6
AD		 7339 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
7340 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7341 DRM_DEBUG("si_irq_set: sw int cp1\n");
7342 if (ring->me == 1) {
7343 switch (ring->pipe) {
7344 case 0:
7345 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7346 break;
7347 case 1:
7348 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7349 break;
7350 case 2:
7351 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7352 break;
7353 case 3:
7354 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7355 break;
7356 default:
7357 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7358 break;
7359 }
7360 } else if (ring->me == 2) {
7361 switch (ring->pipe) {
7362 case 0:
7363 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7364 break;
7365 case 1:
7366 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7367 break;
7368 case 2:
7369 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7370 break;
7371 case 3:
7372 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7373 break;
7374 default:
7375 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
7376 break;
7377 }
7378 } else {
7379 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
7380 }
7381 }
7382 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
7383 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7384 DRM_DEBUG("si_irq_set: sw int cp2\n");
7385 if (ring->me == 1) {
7386 switch (ring->pipe) {
7387 case 0:
7388 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
7389 break;
7390 case 1:
7391 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
7392 break;
7393 case 2:
7394 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7395 break;
7396 case 3:
7397 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
7398 break;
7399 default:
7400 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7401 break;
7402 }
7403 } else if (ring->me == 2) {
7404 switch (ring->pipe) {
7405 case 0:
7406 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
7407 break;
7408 case 1:
7409 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
7410 break;
7411 case 2:
7412 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7413 break;
7414 case 3:
7415 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
7416 break;
7417 default:
7418 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
7419 break;
7420 }
7421 } else {
7422 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
7423 }
7424 }
a59781bb 7425
21a93e13
AD		 7426 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
7427 DRM_DEBUG("cik_irq_set: sw int dma\n");
7428 dma_cntl |= TRAP_ENABLE;
7429 }
7430
7431 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
7432 DRM_DEBUG("cik_irq_set: sw int dma1\n");
7433 dma_cntl1 |= TRAP_ENABLE;
7434 }
7435
a59781bb
AD		 7436 if (rdev->irq.crtc_vblank_int[0] ||
7437 atomic_read(&rdev->irq.pflip[0])) {
7438 DRM_DEBUG("cik_irq_set: vblank 0\n");
7439 crtc1 |= VBLANK_INTERRUPT_MASK;
7440 }
7441 if (rdev->irq.crtc_vblank_int[1] ||
7442 atomic_read(&rdev->irq.pflip[1])) {
7443 DRM_DEBUG("cik_irq_set: vblank 1\n");
7444 crtc2 |= VBLANK_INTERRUPT_MASK;
7445 }
7446 if (rdev->irq.crtc_vblank_int[2] ||
7447 atomic_read(&rdev->irq.pflip[2])) {
7448 DRM_DEBUG("cik_irq_set: vblank 2\n");
7449 crtc3 |= VBLANK_INTERRUPT_MASK;
7450 }
7451 if (rdev->irq.crtc_vblank_int[3] ||
7452 atomic_read(&rdev->irq.pflip[3])) {
7453 DRM_DEBUG("cik_irq_set: vblank 3\n");
7454 crtc4 |= VBLANK_INTERRUPT_MASK;
7455 }
7456 if (rdev->irq.crtc_vblank_int[4] ||
7457 atomic_read(&rdev->irq.pflip[4])) {
7458 DRM_DEBUG("cik_irq_set: vblank 4\n");
7459 crtc5 |= VBLANK_INTERRUPT_MASK;
7460 }
7461 if (rdev->irq.crtc_vblank_int[5] ||
7462 atomic_read(&rdev->irq.pflip[5])) {
7463 DRM_DEBUG("cik_irq_set: vblank 5\n");
7464 crtc6 |= VBLANK_INTERRUPT_MASK;
7465 }
7466 if (rdev->irq.hpd[0]) {
7467 DRM_DEBUG("cik_irq_set: hpd 1\n");
7468 hpd1 |= DC_HPDx_INT_EN;
7469 }
7470 if (rdev->irq.hpd[1]) {
7471 DRM_DEBUG("cik_irq_set: hpd 2\n");
7472 hpd2 |= DC_HPDx_INT_EN;
7473 }
7474 if (rdev->irq.hpd[2]) {
7475 DRM_DEBUG("cik_irq_set: hpd 3\n");
7476 hpd3 |= DC_HPDx_INT_EN;
7477 }
7478 if (rdev->irq.hpd[3]) {
7479 DRM_DEBUG("cik_irq_set: hpd 4\n");
7480 hpd4 |= DC_HPDx_INT_EN;
7481 }
7482 if (rdev->irq.hpd[4]) {
7483 DRM_DEBUG("cik_irq_set: hpd 5\n");
7484 hpd5 |= DC_HPDx_INT_EN;
7485 }
7486 if (rdev->irq.hpd[5]) {
7487 DRM_DEBUG("cik_irq_set: hpd 6\n");
7488 hpd6 |= DC_HPDx_INT_EN;
7489 }
7490
41a524ab
AD		 7491 if (rdev->irq.dpm_thermal) {
7492 DRM_DEBUG("dpm thermal\n");
cc8dbbb4
AD		 7493 if (rdev->flags & RADEON_IS_IGP)
7494 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7495 else
7496 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
41a524ab
AD		 7497 }
7498
a59781bb
AD		 7499 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7500
21a93e13
AD		 7501 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7502 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7503
2b0781a6
AD		 7504 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7505 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7506 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7507 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7508 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7509 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7510 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7511 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7512
a59781bb
AD		 7513 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7514
7515 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7516 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7517 if (rdev->num_crtc >= 4) {
7518 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7519 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7520 }
7521 if (rdev->num_crtc >= 6) {
7522 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7523 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7524 }
7525
f5d636d2
CK		 7526 if (rdev->num_crtc >= 2) {
7527 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7528 GRPH_PFLIP_INT_MASK);
7529 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7530 GRPH_PFLIP_INT_MASK);
7531 }
7532 if (rdev->num_crtc >= 4) {
7533 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7534 GRPH_PFLIP_INT_MASK);
7535 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7536 GRPH_PFLIP_INT_MASK);
7537 }
7538 if (rdev->num_crtc >= 6) {
7539 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7540 GRPH_PFLIP_INT_MASK);
7541 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7542 GRPH_PFLIP_INT_MASK);
7543 }
7544
a59781bb
AD		 7545 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7546 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7547 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7548 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7549 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7550 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7551
cc8dbbb4
AD		 7552 if (rdev->flags & RADEON_IS_IGP)
7553 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7554 else
7555 WREG32_SMC(CG_THERMAL_INT, thermal_int);
41a524ab 7556
a59781bb
AD		 7557 return 0;
7558}
7559
7560/**
7561 * cik_irq_ack - ack interrupt sources
7562 *
7563 * @rdev: radeon_device pointer
7564 *
7565 * Ack interrupt sources on the GPU (vblanks, hpd,
7566 * etc.) (CIK). Certain interrupts sources are sw
7567 * generated and do not require an explicit ack.
7568 */
7569static inline void cik_irq_ack(struct radeon_device *rdev)
7570{
7571 u32 tmp;
7572
7573 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7574 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7575 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7576 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7577 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7578 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7579 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7580
f5d636d2
CK		 7581 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7582 EVERGREEN_CRTC0_REGISTER_OFFSET);
7583 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7584 EVERGREEN_CRTC1_REGISTER_OFFSET);
7585 if (rdev->num_crtc >= 4) {
7586 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7587 EVERGREEN_CRTC2_REGISTER_OFFSET);
7588 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7589 EVERGREEN_CRTC3_REGISTER_OFFSET);
7590 }
7591 if (rdev->num_crtc >= 6) {
7592 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7593 EVERGREEN_CRTC4_REGISTER_OFFSET);
7594 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7595 EVERGREEN_CRTC5_REGISTER_OFFSET);
7596 }
7597
7598 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7599 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7600 GRPH_PFLIP_INT_CLEAR);
7601 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7602 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7603 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7604 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7605 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7606 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7607 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7608 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7609 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7610 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7611 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7612
7613 if (rdev->num_crtc >= 4) {
f5d636d2
CK		 7614 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7615 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7616 GRPH_PFLIP_INT_CLEAR);
7617 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7618 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7619 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7620 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7621 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7622 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7623 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7624 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7625 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7626 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7627 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7628 }
7629
7630 if (rdev->num_crtc >= 6) {
f5d636d2
CK		 7631 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7632 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7633 GRPH_PFLIP_INT_CLEAR);
7634 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7635 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7636 GRPH_PFLIP_INT_CLEAR);
a59781bb
AD		 7637 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7638 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7639 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7640 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7641 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7642 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7643 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7644 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7645 }
7646
7647 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7648 tmp = RREG32(DC_HPD1_INT_CONTROL);
7649 tmp |= DC_HPDx_INT_ACK;
7650 WREG32(DC_HPD1_INT_CONTROL, tmp);
7651 }
7652 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7653 tmp = RREG32(DC_HPD2_INT_CONTROL);
7654 tmp |= DC_HPDx_INT_ACK;
7655 WREG32(DC_HPD2_INT_CONTROL, tmp);
7656 }
7657 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7658 tmp = RREG32(DC_HPD3_INT_CONTROL);
7659 tmp |= DC_HPDx_INT_ACK;
7660 WREG32(DC_HPD3_INT_CONTROL, tmp);
7661 }
7662 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7663 tmp = RREG32(DC_HPD4_INT_CONTROL);
7664 tmp |= DC_HPDx_INT_ACK;
7665 WREG32(DC_HPD4_INT_CONTROL, tmp);
7666 }
7667 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7668 tmp = RREG32(DC_HPD5_INT_CONTROL);
7669 tmp |= DC_HPDx_INT_ACK;
7670 WREG32(DC_HPD5_INT_CONTROL, tmp);
7671 }
7672 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7673 tmp = RREG32(DC_HPD5_INT_CONTROL);
7674 tmp |= DC_HPDx_INT_ACK;
7675 WREG32(DC_HPD6_INT_CONTROL, tmp);
7676 }
7677}
7678
7679/**
7680 * cik_irq_disable - disable interrupts
7681 *
7682 * @rdev: radeon_device pointer
7683 *
7684 * Disable interrupts on the hw (CIK).
7685 */
7686static void cik_irq_disable(struct radeon_device *rdev)
7687{
7688 cik_disable_interrupts(rdev);
7689 /* Wait and acknowledge irq */
7690 mdelay(1);
7691 cik_irq_ack(rdev);
7692 cik_disable_interrupt_state(rdev);
7693}
7694
7695/**
7696 * cik_irq_disable - disable interrupts for suspend
7697 *
7698 * @rdev: radeon_device pointer
7699 *
7700 * Disable interrupts and stop the RLC (CIK).
7701 * Used for suspend.
7702 */
7703static void cik_irq_suspend(struct radeon_device *rdev)
7704{
7705 cik_irq_disable(rdev);
7706 cik_rlc_stop(rdev);
7707}
7708
7709/**
7710 * cik_irq_fini - tear down interrupt support
7711 *
7712 * @rdev: radeon_device pointer
7713 *
7714 * Disable interrupts on the hw and free the IH ring
7715 * buffer (CIK).
7716 * Used for driver unload.
7717 */
7718static void cik_irq_fini(struct radeon_device *rdev)
7719{
7720 cik_irq_suspend(rdev);
7721 r600_ih_ring_fini(rdev);
7722}
7723
7724/**
7725 * cik_get_ih_wptr - get the IH ring buffer wptr
7726 *
7727 * @rdev: radeon_device pointer
7728 *
7729 * Get the IH ring buffer wptr from either the register
7730 * or the writeback memory buffer (CIK). Also check for
7731 * ring buffer overflow and deal with it.
7732 * Used by cik_irq_process().
7733 * Returns the value of the wptr.
7734 */
7735static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7736{
7737 u32 wptr, tmp;
7738
7739 if (rdev->wb.enabled)
7740 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7741 else
7742 wptr = RREG32(IH_RB_WPTR);
7743
7744 if (wptr & RB_OVERFLOW) {
7745 /* When a ring buffer overflow happen start parsing interrupt
7746 * from the last not overwritten vector (wptr + 16). Hopefully
7747 * this should allow us to catchup.
7748 */
7749 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7750 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7751 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7752 tmp = RREG32(IH_RB_CNTL);
7753 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7754 WREG32(IH_RB_CNTL, tmp);
e8c214d2 7755 wptr &= ~RB_OVERFLOW;
a59781bb
AD		 7756 }
7757 return (wptr & rdev->ih.ptr_mask);
7758}
7759
7760/* CIK IV Ring
7761 * Each IV ring entry is 128 bits:
7762 * [7:0] - interrupt source id
7763 * [31:8] - reserved
7764 * [59:32] - interrupt source data
7765 * [63:60] - reserved
21a93e13
AD		 7766 * [71:64] - RINGID
7767 * CP:
7768 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
a59781bb
AD		 7769 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7770 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7771 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7772 * PIPE_ID - ME0 0=3D
7773 * - ME1&2 compute dispatcher (4 pipes each)
21a93e13
AD		 7774 * SDMA:
7775 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7776 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7777 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
a59781bb
AD		 7778 * [79:72] - VMID
7779 * [95:80] - PASID
7780 * [127:96] - reserved
7781 */
7782/**
7783 * cik_irq_process - interrupt handler
7784 *
7785 * @rdev: radeon_device pointer
7786 *
7787 * Interrupt hander (CIK). Walk the IH ring,
7788 * ack interrupts and schedule work to handle
7789 * interrupt events.
7790 * Returns irq process return code.
7791 */
7792int cik_irq_process(struct radeon_device *rdev)
7793{
2b0781a6
AD		 7794 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7795 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
a59781bb
AD		 7796 u32 wptr;
7797 u32 rptr;
7798 u32 src_id, src_data, ring_id;
7799 u8 me_id, pipe_id, queue_id;
7800 u32 ring_index;
7801 bool queue_hotplug = false;
7802 bool queue_reset = false;
3ec7d11b 7803 u32 addr, status, mc_client;
41a524ab 7804 bool queue_thermal = false;
a59781bb
AD		 7805
7806 if (!rdev->ih.enabled || rdev->shutdown)
7807 return IRQ_NONE;
7808
7809 wptr = cik_get_ih_wptr(rdev);
7810
7811restart_ih:
7812 /* is somebody else already processing irqs? */
7813 if (atomic_xchg(&rdev->ih.lock, 1))
7814 return IRQ_NONE;
7815
7816 rptr = rdev->ih.rptr;
7817 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7818
7819 /* Order reading of wptr vs. reading of IH ring data */
7820 rmb();
7821
7822 /* display interrupts */
7823 cik_irq_ack(rdev);
7824
7825 while (rptr != wptr) {
7826 /* wptr/rptr are in bytes! */
7827 ring_index = rptr / 4;
7828 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7829 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7830 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
a59781bb
AD		 7831
7832 switch (src_id) {
7833 case 1: /* D1 vblank/vline */
7834 switch (src_data) {
7835 case 0: /* D1 vblank */
7836 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7837 if (rdev->irq.crtc_vblank_int[0]) {
7838 drm_handle_vblank(rdev->ddev, 0);
7839 rdev->pm.vblank_sync = true;
7840 wake_up(&rdev->irq.vblank_queue);
7841 }
7842 if (atomic_read(&rdev->irq.pflip[0]))
1a0e7918 7843 radeon_crtc_handle_vblank(rdev, 0);
a59781bb
AD		 7844 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7845 DRM_DEBUG("IH: D1 vblank\n");
7846 }
7847 break;
7848 case 1: /* D1 vline */
7849 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7850 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7851 DRM_DEBUG("IH: D1 vline\n");
7852 }
7853 break;
7854 default:
7855 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7856 break;
7857 }
7858 break;
7859 case 2: /* D2 vblank/vline */
7860 switch (src_data) {
7861 case 0: /* D2 vblank */
7862 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7863 if (rdev->irq.crtc_vblank_int[1]) {
7864 drm_handle_vblank(rdev->ddev, 1);
7865 rdev->pm.vblank_sync = true;
7866 wake_up(&rdev->irq.vblank_queue);
7867 }
7868 if (atomic_read(&rdev->irq.pflip[1]))
1a0e7918 7869 radeon_crtc_handle_vblank(rdev, 1);
a59781bb
AD		 7870 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7871 DRM_DEBUG("IH: D2 vblank\n");
7872 }
7873 break;
7874 case 1: /* D2 vline */
7875 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7876 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7877 DRM_DEBUG("IH: D2 vline\n");
7878 }
7879 break;
7880 default:
7881 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7882 break;
7883 }
7884 break;
7885 case 3: /* D3 vblank/vline */
7886 switch (src_data) {
7887 case 0: /* D3 vblank */
7888 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7889 if (rdev->irq.crtc_vblank_int[2]) {
7890 drm_handle_vblank(rdev->ddev, 2);
7891 rdev->pm.vblank_sync = true;
7892 wake_up(&rdev->irq.vblank_queue);
7893 }
7894 if (atomic_read(&rdev->irq.pflip[2]))
1a0e7918 7895 radeon_crtc_handle_vblank(rdev, 2);
a59781bb
AD		 7896 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7897 DRM_DEBUG("IH: D3 vblank\n");
7898 }
7899 break;
7900 case 1: /* D3 vline */
7901 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7902 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7903 DRM_DEBUG("IH: D3 vline\n");
7904 }
7905 break;
7906 default:
7907 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7908 break;
7909 }
7910 break;
7911 case 4: /* D4 vblank/vline */
7912 switch (src_data) {
7913 case 0: /* D4 vblank */
7914 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7915 if (rdev->irq.crtc_vblank_int[3]) {
7916 drm_handle_vblank(rdev->ddev, 3);
7917 rdev->pm.vblank_sync = true;
7918 wake_up(&rdev->irq.vblank_queue);
7919 }
7920 if (atomic_read(&rdev->irq.pflip[3]))
1a0e7918 7921 radeon_crtc_handle_vblank(rdev, 3);
a59781bb
AD		 7922 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7923 DRM_DEBUG("IH: D4 vblank\n");
7924 }
7925 break;
7926 case 1: /* D4 vline */
7927 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7928 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7929 DRM_DEBUG("IH: D4 vline\n");
7930 }
7931 break;
7932 default:
7933 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7934 break;
7935 }
7936 break;
7937 case 5: /* D5 vblank/vline */
7938 switch (src_data) {
7939 case 0: /* D5 vblank */
7940 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7941 if (rdev->irq.crtc_vblank_int[4]) {
7942 drm_handle_vblank(rdev->ddev, 4);
7943 rdev->pm.vblank_sync = true;
7944 wake_up(&rdev->irq.vblank_queue);
7945 }
7946 if (atomic_read(&rdev->irq.pflip[4]))
1a0e7918 7947 radeon_crtc_handle_vblank(rdev, 4);
a59781bb
AD		 7948 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7949 DRM_DEBUG("IH: D5 vblank\n");
7950 }
7951 break;
7952 case 1: /* D5 vline */
7953 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7954 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7955 DRM_DEBUG("IH: D5 vline\n");
7956 }
7957 break;
7958 default:
7959 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7960 break;
7961 }
7962 break;
7963 case 6: /* D6 vblank/vline */
7964 switch (src_data) {
7965 case 0: /* D6 vblank */
7966 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7967 if (rdev->irq.crtc_vblank_int[5]) {
7968 drm_handle_vblank(rdev->ddev, 5);
7969 rdev->pm.vblank_sync = true;
7970 wake_up(&rdev->irq.vblank_queue);
7971 }
7972 if (atomic_read(&rdev->irq.pflip[5]))
1a0e7918 7973 radeon_crtc_handle_vblank(rdev, 5);
a59781bb
AD		 7974 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7975 DRM_DEBUG("IH: D6 vblank\n");
7976 }
7977 break;
7978 case 1: /* D6 vline */
7979 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7980 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7981 DRM_DEBUG("IH: D6 vline\n");
7982 }
7983 break;
7984 default:
7985 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7986 break;
7987 }
7988 break;
f5d636d2
CK		 7989 case 8: /* D1 page flip */
7990 case 10: /* D2 page flip */
7991 case 12: /* D3 page flip */
7992 case 14: /* D4 page flip */
7993 case 16: /* D5 page flip */
7994 case 18: /* D6 page flip */
7995 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
39dc5454
MK		 7996 if (radeon_use_pflipirq > 0)
7997 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
f5d636d2 7998 break;
a59781bb
AD		 7999 case 42: /* HPD hotplug */
8000 switch (src_data) {
8001 case 0:
8002 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
8003 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
8004 queue_hotplug = true;
8005 DRM_DEBUG("IH: HPD1\n");
8006 }
8007 break;
8008 case 1:
8009 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
8010 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
8011 queue_hotplug = true;
8012 DRM_DEBUG("IH: HPD2\n");
8013 }
8014 break;
8015 case 2:
8016 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
8017 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
8018 queue_hotplug = true;
8019 DRM_DEBUG("IH: HPD3\n");
8020 }
8021 break;
8022 case 3:
8023 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
8024 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
8025 queue_hotplug = true;
8026 DRM_DEBUG("IH: HPD4\n");
8027 }
8028 break;
8029 case 4:
8030 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
8031 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
8032 queue_hotplug = true;
8033 DRM_DEBUG("IH: HPD5\n");
8034 }
8035 break;
8036 case 5:
8037 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
8038 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
8039 queue_hotplug = true;
8040 DRM_DEBUG("IH: HPD6\n");
8041 }
8042 break;
8043 default:
8044 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8045 break;
8046 }
8047 break;
6a3808b8
CK		 8048 case 124: /* UVD */
8049 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
8050 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
8051 break;
9d97c99b
AD		 8052 case 146:
8053 case 147:
3ec7d11b
AD		 8054 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
8055 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
8056 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
9b7d786b
CK		 8057 /* reset addr and status */
8058 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
8059 if (addr == 0x0 && status == 0x0)
8060 break;
9d97c99b
AD		 8061 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
8062 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3ec7d11b 8063 addr);
9d97c99b 8064 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3ec7d11b
AD		 8065 status);
8066 cik_vm_decode_fault(rdev, status, addr, mc_client);
9d97c99b 8067 break;
d93f7937
CK		 8068 case 167: /* VCE */
8069 DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
8070 switch (src_data) {
8071 case 0:
8072 radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
8073 break;
8074 case 1:
8075 radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
8076 break;
8077 default:
8078 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
8079 break;
8080 }
8081 break;
a59781bb
AD		 8082 case 176: /* GFX RB CP_INT */
8083 case 177: /* GFX IB CP_INT */
8084 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8085 break;
8086 case 181: /* CP EOP event */
8087 DRM_DEBUG("IH: CP EOP\n");
21a93e13
AD		 8088 /* XXX check the bitfield order! */
8089 me_id = (ring_id & 0x60) >> 5;
8090 pipe_id = (ring_id & 0x18) >> 3;
8091 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 8092 switch (me_id) {
8093 case 0:
8094 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
8095 break;
8096 case 1:
a59781bb 8097 case 2:
2b0781a6
AD		 8098 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
8099 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8100 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
8101 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
a59781bb
AD		 8102 break;
8103 }
8104 break;
8105 case 184: /* CP Privileged reg access */
8106 DRM_ERROR("Illegal register access in command stream\n");
8107 /* XXX check the bitfield order! */
8108 me_id = (ring_id & 0x60) >> 5;
8109 pipe_id = (ring_id & 0x18) >> 3;
8110 queue_id = (ring_id & 0x7) >> 0;
8111 switch (me_id) {
8112 case 0:
8113 /* This results in a full GPU reset, but all we need to do is soft
8114 * reset the CP for gfx
8115 */
8116 queue_reset = true;
8117 break;
8118 case 1:
8119 /* XXX compute */
2b0781a6 8120 queue_reset = true;
a59781bb
AD		 8121 break;
8122 case 2:
8123 /* XXX compute */
2b0781a6 8124 queue_reset = true;
a59781bb
AD		 8125 break;
8126 }
8127 break;
8128 case 185: /* CP Privileged inst */
8129 DRM_ERROR("Illegal instruction in command stream\n");
21a93e13
AD		 8130 /* XXX check the bitfield order! */
8131 me_id = (ring_id & 0x60) >> 5;
8132 pipe_id = (ring_id & 0x18) >> 3;
8133 queue_id = (ring_id & 0x7) >> 0;
a59781bb
AD		 8134 switch (me_id) {
8135 case 0:
8136 /* This results in a full GPU reset, but all we need to do is soft
8137 * reset the CP for gfx
8138 */
8139 queue_reset = true;
8140 break;
8141 case 1:
8142 /* XXX compute */
2b0781a6 8143 queue_reset = true;
a59781bb
AD		 8144 break;
8145 case 2:
8146 /* XXX compute */
2b0781a6 8147 queue_reset = true;
a59781bb
AD		 8148 break;
8149 }
8150 break;
21a93e13
AD		 8151 case 224: /* SDMA trap event */
8152 /* XXX check the bitfield order! */
8153 me_id = (ring_id & 0x3) >> 0;
8154 queue_id = (ring_id & 0xc) >> 2;
8155 DRM_DEBUG("IH: SDMA trap\n");
8156 switch (me_id) {
8157 case 0:
8158 switch (queue_id) {
8159 case 0:
8160 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
8161 break;
8162 case 1:
8163 /* XXX compute */
8164 break;
8165 case 2:
8166 /* XXX compute */
8167 break;
8168 }
8169 break;
8170 case 1:
8171 switch (queue_id) {
8172 case 0:
8173 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8174 break;
8175 case 1:
8176 /* XXX compute */
8177 break;
8178 case 2:
8179 /* XXX compute */
8180 break;
8181 }
8182 break;
8183 }
8184 break;
41a524ab
AD		 8185 case 230: /* thermal low to high */
8186 DRM_DEBUG("IH: thermal low to high\n");
8187 rdev->pm.dpm.thermal.high_to_low = false;
8188 queue_thermal = true;
8189 break;
8190 case 231: /* thermal high to low */
8191 DRM_DEBUG("IH: thermal high to low\n");
8192 rdev->pm.dpm.thermal.high_to_low = true;
8193 queue_thermal = true;
8194 break;
8195 case 233: /* GUI IDLE */
8196 DRM_DEBUG("IH: GUI idle\n");
8197 break;
21a93e13
AD		 8198 case 241: /* SDMA Privileged inst */
8199 case 247: /* SDMA Privileged inst */
8200 DRM_ERROR("Illegal instruction in SDMA command stream\n");
8201 /* XXX check the bitfield order! */
8202 me_id = (ring_id & 0x3) >> 0;
8203 queue_id = (ring_id & 0xc) >> 2;
8204 switch (me_id) {
8205 case 0:
8206 switch (queue_id) {
8207 case 0:
8208 queue_reset = true;
8209 break;
8210 case 1:
8211 /* XXX compute */
8212 queue_reset = true;
8213 break;
8214 case 2:
8215 /* XXX compute */
8216 queue_reset = true;
8217 break;
8218 }
8219 break;
8220 case 1:
8221 switch (queue_id) {
8222 case 0:
8223 queue_reset = true;
8224 break;
8225 case 1:
8226 /* XXX compute */
8227 queue_reset = true;
8228 break;
8229 case 2:
8230 /* XXX compute */
8231 queue_reset = true;
8232 break;
8233 }
8234 break;
8235 }
8236 break;
a59781bb
AD		 8237 default:
8238 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
8239 break;
8240 }
8241
8242 /* wptr/rptr are in bytes! */
8243 rptr += 16;
8244 rptr &= rdev->ih.ptr_mask;
8245 }
8246 if (queue_hotplug)
8247 schedule_work(&rdev->hotplug_work);
8248 if (queue_reset)
8249 schedule_work(&rdev->reset_work);
41a524ab
AD		 8250 if (queue_thermal)
8251 schedule_work(&rdev->pm.dpm.thermal.work);
a59781bb
AD		 8252 rdev->ih.rptr = rptr;
8253 WREG32(IH_RB_RPTR, rdev->ih.rptr);
8254 atomic_set(&rdev->ih.lock, 0);
8255
8256 /* make sure wptr hasn't changed while processing */
8257 wptr = cik_get_ih_wptr(rdev);
8258 if (wptr != rptr)
8259 goto restart_ih;
8260
8261 return IRQ_HANDLED;
8262}
7bf94a2c
AD		 8263
8264/*
8265 * startup/shutdown callbacks
8266 */
8267/**
8268 * cik_startup - program the asic to a functional state
8269 *
8270 * @rdev: radeon_device pointer
8271 *
8272 * Programs the asic to a functional state (CIK).
8273 * Called by cik_init() and cik_resume().
8274 * Returns 0 for success, error for failure.
8275 */
8276static int cik_startup(struct radeon_device *rdev)
8277{
8278 struct radeon_ring *ring;
0e16e4cf 8279 u32 nop;
7bf94a2c
AD		 8280 int r;
8281
8a7cd276
AD		 8282 /* enable pcie gen2/3 link */
8283 cik_pcie_gen3_enable(rdev);
7235711a
AD		 8284 /* enable aspm */
8285 cik_program_aspm(rdev);
8a7cd276 8286
e5903d39
AD		 8287 /* scratch needs to be initialized before MC */
8288 r = r600_vram_scratch_init(rdev);
8289 if (r)
8290 return r;
8291
6fab3feb
AD		 8292 cik_mc_program(rdev);
8293
6c7bccea 8294 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7bf94a2c
AD		 8295 r = ci_mc_load_microcode(rdev);
8296 if (r) {
8297 DRM_ERROR("Failed to load MC firmware!\n");
8298 return r;
8299 }
8300 }
8301
7bf94a2c
AD		 8302 r = cik_pcie_gart_enable(rdev);
8303 if (r)
8304 return r;
8305 cik_gpu_init(rdev);
8306
8307 /* allocate rlc buffers */
22c775ce
AD		 8308 if (rdev->flags & RADEON_IS_IGP) {
8309 if (rdev->family == CHIP_KAVERI) {
8310 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
8311 rdev->rlc.reg_list_size =
8312 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
8313 } else {
8314 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
8315 rdev->rlc.reg_list_size =
8316 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
8317 }
8318 }
8319 rdev->rlc.cs_data = ci_cs_data;
8320 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
1fd11777 8321 r = sumo_rlc_init(rdev);
7bf94a2c
AD		 8322 if (r) {
8323 DRM_ERROR("Failed to init rlc BOs!\n");
8324 return r;
8325 }
8326
8327 /* allocate wb buffer */
8328 r = radeon_wb_init(rdev);
8329 if (r)
8330 return r;
8331
963e81f9
AD		 8332 /* allocate mec buffers */
8333 r = cik_mec_init(rdev);
8334 if (r) {
8335 DRM_ERROR("Failed to init MEC BOs!\n");
8336 return r;
8337 }
8338
7bf94a2c
AD		 8339 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
8340 if (r) {
8341 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8342 return r;
8343 }
8344
963e81f9
AD		 8345 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
8346 if (r) {
8347 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8348 return r;
8349 }
8350
8351 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
8352 if (r) {
8353 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
8354 return r;
8355 }
8356
7bf94a2c
AD		 8357 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
8358 if (r) {
8359 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8360 return r;
8361 }
8362
8363 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
8364 if (r) {
8365 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
8366 return r;
8367 }
8368
2ce529da 8369 r = radeon_uvd_resume(rdev);
87167bb1 8370 if (!r) {
2ce529da
AD		 8371 r = uvd_v4_2_resume(rdev);
8372 if (!r) {
8373 r = radeon_fence_driver_start_ring(rdev,
8374 R600_RING_TYPE_UVD_INDEX);
8375 if (r)
8376 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
8377 }
87167bb1
CK		 8378 }
8379 if (r)
8380 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
8381
d93f7937
CK		 8382 r = radeon_vce_resume(rdev);
8383 if (!r) {
8384 r = vce_v2_0_resume(rdev);
8385 if (!r)
8386 r = radeon_fence_driver_start_ring(rdev,
8387 TN_RING_TYPE_VCE1_INDEX);
8388 if (!r)
8389 r = radeon_fence_driver_start_ring(rdev,
8390 TN_RING_TYPE_VCE2_INDEX);
8391 }
8392 if (r) {
8393 dev_err(rdev->dev, "VCE init error (%d).\n", r);
8394 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
8395 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
8396 }
8397
7bf94a2c
AD		 8398 /* Enable IRQ */
8399 if (!rdev->irq.installed) {
8400 r = radeon_irq_kms_init(rdev);
8401 if (r)
8402 return r;
8403 }
8404
8405 r = cik_irq_init(rdev);
8406 if (r) {
8407 DRM_ERROR("radeon: IH init failed (%d).\n", r);
8408 radeon_irq_kms_fini(rdev);
8409 return r;
8410 }
8411 cik_irq_set(rdev);
8412
0e16e4cf 8413 if (rdev->family == CHIP_HAWAII) {
78cd3661
AD		 8414 if (rdev->new_fw)
8415 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8416 else
8417 nop = RADEON_CP_PACKET2;
0e16e4cf
AD		 8418 } else {
8419 nop = PACKET3(PACKET3_NOP, 0x3FFF);
8420 }
8421
7bf94a2c
AD		 8422 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8423 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
0e16e4cf 8424 nop);
7bf94a2c
AD		 8425 if (r)
8426 return r;
8427
963e81f9 8428 /* set up the compute queues */
2615b53a 8429 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8430 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8431 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
0e16e4cf 8432 nop);
963e81f9
AD		 8433 if (r)
8434 return r;
8435 ring->me = 1; /* first MEC */
8436 ring->pipe = 0; /* first pipe */
8437 ring->queue = 0; /* first queue */
8438 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
8439
2615b53a 8440 /* type-2 packets are deprecated on MEC, use type-3 instead */
963e81f9
AD		 8441 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8442 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
0e16e4cf 8443 nop);
963e81f9
AD		 8444 if (r)
8445 return r;
8446 /* dGPU only have 1 MEC */
8447 ring->me = 1; /* first MEC */
8448 ring->pipe = 0; /* first pipe */
8449 ring->queue = 1; /* second queue */
8450 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
8451
7bf94a2c
AD		 8452 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8453 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2e1e6dad 8454 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8455 if (r)
8456 return r;
8457
8458 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8459 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2e1e6dad 8460 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7bf94a2c
AD		 8461 if (r)
8462 return r;
8463
8464 r = cik_cp_resume(rdev);
8465 if (r)
8466 return r;
8467
8468 r = cik_sdma_resume(rdev);
8469 if (r)
8470 return r;
8471
87167bb1
CK		 8472 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8473 if (ring->ring_size) {
02c9f7fa 8474 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2e1e6dad 8475 RADEON_CP_PACKET2);
87167bb1 8476 if (!r)
e409b128 8477 r = uvd_v1_0_init(rdev);
87167bb1
CK		 8478 if (r)
8479 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
8480 }
8481
d93f7937
CK		 8482 r = -ENOENT;
8483
8484 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8485 if (ring->ring_size)
8486 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8487 VCE_CMD_NO_OP);
8488
8489 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8490 if (ring->ring_size)
8491 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
8492 VCE_CMD_NO_OP);
8493
8494 if (!r)
8495 r = vce_v1_0_init(rdev);
8496 else if (r != -ENOENT)
8497 DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
8498
7bf94a2c
AD		 8499 r = radeon_ib_pool_init(rdev);
8500 if (r) {
8501 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
8502 return r;
8503 }
8504
8505 r = radeon_vm_manager_init(rdev);
8506 if (r) {
8507 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
8508 return r;
8509 }
8510
b530602f
AD		 8511 r = dce6_audio_init(rdev);
8512 if (r)
8513 return r;
8514
7bf94a2c
AD		 8515 return 0;
8516}
8517
8518/**
8519 * cik_resume - resume the asic to a functional state
8520 *
8521 * @rdev: radeon_device pointer
8522 *
8523 * Programs the asic to a functional state (CIK).
8524 * Called at resume.
8525 * Returns 0 for success, error for failure.
8526 */
8527int cik_resume(struct radeon_device *rdev)
8528{
8529 int r;
8530
8531 /* post card */
8532 atom_asic_init(rdev->mode_info.atom_context);
8533
0aafd313
AD		 8534 /* init golden registers */
8535 cik_init_golden_registers(rdev);
8536
bc6a6295
AD		 8537 if (rdev->pm.pm_method == PM_METHOD_DPM)
8538 radeon_pm_resume(rdev);
6c7bccea 8539
7bf94a2c
AD		 8540 rdev->accel_working = true;
8541 r = cik_startup(rdev);
8542 if (r) {
8543 DRM_ERROR("cik startup failed on resume\n");
8544 rdev->accel_working = false;
8545 return r;
8546 }
8547
8548 return r;
8549
8550}
8551
8552/**
8553 * cik_suspend - suspend the asic
8554 *
8555 * @rdev: radeon_device pointer
8556 *
8557 * Bring the chip into a state suitable for suspend (CIK).
8558 * Called at suspend.
8559 * Returns 0 for success.
8560 */
8561int cik_suspend(struct radeon_device *rdev)
8562{
6c7bccea 8563 radeon_pm_suspend(rdev);
b530602f 8564 dce6_audio_fini(rdev);
7bf94a2c
AD		 8565 radeon_vm_manager_fini(rdev);
8566 cik_cp_enable(rdev, false);
8567 cik_sdma_enable(rdev, false);
e409b128 8568 uvd_v1_0_fini(rdev);
87167bb1 8569 radeon_uvd_suspend(rdev);
d93f7937 8570 radeon_vce_suspend(rdev);
473359bc
AD		 8571 cik_fini_pg(rdev);
8572 cik_fini_cg(rdev);
7bf94a2c
AD		 8573 cik_irq_suspend(rdev);
8574 radeon_wb_disable(rdev);
8575 cik_pcie_gart_disable(rdev);
8576 return 0;
8577}
8578
8579/* Plan is to move initialization in that function and use
8580 * helper function so that radeon_device_init pretty much
8581 * do nothing more than calling asic specific function. This
8582 * should also allow to remove a bunch of callback function
8583 * like vram_info.
8584 */
8585/**
8586 * cik_init - asic specific driver and hw init
8587 *
8588 * @rdev: radeon_device pointer
8589 *
8590 * Setup asic specific driver variables and program the hw
8591 * to a functional state (CIK).
8592 * Called at driver startup.
8593 * Returns 0 for success, errors for failure.
8594 */
8595int cik_init(struct radeon_device *rdev)
8596{
8597 struct radeon_ring *ring;
8598 int r;
8599
8600 /* Read BIOS */
8601 if (!radeon_get_bios(rdev)) {
8602 if (ASIC_IS_AVIVO(rdev))
8603 return -EINVAL;
8604 }
8605 /* Must be an ATOMBIOS */
8606 if (!rdev->is_atom_bios) {
8607 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8608 return -EINVAL;
8609 }
8610 r = radeon_atombios_init(rdev);
8611 if (r)
8612 return r;
8613
8614 /* Post card if necessary */
8615 if (!radeon_card_posted(rdev)) {
8616 if (!rdev->bios) {
8617 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8618 return -EINVAL;
8619 }
8620 DRM_INFO("GPU not posted. posting now...\n");
8621 atom_asic_init(rdev->mode_info.atom_context);
8622 }
0aafd313
AD		 8623 /* init golden registers */
8624 cik_init_golden_registers(rdev);
7bf94a2c
AD		 8625 /* Initialize scratch registers */
8626 cik_scratch_init(rdev);
8627 /* Initialize surface registers */
8628 radeon_surface_init(rdev);
8629 /* Initialize clocks */
8630 radeon_get_clock_info(rdev->ddev);
8631
8632 /* Fence driver */
8633 r = radeon_fence_driver_init(rdev);
8634 if (r)
8635 return r;
8636
8637 /* initialize memory controller */
8638 r = cik_mc_init(rdev);
8639 if (r)
8640 return r;
8641 /* Memory manager */
8642 r = radeon_bo_init(rdev);
8643 if (r)
8644 return r;
8645
01ac8794
AD		 8646 if (rdev->flags & RADEON_IS_IGP) {
8647 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8648 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8649 r = cik_init_microcode(rdev);
8650 if (r) {
8651 DRM_ERROR("Failed to load firmware!\n");
8652 return r;
8653 }
8654 }
8655 } else {
8656 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8657 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8658 !rdev->mc_fw) {
8659 r = cik_init_microcode(rdev);
8660 if (r) {
8661 DRM_ERROR("Failed to load firmware!\n");
8662 return r;
8663 }
8664 }
8665 }
8666
6c7bccea
AD		 8667 /* Initialize power management */
8668 radeon_pm_init(rdev);
8669
7bf94a2c
AD		 8670 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8671 ring->ring_obj = NULL;
8672 r600_ring_init(rdev, ring, 1024 * 1024);
8673
963e81f9
AD		 8674 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8675 ring->ring_obj = NULL;
8676 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8677 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8678 if (r)
8679 return r;
8680
8681 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8682 ring->ring_obj = NULL;
8683 r600_ring_init(rdev, ring, 1024 * 1024);
d5754ab8 8684 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
963e81f9
AD		 8685 if (r)
8686 return r;
8687
7bf94a2c
AD		 8688 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8689 ring->ring_obj = NULL;
8690 r600_ring_init(rdev, ring, 256 * 1024);
8691
8692 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8693 ring->ring_obj = NULL;
8694 r600_ring_init(rdev, ring, 256 * 1024);
8695
87167bb1
CK		 8696 r = radeon_uvd_init(rdev);
8697 if (!r) {
8698 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8699 ring->ring_obj = NULL;
8700 r600_ring_init(rdev, ring, 4096);
8701 }
8702
d93f7937
CK		 8703 r = radeon_vce_init(rdev);
8704 if (!r) {
8705 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
8706 ring->ring_obj = NULL;
8707 r600_ring_init(rdev, ring, 4096);
8708
8709 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
8710 ring->ring_obj = NULL;
8711 r600_ring_init(rdev, ring, 4096);
8712 }
8713
7bf94a2c
AD		 8714 rdev->ih.ring_obj = NULL;
8715 r600_ih_ring_init(rdev, 64 * 1024);
8716
8717 r = r600_pcie_gart_init(rdev);
8718 if (r)
8719 return r;
8720
8721 rdev->accel_working = true;
8722 r = cik_startup(rdev);
8723 if (r) {
8724 dev_err(rdev->dev, "disabling GPU acceleration\n");
8725 cik_cp_fini(rdev);
8726 cik_sdma_fini(rdev);
8727 cik_irq_fini(rdev);
1fd11777 8728 sumo_rlc_fini(rdev);
963e81f9 8729 cik_mec_fini(rdev);
7bf94a2c
AD		 8730 radeon_wb_fini(rdev);
8731 radeon_ib_pool_fini(rdev);
8732 radeon_vm_manager_fini(rdev);
8733 radeon_irq_kms_fini(rdev);
8734 cik_pcie_gart_fini(rdev);
8735 rdev->accel_working = false;
8736 }
8737
8738 /* Don't start up if the MC ucode is missing.
8739 * The default clocks and voltages before the MC ucode
8740 * is loaded are not suffient for advanced operations.
8741 */
8742 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8743 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8744 return -EINVAL;
8745 }
8746
8747 return 0;
8748}
8749
8750/**
8751 * cik_fini - asic specific driver and hw fini
8752 *
8753 * @rdev: radeon_device pointer
8754 *
8755 * Tear down the asic specific driver variables and program the hw
8756 * to an idle state (CIK).
8757 * Called at driver unload.
8758 */
8759void cik_fini(struct radeon_device *rdev)
8760{
6c7bccea 8761 radeon_pm_fini(rdev);
7bf94a2c
AD		 8762 cik_cp_fini(rdev);
8763 cik_sdma_fini(rdev);
473359bc
AD		 8764 cik_fini_pg(rdev);
8765 cik_fini_cg(rdev);
7bf94a2c 8766 cik_irq_fini(rdev);
1fd11777 8767 sumo_rlc_fini(rdev);
963e81f9 8768 cik_mec_fini(rdev);
7bf94a2c
AD		 8769 radeon_wb_fini(rdev);
8770 radeon_vm_manager_fini(rdev);
8771 radeon_ib_pool_fini(rdev);
8772 radeon_irq_kms_fini(rdev);
e409b128 8773 uvd_v1_0_fini(rdev);
87167bb1 8774 radeon_uvd_fini(rdev);
d93f7937 8775 radeon_vce_fini(rdev);
7bf94a2c
AD		 8776 cik_pcie_gart_fini(rdev);
8777 r600_vram_scratch_fini(rdev);
8778 radeon_gem_fini(rdev);
8779 radeon_fence_driver_fini(rdev);
8780 radeon_bo_fini(rdev);
8781 radeon_atombios_fini(rdev);
8782 kfree(rdev->bios);
8783 rdev->bios = NULL;
8784}
cd84a27d 8785
134b480f
AD		 8786void dce8_program_fmt(struct drm_encoder *encoder)
8787{
8788 struct drm_device *dev = encoder->dev;
8789 struct radeon_device *rdev = dev->dev_private;
8790 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8791 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8792 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8793 int bpc = 0;
8794 u32 tmp = 0;
6214bb74 8795 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
134b480f 8796
6214bb74
AD		 8797 if (connector) {
8798 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
134b480f 8799 bpc = radeon_get_monitor_bpc(connector);
6214bb74
AD		 8800 dither = radeon_connector->dither;
8801 }
134b480f
AD		 8802
8803 /* LVDS/eDP FMT is set up by atom */
8804 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8805 return;
8806
8807 /* not needed for analog */
8808 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8809 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8810 return;
8811
8812 if (bpc == 0)
8813 return;
8814
8815 switch (bpc) {
8816 case 6:
6214bb74 8817 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8818 /* XXX sort out optimal dither settings */
8819 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8820 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8821 else
8822 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8823 break;
8824 case 8:
6214bb74 8825 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8826 /* XXX sort out optimal dither settings */
8827 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8828 FMT_RGB_RANDOM_ENABLE |
8829 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8830 else
8831 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8832 break;
8833 case 10:
6214bb74 8834 if (dither == RADEON_FMT_DITHER_ENABLE)
134b480f
AD		 8835 /* XXX sort out optimal dither settings */
8836 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8837 FMT_RGB_RANDOM_ENABLE |
8838 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8839 else
8840 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8841 break;
8842 default:
8843 /* not needed */
8844 break;
8845 }
8846
8847 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8848}
8849
cd84a27d
AD		 8850/* display watermark setup */
8851/**
8852 * dce8_line_buffer_adjust - Set up the line buffer
8853 *
8854 * @rdev: radeon_device pointer
8855 * @radeon_crtc: the selected display controller
8856 * @mode: the current display mode on the selected display
8857 * controller
8858 *
8859 * Setup up the line buffer allocation for
8860 * the selected display controller (CIK).
8861 * Returns the line buffer size in pixels.
8862 */
8863static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8864 struct radeon_crtc *radeon_crtc,
8865 struct drm_display_mode *mode)
8866{
bc01a8c7
AD		 8867 u32 tmp, buffer_alloc, i;
8868 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
cd84a27d
AD		 8869 /*
8870 * Line Buffer Setup
8871 * There are 6 line buffers, one for each display controllers.
8872 * There are 3 partitions per LB. Select the number of partitions
8873 * to enable based on the display width. For display widths larger
8874 * than 4096, you need use to use 2 display controllers and combine
8875 * them using the stereo blender.
8876 */
8877 if (radeon_crtc->base.enabled && mode) {
bc01a8c7 8878 if (mode->crtc_hdisplay < 1920) {
cd84a27d 8879 tmp = 1;
bc01a8c7
AD		 8880 buffer_alloc = 2;
8881 } else if (mode->crtc_hdisplay < 2560) {
cd84a27d 8882 tmp = 2;
bc01a8c7
AD		 8883 buffer_alloc = 2;
8884 } else if (mode->crtc_hdisplay < 4096) {
cd84a27d 8885 tmp = 0;
bc01a8c7
AD		 8886 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8887 } else {
cd84a27d
AD		 8888 DRM_DEBUG_KMS("Mode too big for LB!\n");
8889 tmp = 0;
bc01a8c7 8890 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
cd84a27d 8891 }
bc01a8c7 8892 } else {
cd84a27d 8893 tmp = 1;
bc01a8c7
AD		 8894 buffer_alloc = 0;
8895 }
cd84a27d
AD		 8896
8897 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8898 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8899
bc01a8c7
AD		 8900 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8901 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8902 for (i = 0; i < rdev->usec_timeout; i++) {
8903 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8904 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8905 break;
8906 udelay(1);
8907 }
8908
cd84a27d
AD		 8909 if (radeon_crtc->base.enabled && mode) {
8910 switch (tmp) {
8911 case 0:
8912 default:
8913 return 4096 * 2;
8914 case 1:
8915 return 1920 * 2;
8916 case 2:
8917 return 2560 * 2;
8918 }
8919 }
8920
8921 /* controller not enabled, so no lb used */
8922 return 0;
8923}
8924
8925/**
8926 * cik_get_number_of_dram_channels - get the number of dram channels
8927 *
8928 * @rdev: radeon_device pointer
8929 *
8930 * Look up the number of video ram channels (CIK).
8931 * Used for display watermark bandwidth calculations
8932 * Returns the number of dram channels
8933 */
8934static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8935{
8936 u32 tmp = RREG32(MC_SHARED_CHMAP);
8937
8938 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8939 case 0:
8940 default:
8941 return 1;
8942 case 1:
8943 return 2;
8944 case 2:
8945 return 4;
8946 case 3:
8947 return 8;
8948 case 4:
8949 return 3;
8950 case 5:
8951 return 6;
8952 case 6:
8953 return 10;
8954 case 7:
8955 return 12;
8956 case 8:
8957 return 16;
8958 }
8959}
8960
8961struct dce8_wm_params {
8962 u32 dram_channels; /* number of dram channels */
8963 u32 yclk; /* bandwidth per dram data pin in kHz */
8964 u32 sclk; /* engine clock in kHz */
8965 u32 disp_clk; /* display clock in kHz */
8966 u32 src_width; /* viewport width */
8967 u32 active_time; /* active display time in ns */
8968 u32 blank_time; /* blank time in ns */
8969 bool interlaced; /* mode is interlaced */
8970 fixed20_12 vsc; /* vertical scale ratio */
8971 u32 num_heads; /* number of active crtcs */
8972 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8973 u32 lb_size; /* line buffer allocated to pipe */
8974 u32 vtaps; /* vertical scaler taps */
8975};
8976
8977/**
8978 * dce8_dram_bandwidth - get the dram bandwidth
8979 *
8980 * @wm: watermark calculation data
8981 *
8982 * Calculate the raw dram bandwidth (CIK).
8983 * Used for display watermark bandwidth calculations
8984 * Returns the dram bandwidth in MBytes/s
8985 */
8986static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8987{
8988 /* Calculate raw DRAM Bandwidth */
8989 fixed20_12 dram_efficiency; /* 0.7 */
8990 fixed20_12 yclk, dram_channels, bandwidth;
8991 fixed20_12 a;
8992
8993 a.full = dfixed_const(1000);
8994 yclk.full = dfixed_const(wm->yclk);
8995 yclk.full = dfixed_div(yclk, a);
8996 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8997 a.full = dfixed_const(10);
8998 dram_efficiency.full = dfixed_const(7);
8999 dram_efficiency.full = dfixed_div(dram_efficiency, a);
9000 bandwidth.full = dfixed_mul(dram_channels, yclk);
9001 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
9002
9003 return dfixed_trunc(bandwidth);
9004}
9005
9006/**
9007 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
9008 *
9009 * @wm: watermark calculation data
9010 *
9011 * Calculate the dram bandwidth used for display (CIK).
9012 * Used for display watermark bandwidth calculations
9013 * Returns the dram bandwidth for display in MBytes/s
9014 */
9015static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9016{
9017 /* Calculate DRAM Bandwidth and the part allocated to display. */
9018 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
9019 fixed20_12 yclk, dram_channels, bandwidth;
9020 fixed20_12 a;
9021
9022 a.full = dfixed_const(1000);
9023 yclk.full = dfixed_const(wm->yclk);
9024 yclk.full = dfixed_div(yclk, a);
9025 dram_channels.full = dfixed_const(wm->dram_channels * 4);
9026 a.full = dfixed_const(10);
9027 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
9028 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
9029 bandwidth.full = dfixed_mul(dram_channels, yclk);
9030 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
9031
9032 return dfixed_trunc(bandwidth);
9033}
9034
9035/**
9036 * dce8_data_return_bandwidth - get the data return bandwidth
9037 *
9038 * @wm: watermark calculation data
9039 *
9040 * Calculate the data return bandwidth used for display (CIK).
9041 * Used for display watermark bandwidth calculations
9042 * Returns the data return bandwidth in MBytes/s
9043 */
9044static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
9045{
9046 /* Calculate the display Data return Bandwidth */
9047 fixed20_12 return_efficiency; /* 0.8 */
9048 fixed20_12 sclk, bandwidth;
9049 fixed20_12 a;
9050
9051 a.full = dfixed_const(1000);
9052 sclk.full = dfixed_const(wm->sclk);
9053 sclk.full = dfixed_div(sclk, a);
9054 a.full = dfixed_const(10);
9055 return_efficiency.full = dfixed_const(8);
9056 return_efficiency.full = dfixed_div(return_efficiency, a);
9057 a.full = dfixed_const(32);
9058 bandwidth.full = dfixed_mul(a, sclk);
9059 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
9060
9061 return dfixed_trunc(bandwidth);
9062}
9063
9064/**
9065 * dce8_dmif_request_bandwidth - get the dmif bandwidth
9066 *
9067 * @wm: watermark calculation data
9068 *
9069 * Calculate the dmif bandwidth used for display (CIK).
9070 * Used for display watermark bandwidth calculations
9071 * Returns the dmif bandwidth in MBytes/s
9072 */
9073static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
9074{
9075 /* Calculate the DMIF Request Bandwidth */
9076 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
9077 fixed20_12 disp_clk, bandwidth;
9078 fixed20_12 a, b;
9079
9080 a.full = dfixed_const(1000);
9081 disp_clk.full = dfixed_const(wm->disp_clk);
9082 disp_clk.full = dfixed_div(disp_clk, a);
9083 a.full = dfixed_const(32);
9084 b.full = dfixed_mul(a, disp_clk);
9085
9086 a.full = dfixed_const(10);
9087 disp_clk_request_efficiency.full = dfixed_const(8);
9088 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
9089
9090 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
9091
9092 return dfixed_trunc(bandwidth);
9093}
9094
9095/**
9096 * dce8_available_bandwidth - get the min available bandwidth
9097 *
9098 * @wm: watermark calculation data
9099 *
9100 * Calculate the min available bandwidth used for display (CIK).
9101 * Used for display watermark bandwidth calculations
9102 * Returns the min available bandwidth in MBytes/s
9103 */
9104static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
9105{
9106 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
9107 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
9108 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
9109 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
9110
9111 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
9112}
9113
9114/**
9115 * dce8_average_bandwidth - get the average available bandwidth
9116 *
9117 * @wm: watermark calculation data
9118 *
9119 * Calculate the average available bandwidth used for display (CIK).
9120 * Used for display watermark bandwidth calculations
9121 * Returns the average available bandwidth in MBytes/s
9122 */
9123static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
9124{
9125 /* Calculate the display mode Average Bandwidth
9126 * DisplayMode should contain the source and destination dimensions,
9127 * timing, etc.
9128 */
9129 fixed20_12 bpp;
9130 fixed20_12 line_time;
9131 fixed20_12 src_width;
9132 fixed20_12 bandwidth;
9133 fixed20_12 a;
9134
9135 a.full = dfixed_const(1000);
9136 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
9137 line_time.full = dfixed_div(line_time, a);
9138 bpp.full = dfixed_const(wm->bytes_per_pixel);
9139 src_width.full = dfixed_const(wm->src_width);
9140 bandwidth.full = dfixed_mul(src_width, bpp);
9141 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
9142 bandwidth.full = dfixed_div(bandwidth, line_time);
9143
9144 return dfixed_trunc(bandwidth);
9145}
9146
9147/**
9148 * dce8_latency_watermark - get the latency watermark
9149 *
9150 * @wm: watermark calculation data
9151 *
9152 * Calculate the latency watermark (CIK).
9153 * Used for display watermark bandwidth calculations
9154 * Returns the latency watermark in ns
9155 */
9156static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
9157{
9158 /* First calculate the latency in ns */
9159 u32 mc_latency = 2000; /* 2000 ns. */
9160 u32 available_bandwidth = dce8_available_bandwidth(wm);
9161 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
9162 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
9163 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
9164 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
9165 (wm->num_heads * cursor_line_pair_return_time);
9166 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
9167 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
9168 u32 tmp, dmif_size = 12288;
9169 fixed20_12 a, b, c;
9170
9171 if (wm->num_heads == 0)
9172 return 0;
9173
9174 a.full = dfixed_const(2);
9175 b.full = dfixed_const(1);
9176 if ((wm->vsc.full > a.full) ||
9177 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
9178 (wm->vtaps >= 5) ||
9179 ((wm->vsc.full >= a.full) && wm->interlaced))
9180 max_src_lines_per_dst_line = 4;
9181 else
9182 max_src_lines_per_dst_line = 2;
9183
9184 a.full = dfixed_const(available_bandwidth);
9185 b.full = dfixed_const(wm->num_heads);
9186 a.full = dfixed_div(a, b);
9187
9188 b.full = dfixed_const(mc_latency + 512);
9189 c.full = dfixed_const(wm->disp_clk);
9190 b.full = dfixed_div(b, c);
9191
9192 c.full = dfixed_const(dmif_size);
9193 b.full = dfixed_div(c, b);
9194
9195 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
9196
9197 b.full = dfixed_const(1000);
9198 c.full = dfixed_const(wm->disp_clk);
9199 b.full = dfixed_div(c, b);
9200 c.full = dfixed_const(wm->bytes_per_pixel);
9201 b.full = dfixed_mul(b, c);
9202
9203 lb_fill_bw = min(tmp, dfixed_trunc(b));
9204
9205 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
9206 b.full = dfixed_const(1000);
9207 c.full = dfixed_const(lb_fill_bw);
9208 b.full = dfixed_div(c, b);
9209 a.full = dfixed_div(a, b);
9210 line_fill_time = dfixed_trunc(a);
9211
9212 if (line_fill_time < wm->active_time)
9213 return latency;
9214 else
9215 return latency + (line_fill_time - wm->active_time);
9216
9217}
9218
9219/**
9220 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
9221 * average and available dram bandwidth
9222 *
9223 * @wm: watermark calculation data
9224 *
9225 * Check if the display average bandwidth fits in the display
9226 * dram bandwidth (CIK).
9227 * Used for display watermark bandwidth calculations
9228 * Returns true if the display fits, false if not.
9229 */
9230static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
9231{
9232 if (dce8_average_bandwidth(wm) <=
9233 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
9234 return true;
9235 else
9236 return false;
9237}
9238
9239/**
9240 * dce8_average_bandwidth_vs_available_bandwidth - check
9241 * average and available bandwidth
9242 *
9243 * @wm: watermark calculation data
9244 *
9245 * Check if the display average bandwidth fits in the display
9246 * available bandwidth (CIK).
9247 * Used for display watermark bandwidth calculations
9248 * Returns true if the display fits, false if not.
9249 */
9250static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
9251{
9252 if (dce8_average_bandwidth(wm) <=
9253 (dce8_available_bandwidth(wm) / wm->num_heads))
9254 return true;
9255 else
9256 return false;
9257}
9258
9259/**
9260 * dce8_check_latency_hiding - check latency hiding
9261 *
9262 * @wm: watermark calculation data
9263 *
9264 * Check latency hiding (CIK).
9265 * Used for display watermark bandwidth calculations
9266 * Returns true if the display fits, false if not.
9267 */
9268static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
9269{
9270 u32 lb_partitions = wm->lb_size / wm->src_width;
9271 u32 line_time = wm->active_time + wm->blank_time;
9272 u32 latency_tolerant_lines;
9273 u32 latency_hiding;
9274 fixed20_12 a;
9275
9276 a.full = dfixed_const(1);
9277 if (wm->vsc.full > a.full)
9278 latency_tolerant_lines = 1;
9279 else {
9280 if (lb_partitions <= (wm->vtaps + 1))
9281 latency_tolerant_lines = 1;
9282 else
9283 latency_tolerant_lines = 2;
9284 }
9285
9286 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
9287
9288 if (dce8_latency_watermark(wm) <= latency_hiding)
9289 return true;
9290 else
9291 return false;
9292}
9293
9294/**
9295 * dce8_program_watermarks - program display watermarks
9296 *
9297 * @rdev: radeon_device pointer
9298 * @radeon_crtc: the selected display controller
9299 * @lb_size: line buffer size
9300 * @num_heads: number of display controllers in use
9301 *
9302 * Calculate and program the display watermarks for the
9303 * selected display controller (CIK).
9304 */
9305static void dce8_program_watermarks(struct radeon_device *rdev,
9306 struct radeon_crtc *radeon_crtc,
9307 u32 lb_size, u32 num_heads)
9308{
9309 struct drm_display_mode *mode = &radeon_crtc->base.mode;
58ea2dea 9310 struct dce8_wm_params wm_low, wm_high;
cd84a27d
AD		 9311 u32 pixel_period;
9312 u32 line_time = 0;
9313 u32 latency_watermark_a = 0, latency_watermark_b = 0;
9314 u32 tmp, wm_mask;
9315
9316 if (radeon_crtc->base.enabled && num_heads && mode) {
9317 pixel_period = 1000000 / (u32)mode->clock;
9318 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
9319
58ea2dea
AD		 9320 /* watermark for high clocks */
9321 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9322 rdev->pm.dpm_enabled) {
9323 wm_high.yclk =
9324 radeon_dpm_get_mclk(rdev, false) * 10;
9325 wm_high.sclk =
9326 radeon_dpm_get_sclk(rdev, false) * 10;
9327 } else {
9328 wm_high.yclk = rdev->pm.current_mclk * 10;
9329 wm_high.sclk = rdev->pm.current_sclk * 10;
9330 }
9331
9332 wm_high.disp_clk = mode->clock;
9333 wm_high.src_width = mode->crtc_hdisplay;
9334 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
9335 wm_high.blank_time = line_time - wm_high.active_time;
9336 wm_high.interlaced = false;
cd84a27d 9337 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
58ea2dea
AD		 9338 wm_high.interlaced = true;
9339 wm_high.vsc = radeon_crtc->vsc;
9340 wm_high.vtaps = 1;
cd84a27d 9341 if (radeon_crtc->rmx_type != RMX_OFF)
58ea2dea
AD		 9342 wm_high.vtaps = 2;
9343 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
9344 wm_high.lb_size = lb_size;
9345 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
9346 wm_high.num_heads = num_heads;
cd84a27d
AD		 9347
9348 /* set for high clocks */
58ea2dea
AD		 9349 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
9350
9351 /* possibly force display priority to high */
9352 /* should really do this at mode validation time... */
9353 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
9354 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
9355 !dce8_check_latency_hiding(&wm_high) ||
9356 (rdev->disp_priority == 2)) {
9357 DRM_DEBUG_KMS("force priority to high\n");
9358 }
9359
9360 /* watermark for low clocks */
9361 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
9362 rdev->pm.dpm_enabled) {
9363 wm_low.yclk =
9364 radeon_dpm_get_mclk(rdev, true) * 10;
9365 wm_low.sclk =
9366 radeon_dpm_get_sclk(rdev, true) * 10;
9367 } else {
9368 wm_low.yclk = rdev->pm.current_mclk * 10;
9369 wm_low.sclk = rdev->pm.current_sclk * 10;
9370 }
9371
9372 wm_low.disp_clk = mode->clock;
9373 wm_low.src_width = mode->crtc_hdisplay;
9374 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
9375 wm_low.blank_time = line_time - wm_low.active_time;
9376 wm_low.interlaced = false;
9377 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
9378 wm_low.interlaced = true;
9379 wm_low.vsc = radeon_crtc->vsc;
9380 wm_low.vtaps = 1;
9381 if (radeon_crtc->rmx_type != RMX_OFF)
9382 wm_low.vtaps = 2;
9383 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
9384 wm_low.lb_size = lb_size;
9385 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
9386 wm_low.num_heads = num_heads;
9387
cd84a27d 9388 /* set for low clocks */
58ea2dea 9389 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
cd84a27d
AD		 9390
9391 /* possibly force display priority to high */
9392 /* should really do this at mode validation time... */
58ea2dea
AD		 9393 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
9394 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
9395 !dce8_check_latency_hiding(&wm_low) ||
cd84a27d
AD		 9396 (rdev->disp_priority == 2)) {
9397 DRM_DEBUG_KMS("force priority to high\n");
9398 }
9399 }
9400
9401 /* select wm A */
9402 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9403 tmp = wm_mask;
9404 tmp &= ~LATENCY_WATERMARK_MASK(3);
9405 tmp |= LATENCY_WATERMARK_MASK(1);
9406 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9407 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9408 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
9409 LATENCY_HIGH_WATERMARK(line_time)));
9410 /* select wm B */
9411 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
9412 tmp &= ~LATENCY_WATERMARK_MASK(3);
9413 tmp |= LATENCY_WATERMARK_MASK(2);
9414 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
9415 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
9416 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
9417 LATENCY_HIGH_WATERMARK(line_time)));
9418 /* restore original selection */
9419 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
58ea2dea
AD		 9420
9421 /* save values for DPM */
9422 radeon_crtc->line_time = line_time;
9423 radeon_crtc->wm_high = latency_watermark_a;
9424 radeon_crtc->wm_low = latency_watermark_b;
cd84a27d
AD		 9425}
9426
9427/**
9428 * dce8_bandwidth_update - program display watermarks
9429 *
9430 * @rdev: radeon_device pointer
9431 *
9432 * Calculate and program the display watermarks and line
9433 * buffer allocation (CIK).
9434 */
9435void dce8_bandwidth_update(struct radeon_device *rdev)
9436{
9437 struct drm_display_mode *mode = NULL;
9438 u32 num_heads = 0, lb_size;
9439 int i;
9440
9441 radeon_update_display_priority(rdev);
9442
9443 for (i = 0; i < rdev->num_crtc; i++) {
9444 if (rdev->mode_info.crtcs[i]->base.enabled)
9445 num_heads++;
9446 }
9447 for (i = 0; i < rdev->num_crtc; i++) {
9448 mode = &rdev->mode_info.crtcs[i]->base.mode;
9449 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
9450 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
9451 }
9452}
44fa346f
AD		 9453
9454/**
9455 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
9456 *
9457 * @rdev: radeon_device pointer
9458 *
9459 * Fetches a GPU clock counter snapshot (SI).
9460 * Returns the 64 bit clock counter snapshot.
9461 */
9462uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
9463{
9464 uint64_t clock;
9465
9466 mutex_lock(&rdev->gpu_clock_mutex);
9467 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
9468 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
9469 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
9470 mutex_unlock(&rdev->gpu_clock_mutex);
9471 return clock;
9472}
9473
87167bb1
CK		 9474static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
9475 u32 cntl_reg, u32 status_reg)
9476{
9477 int r, i;
9478 struct atom_clock_dividers dividers;
9479 uint32_t tmp;
9480
9481 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9482 clock, false, &dividers);
9483 if (r)
9484 return r;
9485
9486 tmp = RREG32_SMC(cntl_reg);
9487 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
9488 tmp |= dividers.post_divider;
9489 WREG32_SMC(cntl_reg, tmp);
9490
9491 for (i = 0; i < 100; i++) {
9492 if (RREG32_SMC(status_reg) & DCLK_STATUS)
9493 break;
9494 mdelay(10);
9495 }
9496 if (i == 100)
9497 return -ETIMEDOUT;
9498
9499 return 0;
9500}
9501
9502int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
9503{
9504 int r = 0;
9505
9506 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
9507 if (r)
9508 return r;
9509
9510 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
9511 return r;
9512}
9513
5ad6bf91
AD		 9514int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
9515{
9516 int r, i;
9517 struct atom_clock_dividers dividers;
9518 u32 tmp;
9519
9520 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
9521 ecclk, false, &dividers);
9522 if (r)
9523 return r;
9524
9525 for (i = 0; i < 100; i++) {
9526 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9527 break;
9528 mdelay(10);
9529 }
9530 if (i == 100)
9531 return -ETIMEDOUT;
9532
9533 tmp = RREG32_SMC(CG_ECLK_CNTL);
9534 tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
9535 tmp |= dividers.post_divider;
9536 WREG32_SMC(CG_ECLK_CNTL, tmp);
9537
9538 for (i = 0; i < 100; i++) {
9539 if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
9540 break;
9541 mdelay(10);
9542 }
9543 if (i == 100)
9544 return -ETIMEDOUT;
9545
9546 return 0;
9547}
9548
8a7cd276 9549static void cik_pcie_gen3_enable(struct radeon_device *rdev)
87167bb1 9550{
8a7cd276
AD		 9551 struct pci_dev *root = rdev->pdev->bus->self;
9552 int bridge_pos, gpu_pos;
9553 u32 speed_cntl, mask, current_data_rate;
9554 int ret, i;
9555 u16 tmp16;
87167bb1 9556
8a7cd276
AD		 9557 if (radeon_pcie_gen2 == 0)
9558 return;
87167bb1 9559
8a7cd276
AD		 9560 if (rdev->flags & RADEON_IS_IGP)
9561 return;
87167bb1 9562
8a7cd276
AD		 9563 if (!(rdev->flags & RADEON_IS_PCIE))
9564 return;
87167bb1 9565
8a7cd276
AD		 9566 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
9567 if (ret != 0)
9568 return;
87167bb1 9569
8a7cd276
AD		 9570 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
9571 return;
87167bb1 9572
8a7cd276
AD		 9573 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9574 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
9575 LC_CURRENT_DATA_RATE_SHIFT;
9576 if (mask & DRM_PCIE_SPEED_80) {
9577 if (current_data_rate == 2) {
9578 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9579 return;
9580 }
9581 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9582 } else if (mask & DRM_PCIE_SPEED_50) {
9583 if (current_data_rate == 1) {
9584 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9585 return;
9586 }
9587 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9588 }
87167bb1 9589
8a7cd276
AD		 9590 bridge_pos = pci_pcie_cap(root);
9591 if (!bridge_pos)
9592 return;
9593
9594 gpu_pos = pci_pcie_cap(rdev->pdev);
9595 if (!gpu_pos)
9596 return;
9597
9598 if (mask & DRM_PCIE_SPEED_80) {
9599 /* re-try equalization if gen3 is not already enabled */
9600 if (current_data_rate != 2) {
9601 u16 bridge_cfg, gpu_cfg;
9602 u16 bridge_cfg2, gpu_cfg2;
9603 u32 max_lw, current_lw, tmp;
9604
9605 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9606 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9607
9608 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9609 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9610
9611 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9612 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9613
9614 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9615 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9616 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9617
9618 if (current_lw < max_lw) {
9619 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9620 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9621 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9622 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9623 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9624 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9625 }
9626 }
9627
9628 for (i = 0; i < 10; i++) {
9629 /* check status */
9630 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9631 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9632 break;
9633
9634 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9635 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9636
9637 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9638 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9639
9640 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9641 tmp |= LC_SET_QUIESCE;
9642 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9643
9644 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9645 tmp |= LC_REDO_EQ;
9646 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9647
9648 mdelay(100);
9649
9650 /* linkctl */
9651 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9652 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9653 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9654 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9655
9656 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9657 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9658 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9659 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9660
9661 /* linkctl2 */
9662 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9663 tmp16 &= ~((1 << 4) | (7 << 9));
9664 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9665 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9666
9667 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9668 tmp16 &= ~((1 << 4) | (7 << 9));
9669 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9670 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9671
9672 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9673 tmp &= ~LC_SET_QUIESCE;
9674 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9675 }
9676 }
9677 }
9678
9679 /* set the link speed */
9680 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9681 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9682 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9683
9684 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9685 tmp16 &= ~0xf;
9686 if (mask & DRM_PCIE_SPEED_80)
9687 tmp16 |= 3; /* gen3 */
9688 else if (mask & DRM_PCIE_SPEED_50)
9689 tmp16 |= 2; /* gen2 */
9690 else
9691 tmp16 |= 1; /* gen1 */
9692 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9693
9694 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9695 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9696 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9697
9698 for (i = 0; i < rdev->usec_timeout; i++) {
9699 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9700 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9701 break;
9702 udelay(1);
9703 }
9704}
7235711a
AD		 9705
9706static void cik_program_aspm(struct radeon_device *rdev)
9707{
9708 u32 data, orig;
9709 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9710 bool disable_clkreq = false;
9711
9712 if (radeon_aspm == 0)
9713 return;
9714
9715 /* XXX double check IGPs */
9716 if (rdev->flags & RADEON_IS_IGP)
9717 return;
9718
9719 if (!(rdev->flags & RADEON_IS_PCIE))
9720 return;
9721
9722 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9723 data &= ~LC_XMIT_N_FTS_MASK;
9724 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9725 if (orig != data)
9726 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9727
9728 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9729 data |= LC_GO_TO_RECOVERY;
9730 if (orig != data)
9731 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9732
9733 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9734 data |= P_IGNORE_EDB_ERR;
9735 if (orig != data)
9736 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9737
9738 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9739 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9740 data |= LC_PMI_TO_L1_DIS;
9741 if (!disable_l0s)
9742 data |= LC_L0S_INACTIVITY(7);
9743
9744 if (!disable_l1) {
9745 data |= LC_L1_INACTIVITY(7);
9746 data &= ~LC_PMI_TO_L1_DIS;
9747 if (orig != data)
9748 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9749
9750 if (!disable_plloff_in_l1) {
9751 bool clk_req_support;
9752
9753 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9754 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9755 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9756 if (orig != data)
9757 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9758
9759 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9760 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9761 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9762 if (orig != data)
9763 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9764
9765 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9766 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9767 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9768 if (orig != data)
9769 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9770
9771 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9772 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9773 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9774 if (orig != data)
9775 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9776
9777 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9778 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9779 data |= LC_DYN_LANES_PWR_STATE(3);
9780 if (orig != data)
9781 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9782
9783 if (!disable_clkreq) {
9784 struct pci_dev *root = rdev->pdev->bus->self;
9785 u32 lnkcap;
9786
9787 clk_req_support = false;
9788 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9789 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9790 clk_req_support = true;
9791 } else {
9792 clk_req_support = false;
9793 }
9794
9795 if (clk_req_support) {
9796 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9797 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9798 if (orig != data)
9799 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9800
9801 orig = data = RREG32_SMC(THM_CLK_CNTL);
9802 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9803 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9804 if (orig != data)
9805 WREG32_SMC(THM_CLK_CNTL, data);
9806
9807 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9808 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9809 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9810 if (orig != data)
9811 WREG32_SMC(MISC_CLK_CTRL, data);
9812
9813 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9814 data &= ~BCLK_AS_XCLK;
9815 if (orig != data)
9816 WREG32_SMC(CG_CLKPIN_CNTL, data);
9817
9818 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9819 data &= ~FORCE_BIF_REFCLK_EN;
9820 if (orig != data)
9821 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9822
9823 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9824 data &= ~MPLL_CLKOUT_SEL_MASK;
9825 data |= MPLL_CLKOUT_SEL(4);
9826 if (orig != data)
9827 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9828 }
9829 }
9830 } else {
9831 if (orig != data)
9832 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9833 }
9834
9835 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9836 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9837 if (orig != data)
9838 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9839
9840 if (!disable_l0s) {
9841 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9842 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9843 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9844 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9845 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9846 data &= ~LC_L0S_INACTIVITY_MASK;
9847 if (orig != data)
9848 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9849 }
9850 }
9851 }
87167bb1 9852}
Linux 6.x block layer and io_uring tree(s)